def put_dataset(dataset): return { 'results': dataset_service().update_dataset(dataset.id, Dataset.from_dict( request.get_json())).to_dict() }
def _resolve_and_save_dataset(self, entity_id, entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id): actual_id, unsaved_id = self._resolve(EntityType.dataset, entity_id, entity_map, actual_entities_by_unsaved_id) if actual_id: return actual_id node_id = self._node_id(EntityType.dataset, unsaved_id) dataset = Dataset.from_dict(entity_map['unsaved_entities'][node_id]) dataset = self._dataset_service.save_dataset(dataset, commit=False, flush=True) actual_entities_by_node_id[node_id] = dataset actual_entities_by_unsaved_id[unsaved_id] = dataset return dataset.id
def prepare_load_dataset_steps(dry_run, args_by_name, datastore, dataset, action_id, s3_path_and_file_size_gen, target_is_dynamodb=False): """ :type dataset: dart.model.dataset.Dataset """ def add_to(step_partials, step_num, func, *args): # add all params except the last one, which is the total steps (known at the end) step_partials.append( functools.partial(func, *(list(args) + [step_num]))) return step_num + 1 def stage_table_not_needed(ds, file_format, row_format, compression, delimited_by, quoted_by, escaped_by, null_string): """ :type ds: dart.model.dataset.Dataset """ return file_format == ds.data.data_format.file_format\ and row_format == ds.data.data_format.row_format\ and compression == ds.data.compression\ and delimited_by == ds.data.data_format.delimited_by\ and quoted_by == ds.data.data_format.quoted_by\ and escaped_by == ds.data.data_format.escaped_by\ and null_string == ds.data.data_format.null_string # some steps require producing a dataset specific file based on a template, so we will copy all to a tempdir tempdir = tempfile.mkdtemp() try: local_step_path, s3_step_path, s3_temp_path = prepare_step_paths( datastore, tempdir) target_table_name = args_by_name.get( 'target_table_name') or dataset.data.table_name target_file_format = args_by_name.get( 'target_file_format') or dataset.data.data_format.file_format target_row_format = args_by_name.get( 'target_row_format') or dataset.data.data_format.row_format target_compression = args_by_name.get( 'target_compression') or dataset.data.compression target_delimited_by = args_by_name.get( 'target_delimited_by') or dataset.data.data_format.delimited_by target_quoted_by = args_by_name.get( 'target_quoted_by') or dataset.data.data_format.quoted_by target_escaped_by = args_by_name.get( 'target_escaped_by') or dataset.data.data_format.escaped_by target_null_string = args_by_name.get( 'target_null_string') or dataset.data.data_format.null_string stage_table_name = target_table_name + '_stage_for_action_' + action_id staging_not_needed = stage_table_not_needed( dataset, target_file_format, target_row_format, target_compression, target_delimited_by, target_quoted_by, target_escaped_by, target_null_string) first_table_name = target_table_name if staging_not_needed and not target_is_dynamodb else stage_table_name drop_table_names = [] step_funcs = [] i = 1 # ------------------------------------------------------------------------------------------------------------ # all code paths below require copying the data to HDFS, and lowercasing the table is required because of hive # ------------------------------------------------------------------------------------------------------------ i = add_to(step_funcs, i, s3distcp_files_step, s3_path_and_file_size_gen, first_table_name.lower(), dataset, s3_step_path, local_step_path, action_id) # ------------------------------------------------------------------------------------------------------------ # not all folder structures on s3 are hive compatible... if not, rename directories after copying # ------------------------------------------------------------------------------------------------------------ if dataset.data.partitions and not dataset.data.hive_compatible_partition_folders: i = add_to(step_funcs, i, python_fix_partition_folder_names, first_table_name.lower(), dataset.data.partitions, s3_step_path, action_id) # ------------------------------------------------------------------------------------------------------------ # special case to share functionality with the dynamodb_engine # ------------------------------------------------------------------------------------------------------------ if target_is_dynamodb: dyn_dataset = Dataset.from_dict(dataset.to_dict()) assert isinstance(dyn_dataset, Dataset) dyn_dataset.data.data_format = DataFormat('DYNAMODB_TABLE', RowFormat.NONE, 0) dyn_dataset.data.compression = Compression.NONE dyn_dataset.data.columns = [ Column(c.name, dynamodb_column_type(c)) for c in dataset.data.columns ] set_hive_vars = 'SET dynamodb.retry.duration = 0;\nSET dynamodb.throughput.write.percent = %s;' set_hive_vars = set_hive_vars % args_by_name[ 'write_capacity_utilization_percent'] i = add_to(step_funcs, i, hive_table_definition_step, stage_table_name, dataset, s3_step_path, local_step_path, action_id, False) i = add_to(step_funcs, i, hive_table_definition_step, target_table_name, dyn_dataset, s3_step_path, local_step_path, action_id, True) i = add_to(step_funcs, i, hive_msck_repair_table_step, stage_table_name, s3_step_path, action_id) i = add_to(step_funcs, i, hive_copy_to_table, dataset, stage_table_name, dyn_dataset, target_table_name, s3_step_path, local_step_path, action_id, set_hive_vars) # ------------------------------------------------------------------------------------------------------------ # if no stage tables are needed, much complexity can be skipped # ------------------------------------------------------------------------------------------------------------ elif staging_not_needed: i = add_to(step_funcs, i, hive_table_definition_step, target_table_name, dataset, s3_step_path, local_step_path, action_id, False) i = add_to(step_funcs, i, hive_msck_repair_table_step, target_table_name, s3_step_path, action_id) # ------------------------------------------------------------------------------------------------------------ # one or more staging tables are needed # ------------------------------------------------------------------------------------------------------------ else: stage_dataset = dataset target_dataset = Dataset.from_dict(dataset.to_dict()) target_dataset.data.data_format = DataFormat( target_file_format, target_row_format, 0, target_delimited_by, target_quoted_by, target_escaped_by, target_null_string) target_dataset.data.compression = target_compression drop_table_names.append(stage_table_name) # -------------------------------------------------------------------------------------------------------- # define string types for JSON/REGEX based datasets (safe), and we will cast appropriately during insert # -------------------------------------------------------------------------------------------------------- if stage_dataset.data.data_format.row_format in [ RowFormat.JSON, RowFormat.REGEX ]: # make a copy since we are modifying the columns stage_dataset = Dataset.from_dict(dataset.to_dict()) assert isinstance(stage_dataset, Dataset) for c in stage_dataset.data.columns: c.data_type = DataType.STRING i = add_to(step_funcs, i, hive_table_definition_step, stage_table_name, stage_dataset, s3_step_path, local_step_path, action_id, False) i = add_to(step_funcs, i, hive_table_definition_step, target_table_name, target_dataset, s3_step_path, local_step_path, action_id, False) i = add_to(step_funcs, i, hive_msck_repair_table_step, stage_table_name, s3_step_path, action_id) # -------------------------------------------------------------------------------------------------------- # hive has issues creating parquet files # -------------------------------------------------------------------------------------------------------- if target_file_format != FileFormat.PARQUET: i = add_to(step_funcs, i, hive_copy_to_table, stage_dataset, stage_table_name, target_dataset, target_table_name, s3_step_path, local_step_path, action_id, None) # -------------------------------------------------------------------------------------------------------- # impala is better for creating parquet files # -------------------------------------------------------------------------------------------------------- else: # ---------------------------------------------------------------------------------------------------- # no additional staging tables needed if the source dataset file format is RCFILE (impala friendly) # ---------------------------------------------------------------------------------------------------- if dataset.data.data_format.file_format == FileFormat.RCFILE: i = add_to(step_funcs, i, hive_copy_to_table, stage_dataset, stage_table_name, target_dataset, target_table_name, s3_step_path, local_step_path, action_id, None) # ---------------------------------------------------------------------------------------------------- # impala cannot read all hive formats, so we will introduce another staging table # ---------------------------------------------------------------------------------------------------- else: rc_table_name = target_table_name + '_rcfile_stage_for_action_' + action_id rc_dataset = Dataset.from_dict(target_dataset.to_dict()) rc_dataset.data.data_format = DataFormat( FileFormat.RCFILE, RowFormat.NONE, 0) rc_dataset.data.compression = Compression.NONE drop_table_names.append(rc_table_name) i = add_to(step_funcs, i, hive_table_definition_step, rc_table_name, rc_dataset, s3_step_path, local_step_path, action_id, False) i = add_to(step_funcs, i, hive_copy_to_table, stage_dataset, stage_table_name, rc_dataset, rc_table_name, s3_step_path, local_step_path, action_id, None) i = add_to(step_funcs, i, impala_copy_to_table, rc_dataset, rc_table_name, target_dataset, target_table_name, s3_step_path, local_step_path, action_id) # ------------------------------------------------------------------------------------------------------------ # at this point, the load should be considered complete even if something goes wrong in the steps below, # so we will indicate this in the step wrapper # ------------------------------------------------------------------------------------------------------------ considered_successful_at_this_index = i # ------------------------------------------------------------------------------------------------------------ # drop any staging tables created # ------------------------------------------------------------------------------------------------------------ if drop_table_names: script = '\n'.join( ['DROP TABLE %s;' % name for name in drop_table_names]) i = add_to(step_funcs, i, hive_run_script_contents_step, script, s3_step_path, local_step_path, action_id) # ------------------------------------------------------------------------------------------------------------ # inform impala about changes # ------------------------------------------------------------------------------------------------------------ if not target_is_dynamodb: i = add_to(step_funcs, i, impala_invalidate_metadata_step, s3_step_path, action_id) total_steps = i - 1 steps = [] for index, f in enumerate(step_funcs, 1): step_wrapper = f(total_steps) assert isinstance(step_wrapper, StepWrapper) if index >= considered_successful_at_this_index: step_wrapper.action_considered_successful = True steps.append(step_wrapper) if not dry_run: s3_copy_recursive(local_step_path, s3_step_path) return steps finally: shutil.rmtree(tempdir)
def prepare_load_dataset_steps(dry_run, args_by_name, datastore, dataset, action_id, s3_path_and_file_size_gen, target_is_dynamodb=False): """ :type dataset: dart.model.dataset.Dataset """ def add_to(step_partials, step_num, func, *args): # add all params except the last one, which is the total steps (known at the end) step_partials.append(functools.partial(func, *(list(args) + [step_num]))) return step_num + 1 def stage_table_not_needed(ds, file_format, row_format, compression, delimited_by, quoted_by, escaped_by, null_string): """ :type ds: dart.model.dataset.Dataset """ return file_format == ds.data.data_format.file_format\ and row_format == ds.data.data_format.row_format\ and compression == ds.data.compression\ and delimited_by == ds.data.data_format.delimited_by\ and quoted_by == ds.data.data_format.quoted_by\ and escaped_by == ds.data.data_format.escaped_by\ and null_string == ds.data.data_format.null_string # some steps require producing a dataset specific file based on a template, so we will copy all to a tempdir tempdir = tempfile.mkdtemp() try: local_step_path, s3_step_path, s3_temp_path = prepare_step_paths(datastore, tempdir) target_table_name = args_by_name.get('target_table_name') or dataset.data.table_name target_file_format = args_by_name.get('target_file_format') or dataset.data.data_format.file_format target_row_format = args_by_name.get('target_row_format') or dataset.data.data_format.row_format target_compression = args_by_name.get('target_compression') or dataset.data.compression target_delimited_by = args_by_name.get('target_delimited_by') or dataset.data.data_format.delimited_by target_quoted_by = args_by_name.get('target_quoted_by') or dataset.data.data_format.quoted_by target_escaped_by = args_by_name.get('target_escaped_by') or dataset.data.data_format.escaped_by target_null_string = args_by_name.get('target_null_string') or dataset.data.data_format.null_string stage_table_name = target_table_name + '_stage_for_action_' + action_id staging_not_needed = stage_table_not_needed(dataset, target_file_format, target_row_format, target_compression, target_delimited_by, target_quoted_by, target_escaped_by, target_null_string) first_table_name = target_table_name if staging_not_needed and not target_is_dynamodb else stage_table_name drop_table_names = [] step_funcs = [] i = 1 # ------------------------------------------------------------------------------------------------------------ # all code paths below require copying the data to HDFS, and lowercasing the table is required because of hive # ------------------------------------------------------------------------------------------------------------ i = add_to(step_funcs, i, s3distcp_files_step, s3_path_and_file_size_gen, first_table_name.lower(), dataset, s3_step_path, local_step_path, action_id) # ------------------------------------------------------------------------------------------------------------ # not all folder structures on s3 are hive compatible... if not, rename directories after copying # ------------------------------------------------------------------------------------------------------------ if dataset.data.partitions and not dataset.data.hive_compatible_partition_folders: i = add_to(step_funcs, i, python_fix_partition_folder_names, first_table_name.lower(), dataset.data.partitions, s3_step_path, action_id) # ------------------------------------------------------------------------------------------------------------ # special case to share functionality with the dynamodb_engine # ------------------------------------------------------------------------------------------------------------ if target_is_dynamodb: dyn_dataset = Dataset.from_dict(dataset.to_dict()) assert isinstance(dyn_dataset, Dataset) dyn_dataset.data.data_format = DataFormat('DYNAMODB_TABLE', RowFormat.NONE, 0) dyn_dataset.data.compression = Compression.NONE dyn_dataset.data.columns = [Column(c.name, dynamodb_column_type(c)) for c in dataset.data.columns] set_hive_vars = 'SET dynamodb.retry.duration = 0;\nSET dynamodb.throughput.write.percent = %s;' set_hive_vars = set_hive_vars % args_by_name['write_capacity_utilization_percent'] i = add_to(step_funcs, i, hive_table_definition_step, stage_table_name, dataset, s3_step_path, local_step_path, action_id, False) i = add_to(step_funcs, i, hive_table_definition_step, target_table_name, dyn_dataset, s3_step_path, local_step_path, action_id, True) i = add_to(step_funcs, i, hive_msck_repair_table_step, stage_table_name, s3_step_path, action_id) i = add_to(step_funcs, i, hive_copy_to_table, dataset, stage_table_name, dyn_dataset, target_table_name, s3_step_path, local_step_path, action_id, set_hive_vars) # ------------------------------------------------------------------------------------------------------------ # if no stage tables are needed, much complexity can be skipped # ------------------------------------------------------------------------------------------------------------ elif staging_not_needed: i = add_to(step_funcs, i, hive_table_definition_step, target_table_name, dataset, s3_step_path, local_step_path, action_id, False) i = add_to(step_funcs, i, hive_msck_repair_table_step, target_table_name, s3_step_path, action_id) # ------------------------------------------------------------------------------------------------------------ # one or more staging tables are needed # ------------------------------------------------------------------------------------------------------------ else: stage_dataset = dataset target_dataset = Dataset.from_dict(dataset.to_dict()) target_dataset.data.data_format = DataFormat(target_file_format, target_row_format, 0, target_delimited_by, target_quoted_by, target_escaped_by, target_null_string) target_dataset.data.compression = target_compression drop_table_names.append(stage_table_name) # -------------------------------------------------------------------------------------------------------- # define string types for JSON/REGEX based datasets (safe), and we will cast appropriately during insert # -------------------------------------------------------------------------------------------------------- if stage_dataset.data.data_format.row_format in [RowFormat.JSON, RowFormat.REGEX]: # make a copy since we are modifying the columns stage_dataset = Dataset.from_dict(dataset.to_dict()) assert isinstance(stage_dataset, Dataset) for c in stage_dataset.data.columns: c.data_type = DataType.STRING i = add_to(step_funcs, i, hive_table_definition_step, stage_table_name, stage_dataset, s3_step_path, local_step_path, action_id, False) i = add_to(step_funcs, i, hive_table_definition_step, target_table_name, target_dataset, s3_step_path, local_step_path, action_id, False) i = add_to(step_funcs, i, hive_msck_repair_table_step, stage_table_name, s3_step_path, action_id) # -------------------------------------------------------------------------------------------------------- # hive has issues creating parquet files # -------------------------------------------------------------------------------------------------------- if target_file_format != FileFormat.PARQUET: i = add_to(step_funcs, i, hive_copy_to_table, stage_dataset, stage_table_name, target_dataset, target_table_name, s3_step_path, local_step_path, action_id, None) # -------------------------------------------------------------------------------------------------------- # impala is better for creating parquet files # -------------------------------------------------------------------------------------------------------- else: # ---------------------------------------------------------------------------------------------------- # no additional staging tables needed if the source dataset file format is RCFILE (impala friendly) # ---------------------------------------------------------------------------------------------------- if dataset.data.data_format.file_format == FileFormat.RCFILE: i = add_to(step_funcs, i, hive_copy_to_table, stage_dataset, stage_table_name, target_dataset, target_table_name, s3_step_path, local_step_path, action_id, None) # ---------------------------------------------------------------------------------------------------- # impala cannot read all hive formats, so we will introduce another staging table # ---------------------------------------------------------------------------------------------------- else: rc_table_name = target_table_name + '_rcfile_stage_for_action_' + action_id rc_dataset = Dataset.from_dict(target_dataset.to_dict()) rc_dataset.data.data_format = DataFormat(FileFormat.RCFILE, RowFormat.NONE, 0) rc_dataset.data.compression = Compression.NONE drop_table_names.append(rc_table_name) i = add_to(step_funcs, i, hive_table_definition_step, rc_table_name, rc_dataset, s3_step_path, local_step_path, action_id, False) i = add_to(step_funcs, i, hive_copy_to_table, stage_dataset, stage_table_name, rc_dataset, rc_table_name, s3_step_path, local_step_path, action_id, None) i = add_to(step_funcs, i, impala_copy_to_table, rc_dataset, rc_table_name, target_dataset, target_table_name, s3_step_path, local_step_path, action_id) # ------------------------------------------------------------------------------------------------------------ # at this point, the load should be considered complete even if something goes wrong in the steps below, # so we will indicate this in the step wrapper # ------------------------------------------------------------------------------------------------------------ considered_successful_at_this_index = i # ------------------------------------------------------------------------------------------------------------ # drop any staging tables created # ------------------------------------------------------------------------------------------------------------ if drop_table_names: script = '\n'.join(['DROP TABLE %s;' % name for name in drop_table_names]) i = add_to(step_funcs, i, hive_run_script_contents_step, script, s3_step_path, local_step_path, action_id) # ------------------------------------------------------------------------------------------------------------ # inform impala about changes # ------------------------------------------------------------------------------------------------------------ if not target_is_dynamodb: i = add_to(step_funcs, i, impala_invalidate_metadata_step, s3_step_path, action_id) total_steps = i - 1 steps = [] for index, f in enumerate(step_funcs, 1): step_wrapper = f(total_steps) assert isinstance(step_wrapper, StepWrapper) if index >= considered_successful_at_this_index: step_wrapper.action_considered_successful = True steps.append(step_wrapper) if not dry_run: s3_copy_recursive(local_step_path, s3_step_path) return steps finally: shutil.rmtree(tempdir)
def put_dataset(dataset): return {'results': dataset_service().update_dataset(dataset.id, Dataset.from_dict(request.get_json())).to_dict()}
def post_dataset(): return {'results': dataset_service().save_dataset(Dataset.from_dict(request.get_json())).to_dict()}
def post_dataset(): return { 'results': dataset_service().save_dataset(Dataset.from_dict( request.get_json())).to_dict() }
def test_hive_table_definition_step(self): ds = Dataset(data=DatasetData( name='owen_eu_v01', table_name='owen_eu', location='s3://s3-rpt-uss-dat-warehouse/prd/inbound/overlord/eu-all-events', data_format=DataFormat( file_format=FileFormat.TEXTFILE, row_format=RowFormat.JSON, ), columns=[ Column('host', 'STRING', path='metadata.host'), Column('pageName', 'STRING', path='owen.context.pageName'), Column('viewInstanceUuid', 'STRING', path='owen.context.viewInstanceUuid'), Column('previousPageName', 'STRING', path='owen.context.previousPageName'), Column('previousViewInstanceUuid', 'STRING', path='owen.context.previousViewInstanceUuid'), Column('session', 'STRING', path='owen.context.session'), Column('pageType', 'STRING', path='owen.context.pageType'), Column('propertyName', 'STRING', path='owen.context.propertyName'), Column('enviroment', 'STRING', path='owen.context.environment'), Column('appForegroundFlag', 'BOOLEAN', path='owen.context.appForegroundFlag'), Column('bluetoothEnabledFlag', 'BOOLEAN', path='owen.context.bluetoothEnabledFlag'), Column('favoriteFlag', 'BOOLEAN', path='owen.context.favoriteFlag'), Column('locationEnabledFlag', 'BOOLEAN', path='owen.context.locationEnabledFlag'), Column('loggedInFlag', 'BOOLEAN', path='owen.context.loggedInFlag'), Column('notificationEnabledFlag', 'BOOLEAN', path='owen.context.notificationEnabledFlag'), Column('personalizationFlag', 'BOOLEAN', path='owen.context.personalizationFlag'), Column('advertiserUuid', 'STRING', path='owen.context.advertiserUuid'), Column('udid', 'STRING', path='owen.context.udid'), Column('userQualifier', 'STRING', path='owen.context.userQualifier'), Column('userId', 'STRING', path='owen.context.custom.legacy.userId'), Column('userUuid', 'STRING', path='owen.context.userUuid'), Column('macAddress', 'STRING', path='owen.context.macAddress'), Column('ipAddress', 'STRING', path='owen.context.ipAddress'), Column('osVersion', 'STRING', path='owen.context.osVersion'), Column('osFamily', 'STRING', path='owen.context.osFamily'), Column('osName', 'STRING', path='owen.context.osName'), Column('browserFamily', 'STRING', path='owen.context.browserFamily'), Column('deviceCategory', 'STRING', path='owen.context.deviceCategory'), Column('deviceMake', 'STRING', path='owen.context.mobileDeviceMake'), Column('deviceModel', 'STRING', path='owen.context.mobileDeviceModel'), Column('connectionType', 'STRING', path='owen.context.connectionType'), Column('userAgent', 'STRING', path='owen.context.userAgent'), Column('geofenceId', 'STRING', path='owen.context.custom.legacy.geofenceId'), Column('eventTimestamp', 'TIMESTAMP', path='owen.event.eventTimestamp', date_pattern="yyyy-MM-dd'T'HH:mm:ssZ"), Column('eventInstanceUuid', 'STRING', path='owen.event.eventInstanceUuid'), Column('eventPlatformVersion', 'STRING', path='owen.event.eventPlatformVersion'), Column('eventVersion', 'STRING', path='owen.event.eventVersion'), Column('eventCategory', 'STRING', path='owen.event.eventCategory'), Column('eventName', 'STRING', path='owen.event.eventName'), Column('eventAction', 'STRING', path='owen.event.eventAction'), Column('eventPlatform', 'STRING', path='owen.event.eventPlatform'), Column('testUnixTimestampSecondsPattern', 'TIMESTAMP', path='some.fake.path.testUnixTimestampSecondsPattern', date_pattern='UNIX_TIMESTAMP_SECONDS'), Column('testUnixTimestampMillisPattern', 'TIMESTAMP', path='some.fake.path.testUnixTimestampMillisPattern', date_pattern='UNIX_TIMESTAMP_MILLIS'), ], compression='GZIP', partitions=[ Column('year', 'STRING'), Column('week', 'STRING'), Column('day', 'STRING'), ], )) call('mkdir -p /tmp/dart-emr-test/hive/') this_path = os.path.dirname(os.path.abspath(__file__)) shutil.copyfile(this_path + '/../../../engine/emr/steps/hive/copy_to_table.hql', '/tmp/dart-emr-test/hive/copy_to_table.hql') action_id = 'actionid123' target_dataset = Dataset.from_dict(ds.to_dict()) target_dataset.data.data_format.num_header_rows = 0 target_dataset.data.data_format = DataFormat(FileFormat.RCFILE, RowFormat.NONE) stage_dataset = Dataset.from_dict(ds.to_dict()) assert isinstance(stage_dataset, Dataset) for c in stage_dataset.data.columns: c.data_type = DataType.STRING hive_copy_to_table(stage_dataset, 'owen_eu_stage', target_dataset, 'owen_eu', 's3://test', '/tmp/dart-emr-test/', action_id, None, 1, 1) with open(os.path.join(this_path, 'copy_to_table_owen_eu.hql')) as f: expected_contents = f.read() with open('/tmp/dart-emr-test/hive/copy_to_table_owen_eu.hql') as f: actual_contents = f.read() self.assertEqual(expected_contents, actual_contents)