def put_dataset(dataset):
return {
'results':
dataset_service().update_dataset(dataset.id,
Dataset.from_dict(
request.get_json())).to_dict()
}
def _resolve_and_save_dataset(self, entity_id, entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id):
actual_id, unsaved_id = self._resolve(EntityType.dataset, entity_id, entity_map, actual_entities_by_unsaved_id)
if actual_id:
return actual_id
node_id = self._node_id(EntityType.dataset, unsaved_id)
dataset = Dataset.from_dict(entity_map['unsaved_entities'][node_id])
dataset = self._dataset_service.save_dataset(dataset, commit=False, flush=True)
actual_entities_by_node_id[node_id] = dataset
actual_entities_by_unsaved_id[unsaved_id] = dataset
return dataset.id
def _resolve_and_save_dataset(self, entity_id, entity_map, actual_entities_by_node_id, actual_entities_by_unsaved_id):
actual_id, unsaved_id = self._resolve(EntityType.dataset, entity_id, entity_map, actual_entities_by_unsaved_id)
if actual_id:
return actual_id
node_id = self._node_id(EntityType.dataset, unsaved_id)
dataset = Dataset.from_dict(entity_map['unsaved_entities'][node_id])
dataset = self._dataset_service.save_dataset(dataset, commit=False, flush=True)
actual_entities_by_node_id[node_id] = dataset
actual_entities_by_unsaved_id[unsaved_id] = dataset
return dataset.id
def prepare_load_dataset_steps(dry_run,
args_by_name,
datastore,
dataset,
action_id,
s3_path_and_file_size_gen,
target_is_dynamodb=False):
""" :type dataset: dart.model.dataset.Dataset """
def add_to(step_partials, step_num, func, *args):
# add all params except the last one, which is the total steps (known at the end)
step_partials.append(
functools.partial(func, *(list(args) + [step_num])))
return step_num + 1
def stage_table_not_needed(ds, file_format, row_format, compression,
delimited_by, quoted_by, escaped_by,
null_string):
""" :type ds: dart.model.dataset.Dataset """
return file_format == ds.data.data_format.file_format\
and row_format == ds.data.data_format.row_format\
and compression == ds.data.compression\
and delimited_by == ds.data.data_format.delimited_by\
and quoted_by == ds.data.data_format.quoted_by\
and escaped_by == ds.data.data_format.escaped_by\
and null_string == ds.data.data_format.null_string
# some steps require producing a dataset specific file based on a template, so we will copy all to a tempdir
tempdir = tempfile.mkdtemp()
try:
local_step_path, s3_step_path, s3_temp_path = prepare_step_paths(
datastore, tempdir)
target_table_name = args_by_name.get(
'target_table_name') or dataset.data.table_name
target_file_format = args_by_name.get(
'target_file_format') or dataset.data.data_format.file_format
target_row_format = args_by_name.get(
'target_row_format') or dataset.data.data_format.row_format
target_compression = args_by_name.get(
'target_compression') or dataset.data.compression
target_delimited_by = args_by_name.get(
'target_delimited_by') or dataset.data.data_format.delimited_by
target_quoted_by = args_by_name.get(
'target_quoted_by') or dataset.data.data_format.quoted_by
target_escaped_by = args_by_name.get(
'target_escaped_by') or dataset.data.data_format.escaped_by
target_null_string = args_by_name.get(
'target_null_string') or dataset.data.data_format.null_string
stage_table_name = target_table_name + '_stage_for_action_' + action_id
staging_not_needed = stage_table_not_needed(
dataset, target_file_format, target_row_format, target_compression,
target_delimited_by, target_quoted_by, target_escaped_by,
target_null_string)
first_table_name = target_table_name if staging_not_needed and not target_is_dynamodb else stage_table_name
drop_table_names = []
step_funcs = []
i = 1
# ------------------------------------------------------------------------------------------------------------
# all code paths below require copying the data to HDFS, and lowercasing the table is required because of hive
# ------------------------------------------------------------------------------------------------------------
i = add_to(step_funcs, i,
s3distcp_files_step, s3_path_and_file_size_gen,
first_table_name.lower(), dataset, s3_step_path,
local_step_path, action_id)
# ------------------------------------------------------------------------------------------------------------
# not all folder structures on s3 are hive compatible... if not, rename directories after copying
# ------------------------------------------------------------------------------------------------------------
if dataset.data.partitions and not dataset.data.hive_compatible_partition_folders:
i = add_to(step_funcs, i, python_fix_partition_folder_names,
first_table_name.lower(), dataset.data.partitions,
s3_step_path, action_id)
# ------------------------------------------------------------------------------------------------------------
# special case to share functionality with the dynamodb_engine
# ------------------------------------------------------------------------------------------------------------
if target_is_dynamodb:
dyn_dataset = Dataset.from_dict(dataset.to_dict())
assert isinstance(dyn_dataset, Dataset)
dyn_dataset.data.data_format = DataFormat('DYNAMODB_TABLE',
RowFormat.NONE, 0)
dyn_dataset.data.compression = Compression.NONE
dyn_dataset.data.columns = [
Column(c.name, dynamodb_column_type(c))
for c in dataset.data.columns
]
set_hive_vars = 'SET dynamodb.retry.duration = 0;\nSET dynamodb.throughput.write.percent = %s;'
set_hive_vars = set_hive_vars % args_by_name[
'write_capacity_utilization_percent']
i = add_to(step_funcs, i, hive_table_definition_step,
stage_table_name, dataset, s3_step_path,
local_step_path, action_id, False)
i = add_to(step_funcs, i, hive_table_definition_step,
target_table_name, dyn_dataset, s3_step_path,
local_step_path, action_id, True)
i = add_to(step_funcs, i, hive_msck_repair_table_step,
stage_table_name, s3_step_path, action_id)
i = add_to(step_funcs, i, hive_copy_to_table, dataset,
stage_table_name, dyn_dataset, target_table_name,
s3_step_path, local_step_path, action_id, set_hive_vars)
# ------------------------------------------------------------------------------------------------------------
# if no stage tables are needed, much complexity can be skipped
# ------------------------------------------------------------------------------------------------------------
elif staging_not_needed:
i = add_to(step_funcs, i, hive_table_definition_step,
target_table_name, dataset, s3_step_path,
local_step_path, action_id, False)
i = add_to(step_funcs, i, hive_msck_repair_table_step,
target_table_name, s3_step_path, action_id)
# ------------------------------------------------------------------------------------------------------------
# one or more staging tables are needed
# ------------------------------------------------------------------------------------------------------------
else:
stage_dataset = dataset
target_dataset = Dataset.from_dict(dataset.to_dict())
target_dataset.data.data_format = DataFormat(
target_file_format, target_row_format, 0, target_delimited_by,
target_quoted_by, target_escaped_by, target_null_string)
target_dataset.data.compression = target_compression
drop_table_names.append(stage_table_name)
# --------------------------------------------------------------------------------------------------------
# define string types for JSON/REGEX based datasets (safe), and we will cast appropriately during insert
# --------------------------------------------------------------------------------------------------------
if stage_dataset.data.data_format.row_format in [
RowFormat.JSON, RowFormat.REGEX
]:
# make a copy since we are modifying the columns
stage_dataset = Dataset.from_dict(dataset.to_dict())
assert isinstance(stage_dataset, Dataset)
for c in stage_dataset.data.columns:
c.data_type = DataType.STRING
i = add_to(step_funcs, i, hive_table_definition_step,
stage_table_name, stage_dataset, s3_step_path,
local_step_path, action_id, False)
i = add_to(step_funcs, i, hive_table_definition_step,
target_table_name, target_dataset, s3_step_path,
local_step_path, action_id, False)
i = add_to(step_funcs, i, hive_msck_repair_table_step,
stage_table_name, s3_step_path, action_id)
# --------------------------------------------------------------------------------------------------------
# hive has issues creating parquet files
# --------------------------------------------------------------------------------------------------------
if target_file_format != FileFormat.PARQUET:
i = add_to(step_funcs, i, hive_copy_to_table, stage_dataset,
stage_table_name, target_dataset, target_table_name,
s3_step_path, local_step_path, action_id, None)
# --------------------------------------------------------------------------------------------------------
# impala is better for creating parquet files
# --------------------------------------------------------------------------------------------------------
else:
# ----------------------------------------------------------------------------------------------------
# no additional staging tables needed if the source dataset file format is RCFILE (impala friendly)
# ----------------------------------------------------------------------------------------------------
if dataset.data.data_format.file_format == FileFormat.RCFILE:
i = add_to(step_funcs, i, hive_copy_to_table,
stage_dataset, stage_table_name, target_dataset,
target_table_name, s3_step_path,
local_step_path, action_id, None)
# ----------------------------------------------------------------------------------------------------
# impala cannot read all hive formats, so we will introduce another staging table
# ----------------------------------------------------------------------------------------------------
else:
rc_table_name = target_table_name + '_rcfile_stage_for_action_' + action_id
rc_dataset = Dataset.from_dict(target_dataset.to_dict())
rc_dataset.data.data_format = DataFormat(
FileFormat.RCFILE, RowFormat.NONE, 0)
rc_dataset.data.compression = Compression.NONE
drop_table_names.append(rc_table_name)
i = add_to(step_funcs, i, hive_table_definition_step,
rc_table_name, rc_dataset, s3_step_path,
local_step_path, action_id, False)
i = add_to(step_funcs, i, hive_copy_to_table,
stage_dataset, stage_table_name, rc_dataset,
rc_table_name, s3_step_path, local_step_path,
action_id, None)
i = add_to(step_funcs, i, impala_copy_to_table, rc_dataset,
rc_table_name, target_dataset,
target_table_name, s3_step_path,
local_step_path, action_id)
# ------------------------------------------------------------------------------------------------------------
# at this point, the load should be considered complete even if something goes wrong in the steps below,
# so we will indicate this in the step wrapper
# ------------------------------------------------------------------------------------------------------------
considered_successful_at_this_index = i
# ------------------------------------------------------------------------------------------------------------
# drop any staging tables created
# ------------------------------------------------------------------------------------------------------------
if drop_table_names:
script = '\n'.join(
['DROP TABLE %s;' % name for name in drop_table_names])
i = add_to(step_funcs, i, hive_run_script_contents_step, script,
s3_step_path, local_step_path, action_id)
# ------------------------------------------------------------------------------------------------------------
# inform impala about changes
# ------------------------------------------------------------------------------------------------------------
if not target_is_dynamodb:
i = add_to(step_funcs, i, impala_invalidate_metadata_step,
s3_step_path, action_id)
total_steps = i - 1
steps = []
for index, f in enumerate(step_funcs, 1):
step_wrapper = f(total_steps)
assert isinstance(step_wrapper, StepWrapper)
if index >= considered_successful_at_this_index:
step_wrapper.action_considered_successful = True
steps.append(step_wrapper)
if not dry_run:
s3_copy_recursive(local_step_path, s3_step_path)
return steps
finally:
shutil.rmtree(tempdir)
def prepare_load_dataset_steps(dry_run, args_by_name, datastore, dataset, action_id, s3_path_and_file_size_gen,
target_is_dynamodb=False):
""" :type dataset: dart.model.dataset.Dataset """
def add_to(step_partials, step_num, func, *args):
# add all params except the last one, which is the total steps (known at the end)
step_partials.append(functools.partial(func, *(list(args) + [step_num])))
return step_num + 1
def stage_table_not_needed(ds, file_format, row_format, compression, delimited_by, quoted_by,
escaped_by, null_string):
""" :type ds: dart.model.dataset.Dataset """
return file_format == ds.data.data_format.file_format\
and row_format == ds.data.data_format.row_format\
and compression == ds.data.compression\
and delimited_by == ds.data.data_format.delimited_by\
and quoted_by == ds.data.data_format.quoted_by\
and escaped_by == ds.data.data_format.escaped_by\
and null_string == ds.data.data_format.null_string
# some steps require producing a dataset specific file based on a template, so we will copy all to a tempdir
tempdir = tempfile.mkdtemp()
try:
local_step_path, s3_step_path, s3_temp_path = prepare_step_paths(datastore, tempdir)
target_table_name = args_by_name.get('target_table_name') or dataset.data.table_name
target_file_format = args_by_name.get('target_file_format') or dataset.data.data_format.file_format
target_row_format = args_by_name.get('target_row_format') or dataset.data.data_format.row_format
target_compression = args_by_name.get('target_compression') or dataset.data.compression
target_delimited_by = args_by_name.get('target_delimited_by') or dataset.data.data_format.delimited_by
target_quoted_by = args_by_name.get('target_quoted_by') or dataset.data.data_format.quoted_by
target_escaped_by = args_by_name.get('target_escaped_by') or dataset.data.data_format.escaped_by
target_null_string = args_by_name.get('target_null_string') or dataset.data.data_format.null_string
stage_table_name = target_table_name + '_stage_for_action_' + action_id
staging_not_needed = stage_table_not_needed(dataset, target_file_format, target_row_format, target_compression,
target_delimited_by, target_quoted_by, target_escaped_by, target_null_string)
first_table_name = target_table_name if staging_not_needed and not target_is_dynamodb else stage_table_name
drop_table_names = []
step_funcs = []
i = 1
# ------------------------------------------------------------------------------------------------------------
# all code paths below require copying the data to HDFS, and lowercasing the table is required because of hive
# ------------------------------------------------------------------------------------------------------------
i = add_to(step_funcs, i, s3distcp_files_step, s3_path_and_file_size_gen, first_table_name.lower(), dataset, s3_step_path, local_step_path, action_id)
# ------------------------------------------------------------------------------------------------------------
# not all folder structures on s3 are hive compatible... if not, rename directories after copying
# ------------------------------------------------------------------------------------------------------------
if dataset.data.partitions and not dataset.data.hive_compatible_partition_folders:
i = add_to(step_funcs, i, python_fix_partition_folder_names, first_table_name.lower(), dataset.data.partitions, s3_step_path, action_id)
# ------------------------------------------------------------------------------------------------------------
# special case to share functionality with the dynamodb_engine
# ------------------------------------------------------------------------------------------------------------
if target_is_dynamodb:
dyn_dataset = Dataset.from_dict(dataset.to_dict())
assert isinstance(dyn_dataset, Dataset)
dyn_dataset.data.data_format = DataFormat('DYNAMODB_TABLE', RowFormat.NONE, 0)
dyn_dataset.data.compression = Compression.NONE
dyn_dataset.data.columns = [Column(c.name, dynamodb_column_type(c)) for c in dataset.data.columns]
set_hive_vars = 'SET dynamodb.retry.duration = 0;\nSET dynamodb.throughput.write.percent = %s;'
set_hive_vars = set_hive_vars % args_by_name['write_capacity_utilization_percent']
i = add_to(step_funcs, i, hive_table_definition_step, stage_table_name, dataset, s3_step_path, local_step_path, action_id, False)
i = add_to(step_funcs, i, hive_table_definition_step, target_table_name, dyn_dataset, s3_step_path, local_step_path, action_id, True)
i = add_to(step_funcs, i, hive_msck_repair_table_step, stage_table_name, s3_step_path, action_id)
i = add_to(step_funcs, i, hive_copy_to_table, dataset, stage_table_name, dyn_dataset, target_table_name, s3_step_path, local_step_path, action_id, set_hive_vars)
# ------------------------------------------------------------------------------------------------------------
# if no stage tables are needed, much complexity can be skipped
# ------------------------------------------------------------------------------------------------------------
elif staging_not_needed:
i = add_to(step_funcs, i, hive_table_definition_step, target_table_name, dataset, s3_step_path, local_step_path, action_id, False)
i = add_to(step_funcs, i, hive_msck_repair_table_step, target_table_name, s3_step_path, action_id)
# ------------------------------------------------------------------------------------------------------------
# one or more staging tables are needed
# ------------------------------------------------------------------------------------------------------------
else:
stage_dataset = dataset
target_dataset = Dataset.from_dict(dataset.to_dict())
target_dataset.data.data_format = DataFormat(target_file_format, target_row_format, 0, target_delimited_by,
target_quoted_by, target_escaped_by, target_null_string)
target_dataset.data.compression = target_compression
drop_table_names.append(stage_table_name)
# --------------------------------------------------------------------------------------------------------
# define string types for JSON/REGEX based datasets (safe), and we will cast appropriately during insert
# --------------------------------------------------------------------------------------------------------
if stage_dataset.data.data_format.row_format in [RowFormat.JSON, RowFormat.REGEX]:
# make a copy since we are modifying the columns
stage_dataset = Dataset.from_dict(dataset.to_dict())
assert isinstance(stage_dataset, Dataset)
for c in stage_dataset.data.columns:
c.data_type = DataType.STRING
i = add_to(step_funcs, i, hive_table_definition_step, stage_table_name, stage_dataset, s3_step_path, local_step_path, action_id, False)
i = add_to(step_funcs, i, hive_table_definition_step, target_table_name, target_dataset, s3_step_path, local_step_path, action_id, False)
i = add_to(step_funcs, i, hive_msck_repair_table_step, stage_table_name, s3_step_path, action_id)
# --------------------------------------------------------------------------------------------------------
# hive has issues creating parquet files
# --------------------------------------------------------------------------------------------------------
if target_file_format != FileFormat.PARQUET:
i = add_to(step_funcs, i, hive_copy_to_table, stage_dataset, stage_table_name, target_dataset, target_table_name, s3_step_path, local_step_path, action_id, None)
# --------------------------------------------------------------------------------------------------------
# impala is better for creating parquet files
# --------------------------------------------------------------------------------------------------------
else:
# ----------------------------------------------------------------------------------------------------
# no additional staging tables needed if the source dataset file format is RCFILE (impala friendly)
# ----------------------------------------------------------------------------------------------------
if dataset.data.data_format.file_format == FileFormat.RCFILE:
i = add_to(step_funcs, i, hive_copy_to_table, stage_dataset, stage_table_name, target_dataset, target_table_name, s3_step_path, local_step_path, action_id, None)
# ----------------------------------------------------------------------------------------------------
# impala cannot read all hive formats, so we will introduce another staging table
# ----------------------------------------------------------------------------------------------------
else:
rc_table_name = target_table_name + '_rcfile_stage_for_action_' + action_id
rc_dataset = Dataset.from_dict(target_dataset.to_dict())
rc_dataset.data.data_format = DataFormat(FileFormat.RCFILE, RowFormat.NONE, 0)
rc_dataset.data.compression = Compression.NONE
drop_table_names.append(rc_table_name)
i = add_to(step_funcs, i, hive_table_definition_step, rc_table_name, rc_dataset, s3_step_path, local_step_path, action_id, False)
i = add_to(step_funcs, i, hive_copy_to_table, stage_dataset, stage_table_name, rc_dataset, rc_table_name, s3_step_path, local_step_path, action_id, None)
i = add_to(step_funcs, i, impala_copy_to_table, rc_dataset, rc_table_name, target_dataset, target_table_name, s3_step_path, local_step_path, action_id)
# ------------------------------------------------------------------------------------------------------------
# at this point, the load should be considered complete even if something goes wrong in the steps below,
# so we will indicate this in the step wrapper
# ------------------------------------------------------------------------------------------------------------
considered_successful_at_this_index = i
# ------------------------------------------------------------------------------------------------------------
# drop any staging tables created
# ------------------------------------------------------------------------------------------------------------
if drop_table_names:
script = '\n'.join(['DROP TABLE %s;' % name for name in drop_table_names])
i = add_to(step_funcs, i, hive_run_script_contents_step, script, s3_step_path, local_step_path, action_id)
# ------------------------------------------------------------------------------------------------------------
# inform impala about changes
# ------------------------------------------------------------------------------------------------------------
if not target_is_dynamodb:
i = add_to(step_funcs, i, impala_invalidate_metadata_step, s3_step_path, action_id)
total_steps = i - 1
steps = []
for index, f in enumerate(step_funcs, 1):
step_wrapper = f(total_steps)
assert isinstance(step_wrapper, StepWrapper)
if index >= considered_successful_at_this_index:
step_wrapper.action_considered_successful = True
steps.append(step_wrapper)
if not dry_run:
s3_copy_recursive(local_step_path, s3_step_path)
return steps
finally:
shutil.rmtree(tempdir)
def put_dataset(dataset):
return {'results': dataset_service().update_dataset(dataset.id, Dataset.from_dict(request.get_json())).to_dict()}
def post_dataset():
return {'results': dataset_service().save_dataset(Dataset.from_dict(request.get_json())).to_dict()}
def post_dataset():
return {
'results':
dataset_service().save_dataset(Dataset.from_dict(
request.get_json())).to_dict()
}
def test_hive_table_definition_step(self):
ds = Dataset(data=DatasetData(
name='owen_eu_v01',
table_name='owen_eu',
location='s3://s3-rpt-uss-dat-warehouse/prd/inbound/overlord/eu-all-events',
data_format=DataFormat(
file_format=FileFormat.TEXTFILE,
row_format=RowFormat.JSON,
),
columns=[
Column('host', 'STRING', path='metadata.host'),
Column('pageName', 'STRING', path='owen.context.pageName'),
Column('viewInstanceUuid', 'STRING', path='owen.context.viewInstanceUuid'),
Column('previousPageName', 'STRING', path='owen.context.previousPageName'),
Column('previousViewInstanceUuid', 'STRING', path='owen.context.previousViewInstanceUuid'),
Column('session', 'STRING', path='owen.context.session'),
Column('pageType', 'STRING', path='owen.context.pageType'),
Column('propertyName', 'STRING', path='owen.context.propertyName'),
Column('enviroment', 'STRING', path='owen.context.environment'),
Column('appForegroundFlag', 'BOOLEAN', path='owen.context.appForegroundFlag'),
Column('bluetoothEnabledFlag', 'BOOLEAN', path='owen.context.bluetoothEnabledFlag'),
Column('favoriteFlag', 'BOOLEAN', path='owen.context.favoriteFlag'),
Column('locationEnabledFlag', 'BOOLEAN', path='owen.context.locationEnabledFlag'),
Column('loggedInFlag', 'BOOLEAN', path='owen.context.loggedInFlag'),
Column('notificationEnabledFlag', 'BOOLEAN', path='owen.context.notificationEnabledFlag'),
Column('personalizationFlag', 'BOOLEAN', path='owen.context.personalizationFlag'),
Column('advertiserUuid', 'STRING', path='owen.context.advertiserUuid'),
Column('udid', 'STRING', path='owen.context.udid'),
Column('userQualifier', 'STRING', path='owen.context.userQualifier'),
Column('userId', 'STRING', path='owen.context.custom.legacy.userId'),
Column('userUuid', 'STRING', path='owen.context.userUuid'),
Column('macAddress', 'STRING', path='owen.context.macAddress'),
Column('ipAddress', 'STRING', path='owen.context.ipAddress'),
Column('osVersion', 'STRING', path='owen.context.osVersion'),
Column('osFamily', 'STRING', path='owen.context.osFamily'),
Column('osName', 'STRING', path='owen.context.osName'),
Column('browserFamily', 'STRING', path='owen.context.browserFamily'),
Column('deviceCategory', 'STRING', path='owen.context.deviceCategory'),
Column('deviceMake', 'STRING', path='owen.context.mobileDeviceMake'),
Column('deviceModel', 'STRING', path='owen.context.mobileDeviceModel'),
Column('connectionType', 'STRING', path='owen.context.connectionType'),
Column('userAgent', 'STRING', path='owen.context.userAgent'),
Column('geofenceId', 'STRING', path='owen.context.custom.legacy.geofenceId'),
Column('eventTimestamp', 'TIMESTAMP', path='owen.event.eventTimestamp', date_pattern="yyyy-MM-dd'T'HH:mm:ssZ"),
Column('eventInstanceUuid', 'STRING', path='owen.event.eventInstanceUuid'),
Column('eventPlatformVersion', 'STRING', path='owen.event.eventPlatformVersion'),
Column('eventVersion', 'STRING', path='owen.event.eventVersion'),
Column('eventCategory', 'STRING', path='owen.event.eventCategory'),
Column('eventName', 'STRING', path='owen.event.eventName'),
Column('eventAction', 'STRING', path='owen.event.eventAction'),
Column('eventPlatform', 'STRING', path='owen.event.eventPlatform'),
Column('testUnixTimestampSecondsPattern', 'TIMESTAMP', path='some.fake.path.testUnixTimestampSecondsPattern', date_pattern='UNIX_TIMESTAMP_SECONDS'),
Column('testUnixTimestampMillisPattern', 'TIMESTAMP', path='some.fake.path.testUnixTimestampMillisPattern', date_pattern='UNIX_TIMESTAMP_MILLIS'),
],
compression='GZIP',
partitions=[
Column('year', 'STRING'),
Column('week', 'STRING'),
Column('day', 'STRING'),
],
))
call('mkdir -p /tmp/dart-emr-test/hive/')
this_path = os.path.dirname(os.path.abspath(__file__))
shutil.copyfile(this_path + '/../../../engine/emr/steps/hive/copy_to_table.hql', '/tmp/dart-emr-test/hive/copy_to_table.hql')
action_id = 'actionid123'
target_dataset = Dataset.from_dict(ds.to_dict())
target_dataset.data.data_format.num_header_rows = 0
target_dataset.data.data_format = DataFormat(FileFormat.RCFILE, RowFormat.NONE)
stage_dataset = Dataset.from_dict(ds.to_dict())
assert isinstance(stage_dataset, Dataset)
for c in stage_dataset.data.columns:
c.data_type = DataType.STRING
hive_copy_to_table(stage_dataset, 'owen_eu_stage', target_dataset, 'owen_eu', 's3://test', '/tmp/dart-emr-test/', action_id, None, 1, 1)
with open(os.path.join(this_path, 'copy_to_table_owen_eu.hql')) as f:
expected_contents = f.read()
with open('/tmp/dart-emr-test/hive/copy_to_table_owen_eu.hql') as f:
actual_contents = f.read()
self.assertEqual(expected_contents, actual_contents)
