def test_get_target_partitions_string(self):
"""
This method tests the correct functionality of get_partition_column_string of Util class
:return:
"""
assert Util.get_target_partitions_string("year") == "year"
assert Util.get_target_partitions_string("month") == "year,month"
assert Util.get_target_partitions_string("day") == "year,month,day"
assert Util.get_target_partitions_string("") == ""
with pytest.raises(Exception) as exc_info:
Util.get_target_partitions_list("country")
assert "Partition type country not supported" in str(exc_info.value)
def test_get_target_partitions_list(self):
"""
This method tests the correct functionality of get_partition_column_list of Util class
:return:
"""
assert Util.get_target_partitions_list("year") == ["year"]
assert Util.get_target_partitions_list("month") == ["year", "month"]
assert Util.get_target_partitions_list("day") == [
"year", "month", "day"
]
assert Util.get_target_partitions_list("") == []
assert Util.get_target_partitions_list("country") == ["country"]
def get_projection_columns(self, src_column_names,
destination_column_names):
columns = list(
filter(lambda x: x[1],
zip(src_column_names, destination_column_names)))
if self.partitioned_by in Util.defined_partitions:
target_partitions = list(
map(lambda x: (x, x),
Util.get_target_partitions_list(self.partitioned_by)))
return columns + target_partitions
else:
return columns
def test_load_table_delta(self, remove_json_patch, add_tags_patch, _0, _1):
# responses.add_passthru(self.default_server_url)
destination_system = "bdp"
destination_database = "emr_test"
destination_environment = "dev"
destination_active_table = "bi_test101"
destination_changelog_table = "bi_test101_cl"
load_type = "DeltaLoad"
src_tconx_path = "test/resources/test_load_table_delta_s3/tconx-bdp-emr_test-dev-bi_test101.json"
src_tconx_cl_table = "test/resources/test_load_table_delta_s3/tconx-bdp-emr_test-dev-bi_test101_cl.json"
spark_external_parameters = '''{
"spark.driver.memory": "99G",
"spark.executor.instances": "99",
"spark.executor.memory": "90G"
}
'''
# pass desired content of tconx files for active and changelog tables to self.env_setup()
src_tconx_content = py.path.local(src_tconx_path).read()
src_tconx_cl_content = py.path.local(src_tconx_cl_table).read()
m3d_config_file, scon_emr_file, tconx_file, tconx_cl_file, m3d_config_dict, scon_emr_dict = \
self.env_setup(
self.local_run_dir,
destination_system,
destination_database,
destination_environment,
destination_active_table,
src_tconx_content,
src_tconx_cl_content
)
emr_system = EMRSystem(m3d_config_file, destination_system,
destination_database, destination_environment)
s3_table_active = S3Table(emr_system, destination_active_table)
s3_table_changelog = S3Table(emr_system, destination_changelog_table)
# Extract bucket names
bucket_application = scon_emr_dict["environments"][
destination_environment]["s3_buckets"]["application"]
# Put lake data for changelog table, this should be archived
self.dump_data_to_s3(
os.path.join(s3_table_changelog.dir_lake_final,
"changelog.parquet"),
"t|e|s|t|a|d|i|d|a|s|m|3|d|",
)
M3D.load_table(m3d_config_file,
destination_system,
destination_database,
destination_environment,
destination_active_table,
load_type,
self.emr_cluster_id,
spark_params=spark_external_parameters)
filename_json = "delta_load-{environment}-{table}.json".format(
environment=destination_environment,
table=destination_active_table)
# Checking configuration file for m3d-engine
app_files = self.get_child_objects(s3_table_active.dir_apps_delta_load)
assert len(app_files) == 1
assert app_files[
0] == s3_table_active.dir_apps_delta_load + filename_json
delta_load_config_s3 = app_files[0]
delta_load_config_content = self.get_object_content_from_s3(
delta_load_config_s3)
load_table_parameters = json.loads(delta_load_config_content)
assert load_table_parameters[
"active_records_table_lake"] == s3_table_active.db_table_lake
assert load_table_parameters[
"active_records_dir_lake"] == s3_table_active.dir_lake_final
assert load_table_parameters[
"delta_records_file_path"] == s3_table_active.dir_landing_data
assert load_table_parameters["technical_key"] == [
"m3d_timestamp", "datapakid", "partno", "record"
]
assert load_table_parameters[
"business_key"] == s3_table_active.business_key
if s3_table_active.partitioned_by in Util.defined_partitions:
target_partitions = Util.get_target_partitions_list(
s3_table_active.partitioned_by)
else:
target_partitions = s3_table_active.partitioned_by
assert load_table_parameters["target_partitions"] == target_partitions
assert load_table_parameters[
"partition_column"] == s3_table_active.partition_column
assert load_table_parameters[
"partition_column_format"] == s3_table_active.partition_column_format
# Check EMR steps.
fake_cluster = self.mock_emr.backends[
self.default_aws_region].clusters[self.emr_cluster_id]
assert 1 == len(fake_cluster.steps)
expected_algorithms_jar_path = "s3://" + bucket_application + os.path.join(
scon_emr_dict["environments"][destination_environment]
["s3_deployment_dir_base"], destination_environment,
scon_emr_dict["subdir"]["m3d"],
m3d_config_dict["subdir_projects"]["m3d_api"],
scon_emr_dict["spark"]["jar_name"])
delta_load_step = fake_cluster.steps[0]
assert delta_load_step.jar == "command-runner.jar"
assert delta_load_step.args[0] == "spark-submit"
assert delta_load_step.args[
-5] == "com.adidas.analytics.AlgorithmFactory"
assert delta_load_step.args[-4] == expected_algorithms_jar_path
assert delta_load_step.args[-3] == "DeltaLoad"
assert delta_load_step.args[-2] == delta_load_config_s3
assert delta_load_step.args[-1] == "s3"
add_tags_patch_call_args_list = add_tags_patch.call_args_list
assert len(add_tags_patch_call_args_list) == 1
assert sorted(add_tags_patch_call_args_list[0][0][0],
key=lambda x: x["Key"]) == sorted([{
"Key": "ApiMethod",
"Value": "load_table"
}, {
"Key": "LoadType",
"Value": "DeltaLoad"
}, {
"Key": "TargetTable",
"Value": "bi_test101"
}],
key=lambda x: x["Key"])
remove_json_patch.assert_called_once()
assert remove_json_patch.call_args_list[0][0][0] == app_files[0]