def test_check_hql(self, add_tags_patch, _):
logging.info("Starting TestCreateTableS3.test_check_hql()")
destination_system = "bdp"
destination_database = "emr_test"
destination_environment = "dev"
destination_table = "bi_test101"
m3d_config_file, _, _, _, scon_emr_dict = \
self.env_setup(
self.local_run_dir,
destination_system,
destination_database,
destination_environment,
destination_table
)
table_config = [
m3d_config_file,
destination_system,
destination_database,
destination_environment,
destination_table
]
table_config_kwargs = {
"emr_cluster_id": self.emr_cluster_id
}
logging.info("Calling M3D.create_table().")
M3D.create_table(*table_config, **table_config_kwargs)
fake_cluster = self.mock_emr.backends[self.default_aws_region].clusters[self.emr_cluster_id]
assert 1 == len(fake_cluster.steps)
hive_step = fake_cluster.steps[0]
assert hive_step.args[0] == "hive"
assert hive_step.args[1] == "--silent"
assert hive_step.args[2] == "-f"
db_landing = scon_emr_dict["environments"][destination_environment]["schemas"]["landing"]
db_lake = scon_emr_dict["environments"][destination_environment]["schemas"]["lake"]
ddl_landing = "CREATE DATABASE IF NOT EXISTS dev_landing;\n" \
"CREATE DATABASE IF NOT EXISTS dev_lake;\n" \
"CREATE EXTERNAL TABLE dev_landing.bi_test101_stg1(name1 varchar(21), name2 varchar(6), " \
"name3 varchar(4))\n" \
"ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n'\n" \
"LOCATION 's3://m3d-dev-landing/dev/bi/test101/data/'\n" \
"TBLPROPERTIES(\"serialization.encoding\"=\"UTF-8\");"
ddl_lake = "CREATE EXTERNAL TABLE dev_lake.bi_test101(name1 varchar(21), name2 varchar(6), " \
"name3 varchar(4))\n" \
"PARTITIONED BY (year smallint, month smallint)\n" \
"STORED AS PARQUET\n" \
"LOCATION 's3://m3d-dev-lake/dev/bi/test101/data/'\n" \
"TBLPROPERTIES(\"serialization.encoding\"=\"UTF-8\");"
# Get content of hql in s3 bucket
actual_hql_content_in_bucket = self.get_object_content_from_s3(hive_step.args[3])
expected_hql = \
ddl_landing + "\n" + \
"MSCK REPAIR TABLE {}.{}_stg1;".format(db_landing, destination_table) + "\n" + \
ddl_lake + "\n" + \
"MSCK REPAIR TABLE {}.{};".format(db_lake, destination_table)
logging.info("Expected: {0}\n".format(expected_hql))
logging.info("Actual: {0}\n".format(actual_hql_content_in_bucket))
assert actual_hql_content_in_bucket == expected_hql
add_tags_patch_call_args_list = add_tags_patch.call_args_list
assert len(add_tags_patch_call_args_list) == 2
assert add_tags_patch_call_args_list[0][0][0] == [{
"Key": "ApiMethod",
"Value": "create_table"
}]
assert add_tags_patch_call_args_list[1][0][0] == [{
"Key": "TargetTable",
"Value": "dev_lake.bi_test101"
}]
def test_check_s3_cleanup(self, add_tags_patch, _):
logging.info("Starting s3 Checkup cleanup")
destination_system = "bdp"
destination_database = "emr_test"
destination_environment = "dev"
destination_table = "bi_test101"
m3d_config_file, _, _, m3d_config_dict, scon_emr_dict = \
self.env_setup(
self.local_run_dir,
destination_system,
destination_database,
destination_environment,
destination_table
)
table_config_args = [
m3d_config_file,
destination_system,
destination_database,
destination_environment,
destination_table
]
table_config_kwargs = {
"emr_cluster_id": self.emr_cluster_id
}
db_landing = scon_emr_dict["environments"][destination_environment]["schemas"]["landing"]
db_lake = scon_emr_dict["environments"][destination_environment]["schemas"]["lake"]
bucket_landing = scon_emr_dict["environments"][destination_environment]["s3_buckets"]["landing"]
bucket_lake = scon_emr_dict["environments"][destination_environment]["s3_buckets"]["lake"]
test_content = "sample content"
test_lake_key_filename = "test_lake_key"
test_land_key_filename = "test_land_key"
source_system = "bi"
table = "test101"
test_land_key = "{environment}/{source_system}/{table}/data/{obj_name}".format(
environment=destination_environment,
source_system=source_system,
table=table,
obj_name=test_land_key_filename
)
test_lake_key = "{environment}/{source_system}/{table}/data/{obj_name}".format(
environment=destination_environment,
source_system=source_system,
table=table,
obj_name=test_lake_key_filename
)
# adding data to landing and lake directories
self.s3_resource.Bucket(bucket_landing).put_object(Key=test_land_key, Body=test_content)
self.s3_resource.Bucket(bucket_lake).put_object(Key=test_lake_key, Body=test_content)
logging.info("Calling M3D.create_table()")
M3D.create_table(*table_config_args, **table_config_kwargs)
logging.info("Calling M3D.drop_table()")
M3D.drop_table(*table_config_args, **table_config_kwargs)
emr_backend = self.mock_emr.backends[self.default_aws_region]
fake_cluster = emr_backend.clusters[self.emr_cluster_id]
assert 3 == len(fake_cluster.steps)
# Get actual HQL statements
actual_hqls = []
for step in fake_cluster.steps:
assert ["hive", "--silent", "-f"] == step.args[0:3]
hql_file = step.args[3]
hql_content = self.get_object_content_from_s3(hql_file)
actual_hqls.append(hql_content)
expected_hqls = [
'DROP TABLE {}.{}{};'.format(db_landing, destination_table, m3d_config_dict["tags"]["table_suffix_stage"]),
'DROP TABLE {}.{};'.format(db_lake, destination_table)
]
assert expected_hqls == actual_hqls[1:3]
# checking landing directory
landing_files = [k.key for k in self.s3_resource.Bucket(bucket_landing).objects.all()]
assert len(landing_files) == 1
assert landing_files[0] == test_land_key
# checking lake directory
lake_files = [k.key for k in self.s3_resource.Bucket(bucket_lake).objects.all()]
assert len(lake_files) == 1
assert lake_files[0] == test_lake_key
add_tags_patch_call_args_list = add_tags_patch.call_args_list
assert len(add_tags_patch_call_args_list) == 4
assert add_tags_patch_call_args_list[0][0][0] == [{
"Key": "ApiMethod",
"Value": "create_table"
}]
assert add_tags_patch_call_args_list[1][0][0] == [{
"Key": "TargetTable",
"Value": "dev_lake.bi_test101"
}]
assert add_tags_patch_call_args_list[2][0][0] == [{
"Key": "ApiMethod",
"Value": "drop_table"
}]
assert add_tags_patch_call_args_list[3][0][0] == [{
"Key": "TargetTable",
"Value": "dev_lake.bi_test101"
}]
def test_check_hql_single_partitioning(self, add_tags_patch, _):
logging.info(
"Starting TestCreateTableS3.test_check_hql_single_partitioning()")
destination_system = "bdp"
destination_database = "emr_test"
destination_environment = "dev"
destination_table = "bi_test103"
m3d_config_file, _, _, m3d_config_dict, scon_emr_dict = \
self.env_setup(
self.local_run_dir,
destination_system,
destination_database,
destination_environment,
destination_table
)
TconxHelper.setup_tconx_from_file(
m3d_config_dict["tags"]["config"], destination_system,
destination_database, destination_environment, destination_table,
S3TableTestBase.single_partition_tconx)
table_config = [
m3d_config_file, destination_system, destination_database,
destination_environment, destination_table
]
table_config_kwargs = {"emr_cluster_id": self.emr_cluster_id}
logging.info("Calling M3D.create_table().")
M3D.create_table(*table_config, **table_config_kwargs)
fake_cluster = self.mock_emr.backends[
self.default_aws_region].clusters[self.emr_cluster_id]
executed_steps = fake_cluster.steps
assert len(executed_steps) == 1
hive_step = executed_steps[0]
assert hive_step.args[0] == "hive"
assert hive_step.args[1] == "--silent"
assert hive_step.args[2] == "-f"
db_landing = scon_emr_dict["environments"][destination_environment][
"schemas"]["landing"]
db_lake = scon_emr_dict["environments"][destination_environment][
"schemas"]["lake"]
ddl_landing = "CREATE DATABASE IF NOT EXISTS dev_landing;\n" \
"CREATE DATABASE IF NOT EXISTS dev_lake;\n" \
"CREATE EXTERNAL TABLE dev_landing.bi_test103_stg1(name1 varchar(21), name2 varchar(6), " \
"name3 varchar(4))\n" \
"ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n'\n" \
"LOCATION 's3://m3d-dev-landing/dev/bi/test103/data/'\n" \
"TBLPROPERTIES(\"serialization.encoding\"=\"UTF-8\");"
ddl_lake = "CREATE EXTERNAL TABLE dev_lake.bi_test103(name2 varchar(6), name3 varchar(4))\n" \
"PARTITIONED BY (name1 varchar(21))\n" \
"STORED AS PARQUET\n" \
"LOCATION 's3://m3d-dev-lake/dev/bi/test103/data/'\n" \
"TBLPROPERTIES(\"serialization.encoding\"=\"UTF-8\");"
# Get content of hql in s3 bucket
actual_hql_content_in_bucket = self.get_object_content_from_s3(
hive_step.args[3])
expected_hql = \
ddl_landing + "\n" + \
"MSCK REPAIR TABLE {}.{}_stg1;".format(db_landing, destination_table) + "\n" + \
ddl_lake + "\n" + \
"MSCK REPAIR TABLE {}.{};".format(db_lake, destination_table)
print("Expected: {0}\n".format(expected_hql))
print("Actual: {0}\n".format(actual_hql_content_in_bucket))
assert actual_hql_content_in_bucket == expected_hql
