def _create_s3_table(self, s3_resource, hql_validation_function, hql_validation_error=None): s3_resource.create_bucket(Bucket=self.LAKE_SPEC.bucket) for f in self.LAKE_SPEC.keys: logging.info("Creating object s3://{}/{}".format( self.LAKE_SPEC.bucket, f)) s3_resource.Bucket(self.LAKE_SPEC.bucket).put_object(Key=f, Body="") s3_resource.create_bucket(Bucket=self.LANDING_SPEC.bucket) for f in self.LANDING_SPEC.keys: logging.info("Creating object s3://{}/{}".format( self.LANDING_SPEC.bucket, f)) s3_resource.Bucket(self.LANDING_SPEC.bucket).put_object(Key=f, Body="") s3_table = S3Table.__new__(S3Table) # landing s3_table.db_table_landing = self.LANDING_SPEC.table dir_landing_data = "s3://{}/{}".format(self.LANDING_SPEC.bucket, self.LANDING_SPEC.data_dir) s3_table.dir_landing_data = dir_landing_data s3_table.dir_landing_work = dir_landing_data.replace("data", "work") s3_table.dir_landing_archive = dir_landing_data.replace( "data", "archive") s3_table.dir_landing_final = s3_table.dir_landing_data # lake s3_table.db_table_lake = self.LAKE_SPEC.table dir_lake_data = "s3://{}/{}".format(self.LAKE_SPEC.bucket, self.LAKE_SPEC.data_dir) s3_table.dir_lake_final = dir_lake_data s3_table.emr_system = FakeStorageSystem(hql_validation_function, hql_validation_error) s3_table.s3_resource = s3_resource test_landing_bucket_name = self.LANDING_SPEC.bucket test_lake_bucket_name = self.LAKE_SPEC.bucket s3_table.dir_landing_table = "s3://" + test_landing_bucket_name + "/" + self.LANDING_SPEC.data_dir s3_table.dir_lake_table = "s3://" + test_lake_bucket_name + "/" + self.LAKE_SPEC.data_dir s3_table.config_service = ConfigService( TestS3Table.DEFAULT_CONFIG_PATH) s3_table.partitioned_by = "month" s3_table.header_lines = 0 s3_table.delimiter = "|" s3_table.columns_lake = [("name1", "varchar(21)"), ("name2", "varchar(6)"), ("name3", "varchar(4)")] return s3_table
def __init__(self, execution_system, algorithm_instance, algorithm_params): """ Initialize Algorithm Decompression :param execution_system: an instance of EMRSystem object :param algorithm_instance: name of the algorithm instance :param algorithm_params: algorithm configuration """ super(AlgorithmGzipDecompressionEMR, self).__init__(execution_system, algorithm_instance, algorithm_params) destination_table_name = algorithm_params["destination_table"] self._table = S3Table(execution_system, destination_table_name) self._thread_pool_size = self._parameters["thread_pool_size"]
def create_dataset(execution_system, load_type, data_type, dataset_name): if data_type == DataType.STRUCTURED: dataset = S3Table(emr_system=execution_system, destination_table=dataset_name) elif data_type == DataType.SEMISTRUCTURED: if load_type == HiveTable.TableLoadType.APPEND: dataset = SemistructuredDataSet(emr_system=execution_system, dataset_name=dataset_name) else: raise M3DUnsupportedLoadTypeException( load_type=load_type, message="Loading algorithm {} not support for data type {}." .format(load_type, data_type)) else: raise M3DUnsupportedDataTypeException( message="Data Type {} not available.".format(data_type)) return dataset
def __init__(self, test_run_dir, setup_function, partition_columns, regex_filename, file_format=None, null_value=None, quote_character=None, compute_table_statistics=None): self.config_file, _, self.tconx_file, self.config_dict, self.scon_emr_dict = setup_function( *([test_run_dir] + self.destination_params)) self._write_acon(partition_columns, regex_filename, file_format, null_value, quote_character, compute_table_statistics) self._write_tconx() self.table_config = [self.config_file, self.cluster_mode ] + self.destination_params emr_system = EMRSystem(self.config_file, self.cluster_mode, self.destination_system, self.destination_database, self.destination_environment) self.s3_table = S3Table(emr_system, self.destination_table) config_filename = "append_load-{}-{}.json".format( self.destination_environment, self.destination_table) self.config_filepath = os.path.join(self.s3_table.dir_apps_append_load, config_filename) self.db_name_lake = self.scon_emr_dict["environments"][ self.destination_environment]["schemas"]["lake"] self.expected_algorithms_jar_path = "s3://" + os.path.join( (self.scon_emr_dict["environments"][self.destination_environment] ["s3_buckets"]["application"]).strip("/"), (self.scon_emr_dict["environments"][self.destination_environment] ["s3_deployment_dir_base"]).strip("/"), self.destination_environment, self.scon_emr_dict["subdir"]["m3d"], self.config_dict["subdir_projects"]["m3d_api"], self.scon_emr_dict["spark"]["jar_name"])
def __init__(self, execution_system, load_type, destination_table, spark_params_dict): """ Initialize Load Executor :param execution_system: execution system :param load_type: load type :param destination_table: table to load :param spark_params_dict: spark parameters """ super(LoadExecutorHadoop, self).__init__(execution_system) self._destination_table = destination_table self._spark_params_dict = spark_params_dict available_loads = self._get_available_emr_load_types() if load_type not in available_loads: raise M3DUnsupportedLoadTypeException( load_type=load_type, message="Loading algorithm {} not available.".format( load_type)) table = S3Table(emr_system=execution_system, destination_table=destination_table) self._load_wrapper = available_loads[load_type]( execution_system=self._execution_system, table=table) self._execution_system.add_cluster_tags({ EMRSystem.EMRClusterTag.API_METHOD: M3D.load_table.__name__, EMRSystem.EMRClusterTag.LOAD_TYPE: load_type, EMRSystem.EMRClusterTag.TARGET_TABLE: table.db_table_lake })
def test_lakeout_view_hql(self, add_tags_patch): tconx_src_path = "test/resources/test_create_out_view_hive/test_lakeout_view_structure/config/tconx.json" destination_system = "bdp" destination_database = "emr_test" destination_environment = "dev" destination_table = "bi_test101" m3d_config_file, _, tconx_file, m3d_config_dict, scon_emr_dict = \ self.env_setup( self.local_run_dir, destination_system, destination_database, destination_environment, destination_table ) # Use test case specific tconx py.path.local(tconx_file).write(py.path.local(tconx_src_path).read()) table_config = [ m3d_config_file, destination_system, destination_database, destination_environment, destination_table ] table_config_kwargs = { "emr_cluster_id": self.emr_cluster_id } emr_steps_completer = self.create_emr_steps_completer(expected_steps_count=1, timeout_seconds=3) with ConcurrentExecutor(emr_steps_completer, delay_sec=0.4): logging.info("Calling M3D.create_out_view().") M3D.create_out_view(*table_config, **table_config_kwargs) emr_system = EMRSystem(*table_config[:5]) s3_table = S3Table(emr_system, destination_table) mock_cluster = self.mock_emr.backends[self.default_aws_region].clusters[self.emr_cluster_id] assert 1 == len(mock_cluster.steps) hive_step = mock_cluster.steps[0] assert hive_step.args[0] == "hive" assert hive_step.args[1] == "--silent" assert hive_step.args[2] == "-f" actual_hql_content_in_bucket = self.get_object_content_from_s3(hive_step.args[3]) column_name_pairs = [ ("record_date", "v_record_date"), ("p_string", "v_string"), ("p_int", "v_int"), ("p_bigint", "v_bigint"), ("p_float", "v_float"), ("p_varchar_1", "v_varchar_10"), ("p_varchar_2", "v_varchar_100"), ("p_char_1", "v_char"), ("p_boolean", "v_boolean"), ("year", "year"), ("month", "month") ] columns_str = ", ".join(map(lambda x: "{} AS {}".format(x[0], x[1]), column_name_pairs)) drop_view = "DROP VIEW IF EXISTS {};".format(s3_table.db_view_lake_out) # S3Table is partitioned by year and month create_view = "\n".join([ "CREATE VIEW {}".format(s3_table.db_view_lake_out), "AS", "SELECT {}".format(columns_str), "FROM {};".format(s3_table.db_table_lake) ]) expected_hql = "\n".join([drop_view, create_view]) assert actual_hql_content_in_bucket == expected_hql add_tags_patch_call_args_list = add_tags_patch.call_args_list assert len(add_tags_patch_call_args_list) == 2 assert add_tags_patch_call_args_list[0][0][0] == [{ "Key": "ApiMethod", "Value": "create_out_view" }] assert add_tags_patch_call_args_list[1][0][0] == [{ "Key": "TargetView", "Value": "dev_lake_out.bi_test101" }]
def truncate_table(self, destination_table): from m3d.hadoop.emr.s3_table import S3Table full_table_name = "{}.{}".format(self.db_lake, destination_table) self.add_cluster_tag(self.EMRClusterTag.TARGET_TABLE, full_table_name) S3Table(self, destination_table).truncate_tables()
def drop_out_view(self, destination_table): from m3d.hadoop.emr.s3_table import S3Table full_table_name = "{}.{}".format(self.db_lake_out, destination_table) self.add_cluster_tag(self.EMRClusterTag.TARGET_VIEW, full_table_name) S3Table(self, destination_table).drop_out_view()
def test_full_load_emr(self, _0, _1): tconx_src_path = \ "test/resources/test_create_out_view_hive/test_empty_table_lakeout/config/empty_tabl_cd_lakeout.json" destination_system = "bdp" destination_database = "emr_test" destination_environment = "dev" destination_table = "bi_test101" load_type = "FullLoad" landing_dataset = "landing-dataset.psv" spark_external_parameters = '''{ "spark.driver.memory": "99G", "spark.executor.instances": "99", "spark.executor.memory": "90G" } ''' m3d_config_file, scon_emr_file, tconx_file, m3d_config_dict, scon_emr_dict = \ super(TestLoadTableFullS3, self).env_setup( self.local_run_dir, destination_system, destination_database, destination_environment, destination_table ) py.path.local(tconx_file).write(py.path.local(tconx_src_path).read()) table_config = [ m3d_config_file, destination_system, destination_database, destination_environment, destination_table, load_type, self.emr_cluster_id, spark_external_parameters ] # Extract bucket names bucket_application = scon_emr_dict["environments"][ destination_environment]["s3_buckets"]["application"] emr_system = EMRSystem(m3d_config_file, destination_system, destination_database, destination_environment) test_s3_table = S3Table(emr_system, destination_table) # Put landing data self.dump_data_to_s3( os.path.join(test_s3_table.dir_landing_final, landing_dataset), "t|e|s|t|a|d|i|d|a|s|m|3|d|") M3D.load_table(*table_config) # Since we have offloaded data move operations to EMR Steps dir_landing_final will still have # old files in it and dir_landing_archive will not have new files landing_files = self.get_child_objects(test_s3_table.dir_landing_final) assert len(landing_files) == 1 assert landing_files[0] == os.path.join( test_s3_table.dir_landing_final, landing_dataset) landing_archive_files = self.get_child_objects( test_s3_table.dir_landing_archive) assert len(landing_archive_files) == 0 # Check EMR steps. fake_cluster = self.mock_emr.backends[ self.default_aws_region].clusters[self.emr_cluster_id] assert 1 == len(fake_cluster.steps) expected_algorithms_jar_path = "s3://" + bucket_application + os.path.join( scon_emr_dict["environments"][destination_environment] ["s3_deployment_dir_base"], destination_environment, scon_emr_dict["subdir"]["m3d"], m3d_config_dict["subdir_projects"]["m3d_api"], scon_emr_dict["spark"]["jar_name"]) # Check args of spark-submit EMR step spark_step = fake_cluster.steps[0] assert spark_step.jar == "command-runner.jar" assert spark_step.args[0] == "spark-submit" assert spark_step.args[-5] == "com.adidas.analytics.AlgorithmFactory" assert spark_step.args[-4] == expected_algorithms_jar_path assert spark_step.args[-3] == "FullLoad" assert spark_step.args[-2] == "s3://m3d-dev-application/m3d/dev/apps/loading/bdp/test101/" \ "full_load/full_load-dev-bi_test101.json" assert spark_step.args[-1] == "s3"
def test_full_load_emr_external_spark_parameters(self, _0): tconx_src_path = \ "test/resources/test_create_out_view_hive/test_empty_table_lakeout/config/empty_tabl_cd_lakeout.json" acon_src_path = "test/resources/test_load_table_full_s3/acon-emr_test-bi_test101.json" destination_system = "bdp" destination_database = "emr_test" destination_environment = "dev" destination_table = "bi_test101" spark_external_parameters = { "spark.driver.memory": "99G", "spark.executor.instances": "99", "spark.executor.memory": "90G" } load_type = "FullLoad" landing_dataset = "landing-dataset.psv" m3d_config_file, scon_emr_file, tconx_file, m3d_config_dict, scon_emr_dict = \ super(TestLoadTableFullS3, self).env_setup( self.local_run_dir, destination_system, destination_database, destination_environment, destination_table ) AconHelper.setup_acon_from_file(m3d_config_dict["tags"]["config"], destination_database, destination_environment, destination_table, acon_src_path) py.path.local(tconx_file).write(py.path.local(tconx_src_path).read()) table_config = [ m3d_config_file, destination_system, destination_database, destination_environment, destination_table, load_type, self.emr_cluster_id ] # Extract bucket names bucket_application = scon_emr_dict["environments"][ destination_environment]["s3_buckets"]["application"] emr_system = EMRSystem(m3d_config_file, destination_system, destination_database, destination_environment) test_s3_table = S3Table(emr_system, destination_table) # Put landing data self.dump_data_to_s3( os.path.join(test_s3_table.dir_landing_final, landing_dataset), "t|e|s|t|a|d|i|d|a|s|m|3|d|") M3D.load_table(*table_config, spark_params=json.dumps(spark_external_parameters)) # psv file will still be in landing since move operation should be # performed by EMR Step which we mock here. Accordingly archive will # still be empty. landing_files = self.get_child_objects(test_s3_table.dir_landing_final) assert len(landing_files) == 1 assert landing_files[0] == os.path.join( test_s3_table.dir_landing_final, landing_dataset) landing_archive_files = self.get_child_objects( test_s3_table.dir_landing_archive) assert len(landing_archive_files) == 0 # Check EMR steps. fake_cluster = self.mock_emr.backends[ self.default_aws_region].clusters[self.emr_cluster_id] assert 1 == len(fake_cluster.steps) expected_algorithms_jar_path = "s3://" + bucket_application + os.path.join( scon_emr_dict["environments"][destination_environment] ["s3_deployment_dir_base"], destination_environment, scon_emr_dict["subdir"]["m3d"], m3d_config_dict["subdir_projects"]["m3d_api"], scon_emr_dict["spark"]["jar_name"]) spark_step = fake_cluster.steps[0] assert spark_step.jar == "command-runner.jar" assert spark_step.args[0] == "spark-submit" assert spark_step.args[5] == "--conf" assert spark_step.args[7] == "--conf" assert spark_step.args[9] == "--conf" expected_spark_conf_options = set( map(lambda p: "{}={}".format(p[0], p[1]), spark_external_parameters.items())) actual_spark_conf_options = set( map(lambda x: spark_step.args[x], [6, 8, 10])) assert expected_spark_conf_options == actual_spark_conf_options assert spark_step.args[-5] == "com.adidas.analytics.AlgorithmFactory" assert spark_step.args[-4] == expected_algorithms_jar_path assert spark_step.args[-3] == "FullLoad" assert spark_step.args[-2] == "s3://m3d-dev-application/m3d/dev/apps/loading/bdp/test101/" \ "full_load/full_load-dev-bi_test101.json" assert spark_step.args[-1] == "s3"
def test_load_table_delta(self, remove_json_patch, add_tags_patch, _0, _1): # responses.add_passthru(self.default_server_url) destination_system = "bdp" destination_database = "emr_test" destination_environment = "dev" destination_active_table = "bi_test101" destination_changelog_table = "bi_test101_cl" load_type = "DeltaLoad" src_tconx_path = "test/resources/test_load_table_delta_s3/tconx-bdp-emr_test-dev-bi_test101.json" src_tconx_cl_table = "test/resources/test_load_table_delta_s3/tconx-bdp-emr_test-dev-bi_test101_cl.json" spark_external_parameters = '''{ "spark.driver.memory": "99G", "spark.executor.instances": "99", "spark.executor.memory": "90G" } ''' # pass desired content of tconx files for active and changelog tables to self.env_setup() src_tconx_content = py.path.local(src_tconx_path).read() src_tconx_cl_content = py.path.local(src_tconx_cl_table).read() m3d_config_file, scon_emr_file, tconx_file, tconx_cl_file, m3d_config_dict, scon_emr_dict = \ self.env_setup( self.local_run_dir, destination_system, destination_database, destination_environment, destination_active_table, src_tconx_content, src_tconx_cl_content ) emr_system = EMRSystem(m3d_config_file, destination_system, destination_database, destination_environment) s3_table_active = S3Table(emr_system, destination_active_table) s3_table_changelog = S3Table(emr_system, destination_changelog_table) # Extract bucket names bucket_application = scon_emr_dict["environments"][ destination_environment]["s3_buckets"]["application"] # Put lake data for changelog table, this should be archived self.dump_data_to_s3( os.path.join(s3_table_changelog.dir_lake_final, "changelog.parquet"), "t|e|s|t|a|d|i|d|a|s|m|3|d|", ) M3D.load_table(m3d_config_file, destination_system, destination_database, destination_environment, destination_active_table, load_type, self.emr_cluster_id, spark_params=spark_external_parameters) filename_json = "delta_load-{environment}-{table}.json".format( environment=destination_environment, table=destination_active_table) # Checking configuration file for m3d-engine app_files = self.get_child_objects(s3_table_active.dir_apps_delta_load) assert len(app_files) == 1 assert app_files[ 0] == s3_table_active.dir_apps_delta_load + filename_json delta_load_config_s3 = app_files[0] delta_load_config_content = self.get_object_content_from_s3( delta_load_config_s3) load_table_parameters = json.loads(delta_load_config_content) assert load_table_parameters[ "active_records_table_lake"] == s3_table_active.db_table_lake assert load_table_parameters[ "active_records_dir_lake"] == s3_table_active.dir_lake_final assert load_table_parameters[ "delta_records_file_path"] == s3_table_active.dir_landing_data assert load_table_parameters["technical_key"] == [ "m3d_timestamp", "datapakid", "partno", "record" ] assert load_table_parameters[ "business_key"] == s3_table_active.business_key if s3_table_active.partitioned_by in Util.defined_partitions: target_partitions = Util.get_target_partitions_list( s3_table_active.partitioned_by) else: target_partitions = s3_table_active.partitioned_by assert load_table_parameters["target_partitions"] == target_partitions assert load_table_parameters[ "partition_column"] == s3_table_active.partition_column assert load_table_parameters[ "partition_column_format"] == s3_table_active.partition_column_format # Check EMR steps. fake_cluster = self.mock_emr.backends[ self.default_aws_region].clusters[self.emr_cluster_id] assert 1 == len(fake_cluster.steps) expected_algorithms_jar_path = "s3://" + bucket_application + os.path.join( scon_emr_dict["environments"][destination_environment] ["s3_deployment_dir_base"], destination_environment, scon_emr_dict["subdir"]["m3d"], m3d_config_dict["subdir_projects"]["m3d_api"], scon_emr_dict["spark"]["jar_name"]) delta_load_step = fake_cluster.steps[0] assert delta_load_step.jar == "command-runner.jar" assert delta_load_step.args[0] == "spark-submit" assert delta_load_step.args[ -5] == "com.adidas.analytics.AlgorithmFactory" assert delta_load_step.args[-4] == expected_algorithms_jar_path assert delta_load_step.args[-3] == "DeltaLoad" assert delta_load_step.args[-2] == delta_load_config_s3 assert delta_load_step.args[-1] == "s3" add_tags_patch_call_args_list = add_tags_patch.call_args_list assert len(add_tags_patch_call_args_list) == 1 assert sorted(add_tags_patch_call_args_list[0][0][0], key=lambda x: x["Key"]) == sorted([{ "Key": "ApiMethod", "Value": "load_table" }, { "Key": "LoadType", "Value": "DeltaLoad" }, { "Key": "TargetTable", "Value": "bi_test101" }], key=lambda x: x["Key"]) remove_json_patch.assert_called_once() assert remove_json_patch.call_args_list[0][0][0] == app_files[0]