def test_global_config_reader_overwritten_default_value(self): cli_params = ['application_name', 'config_file', 'eu-west-1', '--connection-pre-test', 'False'] config_reader = GlobalConfigParametersReader() default_parameters = config_reader.get_config_key_values_updated_with_cli_args(cli_params) expected_value = False returned_value = default_parameters['connectionPreTest'] self.assertEqual(expected_value, returned_value)
def test_global_config_reader_overwritten_default_value_bool_without_explicit_value_at_end(self): cli_params = ['application_name', 'config_file', 'eu-west-1', '--connection-pre-test', 'False', '--destinationTableAutoCreate'] config_reader = GlobalConfigParametersReader() default_parameters = config_reader.get_config_key_values_updated_with_cli_args(cli_params) expected_value = True returned_value = default_parameters['destinationTableAutoCreate'] self.assertEqual(expected_value, returned_value)
def setUp(self): global_config_reader = GlobalConfigParametersReader() arguments = ['app-name', 'config-file', 'eu-west-1', '--destination-schema-auto-create', '--destination-table-auto-create'] global_config_reader.get_config_key_values_updated_with_cli_args(arguments) self.configure_test_cluster() self.create_test_table()
def main(args): global region global_config_reader = GlobalConfigParametersReader() global_config_values = global_config_reader.get_config_key_values_updated_with_cli_args(args) set_log_level(global_config_values['logLevel']) UnloadCopyTool(global_config_values['s3ConfigFile'], global_config_values['region'], global_config_values)
def test_global_config_reader_unprocessed_values(self): unprocessed_parameters = ['application_name', 'config_file', 'eu-west-1'] full_args = copy.deepcopy(unprocessed_parameters) for parameter in self.processable_parameters: full_args.append(parameter) config_reader = GlobalConfigParametersReader() config_reader.get_config_key_values_updated_with_cli_args(full_args) self.assertEqual(unprocessed_parameters, config_reader.unprocessed_arguments)
def test_global_config_reader_valid_region(self): valid_region_name = 'us-east-2' unprocessed_parameters = [ 'application_name', 'config_file', '--region', valid_region_name ] config_reader = GlobalConfigParametersReader() config_key_values = config_reader.get_config_key_values_updated_with_cli_args( unprocessed_parameters) self.assertEqual(valid_region_name, config_key_values['region'])
def test_global_config_reader_valid_if_boolean_flag_is_used_and_followed_by_no_argument( self): valid_region_name = 'eu-west-1' unprocessed_parameters = [ 'redshift_unload_copy.py', '--log-level', 'debug', '--destination-table-auto-create', 'conf.json', 'eu-west-1' ] config_reader = GlobalConfigParametersReader() config_key_values = config_reader.get_config_key_values_updated_with_cli_args( unprocessed_parameters) self.assertEqual(valid_region_name, config_key_values['region'])
def setUp(self): global_config_reader = GlobalConfigParametersReader() arguments = [ 'app-name', 'config-file', 'eu-west-1', '--destination-schema-auto-create', '--destination-table-auto-create' ] global_config_reader.get_config_key_values_updated_with_cli_args( arguments) self.configure_test_cluster() self.create_test_table()
def test_global_config_reader_unprocessed_values(self): unprocessed_parameters = [ 'application_name', 'config_file', 'eu-west-1' ] full_args = copy.deepcopy(unprocessed_parameters) for parameter in self.processable_parameters: full_args.append(parameter) config_reader = GlobalConfigParametersReader() config_reader.get_config_key_values_updated_with_cli_args(full_args) self.assertEqual(unprocessed_parameters, config_reader.unprocessed_arguments)
def test_global_config_reader_overwritten_default_value(self): cli_params = [ 'application_name', 'config_file', 'eu-west-1', '--connection-pre-test', 'False' ] config_reader = GlobalConfigParametersReader() default_parameters = config_reader.get_config_key_values_updated_with_cli_args( cli_params) expected_value = False returned_value = default_parameters['connectionPreTest'] self.assertEqual(expected_value, returned_value)
def test_global_config_reader_overwritten_default_value_bool_without_explicit_value_at_end( self): cli_params = [ 'application_name', 'config_file', 'eu-west-1', '--connection-pre-test', 'False', '--destinationTableAutoCreate' ] config_reader = GlobalConfigParametersReader() default_parameters = config_reader.get_config_key_values_updated_with_cli_args( cli_params) expected_value = True returned_value = default_parameters['destinationTableAutoCreate'] self.assertEqual(expected_value, returned_value)
def __init__(self, config_file, region_name, global_config_values=GlobalConfigParametersReader(). get_default_config_key_values()): for key, value in global_config_values.items(): config_parameters[key] = value self.region = region_name self.s3_helper = S3Helper(self.region) # load the configuration self.config_helper = ConfigHelper(config_file, self.s3_helper) source = ResourceFactory.get_source_resource_from_config_helper( self.config_helper, self.region) destination = ResourceFactory.get_target_resource_from_config_helper( self.config_helper, self.region) self.task_manager = TaskManager() self.barrier_after_all_cluster_pre_tests = NoOperationTask() self.task_manager.add_task(self.barrier_after_all_cluster_pre_tests) self.barrier_after_all_resource_pre_tests = NoOperationTask() self.task_manager.add_task(self.barrier_after_all_resource_pre_tests) # TODO: Check whether both resources are of type table if that is not the case then perform other scenario's if isinstance(source, TableResource): if isinstance(destination, DBResource): if not isinstance(destination, TableResource): destination = ResourceFactory.get_table_resource_from_merging_2_resources( destination, source) if global_config_values['tableName'] and global_config_values[ 'tableName'] != 'None': destination.set_table(global_config_values['tableName']) self.add_table_migration(source, destination, global_config_values) else: logging.fatal('Destination should be a database resource') raise NotImplementedError elif isinstance(source, SchemaResource): if not isinstance(destination, DBResource): logging.fatal('Destination should be a database resource') raise NotImplementedError self.add_schema_migration(source, destination, global_config_values) elif isinstance(source, DBResource): if not isinstance(destination, DBResource): logging.fatal('Destination should be a database resource') raise NotImplementedError self.add_database_migration(source, destination, global_config_values) else: # TODO: add additional scenario's # For example if both resources are of type schema then create target schema and migrate all tables logging.fatal( 'Source is not a Table, this type of unload-copy is currently not supported.' ) raise NotImplementedError self.task_manager.run()
def __init__(self, config_file, region_name, global_config_values=GlobalConfigParametersReader(). get_default_config_key_values()): for key, value in global_config_values.items(): config_parameters[key] = value self.region = region_name self.s3_helper = S3Helper(self.region) # load the configuration self.config_helper = ConfigHelper(config_file, self.s3_helper) source = ResourceFactory.get_source_resource_from_config_helper( self.config_helper, self.region) destination = ResourceFactory.get_target_resource_from_config_helper( self.config_helper, self.region) self.task_manager = TaskManager() self.barrier_after_all_cluster_pre_tests = NoOperationTask() self.task_manager.add_task(self.barrier_after_all_cluster_pre_tests) self.barrier_after_all_resource_pre_tests = NoOperationTask() self.task_manager.add_task(self.barrier_after_all_resource_pre_tests) # TODO: Check whether both resources are of type table if that is not the case then perform other scenario's # For example if both resources are of type schema then create target schema and migrate all tables self.add_table_migration(source, destination, global_config_values) self.task_manager.run()
def test_global_config_reader_invalid_region(self): unprocessed_parameters = [ 'application_name', 'config_file', '--region', 'europe-west-1' ] config_reader = GlobalConfigParametersReader() self.assertRaises( DefaultConfigParameter.InvalidConfigException, config_reader.get_config_key_values_updated_with_cli_args, unprocessed_parameters)
def __init__(self, config_file, region_name, global_config_values=GlobalConfigParametersReader(). get_default_config_key_values()): for key, value in global_config_values.items(): config_parameters[key] = value self.region = region_name self.s3_helper = S3Helper(self.region) # load the configuration self.config_helper = ConfigHelper(config_file, self.s3_helper) self.task_manager = TaskManager() self.barrier_after_all_cluster_pre_tests = NoOperationTask() self.task_manager.add_task(self.barrier_after_all_cluster_pre_tests) self.barrier_after_all_resource_pre_tests = NoOperationTask() self.task_manager.add_task(self.barrier_after_all_resource_pre_tests) src_config = self.config_helper.config['unloadSource'] dest_config = self.config_helper.config['copyTarget'] if (src_config['tableNames']): src_tables = src_config['tableNames'] dest_tables = dest_config['tableNames'] logging.info("Migrating multiple tables") if (not dest_tables or len(src_tables) != len(dest_tables)): logging.fatal( "When migrating multiple tables 'tableNames' property must be configured in unloadSource and copyTarget, and be the same length" ) raise NotImplementedError for idx in range(0, len(src_tables)): src_config['tableName'] = src_tables[idx] dest_config['tableName'] = dest_tables[idx] source = ResourceFactory.get_source_resource_from_config_helper( self.config_helper, self.region) destination = ResourceFactory.get_target_resource_from_config_helper( self.config_helper, self.region) self.add_src_dest_tasks(source, destination, global_config_values) else: # Migrating a single table source = ResourceFactory.get_source_resource_from_config_helper( self.config_helper, self.region) destination = ResourceFactory.get_target_resource_from_config_helper( self.config_helper, self.region) self.add_src_dest_tasks(source, destination, global_config_values) self.task_manager.run()
def test_global_config_reader_unknown_key_should_raise_parsing_exception( self): unprocessed_parameters = [ 'application_name', 'config_file', 'eu-west-1' ] full_args = copy.deepcopy(unprocessed_parameters) for parameter in self.processable_parameters: full_args.append(parameter) unknown_key = '--unknown-key' unprocessed_parameters.append(unknown_key) full_args.append(unknown_key) config_reader = GlobalConfigParametersReader() self.assertRaises( GlobalConfigParametersReader.ParsingException, config_reader.get_config_key_values_updated_with_cli_args, full_args) self.assertEqual(unprocessed_parameters, config_reader.unprocessed_arguments)
def test_flag_must_start_with_double_dash(self): self.assertIsNone( GlobalConfigParametersReader.get_key_for_cli_flag('-a_value'))
def test_camel_case_flag_multiple_letters(self): cli_flag = '--a-bc' expected_result = 'aBc' self.assertEqual(expected_result, GlobalConfigParametersReader.get_key_for_cli_flag(cli_flag))
def test_simple_flag(self): self.assertEqual('a', GlobalConfigParametersReader.get_key_for_cli_flag('--a'))
def test_flag_must_not_have_double_dash_inside(self): self.assertIsNone(GlobalConfigParametersReader.get_key_for_cli_flag('--a--value'))
def test_global_config_reader_default_value(self): default_parameters = GlobalConfigParametersReader( ).get_default_config_parameters() expected_value = True returned_value = default_parameters['connectionPreTest'].get_value() self.assertEqual(expected_value, returned_value)
def test_global_config_reader_valid_region(self): valid_region_name = 'us-east-2' unprocessed_parameters = ['application_name', 'config_file', '--region', valid_region_name] config_reader = GlobalConfigParametersReader() config_key_values = config_reader.get_config_key_values_updated_with_cli_args(unprocessed_parameters) self.assertEqual(valid_region_name, config_key_values['region'])
def test_simple_flag(self): self.assertEqual( 'a', GlobalConfigParametersReader.get_key_for_cli_flag('--a'))
def test_flag_must_start_with_double_dash(self): self.assertIsNone(GlobalConfigParametersReader.get_key_for_cli_flag('-a_value'))
def test_flag_must_not_have_double_dash_inside(self): self.assertIsNone( GlobalConfigParametersReader.get_key_for_cli_flag('--a--value'))
def test_global_config_reader_valid_if_boolean_flag_is_used_and_followed_by_no_argument(self): valid_region_name = 'eu-west-1' unprocessed_parameters = ['redshift_unload_copy.py', '--log-level', 'debug', '--destination-table-auto-create', 'conf.json', 'eu-west-1'] config_reader = GlobalConfigParametersReader() config_key_values = config_reader.get_config_key_values_updated_with_cli_args(unprocessed_parameters) self.assertEqual(valid_region_name, config_key_values['region'])
def test_camel_case_flag_multiple_letters(self): cli_flag = '--a-bc' expected_result = 'aBc' self.assertEqual( expected_result, GlobalConfigParametersReader.get_key_for_cli_flag(cli_flag))