def test_global_config_reader_overwritten_default_value(self):
cli_params = ['application_name', 'config_file', 'eu-west-1', '--connection-pre-test', 'False']
config_reader = GlobalConfigParametersReader()
default_parameters = config_reader.get_config_key_values_updated_with_cli_args(cli_params)
expected_value = False
returned_value = default_parameters['connectionPreTest']
self.assertEqual(expected_value, returned_value)
def test_global_config_reader_overwritten_default_value_bool_without_explicit_value_at_end(self):
cli_params = ['application_name', 'config_file', 'eu-west-1', '--connection-pre-test', 'False', '--destinationTableAutoCreate']
config_reader = GlobalConfigParametersReader()
default_parameters = config_reader.get_config_key_values_updated_with_cli_args(cli_params)
expected_value = True
returned_value = default_parameters['destinationTableAutoCreate']
self.assertEqual(expected_value, returned_value)
def setUp(self):
global_config_reader = GlobalConfigParametersReader()
arguments = ['app-name', 'config-file', 'eu-west-1', '--destination-schema-auto-create',
'--destination-table-auto-create']
global_config_reader.get_config_key_values_updated_with_cli_args(arguments)
self.configure_test_cluster()
self.create_test_table()
def main(args):
global region
global_config_reader = GlobalConfigParametersReader()
global_config_values = global_config_reader.get_config_key_values_updated_with_cli_args(args)
set_log_level(global_config_values['logLevel'])
UnloadCopyTool(global_config_values['s3ConfigFile'], global_config_values['region'], global_config_values)
def test_global_config_reader_unprocessed_values(self):
unprocessed_parameters = ['application_name', 'config_file', 'eu-west-1']
full_args = copy.deepcopy(unprocessed_parameters)
for parameter in self.processable_parameters:
full_args.append(parameter)
config_reader = GlobalConfigParametersReader()
config_reader.get_config_key_values_updated_with_cli_args(full_args)
self.assertEqual(unprocessed_parameters, config_reader.unprocessed_arguments)
def main(args):
global region
global_config_reader = GlobalConfigParametersReader()
global_config_values = global_config_reader.get_config_key_values_updated_with_cli_args(args)
set_log_level(global_config_values['logLevel'])
UnloadCopyTool(global_config_values['s3ConfigFile'], global_config_values['region'], global_config_values)
def test_global_config_reader_valid_region(self):
valid_region_name = 'us-east-2'
unprocessed_parameters = [
'application_name', 'config_file', '--region', valid_region_name
]
config_reader = GlobalConfigParametersReader()
config_key_values = config_reader.get_config_key_values_updated_with_cli_args(
unprocessed_parameters)
self.assertEqual(valid_region_name, config_key_values['region'])
def test_global_config_reader_valid_if_boolean_flag_is_used_and_followed_by_no_argument(
self):
valid_region_name = 'eu-west-1'
unprocessed_parameters = [
'redshift_unload_copy.py', '--log-level', 'debug',
'--destination-table-auto-create', 'conf.json', 'eu-west-1'
]
config_reader = GlobalConfigParametersReader()
config_key_values = config_reader.get_config_key_values_updated_with_cli_args(
unprocessed_parameters)
self.assertEqual(valid_region_name, config_key_values['region'])
def setUp(self):
global_config_reader = GlobalConfigParametersReader()
arguments = [
'app-name', 'config-file', 'eu-west-1',
'--destination-schema-auto-create',
'--destination-table-auto-create'
]
global_config_reader.get_config_key_values_updated_with_cli_args(
arguments)
self.configure_test_cluster()
self.create_test_table()
def test_global_config_reader_unprocessed_values(self):
unprocessed_parameters = [
'application_name', 'config_file', 'eu-west-1'
]
full_args = copy.deepcopy(unprocessed_parameters)
for parameter in self.processable_parameters:
full_args.append(parameter)
config_reader = GlobalConfigParametersReader()
config_reader.get_config_key_values_updated_with_cli_args(full_args)
self.assertEqual(unprocessed_parameters,
config_reader.unprocessed_arguments)
def test_global_config_reader_overwritten_default_value(self):
cli_params = [
'application_name', 'config_file', 'eu-west-1',
'--connection-pre-test', 'False'
]
config_reader = GlobalConfigParametersReader()
default_parameters = config_reader.get_config_key_values_updated_with_cli_args(
cli_params)
expected_value = False
returned_value = default_parameters['connectionPreTest']
self.assertEqual(expected_value, returned_value)
def test_global_config_reader_overwritten_default_value_bool_without_explicit_value_at_end(
self):
cli_params = [
'application_name', 'config_file', 'eu-west-1',
'--connection-pre-test', 'False', '--destinationTableAutoCreate'
]
config_reader = GlobalConfigParametersReader()
default_parameters = config_reader.get_config_key_values_updated_with_cli_args(
cli_params)
expected_value = True
returned_value = default_parameters['destinationTableAutoCreate']
self.assertEqual(expected_value, returned_value)
def __init__(self,
config_file,
region_name,
global_config_values=GlobalConfigParametersReader().
get_default_config_key_values()):
for key, value in global_config_values.items():
config_parameters[key] = value
self.region = region_name
self.s3_helper = S3Helper(self.region)
# load the configuration
self.config_helper = ConfigHelper(config_file, self.s3_helper)
source = ResourceFactory.get_source_resource_from_config_helper(
self.config_helper, self.region)
destination = ResourceFactory.get_target_resource_from_config_helper(
self.config_helper, self.region)
self.task_manager = TaskManager()
self.barrier_after_all_cluster_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_cluster_pre_tests)
self.barrier_after_all_resource_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_resource_pre_tests)
# TODO: Check whether both resources are of type table if that is not the case then perform other scenario's
if isinstance(source, TableResource):
if isinstance(destination, DBResource):
if not isinstance(destination, TableResource):
destination = ResourceFactory.get_table_resource_from_merging_2_resources(
destination, source)
if global_config_values['tableName'] and global_config_values[
'tableName'] != 'None':
destination.set_table(global_config_values['tableName'])
self.add_table_migration(source, destination,
global_config_values)
else:
logging.fatal('Destination should be a database resource')
raise NotImplementedError
elif isinstance(source, SchemaResource):
if not isinstance(destination, DBResource):
logging.fatal('Destination should be a database resource')
raise NotImplementedError
self.add_schema_migration(source, destination,
global_config_values)
elif isinstance(source, DBResource):
if not isinstance(destination, DBResource):
logging.fatal('Destination should be a database resource')
raise NotImplementedError
self.add_database_migration(source, destination,
global_config_values)
else:
# TODO: add additional scenario's
# For example if both resources are of type schema then create target schema and migrate all tables
logging.fatal(
'Source is not a Table, this type of unload-copy is currently not supported.'
)
raise NotImplementedError
self.task_manager.run()
def __init__(self,
config_file,
region_name,
global_config_values=GlobalConfigParametersReader().
get_default_config_key_values()):
for key, value in global_config_values.items():
config_parameters[key] = value
self.region = region_name
self.s3_helper = S3Helper(self.region)
# load the configuration
self.config_helper = ConfigHelper(config_file, self.s3_helper)
source = ResourceFactory.get_source_resource_from_config_helper(
self.config_helper, self.region)
destination = ResourceFactory.get_target_resource_from_config_helper(
self.config_helper, self.region)
self.task_manager = TaskManager()
self.barrier_after_all_cluster_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_cluster_pre_tests)
self.barrier_after_all_resource_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_resource_pre_tests)
# TODO: Check whether both resources are of type table if that is not the case then perform other scenario's
# For example if both resources are of type schema then create target schema and migrate all tables
self.add_table_migration(source, destination, global_config_values)
self.task_manager.run()
def test_global_config_reader_invalid_region(self):
unprocessed_parameters = [
'application_name', 'config_file', '--region', 'europe-west-1'
]
config_reader = GlobalConfigParametersReader()
self.assertRaises(
DefaultConfigParameter.InvalidConfigException,
config_reader.get_config_key_values_updated_with_cli_args,
unprocessed_parameters)
def __init__(self,
config_file,
region_name,
global_config_values=GlobalConfigParametersReader().
get_default_config_key_values()):
for key, value in global_config_values.items():
config_parameters[key] = value
self.region = region_name
self.s3_helper = S3Helper(self.region)
# load the configuration
self.config_helper = ConfigHelper(config_file, self.s3_helper)
self.task_manager = TaskManager()
self.barrier_after_all_cluster_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_cluster_pre_tests)
self.barrier_after_all_resource_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_resource_pre_tests)
src_config = self.config_helper.config['unloadSource']
dest_config = self.config_helper.config['copyTarget']
if (src_config['tableNames']):
src_tables = src_config['tableNames']
dest_tables = dest_config['tableNames']
logging.info("Migrating multiple tables")
if (not dest_tables or len(src_tables) != len(dest_tables)):
logging.fatal(
"When migrating multiple tables 'tableNames' property must be configured in unloadSource and copyTarget, and be the same length"
)
raise NotImplementedError
for idx in range(0, len(src_tables)):
src_config['tableName'] = src_tables[idx]
dest_config['tableName'] = dest_tables[idx]
source = ResourceFactory.get_source_resource_from_config_helper(
self.config_helper, self.region)
destination = ResourceFactory.get_target_resource_from_config_helper(
self.config_helper, self.region)
self.add_src_dest_tasks(source, destination,
global_config_values)
else:
# Migrating a single table
source = ResourceFactory.get_source_resource_from_config_helper(
self.config_helper, self.region)
destination = ResourceFactory.get_target_resource_from_config_helper(
self.config_helper, self.region)
self.add_src_dest_tasks(source, destination, global_config_values)
self.task_manager.run()
def test_global_config_reader_unknown_key_should_raise_parsing_exception(
self):
unprocessed_parameters = [
'application_name', 'config_file', 'eu-west-1'
]
full_args = copy.deepcopy(unprocessed_parameters)
for parameter in self.processable_parameters:
full_args.append(parameter)
unknown_key = '--unknown-key'
unprocessed_parameters.append(unknown_key)
full_args.append(unknown_key)
config_reader = GlobalConfigParametersReader()
self.assertRaises(
GlobalConfigParametersReader.ParsingException,
config_reader.get_config_key_values_updated_with_cli_args,
full_args)
self.assertEqual(unprocessed_parameters,
config_reader.unprocessed_arguments)
def test_flag_must_start_with_double_dash(self):
self.assertIsNone(
GlobalConfigParametersReader.get_key_for_cli_flag('-a_value'))
def test_camel_case_flag_multiple_letters(self):
cli_flag = '--a-bc'
expected_result = 'aBc'
self.assertEqual(expected_result, GlobalConfigParametersReader.get_key_for_cli_flag(cli_flag))
def test_simple_flag(self):
self.assertEqual('a', GlobalConfigParametersReader.get_key_for_cli_flag('--a'))
def test_flag_must_not_have_double_dash_inside(self):
self.assertIsNone(GlobalConfigParametersReader.get_key_for_cli_flag('--a--value'))
def test_global_config_reader_default_value(self):
default_parameters = GlobalConfigParametersReader(
).get_default_config_parameters()
expected_value = True
returned_value = default_parameters['connectionPreTest'].get_value()
self.assertEqual(expected_value, returned_value)
def test_global_config_reader_valid_region(self):
valid_region_name = 'us-east-2'
unprocessed_parameters = ['application_name', 'config_file', '--region', valid_region_name]
config_reader = GlobalConfigParametersReader()
config_key_values = config_reader.get_config_key_values_updated_with_cli_args(unprocessed_parameters)
self.assertEqual(valid_region_name, config_key_values['region'])
def test_simple_flag(self):
self.assertEqual(
'a', GlobalConfigParametersReader.get_key_for_cli_flag('--a'))
def test_flag_must_start_with_double_dash(self):
self.assertIsNone(GlobalConfigParametersReader.get_key_for_cli_flag('-a_value'))
def test_flag_must_not_have_double_dash_inside(self):
self.assertIsNone(
GlobalConfigParametersReader.get_key_for_cli_flag('--a--value'))
def test_global_config_reader_valid_if_boolean_flag_is_used_and_followed_by_no_argument(self):
valid_region_name = 'eu-west-1'
unprocessed_parameters = ['redshift_unload_copy.py', '--log-level', 'debug', '--destination-table-auto-create', 'conf.json', 'eu-west-1']
config_reader = GlobalConfigParametersReader()
config_key_values = config_reader.get_config_key_values_updated_with_cli_args(unprocessed_parameters)
self.assertEqual(valid_region_name, config_key_values['region'])
def test_camel_case_flag_multiple_letters(self):
cli_flag = '--a-bc'
expected_result = 'aBc'
self.assertEqual(
expected_result,
GlobalConfigParametersReader.get_key_for_cli_flag(cli_flag))
