def __init__(self,
config_file,
region_name,
global_config_values=GlobalConfigParametersReader().
get_default_config_key_values()):
for key, value in global_config_values.items():
config_parameters[key] = value
self.region = region_name
self.s3_helper = S3Helper(self.region)
# load the configuration
self.config_helper = ConfigHelper(config_file, self.s3_helper)
source = ResourceFactory.get_source_resource_from_config_helper(
self.config_helper, self.region)
destination = ResourceFactory.get_target_resource_from_config_helper(
self.config_helper, self.region)
self.task_manager = TaskManager()
self.barrier_after_all_cluster_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_cluster_pre_tests)
self.barrier_after_all_resource_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_resource_pre_tests)
# TODO: Check whether both resources are of type table if that is not the case then perform other scenario's
if isinstance(source, TableResource):
if isinstance(destination, DBResource):
if not isinstance(destination, TableResource):
destination = ResourceFactory.get_table_resource_from_merging_2_resources(
destination, source)
if global_config_values['tableName'] and global_config_values[
'tableName'] != 'None':
destination.set_table(global_config_values['tableName'])
self.add_table_migration(source, destination,
global_config_values)
else:
logging.fatal('Destination should be a database resource')
raise NotImplementedError
elif isinstance(source, SchemaResource):
if not isinstance(destination, DBResource):
logging.fatal('Destination should be a database resource')
raise NotImplementedError
self.add_schema_migration(source, destination,
global_config_values)
elif isinstance(source, DBResource):
if not isinstance(destination, DBResource):
logging.fatal('Destination should be a database resource')
raise NotImplementedError
self.add_database_migration(source, destination,
global_config_values)
else:
# TODO: add additional scenario's
# For example if both resources are of type schema then create target schema and migrate all tables
logging.fatal(
'Source is not a Table, this type of unload-copy is currently not supported.'
)
raise NotImplementedError
self.task_manager.run()
def __init__(self,
config_file,
region_name,
global_config_values=GlobalConfigParametersReader().get_default_config_key_values()):
for key, value in global_config_values.items():
config_parameters[key] = value
self.region = region_name
self.s3_helper = S3Helper(self.region)
# load the configuration
self.config_helper = ConfigHelper(config_file, self.s3_helper)
source = ResourceFactory.get_source_resource_from_config_helper(self.config_helper, self.region)
destination = ResourceFactory.get_target_resource_from_config_helper(self.config_helper, self.region)
self.task_manager = TaskManager()
self.barrier_after_all_cluster_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_cluster_pre_tests)
self.barrier_after_all_resource_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_resource_pre_tests)
# TODO: Check whether both resources are of type table if that is not the case then perform other scenario's
# For example if both resources are of type schema then create target schema and migrate all tables
self.add_table_migration(source, destination, global_config_values)
self.task_manager.run()
def __init__(self,
config_file,
region_name,
global_config_values=GlobalConfigParametersReader().
get_default_config_key_values()):
for key, value in global_config_values.items():
config_parameters[key] = value
self.region = region_name
self.s3_helper = S3Helper(self.region)
# load the configuration
self.config_helper = ConfigHelper(config_file, self.s3_helper)
source = ResourceFactory.get_source_resource_from_config_helper(
self.config_helper, self.region)
destination = ResourceFactory.get_target_resource_from_config_helper(
self.config_helper, self.region)
self.task_manager = TaskManager()
self.barrier_after_all_cluster_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_cluster_pre_tests)
self.barrier_after_all_resource_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_resource_pre_tests)
# TODO: Check whether both resources are of type table if that is not the case then perform other scenario's
# For example if both resources are of type schema then create target schema and migrate all tables
self.add_table_migration(source, destination, global_config_values)
self.task_manager.run()
def __init__(self,
config_file,
region_name,
global_config_values=GlobalConfigParametersReader().
get_default_config_key_values()):
for key, value in global_config_values.items():
config_parameters[key] = value
self.region = region_name
self.s3_helper = S3Helper(self.region)
# load the configuration
self.config_helper = ConfigHelper(config_file, self.s3_helper)
self.task_manager = TaskManager()
self.barrier_after_all_cluster_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_cluster_pre_tests)
self.barrier_after_all_resource_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_resource_pre_tests)
src_config = self.config_helper.config['unloadSource']
dest_config = self.config_helper.config['copyTarget']
if (src_config['tableNames']):
src_tables = src_config['tableNames']
dest_tables = dest_config['tableNames']
logging.info("Migrating multiple tables")
if (not dest_tables or len(src_tables) != len(dest_tables)):
logging.fatal(
"When migrating multiple tables 'tableNames' property must be configured in unloadSource and copyTarget, and be the same length"
)
raise NotImplementedError
for idx in range(0, len(src_tables)):
src_config['tableName'] = src_tables[idx]
dest_config['tableName'] = dest_tables[idx]
source = ResourceFactory.get_source_resource_from_config_helper(
self.config_helper, self.region)
destination = ResourceFactory.get_target_resource_from_config_helper(
self.config_helper, self.region)
self.add_src_dest_tasks(source, destination,
global_config_values)
else:
# Migrating a single table
source = ResourceFactory.get_source_resource_from_config_helper(
self.config_helper, self.region)
destination = ResourceFactory.get_target_resource_from_config_helper(
self.config_helper, self.region)
self.add_src_dest_tasks(source, destination, global_config_values)
self.task_manager.run()
def add_schema_migration(self, source, destination, global_config_values):
tables = source.list_tables()
for table in tables:
source_table = TableResource(source.get_cluster(),
source.get_schema(), table)
target_table = ResourceFactory.get_table_resource_from_merging_2_resources(
destination, source_table)
if 'explicit_ids' in self.config_helper.config['copyTarget']:
if self.config_helper.config['copyTarget']['explicit_ids']:
target_table.set_explicit_ids(True)
self.add_table_migration(source_table, target_table,
global_config_values)
def __init__(self,
config_file,
region_name,
global_config_values=GlobalConfigParametersReader().get_default_config_key_values()):
for key, value in global_config_values.items():
config_parameters[key] = value
self.region = region_name
self.s3_helper = S3Helper(self.region)
# load the configuration
self.config_helper = ConfigHelper(config_file, self.s3_helper)
self.task_manager = TaskManager()
self.barrier_after_all_cluster_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_cluster_pre_tests)
self.barrier_after_all_resource_pre_tests = NoOperationTask()
self.task_manager.add_task(self.barrier_after_all_resource_pre_tests)
src_config = self.config_helper.config['unloadSource']
dest_config = self.config_helper.config['copyTarget']
if(src_config['tableNames']):
src_tables = src_config['tableNames']
dest_tables = dest_config['tableNames']
logging.info("Migrating multiple tables")
if( not dest_tables or len(src_tables) != len(dest_tables) ):
logging.fatal("When migrating multiple tables 'tableNames' property must be configured in unloadSource and copyTarget, and be the same length")
raise NotImplementedError
for idx in range(0,len(src_tables)):
src_config['tableName'] = src_tables[idx]
dest_config['tableName'] = dest_tables[idx]
source = ResourceFactory.get_source_resource_from_config_helper(self.config_helper, self.region)
destination = ResourceFactory.get_target_resource_from_config_helper(self.config_helper, self.region)
self.add_src_dest_tasks(source,destination,global_config_values)
else:
# Migrating a single table
source = ResourceFactory.get_source_resource_from_config_helper(self.config_helper, self.region)
destination = ResourceFactory.get_target_resource_from_config_helper(self.config_helper, self.region)
self.add_src_dest_tasks(source,destination,global_config_values)
self.task_manager.run()
def add_src_dest_tasks(self,source,destination,global_config_values):
# TODO: Check whether both resources are of type table if that is not the case then perform other scenario's
if isinstance(source, TableResource):
if isinstance(destination, DBResource):
if not isinstance(destination, TableResource):
destination = ResourceFactory.get_table_resource_from_merging_2_resources(destination, source)
if global_config_values['tableName'] and global_config_values['tableName'] != 'None':
destination.set_table(global_config_values['tableName'])
self.add_table_migration(source, destination, global_config_values)
else:
logging.fatal('Destination should be a database resource')
raise NotImplementedError
pass
else:
# TODO: add additional scenario's
# For example if both resources are of type schema then create target schema and migrate all tables
logging.fatal('Source is not a Table, this type of unload-copy is currently not supported.')
raise NotImplementedError
def add_src_dest_tasks(self,source,destination,global_config_values):
# TODO: Check whether both resources are of type table if that is not the case then perform other scenario's
if isinstance(source, TableResource):
if isinstance(destination, DBResource):
if not isinstance(destination, TableResource):
destination = ResourceFactory.get_table_resource_from_merging_2_resources(destination, source)
if global_config_values['tableName'] and global_config_values['tableName'] != 'None':
destination.set_table(global_config_values['tableName'])
self.add_table_migration(source, destination, global_config_values)
else:
logging.fatal('Destination should be a database resource')
raise NotImplementedError
pass
else:
# TODO: add additional scenario's
# For example if both resources are of type schema then create target schema and migrate all tables
logging.fatal('Source is not a Table, this type of unload-copy is currently not supported.')
raise NotImplementedError