def test_handle_parent_autodelta_unfinished(self):
self.migration.status = 'finished'
queuer.migrate_partition_shard(
'TID', 'ns', 'src', 'dst',
force=False, requeue=False, migration_type=orm.MigrationType.DELTA,
parent_migration_id='PMID', latest_migration=self.migration)
self.mox.ReplayAll()
self.worker.handle_parent_autodelta(self.migration, self.parent_migration)
def handle_parent_complete(self, migration, parent_migration):
from shinkansen.worker import queuer
if migration.type == orm.MigrationType.AUTODELTA:
self.queue_migration_verification(parent_migration)
else:
self.log('Initial migration finished for complete migration, starting AUTODELTA')
queuer.migrate_partition_shard(
parent_migration.partition_val, parent_migration.namespace,
parent_migration.source_shard, parent_migration.destination_shard,
force=False, requeue=False, migration_type=orm.MigrationType.AUTODELTA,
parent_migration_id=parent_migration.migration_id, latest_migration=migration)
def handle_parent_autodelta(self, migration, parent_migration):
from shinkansen.worker import queuer
if migration.status == 'empty':
self.log('Auto Delta migration finished with an empty delta migration parent_migration_id=%s',
migration.parent_migration_id)
self.queue_migration_verification(parent_migration)
else:
self.log('Auto Delta migration starting another delta migration, previous migration was not empty')
queuer.migrate_partition_shard(
parent_migration.partition_val, parent_migration.namespace,
parent_migration.source_shard, parent_migration.destination_shard,
force=False, requeue=False, migration_type=orm.MigrationType.DELTA,
parent_migration_id=parent_migration.migration_id, latest_migration=migration)
def test_migrate_partition_shard_namespace(self):
partition_val = 13
configs = [
worker.ChunkConfig('mid', data.TableConfig('TableA', 'filter_col_a', 'chunk_col_a', None), None, partition_val,
'ns', 'shn', 'crate', None, None, None,
'ns', 'ns'),
worker.ChunkConfig('mid', data.TableConfig('TableB', 'filter_col_b', 'chunk_col_b', None), None, partition_val,
'ns', 'shn', 'crate', None, None, None,
'ns', 'ns'),
]
self.expected_configs(configs)
self.mox.ReplayAll()
queuer.migrate_partition_shard(partition_val, 'ns', 'shn', 'crate')
def test_migrate_partition_shard_m(self):
partition_val = 13
configs = [
worker.ChunkConfig('mid', data.TableConfig('MTableA', 'm_filter_col_a', 'm_chunk_col_a', None), None,
partition_val,
'', 'shn_m', 'crate_m', None, None, None,
'mysql_m', 'crate_m'),
worker.ChunkConfig('mid', data.TableConfig('MTableB', 'm_filter_col_b', 'm_chunk_col_b', None), None,
partition_val,
'', 'shn_m', 'crate_m', None, None, None,
'mysql_m', 'crate_m'),
]
self.expected_configs(configs)
self.mox.ReplayAll()
queuer.migrate_partition_shard(partition_val, '', 'shn_m', 'crate_m')
def start_migration(source, destination, partition_val, migration_type):
parser = reqparse.RequestParser()
parser.add_argument('namespace', type=str, help='the namespace to prefix the schema with', default='')
parser.add_argument('requeue', type=strbool, help='requeue migrations')
parser.add_argument('force', type=strbool, help='force re-migration (will not delete existing data)')
parser.add_argument('wanted_delta_start', type=int, help='wanted delta start timestamp in seconds')
parser.add_argument('wanted_delta_end', type=int, help='wanted delta end timestamp in seconds')
parser.add_argument(
'chunk_migration_type', type=str, help='indirect or direct',
default=orm.ChunkMigrationType.INDIRECT
)
args = parser.parse_args()
if args['chunk_migration_type'] not in orm.ChunkMigrationType.__ALL__:
raise UnrecoverableError('chunk_migration_type %r is not valid' % (args['chunk_migration_type'],))
if (
migration_type not in orm.MigrationType.__DELTA_TYPES__
and (args['wanted_delta_start'] is not None or args['wanted_delta_end'] is not None)
):
return {'error': 'wanted_delta_start and wanted_delta_end are only allowed for delta migration types'}
(migration, tables) = queuer.migrate_partition_shard(
partition_val, args['namespace'], source, destination,
force=args['force'], requeue=args['requeue'], migration_type=migration_type,
wanted_delta_start=args['wanted_delta_start'],
wanted_delta_end=args['wanted_delta_end'],
chunk_migration_type=args['chunk_migration_type']
)
response = {
'migration_id': migration.migration_id,
'source': source,
'destination': destination,
'namespace': args['namespace'],
'links': [
{
'href': '/v5/migration/%s' % (migration.migration_id,),
'rel': 'migration',
'method': 'GET',
}
],
'tables': {},
}
error = False
for table_name, table in tables.iteritems():
if table is None:
response['tables'][table_name] = {
'status': 'error',
'error': 'Table was not queued due to it being either in progress or previously migrated',
}
error = True
else:
response['tables'][table_name] = {'status': table.status}
if error:
response['error'] = 'Not all tables were queued.'
return response