def test_handle_parent_autodelta_unfinished(self): self.migration.status = 'finished' queuer.migrate_partition_shard( 'TID', 'ns', 'src', 'dst', force=False, requeue=False, migration_type=orm.MigrationType.DELTA, parent_migration_id='PMID', latest_migration=self.migration) self.mox.ReplayAll() self.worker.handle_parent_autodelta(self.migration, self.parent_migration)
def handle_parent_complete(self, migration, parent_migration): from shinkansen.worker import queuer if migration.type == orm.MigrationType.AUTODELTA: self.queue_migration_verification(parent_migration) else: self.log('Initial migration finished for complete migration, starting AUTODELTA') queuer.migrate_partition_shard( parent_migration.partition_val, parent_migration.namespace, parent_migration.source_shard, parent_migration.destination_shard, force=False, requeue=False, migration_type=orm.MigrationType.AUTODELTA, parent_migration_id=parent_migration.migration_id, latest_migration=migration)
def handle_parent_autodelta(self, migration, parent_migration): from shinkansen.worker import queuer if migration.status == 'empty': self.log('Auto Delta migration finished with an empty delta migration parent_migration_id=%s', migration.parent_migration_id) self.queue_migration_verification(parent_migration) else: self.log('Auto Delta migration starting another delta migration, previous migration was not empty') queuer.migrate_partition_shard( parent_migration.partition_val, parent_migration.namespace, parent_migration.source_shard, parent_migration.destination_shard, force=False, requeue=False, migration_type=orm.MigrationType.DELTA, parent_migration_id=parent_migration.migration_id, latest_migration=migration)
def test_migrate_partition_shard_namespace(self): partition_val = 13 configs = [ worker.ChunkConfig('mid', data.TableConfig('TableA', 'filter_col_a', 'chunk_col_a', None), None, partition_val, 'ns', 'shn', 'crate', None, None, None, 'ns', 'ns'), worker.ChunkConfig('mid', data.TableConfig('TableB', 'filter_col_b', 'chunk_col_b', None), None, partition_val, 'ns', 'shn', 'crate', None, None, None, 'ns', 'ns'), ] self.expected_configs(configs) self.mox.ReplayAll() queuer.migrate_partition_shard(partition_val, 'ns', 'shn', 'crate')
def test_migrate_partition_shard_m(self): partition_val = 13 configs = [ worker.ChunkConfig('mid', data.TableConfig('MTableA', 'm_filter_col_a', 'm_chunk_col_a', None), None, partition_val, '', 'shn_m', 'crate_m', None, None, None, 'mysql_m', 'crate_m'), worker.ChunkConfig('mid', data.TableConfig('MTableB', 'm_filter_col_b', 'm_chunk_col_b', None), None, partition_val, '', 'shn_m', 'crate_m', None, None, None, 'mysql_m', 'crate_m'), ] self.expected_configs(configs) self.mox.ReplayAll() queuer.migrate_partition_shard(partition_val, '', 'shn_m', 'crate_m')
def start_migration(source, destination, partition_val, migration_type): parser = reqparse.RequestParser() parser.add_argument('namespace', type=str, help='the namespace to prefix the schema with', default='') parser.add_argument('requeue', type=strbool, help='requeue migrations') parser.add_argument('force', type=strbool, help='force re-migration (will not delete existing data)') parser.add_argument('wanted_delta_start', type=int, help='wanted delta start timestamp in seconds') parser.add_argument('wanted_delta_end', type=int, help='wanted delta end timestamp in seconds') parser.add_argument( 'chunk_migration_type', type=str, help='indirect or direct', default=orm.ChunkMigrationType.INDIRECT ) args = parser.parse_args() if args['chunk_migration_type'] not in orm.ChunkMigrationType.__ALL__: raise UnrecoverableError('chunk_migration_type %r is not valid' % (args['chunk_migration_type'],)) if ( migration_type not in orm.MigrationType.__DELTA_TYPES__ and (args['wanted_delta_start'] is not None or args['wanted_delta_end'] is not None) ): return {'error': 'wanted_delta_start and wanted_delta_end are only allowed for delta migration types'} (migration, tables) = queuer.migrate_partition_shard( partition_val, args['namespace'], source, destination, force=args['force'], requeue=args['requeue'], migration_type=migration_type, wanted_delta_start=args['wanted_delta_start'], wanted_delta_end=args['wanted_delta_end'], chunk_migration_type=args['chunk_migration_type'] ) response = { 'migration_id': migration.migration_id, 'source': source, 'destination': destination, 'namespace': args['namespace'], 'links': [ { 'href': '/v5/migration/%s' % (migration.migration_id,), 'rel': 'migration', 'method': 'GET', } ], 'tables': {}, } error = False for table_name, table in tables.iteritems(): if table is None: response['tables'][table_name] = { 'status': 'error', 'error': 'Table was not queued due to it being either in progress or previously migrated', } error = True else: response['tables'][table_name] = {'status': table.status} if error: response['error'] = 'Not all tables were queued.' return response