def test_finished_is_verified(self):
self.migration.update(lock=self.lock)
status.get_migration_status(migration_id='mid').AndReturn(
status.MigrationStatus('finished', 'pending', {}, {}, {}))
self.mox.StubOutWithMock(self.worker, 'verify_migration')
self.worker.verify_migration(self.migration, self.lock)
self.mox.ReplayAll()
self.worker._run()
def _run(self):
start = datetime.now()
self.log('Pipe worker starting')
self.chunk.status = 'migrating'
self.chunk.num_records_exported = 0
self.chunk.num_records_imported = 0
self.chunk.update()
with db.shard_connection(self.c.source_shard, read=True) as source_conn:
with db.shard_connection(self.c.destination_shard, read=False) as dest_conn:
update_status = self.migrate(source_conn, dest_conn)
if update_status:
self.chunk.import_elapsed_ms = int((datetime.now() - start).total_seconds() * 1000)
self.chunk.status = 'imported'
self.chunk.update()
(migration_status, _, _, _, _) = status.get_migration_status(migration_id=self.c.migration_id)
migration = orm.Migration.get(self.redis_conn, migration_id=self.c.migration_id)
migration.status = migration_status
migration.update()
if config.ENABLE_VERIFIER:
verifier.queue_verification(self.c)
self.log('Pipe worker finished elapsed=%s', datetime.now() - start)
def __run(self):
start = datetime.now()
migration = orm.Migration(self.redis_conn)
migration.migration_id = self.c.migration_id
# TODO: Expose a record's lock via a class method so we don't have to create a dummy object above
lock = migration._lock()
try:
with lock:
migration = orm.Migration.get(self.redis_conn, migration_id=self.c.migration_id)
if migration.verification_status in ('verified', 'failed'):
self.log_debug('This migration has already finished verification, not reverifying')
return
migration_status = status.get_migration_status(migration_id=self.c.migration_id)
migration.status = migration_status.computed_status
migration.update(lock=lock)
if migration_status.computed_status in ('finished', 'empty'):
self.verify_migration(migration, lock)
else:
self.log_debug('Migration not finished or empty (%r), skipping verification for now',
migration_status)
return
except redlock.lock.RedLockError:
msg = 'Failed to acquire lock on migration record, requeueing verification'
self.log_debug(msg)
raise shinkansen.worker.RetryException(msg)
self.log('Finished verify elapsed=%s', datetime.now() - start)
def test_exception_unsets_chunk(self):
status.get_migration_status(migration_id='mid').AndRaise(Exception())
self.mox.ReplayAll()
with self.assertRaises(Exception):
self.worker._run()
self.assertIsNone(self.worker.chunk)
def test_noop_until_finished(self):
self.migration.update(lock=self.lock)
status.get_migration_status(migration_id='mid').AndReturn(
status.MigrationStatus('in_progress', 'pending', {}, {}, {}))
self.mox.ReplayAll()
self.worker._run()
def _test_get_migration_status(
self, table_status_recs, table_verification_status_recs, chunk_status_recs, expected
):
self._test_get_migration_status_base(table_status_recs, table_verification_status_recs, chunk_status_recs)
self.assertEqual(status.get_migration_status('src', 'dst', 42), expected)
def test_get_migration_status_only_destination(self):
with self.assertRaises(shinkansen.UnrecoverableError):
status.get_migration_status(None, 'dst', 42)
def test_get_migration_status_only_source(self):
with self.assertRaises(shinkansen.UnrecoverableError):
status.get_migration_status('src', None, 42)
def test_get_migration_status_no_partition(self):
with self.assertRaises(shinkansen.UnrecoverableError):
status.get_migration_status('src', 'dst', None)