def _run(self):
start = datetime.now()
self.log('Starting table migration queueing')
self.get_table_metadata()
self.table.min_id = self.min_id
self.table.max_id = self.max_id
self.table.num_records = self.num_rows
if self.num_rows == 0 or self.num_rows is None:
self.log('No rows to migrate')
self.table.end_time = int(time.time() * 1000)
self.table.status = 'empty'
self.table.update()
# If the table is empty, jump straight to the verification so end-of-migration logic is run.
verifier.queue_verification(self.c)
return
self.queue_chunks()
self.table.end_time = int(time.time() * 1000)
self.table.status = 'chunks_queued'
self.table.num_chunks = self.num_chunks
self.table.chunk_size = self.c.chunk_size
self.table.update()
self.log('Finished queueing table migration elapsed=%s', datetime.now() - start)
def _run(self):
start = datetime.now()
self.log('Pipe worker starting')
self.chunk.status = 'migrating'
self.chunk.num_records_exported = 0
self.chunk.num_records_imported = 0
self.chunk.update()
with db.shard_connection(self.c.source_shard, read=True) as source_conn:
with db.shard_connection(self.c.destination_shard, read=False) as dest_conn:
update_status = self.migrate(source_conn, dest_conn)
if update_status:
self.chunk.import_elapsed_ms = int((datetime.now() - start).total_seconds() * 1000)
self.chunk.status = 'imported'
self.chunk.update()
(migration_status, _, _, _, _) = status.get_migration_status(migration_id=self.c.migration_id)
migration = orm.Migration.get(self.redis_conn, migration_id=self.c.migration_id)
migration.status = migration_status
migration.update()
if config.ENABLE_VERIFIER:
verifier.queue_verification(self.c)
self.log('Pipe worker finished elapsed=%s', datetime.now() - start)
def _run(self):
if self.c.num_records_converted > 0:
self.chunk.status = 'importing'
self.chunk.update()
start = datetime.now()
if self.c.destination_type == 'crate':
self._import_to_crate()
elif self.c.destination_type == 'mysql':
self._import_to_mysql()
else:
raise Error('Unknown destination type %r' % (self.c.destination_type,))
self.log('Import to destination finished num_records_imported=%s destination_host=%s elapsed=%s',
self.c.num_records_imported, self.c.destination_host, datetime.now() - start)
self.chunk.status = 'imported'
self.chunk.num_records_imported = self.c.num_records_imported
self.chunk.import_elapsed_ms = int((datetime.now() - start).total_seconds() * 1000)
self.chunk.end_time = int(time.time() * 1000)
self.chunk.update()
else:
self.log('No rows to import')
self.chunk.status = 'empty'
self.chunk.end_time = int(time.time() * 1000)
self.chunk.update()
if config.ENABLE_VERIFIER:
verifier.queue_verification(self.c)
# TODO(jpatrin): Refactor this into its own task so it can be independently retried
try:
cmd = (
'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR '
'-i %s -p %r %s@%s "sudo bash -c \'rm -f %s\'"' % (
config.SSH_PRIVKEY, self.c.destination_ssh_port, config.SSH_USER, self.c.destination_host,
ssh.escape_double_quotes(ssh.escape_single_quotes(self.c.import_filename))))
rm_cmd = subprocess.Popen(
cmd,
shell=True,
stdin=subprocess.PIPE)
rm_cmd.stdin.close()
rm_cmd.wait()
if rm_cmd.returncode != 0:
raise CommandException('Removing file on destination server failed with exit code %r' % (
rm_cmd.returncode,))
except Exception, e:
# We catch and log all exceptions here to make this task idempotent. We DO NOT want this task to be
# retried at this point as duplicate imports can fail on mysql and would cause us to corrupt the crate
# chunk import records.
self.log('Exception during removal of destination file, removal will not be retried %r import_filename=%s',
e, self.c.import_filename)
def test_run_no_records(self):
self.config.num_records_converted = 0
import_worker = TestableImportChunkWorker(self.config)
import_worker.chunk = self.mox.CreateMock(orm.Chunk)
import_worker.chunk.update()
self.mox.StubOutWithMock(verifier, 'queue_verification')
verifier.queue_verification(self.config)
self.mox.StubOutClassWithMocks(subprocess, 'Popen')
proc = subprocess.Popen(mox.IgnoreArg(), shell=True, stdin=subprocess.PIPE)
proc.stdin = self.mox.CreateMockAnything()
proc.stdin.close()
proc.wait()
proc.returncode = 0
self.mox.ReplayAll()
import_worker._run()
def test_run_removal_fails(self):
self.config.num_records_converted = 0
import_worker = TestableImportChunkWorker(self.config)
import_worker.chunk = self.mox.CreateMock(orm.Chunk)
import_worker.chunk.update()
self.mox.StubOutWithMock(verifier, 'queue_verification')
verifier.queue_verification(self.config)
self.mox.StubOutClassWithMocks(subprocess, 'Popen')
proc = subprocess.Popen(mox.IgnoreArg(), shell=True, stdin=subprocess.PIPE)
proc.stdin = self.mox.CreateMockAnything()
proc.stdin.close()
proc.wait()
proc.returncode = 1
# Note that we do not get an exception in this case, it is only logged in order to make
# this task idempotent.
self.mox.ReplayAll()
import_worker._run()
def test_queue_migration_verification(self):
self.migration.migration_id = 'MID'
self.migration.partition_val = 'TID'
self.migration.namespace = 'ns'
self.migration.source_shard = 'src'
self.migration.destination_shard = 'dst'
self.migration.type = 'TYPE'
self.mox.StubOutWithMock(verifier, 'queue_verification')
chunk_config = shinkansen.worker.ChunkConfig(
migration_id='MID',
partition_val='TID',
namespace='ns',
source_shard='src',
destination_shard='dst',
source_schema='mysql',
destination_schema='crate',
migration_type='TYPE',
table_config=None, columns=None, chunk_num=None, start_id=None, chunk_size=None,
)
verifier.queue_verification(chunk_config)
self.mox.ReplayAll()
self.worker.queue_migration_verification(self.migration)