コード例 #1
0
ファイル: queuer.py プロジェクト: skyhigh-networks/shinkansen
    def _run(self):
        start = datetime.now()
        self.log('Starting table migration queueing')

        self.get_table_metadata()

        self.table.min_id = self.min_id
        self.table.max_id = self.max_id
        self.table.num_records = self.num_rows

        if self.num_rows == 0 or self.num_rows is None:
            self.log('No rows to migrate')
            self.table.end_time = int(time.time() * 1000)
            self.table.status = 'empty'
            self.table.update()

            # If the table is empty, jump straight to the verification so end-of-migration logic is run.
            verifier.queue_verification(self.c)
            return

        self.queue_chunks()

        self.table.end_time = int(time.time() * 1000)
        self.table.status = 'chunks_queued'
        self.table.num_chunks = self.num_chunks
        self.table.chunk_size = self.c.chunk_size
        self.table.update()

        self.log('Finished queueing table migration elapsed=%s', datetime.now() - start)
コード例 #2
0
ファイル: pipe.py プロジェクト: skyhigh-networks/shinkansen
    def _run(self):
        start = datetime.now()
        self.log('Pipe worker starting')

        self.chunk.status = 'migrating'
        self.chunk.num_records_exported = 0
        self.chunk.num_records_imported = 0
        self.chunk.update()

        with db.shard_connection(self.c.source_shard, read=True) as source_conn:
            with db.shard_connection(self.c.destination_shard, read=False) as dest_conn:
                update_status = self.migrate(source_conn, dest_conn)

        if update_status:
            self.chunk.import_elapsed_ms = int((datetime.now() - start).total_seconds() * 1000)
            self.chunk.status = 'imported'
            self.chunk.update()

            (migration_status, _, _, _, _) = status.get_migration_status(migration_id=self.c.migration_id)
            migration = orm.Migration.get(self.redis_conn, migration_id=self.c.migration_id)
            migration.status = migration_status
            migration.update()

        if config.ENABLE_VERIFIER:
            verifier.queue_verification(self.c)

        self.log('Pipe worker finished elapsed=%s', datetime.now() - start)
コード例 #3
0
    def _run(self):
        if self.c.num_records_converted > 0:
            self.chunk.status = 'importing'
            self.chunk.update()
            start = datetime.now()
            if self.c.destination_type == 'crate':
                self._import_to_crate()
            elif self.c.destination_type == 'mysql':
                self._import_to_mysql()
            else:
                raise Error('Unknown destination type %r' % (self.c.destination_type,))

            self.log('Import to destination finished num_records_imported=%s destination_host=%s elapsed=%s',
                     self.c.num_records_imported, self.c.destination_host, datetime.now() - start)

            self.chunk.status = 'imported'
            self.chunk.num_records_imported = self.c.num_records_imported
            self.chunk.import_elapsed_ms = int((datetime.now() - start).total_seconds() * 1000)
            self.chunk.end_time = int(time.time() * 1000)
            self.chunk.update()
        else:
            self.log('No rows to import')
            self.chunk.status = 'empty'
            self.chunk.end_time = int(time.time() * 1000)
            self.chunk.update()

        if config.ENABLE_VERIFIER:
            verifier.queue_verification(self.c)

        # TODO(jpatrin): Refactor this into its own task so it can be independently retried
        try:
            cmd = (
                'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o LogLevel=ERROR '
                '-i %s -p %r %s@%s "sudo bash -c \'rm -f %s\'"' % (
                    config.SSH_PRIVKEY, self.c.destination_ssh_port, config.SSH_USER, self.c.destination_host,
                    ssh.escape_double_quotes(ssh.escape_single_quotes(self.c.import_filename))))
            rm_cmd = subprocess.Popen(
                cmd,
                shell=True,
                stdin=subprocess.PIPE)
            rm_cmd.stdin.close()
            rm_cmd.wait()
            if rm_cmd.returncode != 0:
                raise CommandException('Removing file on destination server failed with exit code %r' % (
                    rm_cmd.returncode,))
        except Exception, e:
            # We catch and log all exceptions here to make this task idempotent. We DO NOT want this task to be
            # retried at this point as duplicate imports can fail on mysql and would cause us to corrupt the crate
            # chunk import records.
            self.log('Exception during removal of destination file, removal will not be retried %r import_filename=%s',
                     e, self.c.import_filename)
コード例 #4
0
 def test_run_no_records(self):
     self.config.num_records_converted = 0
     import_worker = TestableImportChunkWorker(self.config)
     import_worker.chunk = self.mox.CreateMock(orm.Chunk)
     import_worker.chunk.update()
     self.mox.StubOutWithMock(verifier, 'queue_verification')
     verifier.queue_verification(self.config)
     self.mox.StubOutClassWithMocks(subprocess, 'Popen')
     proc = subprocess.Popen(mox.IgnoreArg(), shell=True, stdin=subprocess.PIPE)
     proc.stdin = self.mox.CreateMockAnything()
     proc.stdin.close()
     proc.wait()
     proc.returncode = 0
     self.mox.ReplayAll()
     import_worker._run()
コード例 #5
0
 def test_run_removal_fails(self):
     self.config.num_records_converted = 0
     import_worker = TestableImportChunkWorker(self.config)
     import_worker.chunk = self.mox.CreateMock(orm.Chunk)
     import_worker.chunk.update()
     self.mox.StubOutWithMock(verifier, 'queue_verification')
     verifier.queue_verification(self.config)
     self.mox.StubOutClassWithMocks(subprocess, 'Popen')
     proc = subprocess.Popen(mox.IgnoreArg(), shell=True, stdin=subprocess.PIPE)
     proc.stdin = self.mox.CreateMockAnything()
     proc.stdin.close()
     proc.wait()
     proc.returncode = 1
     # Note that we do not get an exception in this case, it is only logged in order to make
     # this task idempotent.
     self.mox.ReplayAll()
     import_worker._run()
コード例 #6
0
 def test_queue_migration_verification(self):
     self.migration.migration_id = 'MID'
     self.migration.partition_val = 'TID'
     self.migration.namespace = 'ns'
     self.migration.source_shard = 'src'
     self.migration.destination_shard = 'dst'
     self.migration.type = 'TYPE'
     self.mox.StubOutWithMock(verifier, 'queue_verification')
     chunk_config = shinkansen.worker.ChunkConfig(
         migration_id='MID',
         partition_val='TID',
         namespace='ns',
         source_shard='src',
         destination_shard='dst',
         source_schema='mysql',
         destination_schema='crate',
         migration_type='TYPE',
         table_config=None, columns=None, chunk_num=None, start_id=None, chunk_size=None,
     )
     verifier.queue_verification(chunk_config)
     self.mox.ReplayAll()
     self.worker.queue_migration_verification(self.migration)