class MigrationTask(SequencialTaskBase):
def initialize(self, db_config):
self.db_config = db_config
self.sourceUrl = None
self.migration_req_id = 0
self.block_names = []
self.migration_block_ids = []
self.inserted = True
dbowner = self.db_config.get('dbowner')
connectUrl = self.db_config.get('connectUrl')
print(connectUrl)
dbFactory = DBFactory(MgrLogger, connectUrl, options={})
self.dbi = dbFactory.connect()
self.dbFormatter = DBFormatter(MgrLogger, self.dbi)
self.dbsMigrate = DBSMigrate(MgrLogger, self.dbi, dbowner)
self.DBSBlockInsert = DBSBlockInsert(MgrLogger, self.dbi, dbowner)
def setCallSequence(self):
self._callSequence = [self.getResource, self.insertBlock, self.cleanup]
def getResource(self):
#MgrLogger.info("_"*20+ "getResource")
#query the MIGRATION_REQUESTS table to get a request with the smallest CREATION_DATE
# and MIGRATION_STATUS = 0(pending)
try:
#set MIGRATION_STATUS = 1(in progess) and commit it imeditaly to avoid other threads to touch it.
req = self.dbsMigrate.listMigrationRequests(oldest=True)
if len(req) == 0:
self.sourceUrl = None
return # don't need to go down.
try:
request =req[0]
self.sourceUrl = request['migration_url']
self.migration_req_id = request['migration_request_id']
MgrLogger.error("-"*20+ "getResource-- "+ time.asctime(time.gmtime()) + ' Migration request ID: '+ str(self.migration_req_id))
migration_status = 1
self.dbsMigrate.updateMigrationRequestStatus(migration_status, self.migration_req_id)
except IndexError:
return #No request found. Exit.
except Exception as ex:
MgrLogger.error( time.asctime(time.gmtime()) + str(ex) )
self.sourceUrl = None
return # don't need to go down.
self.block_names = []
self.migration_block_ids = []
#query migration_blocks table to get a list of block_names to be migrated
# and migration_block_id list. Both lists are ordered by MIGRATION_ORDER
try:
blocks = self.dbsMigrate.listMigrationBlocks(self.migration_req_id)
for b in blocks:
self.block_names.append(b['migration_block_name'])
self.migration_block_ids.append(b['migration_block_id'])
if not self.block_names :
logmessage="No migration blocks found under the migration request id %s." %(self.migration_req_id )+ \
"DBS Operator please check it."
#set MIGRATION_STATUS = 3(failed) for MIGRATION_REQUESTS
#if try_count==3, the sql will actually set the migration_status=9 (terminally failed) instead of 3
self.dbsMigrate.updateMigrationRequestStatus(3, self.migration_req_id)
self.sourceUrl = None
MgrLogger.error( time.asctime(time.gmtime()) + logmessage )
else:
#Update MIGRATION_STATUS for all the MIGRATION_BLOCK_IDs in the self.migration_block_id list
#in MIGRATION_BLOCKS table to 1 (in progress)
#set MIGRATION_STATUS = 1 and commit it immediately
#MgrLogger.error("-"*20+ 'Regester ID: %s '%self.migration_req_id + 'Migration Block Names: ')
#MgrLogger.error("block_name: %s. Set all block status=1 " %self.block_names)
self.dbsMigrate.updateMigrationBlockStatus(migration_status=1, migration_block=self.migration_block_ids)
except Exception as ex:
self.sourceUrl = None
#MgrLogger.error( time.asctime(time.gmtime()) + str(ex) )
#print "set MIGRATION_STATUS = 3(failed) for blocks of MIGRATION_REQUESTS_ID=%s" %self.migration_req_id
self.dbsMigrate.updateMigrationRequestStatus(3, self.migration_req_id)
def insertBlock(self):
MgrLogger.info("_"*20+"insertBlock")
self.inserted = True
if self.sourceUrl:
try:
for idx, bName in enumerate(self.block_names):
params={'block_name':bName}
data = self.dbsMigrate.callDBSService(self.sourceUrl, 'blockdump', params)
data = cjson.decode(data)
#MgrLogger.error( "--YG migration server blockdump--")
#MgrLogger.error( data)
migration_status = 0
#idx = self.block_names.index(bName)
MgrLogger.error("-"*20 + time.asctime(time.gmtime()) + " Inserting block: %s for request id: %s" %(bName, self.migration_req_id))
try:
self.DBSBlockInsert.putBlock(data, migration=True)
migration_status = 2
except HTTPError as he:
if "Block %s already exists" % (bName) in str(he):
migration_status = 2
else:
raise
except dbsException as de:
if "Block %s already exists" % (bName) in de.message:
#the block maybe get into the destination by other means.
#skip this block and continue.
migration_status = 2
else:
raise
except Exception as ex:
if "Block %s already exists" % (bName) in str(ex):
migration_status = 2
else:
raise
finally:
if migration_status == 2:
self.dbsMigrate.updateMigrationBlockStatus(migration_status=2,
migration_block=self.migration_block_ids[idx])
MgrLogger.error("-"*20 + time.asctime(time.gmtime()) + " Done insert block: %s for request id: %s" %(bName, self.migration_req_id))
self.dbsMigrate.updateMigrationRequestStatus(2, self.migration_req_id)
except Exception as ex:
self.inserted = False
#handle dbsException
if type(ex) == dbsException:
MgrLogger.error( time.asctime(time.gmtime()) + ex.message + ex.serverError )
MgrLogger.error(time.asctime(time.gmtime()) + str(ex))
#if try_count==3, the sql will actually set the status=9 (terminally failed) instead of 3
#print "set migration_status=3 or 9 for request and blocks ofmigration_request_id=%s" %self.migration_req_id
self.dbsMigrate.updateMigrationRequestStatus(3, self.migration_req_id)
self.dbsMigrate.updateMigrationBlockStatus(migration_status=3, migration_request=self.migration_req_id)
return
def cleanup(self):
#MgrLogger.error("_"*20+"cleanup")
#return to the initial status
self.sourceUrl = None
self.migration_req_id = 0
self.block_names = []
self.migration_block_id = []
self.inserted = True
