class MigrationTask(SequencialTaskBase): def initialize(self, db_config): self.db_config = db_config self.sourceUrl = None self.migration_req_id = 0 self.block_names = [] self.migration_block_ids = [] self.inserted = True dbowner = self.db_config.get('dbowner') connectUrl = self.db_config.get('connectUrl') print(connectUrl) dbFactory = DBFactory(MgrLogger, connectUrl, options={}) self.dbi = dbFactory.connect() self.dbFormatter = DBFormatter(MgrLogger, self.dbi) self.dbsMigrate = DBSMigrate(MgrLogger, self.dbi, dbowner) self.DBSBlockInsert = DBSBlockInsert(MgrLogger, self.dbi, dbowner) def setCallSequence(self): self._callSequence = [self.getResource, self.insertBlock, self.cleanup] def getResource(self): #MgrLogger.info("_"*20+ "getResource") #query the MIGRATION_REQUESTS table to get a request with the smallest CREATION_DATE # and MIGRATION_STATUS = 0(pending) try: #set MIGRATION_STATUS = 1(in progess) and commit it imeditaly to avoid other threads to touch it. req = self.dbsMigrate.listMigrationRequests(oldest=True) if len(req) == 0: self.sourceUrl = None return # don't need to go down. try: request =req[0] self.sourceUrl = request['migration_url'] self.migration_req_id = request['migration_request_id'] MgrLogger.error("-"*20+ "getResource-- "+ time.asctime(time.gmtime()) + ' Migration request ID: '+ str(self.migration_req_id)) migration_status = 1 self.dbsMigrate.updateMigrationRequestStatus(migration_status, self.migration_req_id) except IndexError: return #No request found. Exit. except Exception as ex: MgrLogger.error( time.asctime(time.gmtime()) + str(ex) ) self.sourceUrl = None return # don't need to go down. self.block_names = [] self.migration_block_ids = [] #query migration_blocks table to get a list of block_names to be migrated # and migration_block_id list. Both lists are ordered by MIGRATION_ORDER try: blocks = self.dbsMigrate.listMigrationBlocks(self.migration_req_id) for b in blocks: self.block_names.append(b['migration_block_name']) self.migration_block_ids.append(b['migration_block_id']) if not self.block_names : logmessage="No migration blocks found under the migration request id %s." %(self.migration_req_id )+ \ "DBS Operator please check it." #set MIGRATION_STATUS = 3(failed) for MIGRATION_REQUESTS #if try_count==3, the sql will actually set the migration_status=9 (terminally failed) instead of 3 self.dbsMigrate.updateMigrationRequestStatus(3, self.migration_req_id) self.sourceUrl = None MgrLogger.error( time.asctime(time.gmtime()) + logmessage ) else: #Update MIGRATION_STATUS for all the MIGRATION_BLOCK_IDs in the self.migration_block_id list #in MIGRATION_BLOCKS table to 1 (in progress) #set MIGRATION_STATUS = 1 and commit it immediately #MgrLogger.error("-"*20+ 'Regester ID: %s '%self.migration_req_id + 'Migration Block Names: ') #MgrLogger.error("block_name: %s. Set all block status=1 " %self.block_names) self.dbsMigrate.updateMigrationBlockStatus(migration_status=1, migration_block=self.migration_block_ids) except Exception as ex: self.sourceUrl = None #MgrLogger.error( time.asctime(time.gmtime()) + str(ex) ) #print "set MIGRATION_STATUS = 3(failed) for blocks of MIGRATION_REQUESTS_ID=%s" %self.migration_req_id self.dbsMigrate.updateMigrationRequestStatus(3, self.migration_req_id) def insertBlock(self): MgrLogger.info("_"*20+"insertBlock") self.inserted = True if self.sourceUrl: try: for idx, bName in enumerate(self.block_names): params={'block_name':bName} data = self.dbsMigrate.callDBSService(self.sourceUrl, 'blockdump', params) data = cjson.decode(data) #MgrLogger.error( "--YG migration server blockdump--") #MgrLogger.error( data) migration_status = 0 #idx = self.block_names.index(bName) MgrLogger.error("-"*20 + time.asctime(time.gmtime()) + " Inserting block: %s for request id: %s" %(bName, self.migration_req_id)) try: self.DBSBlockInsert.putBlock(data, migration=True) migration_status = 2 except HTTPError as he: if "Block %s already exists" % (bName) in str(he): migration_status = 2 else: raise except dbsException as de: if "Block %s already exists" % (bName) in de.message: #the block maybe get into the destination by other means. #skip this block and continue. migration_status = 2 else: raise except Exception as ex: if "Block %s already exists" % (bName) in str(ex): migration_status = 2 else: raise finally: if migration_status == 2: self.dbsMigrate.updateMigrationBlockStatus(migration_status=2, migration_block=self.migration_block_ids[idx]) MgrLogger.error("-"*20 + time.asctime(time.gmtime()) + " Done insert block: %s for request id: %s" %(bName, self.migration_req_id)) self.dbsMigrate.updateMigrationRequestStatus(2, self.migration_req_id) except Exception as ex: self.inserted = False #handle dbsException if type(ex) == dbsException: MgrLogger.error( time.asctime(time.gmtime()) + ex.message + ex.serverError ) MgrLogger.error(time.asctime(time.gmtime()) + str(ex)) #if try_count==3, the sql will actually set the status=9 (terminally failed) instead of 3 #print "set migration_status=3 or 9 for request and blocks ofmigration_request_id=%s" %self.migration_req_id self.dbsMigrate.updateMigrationRequestStatus(3, self.migration_req_id) self.dbsMigrate.updateMigrationBlockStatus(migration_status=3, migration_request=self.migration_req_id) return def cleanup(self): #MgrLogger.error("_"*20+"cleanup") #return to the initial status self.sourceUrl = None self.migration_req_id = 0 self.block_names = [] self.migration_block_id = [] self.inserted = True