def getAcquired(self, users): """ Get a number of documents to be submitted (in ACQUIRED status) and return results of the query for logs :return: """ documents = list() for user in users: username = user[0] group = user[1] role = user[2] fileDoc = dict() fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'acquiredTransfers' fileDoc['grouping'] = 1 fileDoc['username'] = username if group == '': group = None if role == '': role = None fileDoc['vogroup'] = group fileDoc['vorole'] = role self.logger.debug("Retrieving users from oracleDB") try: results = self.oracleDB.get(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) documents += oracleOutputMapping(results) except Exception as ex: self.logger.error("Failed to get acquired transfers \ from oracleDB: %s" % ex) return documents
def mark_acquired(self, files=[]): """ Mark the list of files as tranferred """ lfn_in_transfer = [] dash_rep = () if self.config.isOracle: toUpdate = list() for lfn in files: if lfn['value'][0].find('temp') == 7: docId = lfn['key'][5] toUpdate.append(docId) try: docbyId = self.oracleDB.get(self.config.oracleFileTrans.replace('filetransfers','fileusertransfers'), data=encodeRequest({'subresource': 'getById', 'id': docId})) document = oracleOutputMapping(docbyId, None)[0] dash_rep = (document['jobid'], document['job_retry_count'], document['taskname']) lfn_in_transfer.append(lfn) except Exception as ex: self.logger.error("Error during dashboard report update: %s" %ex) return [],() return lfn_in_transfer, dash_rep else: for lfn in files: if lfn['value'][0].find('temp') == 7: docId = getHashLfn(lfn['value'][0]) self.logger.debug("Marking acquired %s" % docId) # Load document to get the retry_count try: document = self.db.document(docId) except Exception as ex: msg = "Error loading document from couch" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue if document['state'] == 'new' or document['state'] == 'retry': data = dict() data['state'] = 'acquired' data['last_update'] = time.time() updateUri = "/" + self.db.name + "/_design/AsyncTransfer/_update/updateJobs/" + docId updateUri += "?" + urllib.urlencode(data) try: self.db.makeRequest(uri=updateUri, type="PUT", decode=False) except Exception as ex: msg = "Error updating document in couch" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue self.logger.debug("Marked acquired %s of %s" % (docId, lfn)) lfn_in_transfer.append(lfn) dash_rep = (document['jobid'], document['job_retry_count'], document['workflow']) else: continue else: good_lfn = lfn['value'][0].replace('store', 'store/temp', 1) self.mark_good([good_lfn]) return lfn_in_transfer, dash_rep
def failed(self, files, failures_reasons=[], max_retry=3, force_fail=False, submission_error=False): """ :param files: tuple (source_lfn, dest_lfn) :param failures_reasons: list(str) with reasons of failure :param max_retry: number of retry before giving up :param force_fail: flag for triggering failure without retry :param submission_error: error during fts submission :return: """ updated_lfn = [] for Lfn in files: lfn = Lfn[0] # Load document and get the retry_count docId = getHashLfn(lfn) self.logger.debug("Marking failed %s" % docId) try: docbyId = self.oracleDB.get(self.config.oracleUserFileTrans.replace('filetransfer','fileusertransfers'), data=encodeRequest({'subresource': 'getById', 'id': docId})) document = oracleOutputMapping(docbyId, None)[0] self.logger.debug("Document: %s" % document) except Exception as ex: self.logger.error("Error updating failed docs: %s" % ex) return 1 fileDoc = dict() fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'updateTransfers' fileDoc['list_of_ids'] = docId if not len(failures_reasons) == 0: try: fileDoc['list_of_failure_reason'] = failures_reasons[files.index(Lfn)] except: fileDoc['list_of_failure_reason'] = "unexcpected error, missing reasons" self.logger.exception("missing reasons") if force_fail or document['transfer_retry_count'] + 1 > max_retry: fileDoc['list_of_transfer_state'] = 'FAILED' fileDoc['list_of_retry_value'] = 1 else: fileDoc['list_of_transfer_state'] = 'RETRY' if submission_error: fileDoc['list_of_failure_reason'] = "Job could not be submitted to FTS: temporary problem of FTS" fileDoc['list_of_retry_value'] = 1 else: fileDoc['list_of_retry_value'] = 1 self.logger.debug("update: %s" % fileDoc) try: updated_lfn.append(docId) self.oracleDB.post(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) except Exception: self.logger.exception('ERROR updating failed documents') return 1 self.logger.debug("failed file updated") return 0
def algorithm(self, parameters=None): """ Performs the doRetries method, loading the appropriate plugin for each job and handling it. """ logging.debug("Running retryManager algorithm") if self.config.isOracle: fileDoc = dict() fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'retryTransfers' fileDoc['time_to'] = self.cooloffTime self.logger.debug('fileDoc: %s' % fileDoc) try: results = self.oracleDB.post(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) except Exception: self.logger.exception("Failed to get retry transfers in oracleDB: %s") return logging.info("Retried files in cooloff: %s,\n now getting transfers to kill" % str(results)) fileDoc = dict() fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'getTransfersToKill' fileDoc['grouping'] = 0 try: results = self.oracleDB.get(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) result = oracleOutputMapping(results) except Exception as ex: self.logger.error("Failed to get killed transfers \ from oracleDB: %s" % ex) return usersToKill = list(set([(x['username'], x['user_group'], x['user_role']) for x in result])) self.logger.debug("Users with transfers to kill: %s" % usersToKill) transfers = Queue() for i in range(self.config.kill_threads): worker = Thread(target=self.killThread, args=(i, transfers,)) worker.setDaemon(True) worker.start() for user in usersToKill: user_trans = [x for x in result if (x['username'], x['user_group'], x['user_role']) == user] self.logger.info("Inserting %s transfers of user %s in the killing queue" % (len(user_trans), user)) transfers.put(user_trans) transfers.join() self.logger.info("Transfers killed.") else: self.doRetries()
def active_tasks(self, db): fileDoc = {} fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'acquirePublication' self.logger.debug("Retrieving publications from oracleDB") results = '' try: results = db.post(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) except Exception as ex: self.logger.error("Failed to acquire publications \ from oracleDB: %s" %ex) return [] fileDoc = dict() fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'acquiredPublication' fileDoc['grouping'] = 0 fileDoc['limit'] = 100000 self.logger.debug("Retrieving max.100000 acquired publications from oracleDB") result = [] try: results = db.get(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) result.extend(oracleOutputMapping(results)) except Exception as ex: self.logger.error("Failed to acquire publications \ from oracleDB: %s" %ex) return [] self.logger.debug("publen: %s" % len(result)) self.logger.debug("%s acquired publications retrieved" % len(result)) #TODO: join query for publisher (same of submitter) unique_tasks = [list(i) for i in set(tuple([x['username'], x['user_group'], x['user_role'], x['taskname']] ) for x in result if x['transfer_state'] == 3)] info = [] for task in unique_tasks: info.append([x for x in result if x['taskname'] == task[3]]) return zip(unique_tasks, info)
def mark_failed(files, oracleDB, logger, failure_reason=""): """ Something failed for these files so increment the retry count """ h = 0 for lfn in files: h += 1 logger.debug("Marking failed %s" % h) source_lfn = lfn docId = getHashLfn(source_lfn) logger.debug("Marking failed %s" % docId) try: docbyId = oracleDB.get(config.General.oracleUserTrans, data=encodeRequest({ 'subresource': 'getById', 'id': docId })) except Exception: logger.exception("Error updating failed docs.") continue document = oracleOutputMapping(docbyId, None)[0] logger.debug("Document: %s" % document) try: fileDoc = dict() fileDoc['asoworker'] = 'asodciangot1' fileDoc['subresource'] = 'updatePublication' fileDoc['list_of_ids'] = docId fileDoc['list_of_publication_state'] = 'FAILED' #if force_failure or document['publish_retry_count'] > self.max_retry: # fileDoc['list_of_publication_state'] = 'FAILED' #else: # fileDoc['list_of_publication_state'] = 'RETRY' # TODO: implement retry fileDoc['list_of_retry_value'] = 1 fileDoc['list_of_failure_reason'] = failure_reason logger.debug("fileDoc: %s " % fileDoc) _ = oracleDB.post(config.General.oracleFileTrans, data=encodeRequest(fileDoc)) logger.debug("updated: %s " % docId) except Exception as ex: msg = "Error updating document: %s" % fileDoc msg += str(ex) msg += str(traceback.format_exc()) logger.error(msg) continue
def getData(subresource): """This function will fetch data from Oracle table""" crabserver = CRABRest(hostname=CMSWEB, localcert=CERTIFICATE, localkey=KEY, retry=3, userAgent='CRABTaskWorker') crabserver.setDbInstance(dbInstance=DBINSTANCE) result = crabserver.get(api='filetransfers', data=encodeRequest({ 'subresource': subresource, 'grouping': 0 })) return oracleOutputMapping(result)
def mark_failed(files, oracleDB, logger, failure_reason=""): """ Something failed for these files so increment the retry count """ h = 0 for lfn in files: h += 1 logger.debug("Marking failed %s" % h) source_lfn = lfn docId = getHashLfn(source_lfn) logger.debug("Marking failed %s" % docId) try: docbyId = oracleDB.get(config.General.oracleUserTrans, data=encodeRequest({'subresource': 'getById', 'id': docId})) except Exception: logger.exception("Error updating failed docs.") continue document = oracleOutputMapping(docbyId, None)[0] logger.debug("Document: %s" % document) try: fileDoc = dict() fileDoc['asoworker'] = config.General.asoworker fileDoc['subresource'] = 'updatePublication' fileDoc['list_of_ids'] = docId fileDoc['list_of_publication_state'] = 'FAILED' #if force_failure or document['publish_retry_count'] > self.max_retry: # fileDoc['list_of_publication_state'] = 'FAILED' #else: # fileDoc['list_of_publication_state'] = 'RETRY' # TODO: implement retry fileDoc['list_of_retry_value'] = 1 fileDoc['list_of_failure_reason'] = failure_reason logger.debug("fileDoc: %s " % fileDoc) _ = oracleDB.post(config.General.oracleFileTrans, data=encodeRequest(fileDoc)) logger.debug("updated: %s " % docId) except Exception as ex: msg = "Error updating document: %s" % fileDoc msg += str(ex) msg += str(traceback.format_exc()) logger.error(msg) continue
def acquire(self): """ Get a number (1k for current oracle rest) of documents and bind them to this aso NEW -> ACQUIRED (asoworker NULL -> config.asoworker) :return: """ self.logger.info('Retrieving users...') fileDoc = dict() fileDoc['subresource'] = 'activeUsers' fileDoc['grouping'] = 0 fileDoc['asoworker'] = self.config.asoworker try: self.oracleDB.get(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) except Exception as ex: self.logger.error("Failed to acquire transfers \ from oracleDB: %s" % ex) return 1 users = list() try: docs = oracleOutputMapping(result) users = [[x['username'], x['user_group'], x['user_role']] for x in docs] self.logger.info('Users to process: %s' % str(users)) except: self.logger.exception('User data malformed. ') for user in users: fileDoc = dict() fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'acquireTransfers' fileDoc['username'] = user[0] self.logger.debug("Retrieving transfers from oracleDB for user: %s " % user) try: self.oracleDB.post(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) except Exception as ex: self.logger.error("Failed to acquire transfers \ from oracleDB: %s" % ex) return users
def searchTask(self, workflow): """ :param workflow: :return: """ data = dict() data['workflow'] = workflow data['subresource'] = 'search' try: result = self.oracleDB.get(self.config.oracleFileTrans.replace('filetransfers', 'task'), data=encodeRequest(data)) self.logger.debug("task: %s " % str(result[0])) self.logger.debug("task: %s " % getColumn(result[0], 'tm_last_publication')) except Exception as ex: self.logger.error("Error during task doc retrieving: %s" % ex) return {} return oracleOutputMapping(result)
def source_destinations_by_user(self): """ Get all the destinations for a user """ if self.config.isOracle: self.logger.debug('Running acquiredTransfers query... ' + self.user) fileDoc = dict() fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'acquiredTransfers' fileDoc['grouping'] = 1 fileDoc['username'] = self.user group = self.group if self.group == '': group = None if self.role == '': role = None fileDoc['vogroup'] = group fileDoc['vorole'] = role fileDoc['limit'] = self.config.max_files_per_transfer result = [] self.logger.debug('Request: ' + str(fileDoc)) try: results = self.oracleDB.get(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) result = oracleOutputMapping(results) res = [[x['source'], x['destination']] for x in result] res.sort() res = list(k for k, _ in itertools.groupby(res)) except Exception as ex: self.logger.error("Failed to get acquired transfers \ from oracleDB: %s" %ex) return [], {} return res, result else: query = {'group': True, 'startkey':[self.user, self.group, self.role], 'endkey':[self.user, self.group, self.role, {}, {}]} try: sites = self.db.loadView(self.config.ftscp_design, 'ftscp_all', query) except: return [] return [[x[4], x[3]] for x in sites['rows']]
def getPub(self): """ :return: """ to_pub_docs = list() filedoc = dict() filedoc['asoworker'] = self.config.asoworker filedoc['subresource'] = 'acquiredPublication' filedoc['grouping'] = 0 try: results = self.oracleDB.get(self.config.oracleFileTrans, data=encodeRequest(filedoc)) to_pub_docs = oracleOutputMapping(results) except Exception as ex: self.logger.error("Failed to get acquired publications \ from oracleDB: %s" % ex) return to_pub_docs return to_pub_docs
def active_tasks(self, crabserver): """ :param crabserver: CRABRest object to access proper REST as createdin __init__ method TODO detail here the strucutre it returns :return: a list tuples [(task,info)]. One element for each task which has jobs to be published each list element has the format (task, info) where task and info are lists task=[] a set fo 4 attributes formatted as a list of strings: [username, usergroup, userrole, taskname] this is a subset of the information present in each element of thee next list ! info=[filedic1, filedic2, filedic3...] a list of dictionaries,one per file with keys: u'username', u'cache_url', u'source_lfn', u'publication_state', u'destination', u'user_role', u'last_update', u'input_dataset', u'dbs_url', u'aso_worker', u'user_group', u'taskname', u'transfer_state', u'destination_lfn' """ self.logger.debug("Retrieving publications from oracleDB") filesToPublish = [] asoworkers = self.config.asoworker # asoworkers can be a string or a list of strings # but if it is a string, do not turn it into a list of chars ! asoworkers = [asoworkers] if isinstance(asoworkers, basestring) else asoworkers for asoworker in asoworkers: self.logger.info( "Processing publication requests for asoworker: %s", asoworker) fileDoc = {} fileDoc['asoworker'] = asoworker fileDoc['subresource'] = 'acquirePublication' data = encodeRequest(fileDoc) try: result = crabserver.post(api='filetransfers', data=data) # pylint: disable=unused-variable except Exception as ex: self.logger.error( "Failed to acquire publications from crabserver: %s", ex) return [] self.logger.debug( "Retrieving max.100000 acquired publications from oracleDB") fileDoc = dict() fileDoc['asoworker'] = asoworker fileDoc['subresource'] = 'acquiredPublication' fileDoc['grouping'] = 0 fileDoc['limit'] = 100000 data = encodeRequest(fileDoc) try: results = crabserver.get(api='filetransfers', data=data) except Exception as ex: self.logger.error( "Failed to acquire publications from crabserver: %s", ex) return [] files = oracleOutputMapping(results) self.logger.info( "%s acquired publications retrieved for asoworker %s", len(files), asoworker) filesToPublish.extend(files) # TODO: join query for publisher (same of submitter) unique_tasks = [ list(i) for i in set( tuple([ x['username'], x['user_group'], x['user_role'], x['taskname'] ]) for x in filesToPublish if x['transfer_state'] == 3) ] info = [] for task in unique_tasks: info.append( [x for x in filesToPublish if x['taskname'] == task[3]]) return zip(unique_tasks, info)
def testFileTransferPUT(self): """ _testFileTransferPUT_ Just test simple testFileTransferPUT with fake data """ # We just sent fake data which is not monitored by dashboard. # Also only the first time to decide is publication ON or NOT for user in self.users: timestamp = time.strftime('%y%m%d_%H%M%S', time.gmtime()) for i in range(self.totalFiles): now = int(time.time()) # Generate a taskname workflowName = "" taskname = "" if user not in self.tasks: workflowName = "".join([ random.choice(string.ascii_lowercase) for _ in range(20) ]) + "_" + str(now) publicationState = random.choice(['NEW', 'NOT_REQUIRED']) else: workflowName = self.tasks[user]['workflowName'] publicationState = self.tasks[user]['publication'] transferState = random.choice(['NEW', 'DONE']) taskname = generateTaskName(user, workflowName, timestamp) finalLfn = self.lfnBase % (user, workflowName, i, random.randint(1, 9999)) idHash = getHashLfn(finalLfn) self.fileDoc['id'] = idHash self.fileDoc['job_id'] = i self.fileDoc['username'] = user self.fileDoc['taskname'] = taskname self.fileDoc['start_time'] = int(time.time()) self.fileDoc['source_lfn'] = finalLfn self.fileDoc['destination_lfn'] = finalLfn self.fileDoc['transfer_state'] = transferState self.fileDoc['publication_state'] = publicationState print(self.fileDoc) self.server.put('/crabserver/dev/fileusertransfers', data=encodeRequest(self.fileDoc)) # if I will put the same doc twice, it should raise an error. # self.server.put('/crabserver/dev/fileusertransfers', data=urllib.urlencode(self.fileDoc)) # This tasks are for the future and next calls if user not in self.tasks: self.tasks[user] = { 'workflowName': workflowName, 'taskname': taskname, 'listOfIds': [], 'publication': publicationState, 'toTransfer': 0, 'toPublish': 0, 'total': self.totalFiles } if self.tasks[user]['publication'] == 'NEW': self.tasks[user]['toPublish'] += 1 if transferState == 'NEW': self.tasks[user]['toTransfer'] += 1 self.tasks[user]['listOfIds'].append(idHash) # This should raise an error for username in self.tasks: taskname = self.tasks[username]['taskname'] for query in ['getTransferStatus', 'getPublicationStatus']: result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({ 'subresource': query, 'username': username, 'taskname': taskname })) print(result) print(result[0]['result']) taskInfoDict = oracleOutputMapping(result, 'id') print(taskInfoDict) for key, docDict in taskInfoDict.items(): result = self.server.get( '/crabserver/dev/fileusertransfers', data=encodeRequest({ 'subresource': 'getById', 'id': key })) randomUsers = random.sample( set(self.users), 3 ) # Take half of the users and kill their transfers for specific task for username in randomUsers: taskname = self.tasks[username]['taskname'] result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({ 'subresource': 'killTransfers', 'username': username, 'taskname': taskname })) print(result) # oneUser is left for killing a list of IDs # leftUsers will be killing transfers one by one for specific id. leftUsers = list(set(self.users) - set(randomUsers)) oneUser = random.sample(set(leftUsers), 1) leftUsers = list(set(leftUsers) - set(oneUser)) for username in leftUsers: # First get all left ids for this users result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({ 'subresource': 'getTransferStatus', 'username': username, 'taskname': self.tasks[username]['taskname'] })) resultOut = oracleOutputMapping(result, None) print("**" * 50) for outDict in resultOut: print(outDict) result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({ 'subresource': 'killTransfersById', 'username': username, 'listOfIds': outDict['id'] })) print(result) print(resultOut) print(result) for username in oneUser: result = self.server.post( '/crabserver/dev/fileusertransfers', data=encodeRequest( { 'subresource': 'killTransfersById', 'username': username, 'listOfIds': self.tasks[username]['listOfIds'] }, ['listOfIds'])) # As it asks to kill all which are in new, need to double check what we submitted before and if the output of killed is correct print(result) print(self.tasks[username])
def oracleSiteUser(self, db): """ 1. Acquire transfers from DB 2. Get acquired users and destination sites """ self.logger.info('Retrieving users...') fileDoc = dict() fileDoc['subresource'] = 'activeUsers' fileDoc['grouping'] = 0 fileDoc['asoworker'] = self.config.asoworker result = dict() try: result = db.get(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) except Exception as ex: self.logger.error("Failed to acquire transfers \ from oracleDB: %s" % ex) return [] self.logger.debug(oracleOutputMapping(result)) # TODO: translate result into list((user,group,role),...) if len(oracleOutputMapping(result)) != 0: self.logger.debug(type( [[x['username'].encode('ascii','ignore'), x['user_group'], x['user_role']] for x in oracleOutputMapping(result)])) try: docs = oracleOutputMapping(result) users = [[x['username'], x['user_group'], x['user_role']] for x in docs] self.logger.info('Users to process: %s' % str(users)) except: self.logger.exception('User data malformed. ') else: self.logger.info('No new user to acquire') return [] actives = list() for user in users: fileDoc = dict() fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'acquireTransfers' fileDoc['username'] = user[0] self.logger.debug("Retrieving transfers from oracleDB for user: %s " % user[0]) try: result = db.post(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) except Exception as ex: self.logger.error("Failed to acquire transfers \ from oracleDB: %s" %ex) continue self.doc_acq = str(result) for i in range(len(user)): if not user[i] or user[i] in ['None', 'NULL']: user[i] = '' user[i] = str(user[i]) actives.append(user) self.logger.debug("Transfers retrieved from oracleDB. %s " % users) return users
def mark_failed(self, files=[], failures_reasons=[], force_fail=False): """ Something failed for these files so increment the retry count """ updated_lfn = [] for lfn in files: data = {} self.logger.debug("Document: %s" % lfn) if not isinstance(lfn, dict): if 'temp' not in lfn: temp_lfn = lfn.replace('store', 'store/temp', 1) else: temp_lfn = lfn else: if 'temp' not in lfn['value']: temp_lfn = lfn['value'].replace('store', 'store/temp', 1) else: temp_lfn = lfn['value'] docId = getHashLfn(temp_lfn) # Load document to get the retry_count if self.config.isOracle: try: self.logger.debug("Document: %s" %docId) docbyId = self.oracleDB.get(self.config.oracleFileTrans.replace('filetransfers', 'fileusertransfers'), data=encodeRequest({'subresource': 'getById', 'id': docId})) document = oracleOutputMapping(docbyId)[0] data = dict() data['asoworker'] = self.config.asoworker data['subresource'] = 'updateTransfers' data['list_of_ids'] = docId if force_fail or document['transfer_retry_count'] + 1 > self.max_retry: data['list_of_transfer_state'] = 'FAILED' data['list_of_retry_value'] = 0 else: data['list_of_transfer_state'] = 'RETRY' fatal_error = self.determine_fatal_error(failures_reasons[files.index(lfn)]) if fatal_error: data['list_of_transfer_state'] = 'FAILED' data['list_of_failure_reason'] = failures_reasons[files.index(lfn)] data['list_of_retry_value'] = 0 self.logger.debug("update: %s" % data) result = self.oracleDB.post(self.config.oracleFileTrans, data=encodeRequest(data)) if not data['list_of_transfer_state'] == 'RETRY': updated_lfn.append(lfn) self.logger.debug("Marked failed %s" % lfn) except Exception as ex: self.logger.error("Error updating document status: %s" %ex) continue else: try: document = self.db.document( docId ) except Exception as ex: msg = "Error loading document from couch" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue if document['state'] != 'killed' and document['state'] != 'done' and document['state'] != 'failed': now = str(datetime.datetime.now()) last_update = time.time() # Prepare data to update the document in couch if force_fail or len(document['retry_count']) + 1 > self.max_retry: data['state'] = 'failed' data['end_time'] = now else: data['state'] = 'retry' fatal_error = self.determine_fatal_error(failures_reasons[files.index(lfn)]) if fatal_error: data['state'] = 'failed' data['end_time'] = now self.logger.debug("Failure list: %s" % failures_reasons) self.logger.debug("Files: %s" % files) self.logger.debug("LFN %s" % lfn) data['failure_reason'] = failures_reasons[files.index(lfn)] data['last_update'] = last_update data['retry'] = now # Update the document in couch self.logger.debug("Marking failed %s" % docId) try: updateUri = "/" + self.db.name + "/_design/AsyncTransfer/_update/updateJobs/" + docId updateUri += "?" + urllib.urlencode(data) self.db.makeRequest(uri = updateUri, type = "PUT", decode = False) updated_lfn.append(docId) self.logger.debug("Marked failed %s" % docId) except Exception as ex: msg = "Error in updating document in couch" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue try: self.db.commit() except Exception as ex: msg = "Error commiting documents in couch" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue else: updated_lfn.append(docId) self.logger.debug("failed file updated") return updated_lfn
def mark_good(self, files): """ Mark the list of files as tranferred """ updated_lfn = [] good_ids = [] if len(files) == 0: return updated_lfn for it, lfn in enumerate(files): hash_lfn = getHashLfn(lfn) self.logger.info("Marking good %s" % hash_lfn) self.logger.debug("Marking good %s" % lfn) if not self.config.isOracle: try: document = self.db.document(hash_lfn) except Exception as ex: msg = "Error loading document from couch" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue self.logger.info("Doc %s Loaded" % hash_lfn) try: now = str(datetime.datetime.now()) last_update = time.time() if self.config.isOracle: docId = getHashLfn(lfn) good_ids.append(docId) updated_lfn.append(lfn) else: if document['state'] != 'killed' and document['state'] != 'done' and document['state'] != 'failed': outputLfn = document['lfn'].replace('store/temp', 'store', 1) data = dict() data['end_time'] = now data['state'] = 'done' data['lfn'] = outputLfn data['last_update'] = last_update updateUri = "/" + self.db.name + "/_design/AsyncTransfer/_update/updateJobs/" + getHashLfn(lfn) updateUri += "?" + urllib.urlencode(data) self.db.makeRequest(uri = updateUri, type = "PUT", decode = False) updated_lfn.append(lfn) self.logger.debug("Marked good %s" % lfn) else: updated_lfn.append(lfn) try: self.db.commit() except Exception as ex: msg = "Error commiting documents in couch" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue except Exception as ex: msg = "Error updating document" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue if self.config.isOracle: try: data = dict() data['asoworker'] = self.config.asoworker data['subresource'] = 'updateTransfers' data['list_of_ids'] = good_ids data['list_of_transfer_state'] = ["DONE" for x in good_ids] result = self.oracleDB.post(self.config.oracleFileTrans, data=encodeRequest(data)) self.logger.debug("Marked good %s" % good_ids) except Exception: self.logger.exception('Error updating document') return {} self.logger.info("Transferred file %s updated, removing now source file" %docId) try: docbyId = self.oracleDB.get(self.config.oracleFileTrans.replace('filetransfers','fileusertransfers'), data=encodeRequest({'subresource': 'getById', 'id': docId})) document = oracleOutputMapping(docbyId, None)[0] except Exception: msg = "Error getting file from source" self.logger.exception(msg) return {} if document["source"] not in self.site_tfc_map: self.logger.debug("site not found... gathering info from phedex") self.site_tfc_map[document["source"]] = self.get_tfc_rules(document["source"]) pfn = self.apply_tfc_to_lfn( '%s:%s' %(document["source"], lfn)) self.logger.debug("File has to be removed now from source site: %s" %pfn) self.remove_files(self.userProxy, pfn) self.logger.debug("Transferred file removed from source") return updated_lfn
def startSlave(self, task): # TODO: lock task! # - process logger logger = setProcessLogger(str(task[0][3])) logger.info("Process %s is starting. PID %s", task[0][3], os.getpid()) self.force_publication = False workflow = str(task[0][3]) wfnamemsg = "%s: " % (workflow) if int(workflow[0:2]) < 18: msg = "Skipped. Ignore tasks created before 2018." logger.info(wfnamemsg+msg) return 0 if len(task[1]) > self.max_files_per_block: self.force_publication = True msg = "All datasets have more than %s ready files." % (self.max_files_per_block) msg += " No need to retrieve task status nor last publication time." logger.info(wfnamemsg+msg) else: msg = "At least one dataset has less than %s ready files." % (self.max_files_per_block) logger.info(wfnamemsg+msg) # Retrieve the workflow status. If the status can not be retrieved, continue # with the next workflow. workflow_status = '' url = '/'.join(self.cache_area.split('/')[:-1]) #+ '/workflow' msg = "Retrieving status from %s" % (url) logger.info(wfnamemsg+msg) header = {"Content-Type":"application/json"} data = {'workflow': workflow, 'subresource': 'taskads'} url = 'cmsweb-testbed.cern.ch' connection = HTTPRequests(url, self.config.opsCert, self.config.opsKey) try: res = connection.get('/crabserver/preprod/workflow', data=encodeRequest(data)) except Exception as ex: #logger.info(wfnamemsg+encodeRequest(data)) logger.warn('Error retrieving status from cache for %s.' % workflow) return 0 try: workflow_status = res[0]['result'][0]['status'] msg = "Task status is %s." % workflow_status logger.info(wfnamemsg+msg) except ValueError: msg = "Workflow removed from WM." logger.error(wfnamemsg+msg) workflow_status = 'REMOVED' except Exception as ex: msg = "Error loading task status!" msg += str(ex) msg += str(traceback.format_exc()) logger.error(wfnamemsg+msg) # If the workflow status is terminal, go ahead and publish all the ready files # in the workflow. if workflow_status in ['COMPLETED', 'FAILED', 'KILLED', 'REMOVED']: self.force_publication = True if workflow_status in ['KILLED', 'REMOVED']: self.force_failure = True msg = "Considering task status as terminal. Will force publication." logger.info(wfnamemsg+msg) # Otherwise... else: msg = "Task status is not considered terminal." logger.info(wfnamemsg+msg) msg = "Getting last publication time." logger.info(wfnamemsg+msg) # Get when was the last time a publication was done for this workflow (this # should be more or less independent of the output dataset in case there are # more than one). last_publication_time = None data = {} data['workflow'] = workflow data['subresource'] = 'search' try: result = self.oracleDB.get(self.config.oracleFileTrans.replace('filetransfers', 'task'), data=encodeRequest(data)) logger.debug("task: %s " % str(result[0])) logger.debug("task: %s " % getColumn(result[0], 'tm_last_publication')) except Exception as ex: logger.error("Error during task doc retrieving: %s" %ex) if last_publication_time: date = oracleOutputMapping(result)['last_publication'] seconds = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f").timetuple() last_publication_time = time.mktime(seconds) msg = "Last publication time: %s." % str(last_publication_time) logger.debug(wfnamemsg+msg) # If this is the first time a publication would be done for this workflow, go # ahead and publish. if not last_publication_time: self.force_publication = True msg = "There was no previous publication. Will force publication." logger.info(wfnamemsg+msg) # Otherwise... else: last = last_publication_time msg = "Last published block: %s" % (last) logger.debug(wfnamemsg+msg) # If the last publication was long time ago (> our block publication timeout), # go ahead and publish. now = int(time.time()) - time.timezone time_since_last_publication = now - last hours = int(time_since_last_publication/60/60) minutes = int((time_since_last_publication - hours*60*60)/60) timeout_hours = int(self.block_publication_timeout/60/60) timeout_minutes = int((self.block_publication_timeout - timeout_hours*60*60)/60) msg = "Last publication was %sh:%sm ago" % (hours, minutes) if time_since_last_publication > self.block_publication_timeout: self.force_publication = True msg += " (more than the timeout of %sh:%sm)." % (timeout_hours, timeout_minutes) msg += " Will force publication." else: msg += " (less than the timeout of %sh:%sm)." % (timeout_hours, timeout_minutes) msg += " Not enough to force publication." logger.info(wfnamemsg+msg) #logger.info(task[1]) try: if self.force_publication: # - get info active_ = [{'key': [x['username'], x['user_group'], x['user_role'], x['taskname']], 'value': [x['destination'], x['source_lfn'], x['destination_lfn'], x['input_dataset'], x['dbs_url'], x['last_update'] ]} for x in task[1] if x['transfer_state'] == 3 and x['publication_state'] not in [2, 3, 5]] lfn_ready = [] wf_jobs_endtime = [] pnn, input_dataset, input_dbs_url = "", "", "" for active_file in active_: job_end_time = active_file['value'][5] if job_end_time and self.config.isOracle: wf_jobs_endtime.append(int(job_end_time) - time.timezone) elif job_end_time: wf_jobs_endtime.append(int(time.mktime(time.strptime(str(job_end_time), '%Y-%m-%d %H:%M:%S'))) - time.timezone) source_lfn = active_file['value'][1] dest_lfn = active_file['value'][2] self.lfn_map[dest_lfn] = source_lfn if not pnn or not input_dataset or not input_dbs_url: pnn = str(active_file['value'][0]) input_dataset = str(active_file['value'][3]) input_dbs_url = str(active_file['value'][4]) lfn_ready.append(dest_lfn) userDN = '' username = task[0][0] user_group = "" if task[0][1]: user_group = task[0][1] user_role = "" if task[0][2]: user_role = task[0][2] logger.debug("Trying to get DN %s %s %s" % (username, user_group, user_role)) try: userDN = getDNFromUserName(username, logger) except Exception as ex: msg = "Error retrieving the user DN" msg += str(ex) msg += str(traceback.format_exc()) logger.error(msg) return 1 # Get metadata toPublish = [] publDescFiles_list = self.getPublDescFiles(workflow, lfn_ready) for file_ in active_: for _, doc in enumerate(publDescFiles_list): #logger.info(type(doc)) #logger.info(doc) if doc["lfn"] == file_["value"][2]: doc["User"] = username doc["Group"] = file_["key"][1] doc["Role"] = file_["key"][2] doc["UserDN"] = userDN doc["Destination"] = file_["value"][0] doc["SourceLFN"] = file_["value"][1] toPublish.append(doc) with open("/tmp/publisher_files/"+workflow+'.json', 'w') as outfile: json.dump(toPublish, outfile) logger.info(". publisher.py %s" % (workflow)) # find the location in the current environment of the script we want to run import Publisher.TaskPublish as tp taskPublishScript = tp.__file__ subprocess.call(["python", taskPublishScript, workflow]) except: logger.exception("Exception!") return 0
def active_tasks(self, db): """ :param db: HTTPRequest object to access proper REST as createdin __init__ method TODO detail here the strucutre it returns :return: a list tuples [(task,info)]. One element for each task which has jobs to be published each list element has the format (task, info) where task and info are lists task=[] a set fo 4 attributes formatted as a list of strings: [username, usergroup, userrole, taskname] this is a subset of the information present in each element of thee next list ! info=[filedic1, filedic2, filedic3...] a list of dictionaries,one per file with keys: u'username', u'cache_url', u'source_lfn', u'publication_state', u'destination', u'user_role', u'last_update', u'input_dataset', u'dbs_url', u'aso_worker', u'user_group', u'taskname', u'transfer_state', u'destination_lfn' """ self.logger.debug("Retrieving publications from oracleDB") uri = self.REST_filetransfers fileDoc = {} fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'acquirePublication' data = encodeRequest(fileDoc) results = '' try: results = db.post(uri=uri, data=data) except Exception as ex: self.logger.error("Failed to acquire publications from %s: %s", uri, ex) return [] self.logger.debug( "Retrieving max.100000 acquired publications from oracleDB") fileDoc = dict() fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'acquiredPublication' fileDoc['grouping'] = 0 fileDoc['limit'] = 100000 result = [] uri = self.REST_filetransfers data = encodeRequest(fileDoc) try: results = db.get(uri=uri, data=data) result.extend(oracleOutputMapping(results)) except Exception as ex: self.logger.error("Failed to acquire publications from %s: %s", uri, ex) return [] self.logger.debug("publen: %s", len(result)) self.logger.debug("%s acquired publications retrieved", len(result)) # TODO: join query for publisher (same of submitter) unique_tasks = [ list(i) for i in set( tuple([ x['username'], x['user_group'], x['user_role'], x['taskname'] ]) for x in result if x['transfer_state'] == 3) ] info = [] for task in unique_tasks: info.append([x for x in result if x['taskname'] == task[3]]) return zip(unique_tasks, info)
def mark_failed(self, files=[], force_fail=False, submission_error=False): """ Something failed for these files so increment the retry count """ updated_lfn = [] for lfn in files: data = {} if not isinstance(lfn, dict): if 'temp' not in lfn: temp_lfn = lfn.replace('store', 'store/temp', 1) else: temp_lfn = lfn else: if 'temp' not in lfn['value'][0]: temp_lfn = lfn['value'][0].replace('store', 'store/temp', 1) else: temp_lfn = lfn['value'][0] # Load document and get the retry_count if self.config.isOracle: docId = getHashLfn(temp_lfn) self.logger.debug("Marking failed %s" % docId) try: docbyId = self.oracleDB.get(self.config.oracleFileTrans.replace('filetransfers', 'fileusertransfers'), data=encodeRequest({'subresource': 'getById', 'id': docId})) except Exception as ex: self.logger.error("Error updating failed docs: %s" %ex) continue document = oracleOutputMapping(docbyId, None)[0] self.logger.debug("Document: %s" % document) fileDoc = dict() fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'updateTransfers' fileDoc['list_of_ids'] = docId if force_fail or document['transfer_retry_count'] + 1 > self.max_retry: fileDoc['list_of_transfer_state'] = 'FAILED' fileDoc['list_of_retry_value'] = 1 else: fileDoc['list_of_transfer_state'] = 'RETRY' if submission_error: fileDoc['list_of_failure_reason'] = "Job could not be submitted to FTS: temporary problem of FTS" fileDoc['list_of_retry_value'] = 1 elif not self.valid_proxy: fileDoc['list_of_failure_reason'] = "Job could not be submitted to FTS: user's proxy expired" fileDoc['list_of_retry_value'] = 1 else: fileDoc['list_of_failure_reason'] = "Site config problem." fileDoc['list_of_retry_value'] = 1 self.logger.debug("update: %s" % fileDoc) try: updated_lfn.append(docId) result = self.oracleDB.post(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) except Exception as ex: msg = "Error updating document" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue else: docId = getHashLfn(temp_lfn) try: document = self.db.document(docId) except Exception as ex: msg = "Error loading document from couch" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue if document['state'] != 'killed' and document['state'] != 'done' and document['state'] != 'failed': now = str(datetime.datetime.now()) last_update = time.time() # Prepare data to update the document in couch if force_fail or len(document['retry_count']) + 1 > self.max_retry: data['state'] = 'failed' else: data['state'] = 'retry' if submission_error: data['failure_reason'] = "Job could not be submitted to FTS: temporary problem of FTS" elif not self.valid_proxy: data['failure_reason'] = "Job could not be submitted to FTS: user's proxy expired" else: data['failure_reason'] = "Site config problem." data['last_update'] = last_update data['retry'] = now # Update the document in couch self.logger.debug("Marking failed %s" % docId) try: updateUri = "/" + self.db.name + "/_design/AsyncTransfer/_update/updateJobs/" + docId updateUri += "?" + urllib.urlencode(data) self.db.makeRequest(uri=updateUri, type="PUT", decode=False) updated_lfn.append(docId) self.logger.debug("Marked failed %s" % docId) except Exception as ex: msg = "Error in updating document in couch" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue try: self.db.commit() except Exception as ex: msg = "Error commiting documents in couch" msg += str(ex) msg += str(traceback.format_exc()) self.logger.error(msg) continue self.logger.debug("failed file updated") return updated_lfn
def active_users(self, db): """ Query a view for users with files to transfer. Get this from the following view: publish?group=true&group_level=1 """ if self.config.isOracle: active_users = [] fileDoc = {} fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'acquirePublication' self.logger.debug("Retrieving publications from oracleDB") results = '' try: results = db.post(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) except Exception as ex: self.logger.error("Failed to acquire publications \ from oracleDB: %s" %ex) fileDoc = dict() fileDoc['asoworker'] = self.config.asoworker fileDoc['subresource'] = 'acquiredPublication' fileDoc['grouping'] = 0 self.logger.debug("Retrieving acquired puclications from oracleDB") try: results = db.get(self.config.oracleFileTrans, data=encodeRequest(fileDoc)) result = oracleOutputMapping(results) except Exception as ex: self.logger.error("Failed to acquire publications \ from oracleDB: %s" %ex) self.logger.debug("%s acquired puclications retrieved" % len(result)) #TODO: join query for publisher (same of submitter) unique_users = [list(i) for i in set(tuple([x['username'], x['user_group'], x['user_role']]) for x in result if x['transfer_state'] == 3)] return unique_users else: # TODO: Remove stale=ok for now until tested # query = {'group': True, 'group_level': 3, 'stale': 'ok'} query = {'group': True, 'group_level': 3} try: users = db.loadView('DBSPublisher', 'publish', query) except Exception as e: self.logger.exception('A problem occured \ when contacting couchDB: %s' % e) return [] if len(users['rows']) <= self.config.publication_pool_size: active_users = users['rows'] active_users = [x['key'] for x in active_users] else: pool_size=self.config.publication_pool_size sorted_users = self.factory.loadObject(self.config.algoName, args=[self.config, self.logger, users['rows'], pool_size], getFromCache=False, listFlag = True) active_users = sorted_users()[:self.config.publication_pool_size] self.logger.info('%s active users' % len(active_users)) self.logger.debug('Active users are: %s' % active_users) return active_users
def testFileTransferPUT(self): """ _testFileTransferPUT_ Just test simple testFileTransferPUT with fake data """ # We just sent fake data which is not monitored by dashboard. # Also only the first time to decide is publication ON or NOT for user in self.users: timestamp = time.strftime('%y%m%d_%H%M%S', time.gmtime()) for i in range(self.totalFiles): now = int(time.time()) # Generate a taskname workflowName = "" taskname = "" if user not in self.tasks: workflowName = "".join([random.choice(string.ascii_lowercase) for _ in range(20)]) + "_" + str(now) publicationState = random.choice(['NEW', 'NOT_REQUIRED']) else: workflowName = self.tasks[user]['workflowName'] publicationState = self.tasks[user]['publication'] transferState = random.choice(['NEW', 'DONE']) taskname = generateTaskName(user, workflowName, timestamp) finalLfn = self.lfnBase % (user, workflowName, i, random.randint(1, 9999)) idHash = getHashLfn(finalLfn) self.fileDoc['id'] = idHash self.fileDoc['job_id'] = i self.fileDoc['username'] = user self.fileDoc['taskname'] = taskname self.fileDoc['start_time'] = int(time.time()) self.fileDoc['source_lfn'] = finalLfn self.fileDoc['destination_lfn'] = finalLfn self.fileDoc['transfer_state'] = transferState self.fileDoc['publication_state'] = publicationState print(self.fileDoc) self.server.put('/crabserver/dev/fileusertransfers', data=encodeRequest(self.fileDoc)) # if I will put the same doc twice, it should raise an error. # self.server.put('/crabserver/dev/fileusertransfers', data=urllib.urlencode(self.fileDoc)) # This tasks are for the future and next calls if user not in self.tasks: self.tasks[user] = {'workflowName': workflowName, 'taskname': taskname, 'listOfIds': [], 'publication': publicationState, 'toTransfer': 0, 'toPublish': 0, 'total': self.totalFiles} if self.tasks[user]['publication'] == 'NEW': self.tasks[user]['toPublish'] += 1 if transferState == 'NEW': self.tasks[user]['toTransfer'] += 1 self.tasks[user]['listOfIds'].append(idHash) # This should raise an error for username in self.tasks: taskname = self.tasks[username]['taskname'] for query in ['getTransferStatus', 'getPublicationStatus']: result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': query, 'username': username, 'taskname': taskname})) print(result) print(result[0]['result']) taskInfoDict = oracleOutputMapping(result, 'id') print(taskInfoDict) for key, docDict in taskInfoDict.items(): result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'getById', 'id': key})) randomUsers = random.sample(set(self.users), 3) # Take half of the users and kill their transfers for specific task for username in randomUsers: taskname = self.tasks[username]['taskname'] result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'killTransfers', 'username': username, 'taskname': taskname})) print(result) # oneUser is left for killing a list of IDs # leftUsers will be killing transfers one by one for specific id. leftUsers = list(set(self.users) - set(randomUsers)) oneUser = random.sample(set(leftUsers), 1) leftUsers = list(set(leftUsers) - set(oneUser)) for username in leftUsers: # First get all left ids for this users result = self.server.get('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'getTransferStatus', 'username': username, 'taskname': self.tasks[username]['taskname']})) resultOut = oracleOutputMapping(result, None) print("**"*50) for outDict in resultOut: print(outDict) result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'killTransfersById', 'username': username, 'listOfIds': outDict['id']})) print(result) print(resultOut) print(result) for username in oneUser: result = self.server.post('/crabserver/dev/fileusertransfers', data=encodeRequest({'subresource': 'killTransfersById', 'username': username, 'listOfIds': self.tasks[username]['listOfIds']}, ['listOfIds'])) # As it asks to kill all which are in new, need to double check what we submitted before and if the output of killed is correct print(result) print(self.tasks[username])
pu = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) break except Exception, e: logger.debug(str(e)) maxi = maxi + 1 continue if __name__ == "__main__": server = HTTPRequests('cmsweb-testbed.cern.ch', '/data/srv/asyncstageout/state/asyncstageout/creds/OpsProxy', '/data/srv/asyncstageout/state/asyncstageout/creds/OpsProxy') result = server.get('/crabserver/preprod/filetransfers', data=encodeRequest({'subresource': 'groupedTransferStatistics', 'grouping': 0})) results = oracleOutputMapping(result) status = {'transfers':{}, 'publications':{}} tmp = {'transfers':{ 'DONE':0, 'ACQUIRED':0, 'SUBMITTED':0, 'FAILED':0, 'RETRY':0 }, 'publications':{'DONE':0, 'ACQUIRED':0, 'NEW':0, 'FAILED':0, 'RETRY':0}} #past = open("tmp_transfer") #tmp = json.load(past) for doc in results: if doc['aso_worker']=="asodciangot1": if 'transfers' in tmp and TRANSFERDB_STATES[doc['transfer_state']] in tmp['transfers']: status['transfers'][TRANSFERDB_STATES[doc['transfer_state']]] = - tmp['transfers'][TRANSFERDB_STATES[doc['transfer_state']]] + doc['nt'] tmp['transfers'][TRANSFERDB_STATES[doc['transfer_state']]] = doc['nt'] else: