def __init__(self, transClient=None, logger=None, requestClient=None, requestClass=None, requestValidator=None): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger('RequestTasks') super(RequestTasks, self).__init__(transClient, logger) if not requestClient: self.requestClient = ReqClient() else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator
def __init__(self, transClient=None, logger=None, requestClient=None, requestClass=None, requestValidator=None, ownerDN=None, ownerGroup=None): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger('RequestTasks') super(RequestTasks, self).__init__(transClient, logger) useCertificates = True if (bool(ownerDN) and bool(ownerGroup)) else False if not requestClient: self.requestClient = ReqClient(useCertificates=useCertificates, delegatedDN=ownerDN, delegatedGroup=ownerGroup) else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator
def __init__( self, transClient = None, logger = None, requestClient = None, requestClass = None, requestValidator = None ): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger( 'RequestTasks' ) super( RequestTasks, self ).__init__( transClient, logger ) if not requestClient: self.requestClient = ReqClient() else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator
def export_putRequest(self, requestJSON): """put a new request into RequestDB :param cls: class ref :param str requestJSON: request serialized to JSON format """ requestDict = json.loads(requestJSON) requestName = requestDict.get("RequestID", requestDict.get("RequestName", "***UNKNOWN***")) request = Request(requestDict) # Check whether the credentials in the Requests are correct and allowed to be set isAuthorized = RequestValidator.setAndCheckRequestOwner(request, self.getRemoteCredentials()) if not isAuthorized: return S_ERROR(DErrno.ENOAUTH, "Credentials in the requests are not allowed") optimized = request.optimize() if optimized.get("Value", False): gLogger.debug("putRequest: request was optimized") else: gLogger.debug("putRequest: request unchanged", optimized.get("Message", "Nothing could be optimized")) valid = self.validate(request) if not valid["OK"]: gLogger.error("putRequest: request %s not valid: %s" % (requestName, valid["Message"])) return valid # If NotBefore is not set or user defined, we calculate its value now = datetime.datetime.utcnow().replace(microsecond=0) extraDelay = datetime.timedelta(0) if request.Status not in Request.FINAL_STATES and (not request.NotBefore or request.NotBefore < now): # We don't delay if it is the first insertion if getattr(request, "RequestID", 0): # If it is a constant delay, just set it if self.constantRequestDelay: extraDelay = datetime.timedelta(minutes=self.constantRequestDelay) else: # If there is a waiting Operation with Files op = request.getWaiting().get("Value") if op and len(op): attemptList = [opFile.Attempt for opFile in op if opFile.Status == "Waiting"] if attemptList: maxWaitingAttempt = max([opFile.Attempt for opFile in op if opFile.Status == "Waiting"]) # In case it is the first attempt, extraDelay is 0 # maxWaitingAttempt can be None if the operation has no File, like the ForwardDiset extraDelay = datetime.timedelta( minutes=2 * math.log(maxWaitingAttempt) if maxWaitingAttempt else 0 ) request.NotBefore = now + extraDelay gLogger.info( "putRequest: request %s not before %s (extra delay %s)" % (request.RequestName, request.NotBefore, extraDelay) ) requestName = request.RequestName gLogger.info("putRequest: Attempting to set request '%s'" % requestName) return self.__requestDB.putRequest(request)
def _sendFailoverRequest(self, request): """Send failover reques per Job. This request would basically be a DISET request for setting the job status. If this fails, it only prints a message. :param Request request: Request() object :return: None """ if len(request): self.log.info("Trying to send the failover request") # The request is ready, send it now isValid = RequestValidator().validate(request) if not isValid["OK"]: self.log.error("Failover request is not valid", isValid["Message"]) self.log.error("Printing out the content of the request") reqToJSON = request.toJSON() if reqToJSON["OK"]: print(str(reqToJSON["Value"])) else: self.log.error( "Something went wrong creating the JSON from request", reqToJSON["Message"]) else: # Now trying to send the request requestClient = ReqClient() result = requestClient.putRequest(request) if not result["OK"]: self.log.error("Failed to set failover request", result["Message"])
def myRequest(): """Create a request and put it to the db""" request = Request() request.RequestName = 'myAwesomeRemovalRequest.xml' request.JobID = 0 request.SourceComponent = "myScript" remove = Operation() remove.Type = "RemoveFile" lfn = "/ilc/user/s/sailer/test.txt" rmFile = File() rmFile.LFN = lfn remove.addFile(rmFile) request.addOperation(remove) isValid = RequestValidator().validate(request) if not isValid['OK']: raise RuntimeError("Failover request is not valid: %s" % isValid['Message']) else: print "It is a GOGOGO" requestClient = ReqClient() result = requestClient.putRequest(request) print result
def generateFailoverFile( self ): """ Retrieve the accumulated reporting request, and produce a JSON file that is consumed by the JobWrapper """ request = self._getRequestContainer() self.log.notice("Request Before: %s" %request) reportRequest = None resultJR = self.jobReport.generateForwardDISET() if not resultJR['OK']: self.log.warn( "Could not generate Operation for job report with result:\n%s" % ( resultJR ) ) else: reportRequest = resultJR['Value'] if reportRequest: self.log.info( "Populating request with job report information" ) request.addOperation( reportRequest ) accountingReport = self.workflow_commons.get( 'AccountingReport', None) if accountingReport: resultAR = accountingReport.commit() if not resultAR['OK']: self.log.error( "!!! Both Accounting and RequestDB are down? !!!" ) return resultAR if not self.workflowStatus['OK'] or not self.stepStatus['OK']: if 'ProductionOutputData' in self.workflow_commons: prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";") self.log.info("There was some kind of error, cleaning up outputData: %s" % prodOutputLFNs) self.setApplicationStatus("Creating Removal Requests") self._cleanUp(prodOutputLFNs) if not len(request): #pylint: disable=len-as-condition self.log.info("No Requests to process ") return S_OK() isValid = RequestValidator().validate( request ) if not isValid['OK']: raise RuntimeError( "Failover request is not valid: %s" % isValid['Message'] ) requestJSON = request.toJSON() if not requestJSON['OK']: raise RuntimeError( requestJSON['Message'] ) self.log.info( "Creating failover request for deferred operations for job %d" % int(self.jobID) ) request_string = str( requestJSON['Value'] ) self.log.debug( request_string ) # Write out the request string fname = '%s_%s_request.json' % ( self.productionID, self.prod_job_id ) with open( fname, 'w' ) as jsonFile: jsonFile.write( request_string ) self.log.info( "Created file containing failover request %s" % fname ) resultDigest = request.getDigest() if resultDigest['OK']: self.log.info( "Digest of the request: %s" % resultDigest['Value'] ) else: self.log.error( "No digest? That's not sooo important, anyway: %s" % resultDigest['Message'] ) self.log.notice("Request After: %s" %request) return S_OK()
def generateFailoverFile(self): """ Retrieve the accumulated reporting request, and produce a JSON file that is consumed by the JobWrapper """ reportRequest = None result = self.jobReport.generateForwardDISET() if not result['OK']: self.log.warn( "Could not generate Operation for job report with result:\n%s" % (result)) else: reportRequest = result['Value'] if reportRequest: self.log.info("Populating request with job report information") self.request.addOperation(reportRequest) accountingReport = None if self.workflow_commons.has_key('AccountingReport'): accountingReport = self.workflow_commons['AccountingReport'] if accountingReport: result = accountingReport.commit() if not result['OK']: self.log.error( "!!! Both accounting and RequestDB are down? !!!") return result if len(self.request): isValid = RequestValidator().validate(self.request) if not isValid['OK']: raise RuntimeError("Failover request is not valid: %s" % isValid['Message']) else: requestJSON = self.request.toJSON() if requestJSON['OK']: self.log.info( "Creating failover request for deferred operations for job %d" % self.jobID) request_string = str(requestJSON['Value']) self.log.debug(request_string) # Write out the request string fname = '%d_%d_request.json' % (self.production_id, self.prod_job_id) jsonFile = open(fname, 'w') jsonFile.write(request_string) jsonFile.close() self.log.info( "Created file containing failover request %s" % fname) result = self.request.getDigest() if result['OK']: self.log.info("Digest of the request: %s" % result['Value']) else: self.log.error( "No digest? That's not sooo important, anyway:", result['Message']) else: raise RuntimeError(requestJSON['Message'])
def commitRequest(self): """Send request to the Request Management Service""" if self.request.isEmpty(): return S_OK() isValid = RequestValidator().validate(self.request) if not isValid["OK"]: return S_ERROR("Failover request is not valid: %s" % isValid["Message"]) else: requestClient = ReqClient() result = requestClient.putRequest(self.request) return result
def export_putRequest(self, requestJSON): """forward request from local RequestDB to central RequestManager :param self: self reference :param str requestType: request type """ gMonitor.addMark("reqReceived", 1) requestDict = json.loads(requestJSON) requestName = requestDict.get( "RequestID", requestDict.get("RequestName", "***UNKNOWN***")) gLogger.info("putRequest: got request '%s'" % requestName) # We only need the object to check the authorization request = Request(requestDict) # Check whether the credentials in the Requests are correct and allowed to be set isAuthorized = RequestValidator.setAndCheckRequestOwner( request, self.getRemoteCredentials()) if not isAuthorized: return S_ERROR(DErrno.ENOAUTH, "Credentials in the requests are not allowed") forwardable = self.__forwardable(requestDict) if not forwardable["OK"]: gLogger.warn("putRequest: %s" % forwardable["Message"]) setRequest = self.requestManager().putRequest(requestJSON) if not setRequest["OK"]: gLogger.error( "setReqeuest: unable to set request '%s' @ RequestManager: %s" % (requestName, setRequest["Message"])) # # put request to the request file cache save = self.__saveRequest(requestName, requestJSON) if not save["OK"]: gLogger.error( "setRequest: unable to save request to the cache: %s" % save["Message"]) return save gLogger.info("setRequest: %s is saved to %s file" % (requestName, save["Value"])) return S_OK({"set": False, "saved": True}) gLogger.info( "setRequest: request '%s' has been set to the ReqManager" % (requestName)) return S_OK({"set": True, "saved": False})
def run(self): """Perform checks and create the request.""" from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator for count, lfnChunk in enumerate(breakListIntoChunks(self.lfnList, 100)): if not lfnChunk: sLog.error('LFN list is empty!!!') return 1 requestName = '%s_%d' % (self.switches.get('Name'), count) request = self.createRequest(requestName, lfnChunk) valid = RequestValidator().validate(request) if not valid['OK']: sLog.error('putRequest: request not valid', '%s' % valid['Message']) return 1 else: self.requests.append(request) self.putOrRunRequests() return 0
def run(self): """Perform checks and create the request.""" if self.switches.get("AutoName"): baseArchiveLFN = archiveLFN = self.switches["AutoName"] tarballName = os.path.basename(archiveLFN) else: baseArchiveLFN = archiveLFN = self.name tarballName = os.path.basename(archiveLFN) baseRequestName = requestName = "Archive_%s" % tarballName.rsplit( ".", 1)[0] from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator self.splitLFNsBySize() for count, lfnChunk in enumerate(self.lfnChunks): if not lfnChunk: sLog.error("LFN list is empty!!!") return 1 if len(self.lfnChunks) > 1: requestName = "%s_%d" % (baseRequestName, count) baseName = os.path.split(baseArchiveLFN.rsplit(".", 1)[0]) archiveLFN = "%s/%s_Tars/%s_%d.tar" % ( baseName[0], baseName[1], baseName[1], count) self.checkArchive(archiveLFN) request = self.createRequest(requestName, archiveLFN, lfnChunk) valid = RequestValidator().validate(request) if not valid["OK"]: sLog.error("putRequest: request not valid", "%s" % valid["Message"]) return 1 else: self.requests.append(request) self.putOrRunRequests() return 0
def export_putRequest(self, requestJSON): """ forward request from local RequestDB to central RequestManager :param self: self reference :param str requestType: request type """ gMonitor.addMark('reqReceived', 1) requestDict = json.loads(requestJSON) requestName = requestDict.get("RequestID", requestDict.get('RequestName', "***UNKNOWN***")) gLogger.info("putRequest: got request '%s'" % requestName) # We only need the object to check the authorization request = Request(requestDict) # Check whether the credentials in the Requests are correct and allowed to be set isAuthorized = RequestValidator.setAndCheckRequestOwner(request, self.getRemoteCredentials()) if not isAuthorized: return S_ERROR(DErrno.ENOAUTH, "Credentials in the requests are not allowed") forwardable = self.__forwardable(requestDict) if not forwardable["OK"]: gLogger.warn("putRequest: %s" % forwardable["Message"]) setRequest = self.requestManager().putRequest(requestJSON) if not setRequest["OK"]: gLogger.error( "setReqeuest: unable to set request '%s' @ RequestManager: %s" % (requestName, setRequest["Message"])) # # put request to the request file cache save = self.__saveRequest(requestName, requestJSON) if not save["OK"]: gLogger.error("setRequest: unable to save request to the cache: %s" % save["Message"]) return save gLogger.info("setRequest: %s is saved to %s file" % (requestName, save["Value"])) return S_OK({"set": False, "saved": True}) gLogger.info("setRequest: request '%s' has been set to the ReqManager" % (requestName)) return S_OK({"set": True, "saved": False})
def run(self): """Perform checks and create the request.""" from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator for count, lfnChunk in enumerate(breakListIntoChunks( self.lfnList, 100)): if not lfnChunk: sLog.error("LFN list is empty!!!") return 1 requestName = "%s_%d" % (self.switches.get("Name"), count) request = self.createRequest(requestName, lfnChunk) valid = RequestValidator().validate(request) if not valid["OK"]: sLog.error("putRequest: request not valid", "%s" % valid["Message"]) return 1 else: self.requests.append(request) self.putOrRunRequests() return 0
def run(self): """Perform checks and create the request.""" if self.switches.get('AutoName'): baseArchiveLFN = archiveLFN = self.switches['AutoName'] tarballName = os.path.basename(archiveLFN) else: baseArchiveLFN = archiveLFN = self.name tarballName = os.path.basename(archiveLFN) baseRequestName = requestName = 'Archive_%s' % tarballName.rsplit('.', 1)[0] from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator self.splitLFNsBySize() for count, lfnChunk in enumerate(self.lfnChunks): if not lfnChunk: LOG.error('LFN list is empty!!!') return 1 if len(self.lfnChunks) > 1: requestName = '%s_%d' % (baseRequestName, count) baseName = os.path.split(baseArchiveLFN.rsplit('.', 1)[0]) archiveLFN = '%s/%s_Tars/%s_%d.tar' % (baseName[0], baseName[1], baseName[1], count) self.checkArchive(archiveLFN) request = self.createRequest(requestName, archiveLFN, lfnChunk) valid = RequestValidator().validate(request) if not valid['OK']: LOG.error('putRequest: request not valid', '%s' % valid['Message']) return 1 else: self.requests.append(request) self.putOrRunRequests() return 0
def requestValidator(self): """ get request validator """ if not self.__requestValidator: self.__requestValidator = RequestValidator() return self.__requestValidator
class RequestTasks(TaskBase): """ Class for handling tasks for the RMS """ def __init__(self, transClient=None, logger=None, requestClient=None, requestClass=None, requestValidator=None, ownerDN=None, ownerGroup=None): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger('RequestTasks') super(RequestTasks, self).__init__(transClient, logger) useCertificates = True if (bool(ownerDN) and bool(ownerGroup)) else False if not requestClient: self.requestClient = ReqClient(useCertificates=useCertificates, delegatedDN=ownerDN, delegatedGroup=ownerGroup) else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN='', bulkSubmissionFlag=False): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername(owner) if not res['OK']: return res ownerDN = res['Value'][0] try: transJson = json.loads(transBody) self._multiOperationsBody(transJson, taskDict, ownerDN, ownerGroup) except ValueError: # #json couldn't load self._singleOperationsBody(transBody, taskDict, ownerDN, ownerGroup) return S_OK(taskDict) def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup): """ deal with a Request that has multiple operations :param transJson: list of lists of string and dictionaries, e.g.: .. code :: python body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }), ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ), ] :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ failedTasks = [] for taskID, task in taskDict.items(): transID = task['TransformationID'] if not task.get('InputData'): self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID) taskDict.pop(taskID) continue files = [] oRequest = Request() if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') # create the operations from the json structure for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] for parameter, value in operationTuple[1].iteritems(): setattr(op, parameter, value) for lfn in files: opFile = File() opFile.LFN = lfn op.addFile(opFile) oRequest.addOperation(op) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID) def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.iteritems(): transID = task['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task['TargetSE'] # If there are input files if task.get('InputData'): if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID) def _assignRequestToTask(self, oRequest, taskDict, transID, taskID, ownerDN, ownerGroup): """set ownerDN and group to request, and add the request to taskDict if it is valid, otherwise remove the task from the taskDict :param oRequest: Request :param dict taskDict: dictionary of tasks, modified in this function :param int transID: Transformation ID :param int taskID: Task ID :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ oRequest.RequestName = self._transTaskName(transID, taskID) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate(oRequest) if not isValid['OK']: self._logError("Error creating request for task", "%s %s" % (taskID, isValid), transID=transID) return S_ERROR('Error creating request') taskDict[taskID]['TaskObject'] = oRequest return S_OK() def submitTransformationTasks(self, taskDict): """ Submit requests one by one """ submitted = 0 failed = 0 startTime = time.time() method = 'submitTransformationTasks' for task in taskDict.itervalues(): # transID is the same for all tasks, so pick it up every time here transID = task['TransformationID'] if not task['TaskObject']: task['Success'] = False failed += 1 continue res = self.submitTaskToExternal(task['TaskObject']) if res['OK']: task['ExternalID'] = res['Value'] task['Success'] = True submitted += 1 else: self._logError("Failed to submit task to RMS", res['Message'], transID=transID) task['Success'] = False failed += 1 if submitted: self._logInfo('Submitted %d tasks to RMS in %.1f seconds' % (submitted, time.time() - startTime), transID=transID, method=method) if failed: self._logWarn('Failed to submit %d tasks to RMS.' % (failed), transID=transID, method=method) return S_OK(taskDict) def submitTaskToExternal(self, oRequest): """ Submits a request to RMS """ if isinstance(oRequest, self.requestClass): return self.requestClient.putRequest(oRequest, useFailoverProxy=False, retryMainService=2) return S_ERROR("Request should be a Request object") def updateTransformationReservedTasks(self, taskDicts): requestNameIDs = {} noTasks = [] for taskDict in taskDicts: requestName = self._transTaskName(taskDict['TransformationID'], taskDict['TaskID']) reqID = taskDict['ExternalID'] if reqID: requestNameIDs[requestName] = reqID else: noTasks.append(requestName) return S_OK({'NoTasks': noTasks, 'TaskNameIDs': requestNameIDs}) def getSubmittedTaskStatus(self, taskDicts): """ Check if tasks changed status, and return a list of tasks per new status """ updateDict = {} badRequestID = 0 for taskDict in taskDicts: oldStatus = taskDict['ExternalStatus'] # ExternalID is normally a string if taskDict['ExternalID'] and int(taskDict['ExternalID']): newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID']) if not newStatus['OK']: log = self._logVerbose if 'not exist' in newStatus[ 'Message'] else self._logWarn log("getSubmittedTaskStatus: Failed to get requestID for request", newStatus['Message'], transID=taskDict['TransformationID']) else: newStatus = newStatus['Value'] # We don't care updating the tasks to Assigned while the request is being processed if newStatus != oldStatus and newStatus != 'Assigned': updateDict.setdefault(newStatus, []).append(taskDict['TaskID']) else: badRequestID += 1 if badRequestID: self._logWarn("%d requests have identifier 0" % badRequestID) return S_OK(updateDict) def getSubmittedFileStatus(self, fileDicts): """ Check if transformation files changed status, and return a list of taskIDs per new status """ # Don't try and get status of not submitted tasks! transID = None taskFiles = {} for fileDict in fileDicts: # There is only one transformation involved, get however the transID in the loop transID = fileDict['TransformationID'] taskID = int(fileDict['TaskID']) taskFiles.setdefault(taskID, []).append(fileDict['LFN']) # Should not happen, but just in case there are no files, return if transID is None: return S_OK({}) res = self.transClient.getTransformationTasks({ 'TransformationID': transID, 'TaskID': taskFiles.keys() }) if not res['OK']: return res requestFiles = {} for taskDict in res['Value']: taskID = taskDict['TaskID'] externalID = taskDict['ExternalID'] # Only consider tasks that are submitted, ExternalID is a string if taskDict['ExternalStatus'] != 'Created' and externalID and int( externalID): requestFiles[externalID] = taskFiles[taskID] updateDict = {} for requestID, lfnList in requestFiles.iteritems(): statusDict = self.requestClient.getRequestFileStatus( requestID, lfnList) if not statusDict['OK']: log = self._logVerbose if 'not exist' in statusDict[ 'Message'] else self._logWarn log("Failed to get files status for request", statusDict['Message'], transID=transID, method='getSubmittedFileStatus') else: for lfn, newStatus in statusDict['Value'].iteritems(): if newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': updateDict[lfn] = 'Problematic' return S_OK(updateDict)
def testValidator(self): """ validator test """ ## create validator validator = RequestValidator() self.assertEqual(isinstance(validator, RequestValidator), True) ## RequestName not set ret = validator.validate(self.request) self.assertEqual(ret, {'Message': 'RequestName not set', 'OK': False}) self.request.RequestName = "test_request" # # no ownerDN ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': "Request 'test_request' is missing OwnerDN value", 'OK': False }) self.request.OwnerDN = "foo/bar=baz" # # no owner group ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': "Request 'test_request' is missing OwnerGroup value", 'OK': False }) self.request.OwnerGroup = "dirac_user" ## no operations ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': "Operations not present in request 'test_request'", 'OK': False }) self.request.addOperation(self.operation) ## type not set ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': "Operation #0 in request 'test_request' hasn't got Type set", 'OK': False }) self.operation.Type = "ReplicateAndRegister" ## files not present ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': "Operation #0 of type 'ReplicateAndRegister' hasn't got files to process.", 'OK': False }) self.operation.addFile(self.file) ## targetSE not set ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': "Operation #0 of type 'ReplicateAndRegister' is missing TargetSE attribute.", 'OK': False }) self.operation.TargetSE = "CERN-USER" ## missing LFN ret = validator.validate(self.request) self.assertEqual( ret, { "Message": "Operation #0 of type 'ReplicateAndRegister' is missing LFN attribute for file.", "OK": False }) self.file.LFN = "/a/b/c" ## Checksum set, ChecksumType not set self.file.Checksum = "abcdef" ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': 'File in operation #0 is missing Checksum (abcdef) or ChecksumType ()', 'OK': False }) ## ChecksumType set, Checksum not set self.file.Checksum = "" self.file.ChecksumType = "adler32" ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': 'File in operation #0 is missing Checksum () or ChecksumType (ADLER32)', 'OK': False }) ## both set self.file.Checksum = "abcdef" self.file.ChecksumType = "adler32" ret = validator.validate(self.request) self.assertEqual(ret, {'OK': True, 'Value': ''}) ## both unset self.file.Checksum = "" self.file.ChecksumType = None ret = validator.validate(self.request) self.assertEqual(ret, {'OK': True, 'Value': ''}) ## all OK ret = validator.validate(self.request) self.assertEqual(ret, {'OK': True, 'Value': ''})
class RequestTasks( TaskBase ): def __init__( self, transClient = None, logger = None, requestClient = None, requestClass = None, requestValidator = None ): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger( 'RequestTasks' ) super( RequestTasks, self ).__init__( transClient, logger ) if not requestClient: self.requestClient = ReqClient() else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '', bulkSubmissionFlag = False): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] try: transJson = json.loads(transBody) self._multiOperationsBody( transJson, taskDict, ownerDN, ownerGroup ) except ValueError: ##json couldn't load self._singleOperationsBody( transBody, taskDict, ownerDN, ownerGroup ) return S_OK( taskDict ) def _multiOperationsBody( self, transJson, taskDict, ownerDN, ownerGroup ): """ deal with a Request that has multiple operations :param transJson: list of lists of string and dictionaries, e.g.: .. code :: python body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }), ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ), ] :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if not paramDict.get('InputData'): self.log.error( "Error creating request for task", "%s, No input data" % taskID ) taskDict.pop( taskID ) continue files = [] transID = paramDict['TransformationID'] oRequest = Request() if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) # create the operations from the json structure for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] for parameter, value in operationTuple[1].iteritems(): setattr( op, parameter, value ) for lfn in files: opFile = File() opFile.LFN = lfn op.addFile( opFile ) oRequest.addOperation( op ) self._assignRequestToTask( oRequest, taskDict, transID, taskID, ownerDN, ownerGroup ) def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup ): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass # Do not remove sorted, we might pop elements in the loop for taskID in sorted( taskDict ): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] # If there are input files if paramDict.get('InputData'): if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) self._assignRequestToTask( oRequest, taskDict, transID, taskID, ownerDN, ownerGroup ) def _assignRequestToTask( self, oRequest, taskDict, transID, taskID, ownerDN, ownerGroup ): """set ownerDN and group to request, and add the request to taskDict if it is valid, otherwise remove the task from the taskDict :param oRequest: Request :param dict taskDict: dictionary of tasks, modified in this function :param int transID: Transformation ID :param int taskID: Task ID :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) ) # This works because we loop over a copy of the keys ! taskDict.pop( taskID ) return taskDict[taskID]['TaskObject'] = oRequest return def submitTransformationTasks( self, taskDict ): """ Submit requests one by one """ submitted = 0 failed = 0 startTime = time.time() for taskID in sorted( taskDict ): if not taskDict[taskID]['TaskObject']: taskDict[taskID]['Success'] = False failed += 1 continue res = self.submitTaskToExternal( taskDict[taskID]['TaskObject'] ) if res['OK']: taskDict[taskID]['ExternalID'] = res['Value'] taskDict[taskID]['Success'] = True submitted += 1 else: self._logError( "Failed to submit task to RMS", res['Message'] ) taskDict[taskID]['Success'] = False failed += 1 self._logInfo( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % ( submitted, time.time() - startTime ) ) if failed: self._logWarn( 'submitTasks: But at the same time failed to submit %d tasks to RMS.' % ( failed ) ) return S_OK( taskDict ) def submitTaskToExternal( self, oRequest ): """ Submits a request using ReqClient """ if isinstance( oRequest, self.requestClass ): return self.requestClient.putRequest( oRequest, useFailoverProxy = False, retryMainService = 2 ) else: return S_ERROR( "Request should be a Request object" ) def updateTransformationReservedTasks( self, taskDicts ): requestNameIDs = {} noTasks = [] for taskDict in taskDicts: requestName = _requestName( taskDict['TransformationID'], taskDict['TaskID'] ) reqID = taskDict['ExternalID'] if reqID: requestNameIDs[requestName] = reqID else: noTasks.append( requestName ) return S_OK( {'NoTasks':noTasks, 'TaskNameIDs':requestNameIDs} ) def getSubmittedTaskStatus( self, taskDicts ): updateDict = {} for taskDict in taskDicts: oldStatus = taskDict['ExternalStatus'] newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID'] ) if not newStatus['OK']: log = self._logVerbose if 'not exist' in newStatus['Message'] else self.log.warn log( "getSubmittedTaskStatus: Failed to get requestID for request", '%s' % newStatus['Message'] ) else: newStatus = newStatus['Value'] if newStatus != oldStatus: updateDict.setdefault( newStatus, [] ).append( taskDict['TaskID'] ) return S_OK( updateDict ) def getSubmittedFileStatus( self, fileDicts ): taskFiles = {} submittedTasks = {} externalIds = {} # Don't try and get status of not submitted tasks! for fileDict in fileDicts: submittedTasks.setdefault( fileDict['TransformationID'], set() ).add( int( fileDict['TaskID'] ) ) for transID in submittedTasks: res = self.transClient.getTransformationTasks( { 'TransformationID':transID, 'TaskID': list( submittedTasks[transID] )} ) if not res['OK']: return res for taskDict in res['Value']: taskID = taskDict['TaskID'] externalIds[taskID] = taskDict['ExternalID'] if taskDict['ExternalStatus'] == 'Created': submittedTasks[transID].remove( taskID ) for fileDict in fileDicts: transID = fileDict['TransformationID'] taskID = int( fileDict['TaskID'] ) if taskID in submittedTasks[transID]: taskFiles.setdefault( externalIds[taskID], [] ).append( fileDict['LFN'] ) updateDict = {} for requestID in sorted( taskFiles ): lfnList = taskFiles[requestID] statusDict = self.requestClient.getRequestFileStatus( requestID, lfnList ) if not statusDict['OK']: log = self._logVerbose if 'not exist' in statusDict['Message'] else self.log.warn log( "getSubmittedFileStatus: Failed to get files status for request", '%s' % statusDict['Message'] ) continue for lfn, newStatus in statusDict['Value'].items(): if newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': updateDict[lfn] = 'Problematic' return S_OK( updateDict )
class RequestTasks( TaskBase ): def __init__( self, transClient = None, logger = None, requestClient = None, requestClass = None, requestValidator = None ): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger( 'RequestTasks' ) super( RequestTasks, self ).__init__( transClient, logger ) if not requestClient: self.requestClient = ReqClient() else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict ) def submitTransformationTasks( self, taskDict ): """ Submit requests one by one """ submitted = 0 failed = 0 startTime = time.time() for taskID in sorted( taskDict ): if not taskDict[taskID]['TaskObject']: taskDict[taskID]['Success'] = False failed += 1 continue res = self.submitTaskToExternal( taskDict[taskID]['TaskObject'] ) if res['OK']: taskDict[taskID]['ExternalID'] = res['Value'] taskDict[taskID]['Success'] = True submitted += 1 else: self._logError( "Failed to submit task to RMS", res['Message'] ) taskDict[taskID]['Success'] = False failed += 1 self._logInfo( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % ( submitted, time.time() - startTime ) ) if failed: self._logWarn( 'submitTasks: But at the same time failed to submit %d tasks to RMS.' % ( failed ) ) return S_OK( taskDict ) def submitTaskToExternal( self, oRequest ): """ Submits a request using ReqClient """ if isinstance( oRequest, self.requestClass ): return self.requestClient.putRequest( oRequest ) else: return S_ERROR( "Request should be a Request object" ) def updateTransformationReservedTasks( self, taskDicts ): requestNameIDs = {} noTasks = [] for taskDict in taskDicts: requestName = _requestName( taskDict['TransformationID'], taskDict['TaskID'] ) reqID = taskDict['ExternalID'] if reqID: requestNameIDs[requestName] = reqID else: noTasks.append( requestName ) return S_OK( {'NoTasks':noTasks, 'TaskNameIDs':requestNameIDs} ) def getSubmittedTaskStatus( self, taskDicts ): updateDict = {} for taskDict in taskDicts: oldStatus = taskDict['ExternalStatus'] newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID'] ) if not newStatus['OK']: log = self._logVerbose if 'not exist' in newStatus['Message'] else self.log.warn log( "getSubmittedTaskStatus: Failed to get requestID for request", '%s' % newStatus['Message'] ) else: newStatus = newStatus['Value'] if newStatus != oldStatus: updateDict.setdefault( newStatus, [] ).append( taskDict['TaskID'] ) return S_OK( updateDict ) def getSubmittedFileStatus( self, fileDicts ): taskFiles = {} submittedTasks = {} externalIds = {} # Don't try and get status of not submitted tasks! for fileDict in fileDicts: submittedTasks.setdefault( fileDict['TransformationID'], set() ).add( int( fileDict['TaskID'] ) ) for transID in submittedTasks: res = self.transClient.getTransformationTasks( { 'TransformationID':transID, 'TaskID': list( submittedTasks[transID] )} ) if not res['OK']: return res for taskDict in res['Value']: taskID = taskDict['TaskID'] externalIds[taskID] = taskDict['ExternalID'] if taskDict['ExternalStatus'] == 'Created': submittedTasks[transID].remove( taskID ) for fileDict in fileDicts: transID = fileDict['TransformationID'] taskID = int( fileDict['TaskID'] ) if taskID in submittedTasks[transID]: requestID = externalIds[taskID] taskFiles.setdefault( requestID, {} )[fileDict['LFN']] = fileDict['Status'] updateDict = {} for requestID in sorted( taskFiles ): lfnDict = taskFiles[requestID] statusDict = self.requestClient.getRequestFileStatus( requestID, lfnDict.keys() ) if not statusDict['OK']: log = self._logVerbose if 'not exist' in statusDict['Message'] else self.log.warn log( "getSubmittedFileStatus: Failed to get files status for request", '%s' % statusDict['Message'] ) continue statusDict = statusDict['Value'] for lfn, newStatus in statusDict.items(): if newStatus == lfnDict[lfn]: pass elif newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': updateDict[lfn] = 'Problematic' return S_OK( updateDict )
def main(): Script.parseCommandLine(ignoreErrors=False) args = Script.getPositionalArgs() if len(args) < 2: Script.showHelp() targetSE = args.pop(0) lfns = [] for inputFileName in args: if os.path.exists(inputFileName): inputFile = open(inputFileName, 'r') string = inputFile.read() inputFile.close() lfns.extend([lfn.strip() for lfn in string.splitlines()]) else: lfns.append(inputFileName) from DIRAC.Resources.Storage.StorageElement import StorageElement import DIRAC # Check is provided SE is OK if targetSE != 'All': se = StorageElement(targetSE) if not se.valid: print(se.errorReason) print() Script.showHelp() from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator from DIRAC.Resources.Catalog.FileCatalog import FileCatalog reqClient = ReqClient() fc = FileCatalog() requestOperation = 'RemoveReplica' if targetSE == 'All': requestOperation = 'RemoveFile' for lfnList in breakListIntoChunks(lfns, 100): oRequest = Request() requestName = "%s_%s" % ( md5(repr(time.time()).encode()).hexdigest()[:16], md5(repr(time.time()).encode()).hexdigest()[:16], ) oRequest.RequestName = requestName oOperation = Operation() oOperation.Type = requestOperation oOperation.TargetSE = targetSE res = fc.getFileMetadata(lfnList) if not res['OK']: print("Can't get file metadata: %s" % res['Message']) DIRAC.exit(1) if res['Value']['Failed']: print( "Could not get the file metadata of the following, so skipping them:" ) for fFile in res['Value']['Failed']: print(fFile) lfnMetadata = res['Value']['Successful'] for lfn in lfnMetadata: rarFile = File() rarFile.LFN = lfn rarFile.Size = lfnMetadata[lfn]['Size'] rarFile.Checksum = lfnMetadata[lfn]['Checksum'] rarFile.GUID = lfnMetadata[lfn]['GUID'] rarFile.ChecksumType = 'ADLER32' oOperation.addFile(rarFile) oRequest.addOperation(oOperation) isValid = RequestValidator().validate(oRequest) if not isValid['OK']: print("Request is not valid: ", isValid['Message']) DIRAC.exit(1) result = reqClient.putRequest(oRequest) if result['OK']: print('Request %d Submitted' % result['Value']) else: print('Failed to submit Request: ', result['Message'])
class RequestTasks(TaskBase): def __init__(self, transClient=None, logger=None, requestClient=None, requestClass=None, requestValidator=None): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger('RequestTasks') super(RequestTasks, self).__init__(transClient, logger) if not requestClient: self.requestClient = ReqClient() else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN=''): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername(owner) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass for taskID in sorted(taskDict): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance(paramDict['InputData'], list): files = paramDict['InputData'] elif isinstance(paramDict['InputData'], basestring): files = paramDict['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) oRequest.RequestName = _requestName(transID, taskID) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate(oRequest) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK(taskDict) def submitTransformationTasks(self, taskDict): """ Submit requests one by one """ submitted = 0 failed = 0 startTime = time.time() for taskID in sorted(taskDict): if not taskDict[taskID]['TaskObject']: taskDict[taskID]['Success'] = False failed += 1 continue res = self.submitTaskToExternal(taskDict[taskID]['TaskObject']) if res['OK']: taskDict[taskID]['ExternalID'] = res['Value'] taskDict[taskID]['Success'] = True submitted += 1 else: self._logError("Failed to submit task to RMS", res['Message']) taskDict[taskID]['Success'] = False failed += 1 self._logInfo( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % (submitted, time.time() - startTime)) if failed: self._logWarn( 'submitTasks: But at the same time failed to submit %d tasks to RMS.' % (failed)) return S_OK(taskDict) def submitTaskToExternal(self, oRequest): """ Submits a request using ReqClient """ if isinstance(oRequest, self.requestClass): return self.requestClient.putRequest(oRequest) else: return S_ERROR("Request should be a Request object") def updateTransformationReservedTasks(self, taskDicts): requestNameIDs = {} noTasks = [] for taskDict in taskDicts: requestName = _requestName(taskDict['TransformationID'], taskDict['TaskID']) reqID = taskDict['ExternalID'] if reqID: requestNameIDs[requestName] = reqID else: noTasks.append(requestName) return S_OK({'NoTasks': noTasks, 'TaskNameIDs': requestNameIDs}) def getSubmittedTaskStatus(self, taskDicts): updateDict = {} for taskDict in taskDicts: oldStatus = taskDict['ExternalStatus'] newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID']) if not newStatus['OK']: log = self._logVerbose if 'not exist' in newStatus[ 'Message'] else self.log.warn log( "getSubmittedTaskStatus: Failed to get requestID for request", '%s' % newStatus['Message']) else: newStatus = newStatus['Value'] if newStatus != oldStatus: updateDict.setdefault(newStatus, []).append(taskDict['TaskID']) return S_OK(updateDict) def getSubmittedFileStatus(self, fileDicts): taskFiles = {} submittedTasks = {} externalIds = {} # Don't try and get status of not submitted tasks! for fileDict in fileDicts: submittedTasks.setdefault(fileDict['TransformationID'], set()).add(int(fileDict['TaskID'])) for transID in submittedTasks: res = self.transClient.getTransformationTasks({ 'TransformationID': transID, 'TaskID': list(submittedTasks[transID]) }) if not res['OK']: return res for taskDict in res['Value']: taskID = taskDict['TaskID'] externalIds[taskID] = taskDict['ExternalID'] if taskDict['ExternalStatus'] == 'Created': submittedTasks[transID].remove(taskID) for fileDict in fileDicts: transID = fileDict['TransformationID'] taskID = int(fileDict['TaskID']) if taskID in submittedTasks[transID]: requestID = externalIds[taskID] taskFiles.setdefault(requestID, {})[fileDict['LFN']] = fileDict['Status'] updateDict = {} for requestID in sorted(taskFiles): lfnDict = taskFiles[requestID] statusDict = self.requestClient.getRequestFileStatus( requestID, lfnDict.keys()) if not statusDict['OK']: log = self._logVerbose if 'not exist' in statusDict[ 'Message'] else self.log.warn log( "getSubmittedFileStatus: Failed to get files status for request", '%s' % statusDict['Message']) continue statusDict = statusDict['Value'] for lfn, newStatus in statusDict.items(): if newStatus == lfnDict[lfn]: pass elif newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': updateDict[lfn] = 'Problematic' return S_OK(updateDict)
class RequestTasks(TaskBase): """ Class for handling tasks for the RMS """ def __init__(self, transClient=None, logger=None, requestClient=None, requestClass=None, requestValidator=None, ownerDN=None, ownerGroup=None): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger('RequestTasks') super(RequestTasks, self).__init__(transClient, logger) useCertificates = True if (bool(ownerDN) and bool(ownerGroup)) else False if not requestClient: self.requestClient = ReqClient(useCertificates=useCertificates, delegatedDN=ownerDN, delegatedGroup=ownerGroup) else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN='', bulkSubmissionFlag=False): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername(owner) if not res['OK']: return res ownerDN = res['Value'][0] try: transJson = json.loads(transBody) self._multiOperationsBody(transJson, taskDict, ownerDN, ownerGroup) except ValueError: # #json couldn't load self._singleOperationsBody(transBody, taskDict, ownerDN, ownerGroup) return S_OK(taskDict) def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup): """ deal with a Request that has multiple operations :param transJson: list of lists of string and dictionaries, e.g.: .. code :: python body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }), ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ), ] :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ failedTasks = [] for taskID, task in taskDict.items(): transID = task['TransformationID'] if not task.get('InputData'): self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID) taskDict.pop(taskID) continue files = [] oRequest = Request() if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') # create the operations from the json structure for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] for parameter, value in operationTuple[1].iteritems(): setattr(op, parameter, value) for lfn in files: opFile = File() opFile.LFN = lfn op.addFile(opFile) oRequest.addOperation(op) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID) def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.iteritems(): transID = task['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task['TargetSE'] # If there are input files if task.get('InputData'): if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID) def _assignRequestToTask(self, oRequest, taskDict, transID, taskID, ownerDN, ownerGroup): """set ownerDN and group to request, and add the request to taskDict if it is valid, otherwise remove the task from the taskDict :param oRequest: Request :param dict taskDict: dictionary of tasks, modified in this function :param int transID: Transformation ID :param int taskID: Task ID :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ oRequest.RequestName = self._transTaskName(transID, taskID) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate(oRequest) if not isValid['OK']: self._logError("Error creating request for task", "%s %s" % (taskID, isValid), transID=transID) return S_ERROR('Error creating request') taskDict[taskID]['TaskObject'] = oRequest return S_OK() def submitTransformationTasks(self, taskDict): """ Submit requests one by one """ submitted = 0 failed = 0 startTime = time.time() method = 'submitTransformationTasks' for task in taskDict.itervalues(): # transID is the same for all tasks, so pick it up every time here transID = task['TransformationID'] if not task['TaskObject']: task['Success'] = False failed += 1 continue res = self.submitTaskToExternal(task['TaskObject']) if res['OK']: task['ExternalID'] = res['Value'] task['Success'] = True submitted += 1 else: self._logError("Failed to submit task to RMS", res['Message'], transID=transID) task['Success'] = False failed += 1 if submitted: self._logInfo('Submitted %d tasks to RMS in %.1f seconds' % (submitted, time.time() - startTime), transID=transID, method=method) if failed: self._logWarn('Failed to submit %d tasks to RMS.' % (failed), transID=transID, method=method) return S_OK(taskDict) def submitTaskToExternal(self, oRequest): """ Submits a request to RMS """ if isinstance(oRequest, self.requestClass): return self.requestClient.putRequest(oRequest, useFailoverProxy=False, retryMainService=2) return S_ERROR("Request should be a Request object") def updateTransformationReservedTasks(self, taskDicts): requestNameIDs = {} noTasks = [] for taskDict in taskDicts: requestName = self._transTaskName(taskDict['TransformationID'], taskDict['TaskID']) reqID = taskDict['ExternalID'] if reqID: requestNameIDs[requestName] = reqID else: noTasks.append(requestName) return S_OK({'NoTasks': noTasks, 'TaskNameIDs': requestNameIDs}) def getSubmittedTaskStatus(self, taskDicts): """ Check if tasks changed status, and return a list of tasks per new status """ updateDict = {} badRequestID = 0 for taskDict in taskDicts: oldStatus = taskDict['ExternalStatus'] # ExternalID is normally a string if taskDict['ExternalID'] and int(taskDict['ExternalID']): newStatus = self.requestClient.getRequestStatus(taskDict['ExternalID']) if not newStatus['OK']: log = self._logVerbose if 'not exist' in newStatus['Message'] else self._logWarn log("getSubmittedTaskStatus: Failed to get requestID for request", newStatus['Message'], transID=taskDict['TransformationID']) else: newStatus = newStatus['Value'] # We don't care updating the tasks to Assigned while the request is being processed if newStatus != oldStatus and newStatus != 'Assigned': updateDict.setdefault(newStatus, []).append(taskDict['TaskID']) else: badRequestID += 1 if badRequestID: self._logWarn("%d requests have identifier 0" % badRequestID) return S_OK(updateDict) def getSubmittedFileStatus(self, fileDicts): """ Check if transformation files changed status, and return a list of taskIDs per new status """ # Don't try and get status of not submitted tasks! transID = None taskFiles = {} for fileDict in fileDicts: # There is only one transformation involved, get however the transID in the loop transID = fileDict['TransformationID'] taskID = int(fileDict['TaskID']) taskFiles.setdefault(taskID, []).append(fileDict['LFN']) # Should not happen, but just in case there are no files, return if transID is None: return S_OK({}) res = self.transClient.getTransformationTasks({'TransformationID': transID, 'TaskID': taskFiles.keys()}) if not res['OK']: return res requestFiles = {} for taskDict in res['Value']: taskID = taskDict['TaskID'] externalID = taskDict['ExternalID'] # Only consider tasks that are submitted, ExternalID is a string if taskDict['ExternalStatus'] != 'Created' and externalID and int(externalID): requestFiles[externalID] = taskFiles[taskID] updateDict = {} for requestID, lfnList in requestFiles.iteritems(): statusDict = self.requestClient.getRequestFileStatus(requestID, lfnList) if not statusDict['OK']: log = self._logVerbose if 'not exist' in statusDict['Message'] else self._logWarn log("Failed to get files status for request", statusDict['Message'], transID=transID, method='getSubmittedFileStatus') else: for lfn, newStatus in statusDict['Value'].iteritems(): if newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': updateDict[lfn] = 'Problematic' return S_OK(updateDict)
class ModuleBase(object): """ Base class for Modules - works only within DIRAC workflows """ ############################################################################# def __init__(self, loggerIn=None, operationsHelperIn=None, bkClientIn=None, dm=None): """ Initialization of module base. """ if loggerIn is None: self.log = gLogger.getSubLogger('ModuleBase') else: self.log = loggerIn if operationsHelperIn is None: self.opsH = Operations() else: self.opsH = operationsHelperIn if bkClientIn is None: self.bkClient = BookkeepingClient() else: self.bkClient = bkClientIn if dm is None: self.dataManager = DataManager() else: self.dataManager = dm self.requestValidator = RequestValidator() self.production_id = '' self.prod_job_id = '' self.jobID = 0 self.step_number = '' self.step_id = '' self.fileReport = None self.jobReport = None self.request = None self.workflowStatus = None self.stepStatus = None self.workflow_commons = None self.step_commons = None self.debugSE = 'CERN-DEBUG' self.executable = 'gaudirun.py' self.applicationName = 'Unknown' self.applicationVersion = 'Unknown' self.applicationLog = '' self.applicationType = None self.systemConfig = None self.extraPackages = None self.bkConfigName = None self.BKstepID = None self.condDBTag = None self.DDDBTag = None self.dqTag = None self.CPUe = None self.eventType = '' self.gaudiSteps = None self.InputData = '' self.inputDataList = [] self.inputDataType = None self.histoName = "Hist.root" self.optionsFile = None self.optionsFormat = None self.optionsLine = None self.extraOptionsLine = None self.jobType = None self.logFilePath = None self.onlineCondDBTag = None self.onlineDDDBTag = None self.outputSEs = {} self.outputDataFileMask = None self.numberOfEvents = -1 self.maxNumberOfEvents = None self.TCK = None self.mcTCK = None self.multicoreJob = None self.multicoreStep = None self.poolXMLCatName = 'pool_xml_catalog.xml' self.persistency = '' self.processingPass = None self.runNumber = 'Unknown' self.runTimeProjectName = None self.runTimeProjectVersion = None self.simDescription = '' self.siteName = None self.stepName = None self.stepInputData = None self.XMLSummary = '' # name of the file, not the object self.stepProcPass = None self.outputFilePrefix = '' ############################################################################# def execute(self, version=None, production_id=None, prod_job_id=None, wms_job_id=None, workflowStatus=None, stepStatus=None, wf_commons=None, step_commons=None, step_number=None, step_id=None): """ Function called by all super classes """ if version: self.log.info('===== Executing ' + version + ' ===== ') self.log.verbose("Executing directory for job is %s" % os.getcwd()) if production_id: self.production_id = production_id else: self.production_id = self.workflow_commons['PRODUCTION_ID'] # This is a string, like '00051753' if prod_job_id: self.prod_job_id = prod_job_id else: self.prod_job_id = self.workflow_commons['JOB_ID'] if 'JOBID' in os.environ: self.jobID = os.environ['JOBID'] if wms_job_id: self.jobID = wms_job_id if workflowStatus: self.workflowStatus = workflowStatus if stepStatus: self.stepStatus = stepStatus if wf_commons: self.workflow_commons = wf_commons if step_commons: self.step_commons = step_commons if step_number: self.step_number = step_number else: self.step_number = self.STEP_NUMBER # pylint: disable=no-member if step_id: self.step_id = step_id else: self.step_id = '%s_%s_%s' % (self.production_id, self.prod_job_id, self.step_number) self.siteName = siteName() ############################################################################# def finalize(self, version=None): """ Just finalizing """ self.log.flushAllMessages(0) if version: self.log.info('===== Terminating ' + version + ' ===== ') ############################################################################# def setApplicationStatus(self, status, sendFlag=True): """Wraps around setJobApplicationStatus of state update client """ if not self._WMSJob(): return 0 # e.g. running locally prior to submission if self._checkWFAndStepStatus(noPrint=True): # The application status won't be updated in case the workflow or the step is failed already if not isinstance(status, str): status = str(status) self.log.verbose('setJobApplicationStatus(%d, %s)' % (self.jobID, status)) jobStatus = self.jobReport.setApplicationStatus(status, sendFlag) if not jobStatus['OK']: self.log.warn(jobStatus['Message']) ############################################################################# def setJobParameter(self, name, value, sendFlag=True): """Wraps around setJobParameter of state update client """ if not self._WMSJob(): return 0 # e.g. running locally prior to submission self.log.verbose('setJobParameter(%d,%s,%s)' % (self.jobID, name, value)) jobParam = self.jobReport.setJobParameter(str(name), str(value), sendFlag) if not jobParam['OK']: self.log.warn(jobParam['Message']) ############################################################################# def _resolveInputVariables(self): """ By convention the module input parameters are resolved here. """ self.log.verbose("workflow_commons = ", self.workflow_commons) self.log.verbose("step_commons = ", self.step_commons) self.fileReport = self._getFileReporter() self.jobReport = self._getJobReporter() self.request = self._getRequestContainer() self.__resolveInputWorkflow() ############################################################################# def __resolveInputWorkflow(self): """ Resolve the input variables that are in the workflow_commons """ self.runNumber = self.workflow_commons.get('runNumber', self.runNumber) self.persistency = self.workflow_commons.get('persistency', self.persistency) self.jobType = self.workflow_commons.get('JobType', self.jobType) self.poolXMLCatName = self.workflow_commons.get('poolXMLCatName', self.poolXMLCatName) self.InputData = self.workflow_commons.get('InputData', self.InputData) if 'ParametricInputData' in self.workflow_commons: pID = copy.deepcopy(self.workflow_commons['ParametricInputData']) if pID: if isinstance(pID, list): pID = ';'.join(pID) # self.InputData += ';' + pID self.InputData = pID self.InputData = self.InputData.rstrip(';') if self.InputData == ';': self.InputData = '' self.inputDataList = [lfn.strip('LFN:') for lfn in self.InputData.split(';') if lfn] # only required until the stripping is the same for MC / data self.bkConfigName = self.workflow_commons.get('configName', self.bkConfigName) self.simDescription = self.workflow_commons.get('simDescription', self.simDescription) if 'runMetadata' in self.workflow_commons: runMetadataDict = eval(self.workflow_commons['runMetadata']) self.onlineDDDBTag = runMetadataDict['DDDB'] self.onlineCondDBTag = runMetadataDict['CondDb'] self.TCK = runMetadataDict['TCK'] if 'outputDataFileMask' in self.workflow_commons: self.outputDataFileMask = self.workflow_commons['outputDataFileMask'] if not isinstance(self.outputDataFileMask, list): self.outputDataFileMask = [i.lower().strip() for i in self.outputDataFileMask.split(';')] self.gaudiSteps = self.workflow_commons.get('gaudiSteps', self.gaudiSteps) if 'CPUe' in self.workflow_commons: self.CPUe = int(round(float(self.workflow_commons['CPUe']))) self.multicoreJob = self.workflow_commons.get('multicore', self.multicoreJob) self.processingPass = self.workflow_commons.get('processingPass', self.processingPass) self.logFilePath = self.workflow_commons.get('LogFilePath', self.logFilePath) if isinstance(self.logFilePath, list): self.logFilePath = self.logFilePath[0] else: if 'PRODUCTION_ID' and 'JOB_ID' and 'configVersion' and 'configName' in self.workflow_commons: self.log.info('LogFilePath parameter not found, creating on the fly') result = getLogPath(self.workflow_commons, self.bkClient) if not result['OK']: self.log.error('Could not create LogFilePath', result['Message']) raise RuntimeError(result['Message']) self.logFilePath = result['Value']['LogFilePath'][0] if 'maxNumberOfEvents' in self.workflow_commons: self.maxNumberOfEvents = int(self.workflow_commons['maxNumberOfEvents']) self.eventType = self.workflow_commons.get('eventType', self.eventType) self.numberOfEvents = int(self.workflow_commons.get('numberOfEvents', self.numberOfEvents)) if 'outputSEs' in self.workflow_commons: self.outputSEs = self.workflow_commons['outputSEs'] else: # this is here for backward compatibility histogramSE = self.opsH.getValue('Productions/HistogramSE', 'CERN-HIST') histoTypes = self.opsH.getValue('Productions/HistogramTypes', ['HIST', 'BRUNELHIST', 'DAVINCIHIST', 'GAUSSHIST']) self.outputSEs = dict((ht, histogramSE) for ht in histoTypes) # for older productions we construct it based on what should be found in the steps if 'listoutput' in self.step_commons: listOutputStep = self.step_commons['listoutput'] for lOutput in listOutputStep: try: for outputDataType in lOutput['outputDataType'].split(';'): if outputDataType: self.outputSEs.setdefault(outputDataType.upper(), lOutput['outputDataSE']) except KeyError: continue self.workflow_commons['outputSEs'] = self.outputSEs ############################################################################# def _resolveInputStep(self): """ Resolve the input variables for an application step """ prodID = self.workflow_commons.get('PRODUCTION_ID', '') jobID = self.workflow_commons.get('JOB_ID', '') stepInstanceNumber = self.step_commons.get('STEP_NUMBER', '') self.stepName = self.step_commons['STEP_INSTANCE_NAME'] self.executable = self.step_commons.get('executable', self.executable) self.applicationName = self.step_commons.get('applicationName', self.applicationName) self.applicationVersion = self.step_commons.get('applicationVersion', self.applicationVersion) self.BKstepID = self.step_commons.get('BKStepID', self.BKstepID) self.stepProcPass = self.step_commons.get('StepProcPass', self.stepProcPass) # this is only for production jobs and for application steps if prodID and jobID and stepInstanceNumber and 'listoutput' in self.step_commons: self.outputFilePrefix = "%s_%s_%s" % (prodID, jobID, stepInstanceNumber) self.applicationLog = self.applicationName + '_' + self.outputFilePrefix + '.log' self.XMLSummary = 'summary' + self.applicationName + '_' + self.outputFilePrefix + '.xml' self.histoName = self.applicationName + '_' + self.outputFilePrefix + '.Hist.root' for fileTypeDict in self.step_commons['listoutput']: # this is a dict like {'outputDataType': 'sim'} # for non histo-merging prods if 'hist' in fileTypeDict['outputDataType'].lower() and self.jobType.lower() != 'merge': # Watch out: this assumes that: # - 'hist' is always in the file type name # - merging jobs won't produce histograms # - the only merging jobs that produce output types with hist are histomerging productions if 'outputDataName' not in fileTypeDict: fileTypeDict['outputDataName'] = self.histoName else: fileTypeDict['outputDataName'] = self.outputFilePrefix + '.' + fileTypeDict['outputDataType'] else: self.applicationLog = self.step_commons.get('applicationLog', self.applicationLog) self.inputDataType = self.step_commons.get('inputDataType', self.inputDataType) self.applicationType = self.step_commons.get('applicationType', self.applicationType) self.optionsFile = self.step_commons.get('optionsFile', self.optionsFile) self.optionsLine = self.step_commons.get('optionsLine', self.optionsLine) self.extraOptionsLine = self.step_commons.get('extraOptionsLine', self.extraOptionsLine) if 'runTimeProjectName' in self.step_commons: self.runTimeProjectName = self.step_commons['runTimeProjectName'] self.runTimeProjectVersion = self.step_commons['runTimeProjectVersion'] if 'extraPackages' in self.step_commons: self.extraPackages = self.step_commons['extraPackages'] if self.extraPackages: if isinstance(self.extraPackages, basestring): eps = self.extraPackages.split(';') # pylint: disable=no-member epList = [] for ep in eps: epList.append(tuple(ep.split('.'))) self.extraPackages = epList stepInputData = [] if 'inputData' in self.step_commons: if self.step_commons['inputData']: stepInputData = self.step_commons['inputData'] elif self.InputData: stepInputData = copy.deepcopy(self.InputData) if stepInputData: stepInputData = self._determineStepInputData(stepInputData, ) self.stepInputData = [sid.strip('LFN:') for sid in stepInputData] self.optionsFormat = self.step_commons.get('optionsFormat', self.optionsFormat) self.multicoreStep = self.step_commons.get('multiCore', self.multicoreStep) self.systemConfig = self.step_commons.get('SystemConfig', self.systemConfig) self.mcTCK = self.step_commons.get('mcTCK', self.mcTCK) self.DDDBTag = self.step_commons.get('DDDBTag', self.DDDBTag) self.condDBTag = self.step_commons.get('CondDBTag', self.condDBTag) self.dqTag = self.step_commons.get('DQTag', self.dqTag) # This is resolved also in __resolveInputWorkflow but can be specialized per step (e.g. LHCbJob().setApplication()) self.numberOfEvents = int(self.step_commons.get('numberOfEvents', self.numberOfEvents)) ############################################################################# def _determineOutputs(self): """ Method that determines the correct outputs. For merging jobs the output has normally to be the same as the input, but there might be exceptions (like for the productions for the merging of histograms) For the others, we use what is in the step definition We always remove the 'HIST'(s), when present, from the list of output file types as these are treated differently. There is anyway also here the special case of histogram merging productions. """ histoTypes = self.opsH.getValue('Productions/HistogramTypes', ['HIST', 'BRUNELHIST', 'DAVINCIHIST', 'GAUSSHIST']) stepOutputs = self.step_commons['listoutput'] stepOutputsT = [x['outputDataType'] for x in stepOutputs] stepOutTypes = [] for fts in stepOutputsT: for ft in fts.split(';'): if ft and ft not in stepOutTypes: stepOutTypes.append(ft.lower()) # corrections for Merge productions if self.jobType.lower() in ('merge', 'histomerge'): if len(stepOutTypes) == 1 and stepOutTypes[0] in [hts.lower() for hts in histoTypes] + ['root']: # If it is root/histo/ntuple merging job, we treat it almost as a stripping job pass else: res = self.bkClient.getFileMetadata(self.stepInputData) if not res['OK']: raise RuntimeError(res['Message']) outputTypes = set() success = res['Value']['Successful'] for mdDict in success.values(): outputTypes.add(mdDict['FileType']) if len(success) != len(self.stepInputData): self.log.warn("Some inputs are not in BKK, trying to parse the file names") for sid in set(self.stepInputData) - set(success): # File types in the BK are upper case fType = '.'.join(os.path.basename(sid).split('.')[1:]).upper() outputTypes.add(fType) outputTypes = list(outputTypes) if len(outputTypes) > 1: raise ValueError("Not all input files have the same type (%s)" % ','.join(outputTypes)) outputType = outputTypes[0].lower() stepOutTypes = [outputType.lower()] stepOutputs = [{'outputDataName': self.step_id + '.' + outputType.lower(), 'outputDataType': outputType.lower(), 'outputBKType': outputType.upper()}] histogram = False # first here treating the special case of root/histo/ntuple merging jobs if self.jobType.lower() in ('merge', 'histomerge') \ and len(stepOutTypes) == 1 \ and stepOutTypes[0] in [hts.lower() for hts in histoTypes] + ['root']: pass else: # This is not a histogram merging production, so we remove the histograms from the step outputs for hist in histoTypes: try: stepOutTypes.remove(hist) histogram = True except ValueError: pass try: stepOutTypes.remove(hist.lower()) histogram = True except ValueError: pass return stepOutputs, stepOutTypes, histogram ############################################################################# def _getJobReporter(self): """ just return the job reporter (object, always defined by dirac-jobexec) """ if 'JobReport' in self.workflow_commons: return self.workflow_commons['JobReport'] jobReport = JobReport(self.jobID) self.workflow_commons['JobReport'] = jobReport return jobReport ############################################################################# def _getFileReporter(self): """ just return the file reporter (object) """ if 'FileReport' in self.workflow_commons: return self.workflow_commons['FileReport'] fileReport = FileReport() self.workflow_commons['FileReport'] = fileReport return fileReport ############################################################################# def _getRequestContainer(self): """ just return the Request reporter (object) """ if 'Request' in self.workflow_commons: return self.workflow_commons['Request'] request = Request() self.workflow_commons['Request'] = request return request ############################################################################# def getCandidateFiles(self, outputList, outputLFNs, fileMask='', stepMask=''): """ Returns list of candidate files to upload, check if some outputs are missing. :param list outputList: list of outputs with the following structure:: [{'outputDataType': '', 'outputDataName': ''} , {...}] :param list outputLFNs: output LFNs for the job :param str fileMask: the output file extensions to restrict the outputs to. Can also be a list of strings :param str stepMask: the step ID to restrict the outputs to. Can also be a list of strings. :returns: dictionary containing type, SE and LFN for files restricted by mask """ fileInfo = {} for outputFile in outputList: if 'outputDataType' in outputFile and 'outputDataName' in outputFile: fname = outputFile['outputDataName'] fileType = outputFile['outputDataType'] fileInfo[fname] = {'type': fileType} else: self.log.error('Ignoring mal-formed output data specification', str(outputFile)) for lfn in outputLFNs: if os.path.basename(lfn).lower() in list(fi.lower() for fi in fileInfo): try: fileInfo[os.path.basename(lfn)]['lfn'] = lfn self.log.verbose("Found LFN for file", "%s -> %s" % (lfn, os.path.basename(lfn))) except KeyError: fileInfo[os.path.basename(lfn).lower()]['lfn'] = lfn self.log.verbose("Found LFN for file", "%s -> %s" % (lfn, os.path.basename(lfn).lower())) elif os.path.basename(lfn).split('_')[-1].lower() in list(fi.lower() for fi in fileInfo): try: fileInfo[os.path.basename(lfn).split('_')[-1]]['lfn'] = lfn self.log.verbose("Found LFN for file", " %s -> %s" % (lfn, os.path.basename(lfn).split('_')[-1])) except KeyError: fileInfo[os.path.basename(lfn).split('_')[-1].lower()]['lfn'] = lfn self.log.verbose("Found LFN for file", " %s -> %s" % (lfn, os.path.basename(lfn).split('_')[-1].lower())) else: self.log.warn("LFN not recognized", "for LFN %s" % lfn) # check local existance fileList = self._checkLocalExistance(list(fileInfo.keys())) # really horrible stuff for updating the name with what's found on the disk # (because maybe the case is not the same as the expected) newFileInfo = {} for fi in fileInfo.iteritems(): for li in fileList: if fi[0].lower() == li.lower(): newFileInfo[li] = fi[1] # Select which files have to be uploaded: in principle all candidateFiles = self._applyMask(newFileInfo, fileMask, stepMask) # Sanity check all final candidate metadata keys are present self._checkSanity(candidateFiles) # Adding the SEs for candidateFile in candidateFiles: try: fType = candidateFiles[candidateFile]['type'] for fType in [fType, fType.lower(), fType.upper(), fType.capitalize()]: try: wfSE = self.outputSEs[fType] candidateFiles[candidateFile]['workflowSE'] = wfSE break except KeyError: continue except AttributeError: break return candidateFiles ############################################################################# def _checkLocalExistance(self, fileList): """ Check that the list of output files are present locally """ notPresentFiles = [] filesOnDisk = set(os.listdir('.')) diff = set(fileList).difference(filesOnDisk) if diff: self.log.warn("Not all files found", "set of what's on the disk: %s" % filesOnDisk) self.log.warn("Looking for files with different case") diffCI = set(fx.lower() for fx in fileList).difference(set(fod.lower() for fod in filesOnDisk)) if diffCI: self.log.error("Output data not found", "File list %s does not exist locally" % notPresentFiles) raise os.error("Output data not found") # now checking what's the actual filename case that's written self.log.warn("Found files with filename with different case, returning those") filesActuallyOnDisk = [] for fod in os.listdir('.'): if fod.lower() in [fl.lower() for fl in fileList]: filesActuallyOnDisk.append(fod) return filesActuallyOnDisk return fileList ############################################################################# def _applyMask(self, candidateFilesIn, fileMask, stepMask): """ Select which files have to be uploaded: in principle all :param dict candidateFilesIn: dictionary like {'00012345_00012345_4.dst': {'lfn': '/lhcb/MC/2010/DST/123/123_45_4.dst', type': 'dst'}, '00012345_00012345_2.digi': {'type': 'digi'}} :param str fileMask: the output file extensions to restrict the outputs to. Can also be a list of strings :param str stepMask: the step ID to restrict the outputs to. Can also be a list of strings. :returns: a dict like the one in candidateFilesIn """ candidateFiles = copy.deepcopy(candidateFilesIn) if fileMask and not isinstance(fileMask, list): fileMask = [fileMask] if isinstance(stepMask, int): stepMask = str(stepMask) if stepMask and not isinstance(stepMask, list): stepMask = [stepMask] if fileMask and fileMask != ['']: for fileName, metadata in list(candidateFiles.iteritems()): if metadata['type'].lower() not in [fm.lower() for fm in fileMask]: del candidateFiles[fileName] self.log.info('Output file %s was produced but will not be treated (fileMask is %s)' % (fileName, ', '.join(fileMask))) else: self.log.info('No outputDataFileMask provided, the files with all the extensions will be considered') if stepMask and stepMask != ['']: for fileName, metadata in list(candidateFiles.iteritems()): if fileName.lower().replace(metadata['type'].lower(), '').split('_')[-1].split('.')[0] not in stepMask: del candidateFiles[fileName] self.log.info('Output file %s was produced but will not be treated (stepMask is %s)' % (fileName, ', '.join(stepMask))) else: self.log.info('No outputDataStep provided, the files output of all the steps will be considered') return candidateFiles ############################################################################# def _checkSanity(self, candidateFiles): """ Sanity check all final candidate metadata keys are present :param dict candidateFiles: dictionary like {'00012345_00012345_4.dst': {'lfn': '/lhcb/MC/2010/DST/123/123_45_4.dst', type': 'dst'}, '00012345_00012345_2.digi': {'type': 'digi'}} :returns: None or raises ValueError """ notPresentKeys = [] mandatoryKeys = ['type', 'lfn'] # filedict is used for requests for fileName, metadata in candidateFiles.iteritems(): for key in mandatoryKeys: if key not in metadata: notPresentKeys.append((fileName, key)) if notPresentKeys: for fileName_keys in notPresentKeys: self.log.error("File %s has missing %s" % (fileName_keys[0], fileName_keys[1])) raise ValueError("Missing requested fileName keys") ############################################################################# def getFileMetadata(self, candidateFiles): """ Returns the candidate file dictionary with associated metadata. The input candidate files dictionary has the structure: {'foo_1.txt': {'lfn': '/lhcb/MC/2010/DST/00012345/0001/foo_1.txt', 'type': 'txt', 'workflowSE': SE1}, 'bar_2.py': {'lfn': '/lhcb/MC/2010/DST/00012345/0001/bar_2.py', 'type': 'py', 'workflowSE': 'SE2'}, } this also assumes the files are in the current working directory. """ # Retrieve the POOL File GUID(s) for any final output files self.log.info('Will search for POOL GUIDs for: %s' % (', '.join(candidateFiles.keys()))) pfnGUID = getGUID(candidateFiles.keys()) if not pfnGUID['OK']: self.log.error('''PoolXMLFile failed to determine POOL GUID(s) for output file list, these will be generated by the DataManager''', pfnGUID['Message']) for fileName in candidateFiles.keys(): candidateFiles[fileName]['guid'] = '' elif pfnGUID['generated']: self.log.warn('PoolXMLFile generated GUID(s) for the following files ', ', '.join(pfnGUID['generated'])) else: self.log.info('GUIDs found for all specified POOL files: %s' % (', '.join(candidateFiles.keys()))) for pfn, guid in pfnGUID['Value'].iteritems(): candidateFiles[pfn]['guid'] = guid # Get all additional metadata about the file necessary for requests final = {} for fileName, metadata in candidateFiles.iteritems(): fileDict = {} fileDict['LFN'] = metadata['lfn'] fileDict['Size'] = os.path.getsize(fileName) fileDict['Checksum'] = fileAdler(fileName) fileDict['ChecksumType'] = 'ADLER32' fileDict['GUID'] = metadata['guid'] fileDict['Status'] = 'Waiting' final[fileName] = metadata final[fileName]['filedict'] = fileDict final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName) # Sanity check all final candidate metadata keys are present (return S_ERROR if not) mandatoryKeys = ['guid', 'filedict'] # filedict is used for requests (this method adds guid and filedict) for fileName, metadata in final.iteritems(): for key in mandatoryKeys: if key not in metadata: raise RuntimeError("File %s has missing %s" % (fileName, key)) return final ############################################################################# def _determineStepInputData(self, inputData): """ determine the input data for the step """ if inputData == 'previousStep': stepIndex = self.gaudiSteps.index(self.stepName) previousStep = self.gaudiSteps[stepIndex - 1] stepInputData = [] for outputF in self.workflow_commons['outputList']: try: if outputF['stepName'] == previousStep and outputF['outputBKType'].lower() == self.inputDataType.lower(): stepInputData.append(outputF['outputDataName']) except KeyError: raise RuntimeError("Can't find output of step %s" % previousStep) return stepInputData return [x.strip('LFN:') for x in inputData.split(';')] ############################################################################# def _manageAppOutput(self, outputs): """ Calls self._findOutputs to find what's produced, then creates the LFNs outputs, as called here, is created starting from step_commons['listoutput'], but enriched with at least the outputDataName. example of outputs: [{'outputDataType': 'bhadron.dst', 'outputBKType': 'BHADRON.DST', 'outputDataName': '00012345_00012345_2.BHADRON.DST'}, {'outputDataType': 'calibration.dst','outputDataType': 'CALIBRATION.DST', 'outputDataName': '00012345_00012345_2.CALIBRATION.DST'}] :params list outputs: list of dicts of step output files descriptions """ if not outputs: self.log.warn('Step outputs are not defined (normal user jobs. Not normal in productions and SAM jobs)') return else: finalOutputs, _bkFileTypes = self._findOutputs(outputs) self.log.info('Final step outputs are: %s' % (finalOutputs)) self.step_commons['listoutput'] = finalOutputs if 'outputList' in self.workflow_commons: for outFile in finalOutputs: if outFile not in self.workflow_commons['outputList']: self.workflow_commons['outputList'].append(outFile) else: self.workflow_commons['outputList'] = finalOutputs if 'PRODUCTION_ID' and 'JOB_ID' and 'configVersion' and 'configName' in self.workflow_commons: self.log.info('Attempting to recreate the production output LFNs...') result = constructProductionLFNs(self.workflow_commons, self.bkClient) if not result['OK']: raise IOError("Could not create production LFNs: %s" % result['Message']) self.workflow_commons['BookkeepingLFNs'] = result['Value']['BookkeepingLFNs'] self.workflow_commons['LogFilePath'] = result['Value']['LogFilePath'] self.workflow_commons['ProductionOutputData'] = result['Value']['ProductionOutputData'] ############################################################################# def _findOutputs(self, stepOutput): """ Find which outputs of those in stepOutput (what are expected to be produced) are effectively produced. stepOutput, as called here, is created starting from step_commons['listoutput'] example of stepOutput: [{'outputDataType': 'bhadron.dst', 'outputBKType': 'BHADRON.DST', 'outputDataName': '00012345_00012345_2.BHADRON.DST'}, {'outputDataType': 'calibration.dst','outputDataType': 'CALIBRATION.DST', 'outputDataName': '00012345_00012345_2.CALIBRATION.DST'}] :params list stepOutput: list of dicts of step output files descriptions :returns: list, list """ bkFileTypes = [] # uppercase list of file types finalOutputs = [] # list of dicts of what's found on the local disk filesFound = [] for output in stepOutput: found = False fileOnDisk = None for fileOnDisk in os.listdir('.'): if output['outputDataName'].lower() == fileOnDisk.lower(): found = True break if found and fileOnDisk: self.log.info('Found output file (case is not considered)', '%s matching %s ' % (fileOnDisk, output['outputDataName'])) output['outputDataName'] = fileOnDisk filesFound.append(output) else: self.log.error('Output not found', output['outputDataName']) raise IOError("OutputData not found") for fileFound in filesFound: bkFileTypes.append(fileFound['outputDataType'].upper()) finalOutputs.append({'outputDataName': fileFound['outputDataName'], 'outputDataType': fileFound['outputDataType'].lower(), 'outputBKType': fileFound['outputDataType'].upper(), 'stepName': self.stepName}) return (finalOutputs, bkFileTypes) ############################################################################# def _WMSJob(self): """ Check if this job is running via WMS """ return True if self.jobID else False ############################################################################# def _enableModule(self): """ Enable module if it's running via WMS """ if not self._WMSJob(): self.log.info('No WMS JobID found, disabling module via control flag') return False self.log.verbose('Found WMS JobID', self.jobID) return True ############################################################################# def _checkWFAndStepStatus(self, noPrint=False): """ Check the WF and Step status """ if not self.workflowStatus['OK'] or not self.stepStatus['OK']: if not noPrint: self.log.info('Skip this module, failure detected in a previous step') self.log.info('Workflow status', self.workflowStatus) self.log.info('Step Status', self.stepStatus) return False else: return True ############################################################################# def _disableWatchdogCPUCheck(self): """ just writes a file to disable the watchdog """ self.log.info("Creating DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK in order to disable the Watchdog") with open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w') as fopen: fopen.write('%s' % time.asctime()) ############################################################################# def generateFailoverFile(self): """ Retrieve the accumulated reporting request, and produce a JSON file that is consumed by the JobWrapper """ reportRequest = None result = self.jobReport.generateForwardDISET() if not result['OK']: self.log.warn("Could not generate Operation for job report", "with result:\n%s" % (result)) else: reportRequest = result['Value'] if reportRequest: self.log.info("Populating request with job report information") self.request.addOperation(reportRequest) if len(self.request): try: optimized = self.request.optimize() except AttributeError: optimized = {'OK': True} if not optimized['OK']: self.log.error("Could not optimize", optimized['Message']) self.log.error("Not failing the job because of that, keep going") isValid = self.requestValidator.validate(self.request) if not isValid['OK']: raise RuntimeError("Failover request is not valid: %s" % isValid['Message']) else: requestJSON = self.request.toJSON() if requestJSON['OK']: self.log.info("Creating failover request for deferred operations", "for job %d" % self.jobID) request_string = str(requestJSON['Value']) self.log.debug(request_string) # Write out the request string fname = '%s_%s_request.json' % (self.production_id, self.prod_job_id) with open(fname, 'w') as jsonFile: jsonFile.write(request_string) self.log.info("Created file containing failover request", fname) result = self.request.getDigest() if result['OK']: self.log.info("Digest of the request", result['Value']) else: self.log.warn("No digest? That's not sooo important", "anyway: %s" % result['Message']) else: raise RuntimeError(requestJSON['Message']) accountingReport = None if 'AccountingReport' in self.workflow_commons: accountingReport = self.workflow_commons['AccountingReport'] if accountingReport: result = accountingReport.commit() if not result['OK']: self.log.error("!!! Both accounting and RequestDB are down? !!!") self.log.error("accountingReport result: %s" % result['Message']) self.log.error("Anyway, the job won't fail for this reason, because this is \"just\" the accounting report") ############################################################################# def setBKRegistrationRequest(self, lfn, error='', metaData={'Checksum': 'justSomething', 'ChecksumType': 'ADLER32', 'GUID': 'aGUID'}): """ Set a BK registration request for changing the replica flag. Uses the global request object (self.request). """ if error: self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error)) else: self.log.info('Setting BK registration request for %s' % (lfn)) regFile = Operation() regFile.Type = 'RegisterFile' regFile.Catalog = 'BookkeepingDB' bkFile = File() bkFile.LFN = lfn # this should NOT be needed... but RMS complains! bkFile.PFN = lfn bkFile.GUID = metaData['GUID'] bkFile.Checksum = metaData['Checksum'] bkFile.ChecksumType = metaData['ChecksumType'] regFile.addFile(bkFile) res = self.request.addOperation(regFile) if not res['OK']: raise RuntimeError(res['Message']) ############################################################################# def createProdConfFile(self, stepOutputTypes, histogram, runNumberGauss, firstEventNumberGauss): """ Utility that creates a ProdConf file, used mostly as input for gaudirun jobs """ # Creating ProdConf file prodConfFileName = 'prodConf_%s_%s_%s_%s.py' % (self.applicationName, self.production_id, self.prod_job_id, self.step_number) optionsDict = {} optionsDict['Application'] = self.applicationName optionsDict['AppVersion'] = self.applicationVersion if self.optionsFormat: optionsDict['OptionFormat'] = self.optionsFormat if self.stepInputData: optionsDict['InputFiles'] = ['LFN:' + sid for sid in self.stepInputData] else: if self.applicationName.lower() != "gauss": raise RuntimeError("No MC, but no input data") if self.outputFilePrefix: optionsDict['OutputFilePrefix'] = self.outputFilePrefix optionsDict['OutputFileTypes'] = stepOutputTypes optionsDict['XMLSummaryFile'] = self.XMLSummary optionsDict['XMLFileCatalog'] = self.poolXMLCatName if histogram: optionsDict['HistogramFile'] = self.histoName if self.DDDBTag: if self.DDDBTag.lower() == 'online': try: optionsDict['DDDBTag'] = self.onlineDDDBTag self.log.debug('Set the online DDDB tag') except NameError as e: self.log.error('Could not find online DDDb Tag', e) raise RuntimeError("Could not find online DDDb Tag") else: optionsDict['DDDBTag'] = self.DDDBTag if self.condDBTag: if self.condDBTag.lower() == 'online': optionsDict['CondDBTag'] = self.onlineCondDBTag self.log.debug('Set the online CondDB tag') else: optionsDict['CondDBTag'] = self.condDBTag if self.dqTag: optionsDict['DQTag'] = self.dqTag if self.applicationName.lower() == 'gauss': if self.CPUe and self.maxNumberOfEvents and self.numberOfEvents <= 0: # Here we set maxCPUTime to 24 hours, which seems reasonable eventsToProduce = getEventsToProduce(self.CPUe, maxNumberOfEvents=self.maxNumberOfEvents, jobMaxCPUTime=86400) else: eventsToProduce = self.numberOfEvents else: eventsToProduce = self.numberOfEvents optionsDict['NOfEvents'] = eventsToProduce if runNumberGauss: optionsDict['RunNumber'] = runNumberGauss if self.runNumber: if self.runNumber not in ('Unknown', 'Multiple'): optionsDict['RunNumber'] = self.runNumber if firstEventNumberGauss: optionsDict['FirstEventNumber'] = firstEventNumberGauss # TCK: can't have both set! if self.TCK and self.mcTCK: raise RuntimeError("%s step: TCK set in step, and should't be!" % self.applicationName) if self.TCK or self.mcTCK: optionsDict['TCK'] = self.TCK if self.TCK else self.mcTCK if self.processingPass: optionsDict['ProcessingPass'] = self.processingPass prodConfFile = ProdConf(prodConfFileName) self.log.debug(optionsDict) prodConfFile.putOptionsIn(optionsDict) return prodConfFileName ############################################################################# # properties def set_jobID(self, value): if isinstance(value, str): if value: value = int(value) else: value = 0 self._jobID = value def get_jobID(self): return self._jobID jobID = property(get_jobID, set_jobID)
requestClient = ReqClient() request.RequestName = 'copy_%s' % os.path.basename(appTar).replace( ".tgz", "").replace(".tar.gz", "") request.SourceComponent = 'ReplicateILCSoft' copies_at = ops.getValue('Software/CopiesAt', []) for copies in copies_at: transfer = Operation() transfer.Type = "ReplicateAndRegister" transfer.TargetSE = copies trFile = File() trFile.LFN = lfnpath trFile.GUID = "" transfer.addFile(trFile) request.addOperation(transfer) res = RequestValidator().validate(request) if not res['OK']: return res if copies_at: res = requestClient.putRequest(request) if not res['OK']: gLogger.error('Could not set replication request', res['Message']) return S_OK('Application uploaded') return S_OK() def fullCopy(srcdir, dstdir, item): """ Copy the item from srcdir to dstdir, creates missing directories if needed """
def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN=''): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername(owner) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass for taskID in sorted(taskDict): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if type(paramDict['InputData']) == type([]): files = paramDict['InputData'] elif type(paramDict['InputData']) == type(''): files = paramDict['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) oRequest.RequestName = _requestName(transID, taskID) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = RequestValidator().validate(oRequest) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK(taskDict)
def validate(cls, request): """ request validation """ if not cls.__validator: cls.__validator = RequestValidator() return cls.__validator.validate(request)
def export_putRequest(self, requestJSON): """ put a new request into RequestDB :param cls: class ref :param str requestJSON: request serialized to JSON format """ requestDict = json.loads(requestJSON) requestName = requestDict.get("RequestID", requestDict.get('RequestName', "***UNKNOWN***")) request = Request(requestDict) # Check whether the credentials in the Requests are correct and allowed to be set isAuthorized = RequestValidator.setAndCheckRequestOwner(request, self.getRemoteCredentials()) if not isAuthorized: return S_ERROR(DErrno.ENOAUTH, "Credentials in the requests are not allowed") optimized = request.optimize() if optimized.get("Value", False): gLogger.debug("putRequest: request was optimized") else: gLogger.debug( "putRequest: request unchanged", optimized.get( "Message", "Nothing could be optimized")) valid = self.validate(request) if not valid["OK"]: gLogger.error("putRequest: request %s not valid: %s" % (requestName, valid["Message"])) return valid # If NotBefore is not set or user defined, we calculate its value now = datetime.datetime.utcnow().replace(microsecond=0) extraDelay = datetime.timedelta(0) if request.Status not in Request.FINAL_STATES and ( not request.NotBefore or request.NotBefore < now): # We don't delay if it is the first insertion if getattr(request, 'RequestID', 0): # If it is a constant delay, just set it if self.constantRequestDelay: extraDelay = datetime.timedelta(minutes=self.constantRequestDelay) else: # If there is a waiting Operation with Files op = request.getWaiting().get('Value') if op and len(op): attemptList = [opFile.Attempt for opFile in op if opFile.Status == "Waiting"] if attemptList: maxWaitingAttempt = max( [opFile.Attempt for opFile in op if opFile.Status == "Waiting"]) # In case it is the first attempt, extraDelay is 0 # maxWaitingAttempt can be None if the operation has no File, like the ForwardDiset extraDelay = datetime.timedelta( minutes=2 * math.log(maxWaitingAttempt) if maxWaitingAttempt else 0) request.NotBefore = now + extraDelay gLogger.info("putRequest: request %s not before %s (extra delay %s)" % (request.RequestName, request.NotBefore, extraDelay)) requestName = request.RequestName gLogger.info("putRequest: Attempting to set request '%s'" % requestName) return self.__requestDB.putRequest(request)
if res['Value']['Failed']: print( "Could not get the file metadata of the following, so skipping them:" ) for fFile in res['Value']['Failed']: print(fFile) lfnMetadata = res['Value']['Successful'] for lfn in lfnMetadata: rarFile = File() rarFile.LFN = lfn rarFile.Size = lfnMetadata[lfn]['Size'] rarFile.Checksum = lfnMetadata[lfn]['Checksum'] rarFile.GUID = lfnMetadata[lfn]['GUID'] rarFile.ChecksumType = 'ADLER32' oOperation.addFile(rarFile) oRequest.addOperation(oOperation) isValid = RequestValidator().validate(oRequest) if not isValid['OK']: print("Request is not valid: ", isValid['Message']) DIRAC.exit(1) result = reqClient.putRequest(oRequest) if result['OK']: print('Request %d Submitted' % result['Value']) else: print('Failed to submit Request: ', result['Message'])
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(" SE: StorageElement|All") Script.registerArgument(["LFN: LFN or file containing a List of LFNs"]) Script.parseCommandLine(ignoreErrors=False) # parseCommandLine show help when mandatory arguments are not specified or incorrect argument args = Script.getPositionalArgs() targetSE = args.pop(0) lfns = [] for inputFileName in args: if os.path.exists(inputFileName): with open(inputFileName, "r") as inputFile: string = inputFile.read() lfns.extend([lfn.strip() for lfn in string.splitlines()]) else: lfns.append(inputFileName) from DIRAC.Resources.Storage.StorageElement import StorageElement import DIRAC # Check is provided SE is OK if targetSE != "All": se = StorageElement(targetSE) if not se.valid: print(se.errorReason) print() Script.showHelp() from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.File import File from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator from DIRAC.Resources.Catalog.FileCatalog import FileCatalog reqClient = ReqClient() fc = FileCatalog() requestOperation = "RemoveReplica" if targetSE == "All": requestOperation = "RemoveFile" for lfnList in breakListIntoChunks(lfns, 100): oRequest = Request() requestName = "%s_%s" % ( md5(repr(time.time()).encode()).hexdigest()[:16], md5(repr(time.time()).encode()).hexdigest()[:16], ) oRequest.RequestName = requestName oOperation = Operation() oOperation.Type = requestOperation oOperation.TargetSE = targetSE res = fc.getFileMetadata(lfnList) if not res["OK"]: print("Can't get file metadata: %s" % res["Message"]) DIRAC.exit(1) if res["Value"]["Failed"]: print( "Could not get the file metadata of the following, so skipping them:" ) for fFile in res["Value"]["Failed"]: print(fFile) lfnMetadata = res["Value"]["Successful"] for lfn in lfnMetadata: rarFile = File() rarFile.LFN = lfn rarFile.Size = lfnMetadata[lfn]["Size"] rarFile.Checksum = lfnMetadata[lfn]["Checksum"] rarFile.GUID = lfnMetadata[lfn]["GUID"] rarFile.ChecksumType = "ADLER32" oOperation.addFile(rarFile) oRequest.addOperation(oOperation) isValid = RequestValidator().validate(oRequest) if not isValid["OK"]: print("Request is not valid: ", isValid["Message"]) DIRAC.exit(1) result = reqClient.putRequest(oRequest) if result["OK"]: print("Request %d Submitted" % result["Value"]) else: print("Failed to submit Request: ", result["Message"])
def __init__(self, loggerIn=None, operationsHelperIn=None, bkClientIn=None, dm=None): """ Initialization of module base. """ if loggerIn is None: self.log = gLogger.getSubLogger('ModuleBase') else: self.log = loggerIn if operationsHelperIn is None: self.opsH = Operations() else: self.opsH = operationsHelperIn if bkClientIn is None: self.bkClient = BookkeepingClient() else: self.bkClient = bkClientIn if dm is None: self.dataManager = DataManager() else: self.dataManager = dm self.requestValidator = RequestValidator() self.production_id = '' self.prod_job_id = '' self.jobID = 0 self.step_number = '' self.step_id = '' self.fileReport = None self.jobReport = None self.request = None self.workflowStatus = None self.stepStatus = None self.workflow_commons = None self.step_commons = None self.debugSE = 'CERN-DEBUG' self.executable = 'gaudirun.py' self.applicationName = 'Unknown' self.applicationVersion = 'Unknown' self.applicationLog = '' self.applicationType = None self.systemConfig = None self.extraPackages = None self.bkConfigName = None self.BKstepID = None self.condDBTag = None self.DDDBTag = None self.dqTag = None self.CPUe = None self.eventType = '' self.gaudiSteps = None self.InputData = '' self.inputDataList = [] self.inputDataType = None self.histoName = "Hist.root" self.optionsFile = None self.optionsFormat = None self.optionsLine = None self.extraOptionsLine = None self.jobType = None self.logFilePath = None self.onlineCondDBTag = None self.onlineDDDBTag = None self.outputSEs = {} self.outputDataFileMask = None self.numberOfEvents = -1 self.maxNumberOfEvents = None self.TCK = None self.mcTCK = None self.multicoreJob = None self.multicoreStep = None self.poolXMLCatName = 'pool_xml_catalog.xml' self.persistency = '' self.processingPass = None self.runNumber = 'Unknown' self.runTimeProjectName = None self.runTimeProjectVersion = None self.simDescription = '' self.siteName = None self.stepName = None self.stepInputData = None self.XMLSummary = '' # name of the file, not the object self.stepProcPass = None self.outputFilePrefix = ''
def testValidator(self): """validator test""" # create validator validator = RequestValidator() self.assertEqual(isinstance(validator, RequestValidator), True) # RequestName not set ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.request.RequestName = "test_request" # # no operations ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.request.addOperation(self.operation) # # type not set ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.operation.Type = "ReplicateAndRegister" # # files not present ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.operation.addFile(self.file) # # targetSE not set ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.operation.TargetSE = "CERN-USER" # # missing LFN ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.file.LFN = "/a/b/c" # # no ownerDN # force no owner DN because it takes the one of the current user self.request.OwnerDN = "" ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.request.OwnerDN = "foo/bar=baz" # # no owner group # same, force it self.request.OwnerGroup = "" ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.request.OwnerGroup = "dirac_user" # Checksum set, ChecksumType not set self.file.Checksum = "abcdef" ret = validator.validate(self.request) self.assertFalse(ret["OK"]) # ChecksumType set, Checksum not set self.file.Checksum = "" self.file.ChecksumType = "adler32" ret = validator.validate(self.request) self.assertFalse(ret["OK"]) # both set self.file.Checksum = "abcdef" self.file.ChecksumType = "adler32" ret = validator.validate(self.request) self.assertEqual(ret, {"OK": True, "Value": None}) # both unset self.file.Checksum = "" self.file.ChecksumType = None ret = validator.validate(self.request) self.assertEqual(ret, {"OK": True, "Value": None}) # all OK ret = validator.validate(self.request) self.assertEqual(ret, {"OK": True, "Value": None})
class RequestTasks(TaskBase): """ Class for handling tasks for the RMS """ def __init__( self, transClient=None, logger=None, requestClient=None, requestClass=None, requestValidator=None, ownerDN=None, ownerGroup=None, ): """c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger(self.__class__.__name__) super(RequestTasks, self).__init__(transClient, logger) useCertificates = True if (bool(ownerDN) and bool(ownerGroup)) else False if not requestClient: self.requestClient = ReqClient(useCertificates=useCertificates, delegatedDN=ownerDN, delegatedGroup=ownerGroup) else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator def prepareTransformationTasks(self, transBody, taskDict, owner="", ownerGroup="", ownerDN="", bulkSubmissionFlag=False): """Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB""" if not taskDict: return S_OK({}) if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res["OK"]: return res proxyInfo = res["Value"] owner = proxyInfo["username"] ownerGroup = proxyInfo["group"] if not ownerDN: res = getDNForUsername(owner) if not res["OK"]: return res ownerDN = res["Value"][0] try: transJson, _decLen = decode(transBody) if isinstance(transJson, BaseBody): self._bodyPlugins(transJson, taskDict, ownerDN, ownerGroup) else: self._multiOperationsBody(transJson, taskDict, ownerDN, ownerGroup) except ValueError: # #json couldn't load self._singleOperationsBody(transBody, taskDict, ownerDN, ownerGroup) return S_OK(taskDict) def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup): """Deal with a Request that has multiple operations :param transJson: list of lists of string and dictionaries, e.g.: .. code :: python body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"TASK:TargetSE" }), ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ), ] If a value of an operation parameter in the body starts with ``TASK:``, we take it from the taskDict. For example ``TASK:TargetSE`` is replaced with ``task['TargetSE']`` :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ for taskID, task in list(taskDict.items()): try: transID = task["TransformationID"] if not task.get("InputData"): raise StopTaskIteration("No input data") files = [] oRequest = Request() if isinstance(task["InputData"], list): files = task["InputData"] elif isinstance(task["InputData"], six.string_types): files = task["InputData"].split(";") # create the operations from the json structure for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] for parameter, value in operationTuple[1].items(): # Here we massage a bit the body to replace some parameters # with what we have in the task. try: taskKey = value.split("TASK:")[1] value = task[taskKey] # Either the attribute is not a string (AttributeError) # or it does not start with 'TASK:' (IndexError) except (AttributeError, IndexError): pass # That happens when the requested substitution is not # a key in the task, and that's a problem except KeyError: raise StopTaskIteration( "Parameter %s does not exist in taskDict" % taskKey) setattr(op, parameter, value) for lfn in files: opFile = File() opFile.LFN = lfn op.addFile(opFile) oRequest.addOperation(op) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result["OK"]: raise StopTaskIteration( "Could not assign request to task: %s" % result["Message"]) except StopTaskIteration as e: self._logError("Error creating request for task", "%s, %s" % (taskID, e), transID=transID) taskDict.pop(taskID) def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = "ReplicateAndRegister" if transBody: try: _requestType, requestOperation = transBody.split(";") except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.items(): transID = task["TransformationID"] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task["TargetSE"] # If there are input files if task.get("InputData"): if isinstance(task["InputData"], list): files = task["InputData"] elif isinstance(task["InputData"], six.string_types): files = task["InputData"].split(";") for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result["OK"]: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID) def _bodyPlugins(self, bodyObj, taskDict, ownerDN, ownerGroup): """Deal with complex body object""" for taskID, task in list(taskDict.items()): try: transID = task["TransformationID"] if not task.get("InputData"): raise StopTaskIteration("No input data") oRequest = bodyObj.taskToRequest(taskID, task, transID) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result["OK"]: raise StopTaskIteration( "Could not assign request to task: %s" % result["Message"]) except StopTaskIteration as e: self._logError("Error creating request for task", "%s, %s" % (taskID, e), transID=transID) taskDict.pop(taskID) def _assignRequestToTask(self, oRequest, taskDict, transID, taskID, ownerDN, ownerGroup): """set ownerDN and group to request, and add the request to taskDict if it is valid, otherwise remove the task from the taskDict :param oRequest: Request :param dict taskDict: dictionary of tasks, modified in this function :param int transID: Transformation ID :param int taskID: Task ID :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ oRequest.RequestName = self._transTaskName(transID, taskID) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate(oRequest) if not isValid["OK"]: self._logError("Error creating request for task", "%s %s" % (taskID, isValid), transID=transID) return S_ERROR("Error creating request") taskDict[taskID]["TaskObject"] = oRequest return S_OK() def submitTransformationTasks(self, taskDict): """Submit requests one by one""" submitted = 0 failed = 0 startTime = time.time() method = "submitTransformationTasks" for task in taskDict.values(): # transID is the same for all tasks, so pick it up every time here transID = task["TransformationID"] if not task["TaskObject"]: task["Success"] = False failed += 1 continue res = self.submitTaskToExternal(task["TaskObject"]) if res["OK"]: task["ExternalID"] = res["Value"] task["Success"] = True submitted += 1 else: self._logError("Failed to submit task to RMS", res["Message"], transID=transID) task["Success"] = False failed += 1 if submitted: self._logInfo( "Submitted %d tasks to RMS in %.1f seconds" % (submitted, time.time() - startTime), transID=transID, method=method, ) if failed: self._logWarn("Failed to submit %d tasks to RMS." % (failed), transID=transID, method=method) return S_OK(taskDict) def submitTaskToExternal(self, oRequest): """ Submits a request to RMS """ if isinstance(oRequest, self.requestClass): return self.requestClient.putRequest(oRequest, useFailoverProxy=False, retryMainService=2) return S_ERROR("Request should be a Request object") def updateTransformationReservedTasks(self, taskDicts): requestNameIDs = {} noTasks = [] for taskDict in taskDicts: requestName = self._transTaskName(taskDict["TransformationID"], taskDict["TaskID"]) reqID = taskDict["ExternalID"] if reqID and int(reqID): requestNameIDs[requestName] = reqID else: noTasks.append(requestName) return S_OK({"NoTasks": noTasks, "TaskNameIDs": requestNameIDs}) def getSubmittedTaskStatus(self, taskDicts): """ Check if tasks changed status, and return a list of tasks per new status """ updateDict = {} badRequestID = 0 for taskDict in taskDicts: oldStatus = taskDict["ExternalStatus"] # ExternalID is normally a string if taskDict["ExternalID"] and int(taskDict["ExternalID"]): newStatus = self.requestClient.getRequestStatus( taskDict["ExternalID"]) if not newStatus["OK"]: log = self._logVerbose if "not exist" in newStatus[ "Message"] else self._logWarn log( "getSubmittedTaskStatus: Failed to get requestID for request", newStatus["Message"], transID=taskDict["TransformationID"], ) else: newStatus = newStatus["Value"] # We don't care updating the tasks to Assigned while the request is being processed if newStatus != oldStatus and newStatus != "Assigned": updateDict.setdefault(newStatus, []).append(taskDict["TaskID"]) else: badRequestID += 1 if badRequestID: self._logWarn("%d requests have identifier 0" % badRequestID) return S_OK(updateDict) def getSubmittedFileStatus(self, fileDicts): """ Check if transformation files changed status, and return a list of taskIDs per new status """ # Don't try and get status of not submitted tasks! transID = None taskFiles = {} for fileDict in fileDicts: # There is only one transformation involved, get however the transID in the loop transID = fileDict["TransformationID"] taskID = int(fileDict["TaskID"]) taskFiles.setdefault(taskID, []).append(fileDict["LFN"]) # Should not happen, but just in case there are no files, return if transID is None: return S_OK({}) res = self.transClient.getTransformationTasks({ "TransformationID": transID, "TaskID": list(taskFiles) }) if not res["OK"]: return res requestFiles = {} for taskDict in res["Value"]: taskID = taskDict["TaskID"] externalID = taskDict["ExternalID"] # Only consider tasks that are submitted, ExternalID is a string if taskDict["ExternalStatus"] != "Created" and externalID and int( externalID): requestFiles[externalID] = taskFiles[taskID] updateDict = {} for requestID, lfnList in requestFiles.items(): statusDict = self.requestClient.getRequestFileStatus( requestID, lfnList) if not statusDict["OK"]: log = self._logVerbose if "not exist" in statusDict[ "Message"] else self._logWarn log( "Failed to get files status for request", statusDict["Message"], transID=transID, method="getSubmittedFileStatus", ) else: for lfn, newStatus in statusDict["Value"].items(): if newStatus == "Done": updateDict[lfn] = TransformationFilesStatus.PROCESSED elif newStatus == "Failed": updateDict[lfn] = TransformationFilesStatus.PROBLEMATIC return S_OK(updateDict)
def testValidator( self ): """ validator test """ ## create validator validator = RequestValidator() self.assertEqual( isinstance( validator, RequestValidator ), True ) ## RequestName not set ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : 'RequestName not set', 'OK' : False } ) self.request.RequestName = "test_request" # # no operations ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : "Operations not present in request 'test_request'", 'OK': False} ) self.request.addOperation( self.operation ) # # type not set ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : "Operation #0 in request 'test_request' hasn't got Type set", 'OK' : False } ) self.operation.Type = "ReplicateAndRegister" # # files not present ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : "Operation #0 of type 'ReplicateAndRegister' hasn't got files to process.", 'OK' : False } ) self.operation.addFile( self.file ) # # targetSE not set ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : "Operation #0 of type 'ReplicateAndRegister' is missing TargetSE attribute.", 'OK': False } ) self.operation.TargetSE = "CERN-USER" # # missing LFN ret = validator.validate( self.request ) self.assertEqual( ret, { "Message" : "Operation #0 of type 'ReplicateAndRegister' is missing LFN attribute for file.", "OK": False } ) self.file.LFN = "/a/b/c" # # no ownerDN # force no owner DN because it takes the one of the current user self.request.OwnerDN = '' ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : "Request 'test_request' is missing OwnerDN value", 'OK': False} ) self.request.OwnerDN = "foo/bar=baz" # # no owner group # same, force it self.request.OwnerGroup = '' ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : "Request 'test_request' is missing OwnerGroup value", 'OK': False} ) self.request.OwnerGroup = "dirac_user" ## Checksum set, ChecksumType not set self.file.Checksum = "abcdef" ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : 'File in operation #0 is missing Checksum (abcdef) or ChecksumType ()', 'OK' : False } ) ## ChecksumType set, Checksum not set self.file.Checksum = "" self.file.ChecksumType = "adler32" ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : 'File in operation #0 is missing Checksum () or ChecksumType (ADLER32)', 'OK' : False } ) ## both set self.file.Checksum = "abcdef" self.file.ChecksumType = "adler32" ret = validator.validate( self.request ) self.assertEqual( ret, {'OK': True, 'Value': None} ) ## both unset self.file.Checksum = "" self.file.ChecksumType = None ret = validator.validate( self.request ) self.assertEqual( ret, {'OK': True, 'Value': None} ) ## all OK ret = validator.validate( self.request ) self.assertEqual( ret, {'OK': True, 'Value': None} )