class RequestTasks(TaskBase): """ Class for handling tasks for the RMS """ def __init__( self, transClient=None, logger=None, requestClient=None, requestClass=None, requestValidator=None, ownerDN=None, ownerGroup=None, ): """c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger(self.__class__.__name__) super(RequestTasks, self).__init__(transClient, logger) useCertificates = True if (bool(ownerDN) and bool(ownerGroup)) else False if not requestClient: self.requestClient = ReqClient(useCertificates=useCertificates, delegatedDN=ownerDN, delegatedGroup=ownerGroup) else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator def prepareTransformationTasks(self, transBody, taskDict, owner="", ownerGroup="", ownerDN="", bulkSubmissionFlag=False): """Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB""" if not taskDict: return S_OK({}) if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res["OK"]: return res proxyInfo = res["Value"] owner = proxyInfo["username"] ownerGroup = proxyInfo["group"] if not ownerDN: res = getDNForUsername(owner) if not res["OK"]: return res ownerDN = res["Value"][0] try: transJson, _decLen = decode(transBody) if isinstance(transJson, BaseBody): self._bodyPlugins(transJson, taskDict, ownerDN, ownerGroup) else: self._multiOperationsBody(transJson, taskDict, ownerDN, ownerGroup) except ValueError: # #json couldn't load self._singleOperationsBody(transBody, taskDict, ownerDN, ownerGroup) return S_OK(taskDict) def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup): """Deal with a Request that has multiple operations :param transJson: list of lists of string and dictionaries, e.g.: .. code :: python body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"TASK:TargetSE" }), ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ), ] If a value of an operation parameter in the body starts with ``TASK:``, we take it from the taskDict. For example ``TASK:TargetSE`` is replaced with ``task['TargetSE']`` :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ for taskID, task in list(taskDict.items()): try: transID = task["TransformationID"] if not task.get("InputData"): raise StopTaskIteration("No input data") files = [] oRequest = Request() if isinstance(task["InputData"], list): files = task["InputData"] elif isinstance(task["InputData"], six.string_types): files = task["InputData"].split(";") # create the operations from the json structure for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] for parameter, value in operationTuple[1].items(): # Here we massage a bit the body to replace some parameters # with what we have in the task. try: taskKey = value.split("TASK:")[1] value = task[taskKey] # Either the attribute is not a string (AttributeError) # or it does not start with 'TASK:' (IndexError) except (AttributeError, IndexError): pass # That happens when the requested substitution is not # a key in the task, and that's a problem except KeyError: raise StopTaskIteration( "Parameter %s does not exist in taskDict" % taskKey) setattr(op, parameter, value) for lfn in files: opFile = File() opFile.LFN = lfn op.addFile(opFile) oRequest.addOperation(op) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result["OK"]: raise StopTaskIteration( "Could not assign request to task: %s" % result["Message"]) except StopTaskIteration as e: self._logError("Error creating request for task", "%s, %s" % (taskID, e), transID=transID) taskDict.pop(taskID) def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = "ReplicateAndRegister" if transBody: try: _requestType, requestOperation = transBody.split(";") except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.items(): transID = task["TransformationID"] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task["TargetSE"] # If there are input files if task.get("InputData"): if isinstance(task["InputData"], list): files = task["InputData"] elif isinstance(task["InputData"], six.string_types): files = task["InputData"].split(";") for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result["OK"]: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID) def _bodyPlugins(self, bodyObj, taskDict, ownerDN, ownerGroup): """Deal with complex body object""" for taskID, task in list(taskDict.items()): try: transID = task["TransformationID"] if not task.get("InputData"): raise StopTaskIteration("No input data") oRequest = bodyObj.taskToRequest(taskID, task, transID) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result["OK"]: raise StopTaskIteration( "Could not assign request to task: %s" % result["Message"]) except StopTaskIteration as e: self._logError("Error creating request for task", "%s, %s" % (taskID, e), transID=transID) taskDict.pop(taskID) def _assignRequestToTask(self, oRequest, taskDict, transID, taskID, ownerDN, ownerGroup): """set ownerDN and group to request, and add the request to taskDict if it is valid, otherwise remove the task from the taskDict :param oRequest: Request :param dict taskDict: dictionary of tasks, modified in this function :param int transID: Transformation ID :param int taskID: Task ID :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ oRequest.RequestName = self._transTaskName(transID, taskID) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate(oRequest) if not isValid["OK"]: self._logError("Error creating request for task", "%s %s" % (taskID, isValid), transID=transID) return S_ERROR("Error creating request") taskDict[taskID]["TaskObject"] = oRequest return S_OK() def submitTransformationTasks(self, taskDict): """Submit requests one by one""" submitted = 0 failed = 0 startTime = time.time() method = "submitTransformationTasks" for task in taskDict.values(): # transID is the same for all tasks, so pick it up every time here transID = task["TransformationID"] if not task["TaskObject"]: task["Success"] = False failed += 1 continue res = self.submitTaskToExternal(task["TaskObject"]) if res["OK"]: task["ExternalID"] = res["Value"] task["Success"] = True submitted += 1 else: self._logError("Failed to submit task to RMS", res["Message"], transID=transID) task["Success"] = False failed += 1 if submitted: self._logInfo( "Submitted %d tasks to RMS in %.1f seconds" % (submitted, time.time() - startTime), transID=transID, method=method, ) if failed: self._logWarn("Failed to submit %d tasks to RMS." % (failed), transID=transID, method=method) return S_OK(taskDict) def submitTaskToExternal(self, oRequest): """ Submits a request to RMS """ if isinstance(oRequest, self.requestClass): return self.requestClient.putRequest(oRequest, useFailoverProxy=False, retryMainService=2) return S_ERROR("Request should be a Request object") def updateTransformationReservedTasks(self, taskDicts): requestNameIDs = {} noTasks = [] for taskDict in taskDicts: requestName = self._transTaskName(taskDict["TransformationID"], taskDict["TaskID"]) reqID = taskDict["ExternalID"] if reqID and int(reqID): requestNameIDs[requestName] = reqID else: noTasks.append(requestName) return S_OK({"NoTasks": noTasks, "TaskNameIDs": requestNameIDs}) def getSubmittedTaskStatus(self, taskDicts): """ Check if tasks changed status, and return a list of tasks per new status """ updateDict = {} badRequestID = 0 for taskDict in taskDicts: oldStatus = taskDict["ExternalStatus"] # ExternalID is normally a string if taskDict["ExternalID"] and int(taskDict["ExternalID"]): newStatus = self.requestClient.getRequestStatus( taskDict["ExternalID"]) if not newStatus["OK"]: log = self._logVerbose if "not exist" in newStatus[ "Message"] else self._logWarn log( "getSubmittedTaskStatus: Failed to get requestID for request", newStatus["Message"], transID=taskDict["TransformationID"], ) else: newStatus = newStatus["Value"] # We don't care updating the tasks to Assigned while the request is being processed if newStatus != oldStatus and newStatus != "Assigned": updateDict.setdefault(newStatus, []).append(taskDict["TaskID"]) else: badRequestID += 1 if badRequestID: self._logWarn("%d requests have identifier 0" % badRequestID) return S_OK(updateDict) def getSubmittedFileStatus(self, fileDicts): """ Check if transformation files changed status, and return a list of taskIDs per new status """ # Don't try and get status of not submitted tasks! transID = None taskFiles = {} for fileDict in fileDicts: # There is only one transformation involved, get however the transID in the loop transID = fileDict["TransformationID"] taskID = int(fileDict["TaskID"]) taskFiles.setdefault(taskID, []).append(fileDict["LFN"]) # Should not happen, but just in case there are no files, return if transID is None: return S_OK({}) res = self.transClient.getTransformationTasks({ "TransformationID": transID, "TaskID": list(taskFiles) }) if not res["OK"]: return res requestFiles = {} for taskDict in res["Value"]: taskID = taskDict["TaskID"] externalID = taskDict["ExternalID"] # Only consider tasks that are submitted, ExternalID is a string if taskDict["ExternalStatus"] != "Created" and externalID and int( externalID): requestFiles[externalID] = taskFiles[taskID] updateDict = {} for requestID, lfnList in requestFiles.items(): statusDict = self.requestClient.getRequestFileStatus( requestID, lfnList) if not statusDict["OK"]: log = self._logVerbose if "not exist" in statusDict[ "Message"] else self._logWarn log( "Failed to get files status for request", statusDict["Message"], transID=transID, method="getSubmittedFileStatus", ) else: for lfn, newStatus in statusDict["Value"].items(): if newStatus == "Done": updateDict[lfn] = TransformationFilesStatus.PROCESSED elif newStatus == "Failed": updateDict[lfn] = TransformationFilesStatus.PROBLEMATIC return S_OK(updateDict)
class RequestTasks(TaskBase): """ Class for handling tasks for the RMS """ def __init__(self, transClient=None, logger=None, requestClient=None, requestClass=None, requestValidator=None, ownerDN=None, ownerGroup=None): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger('RequestTasks') super(RequestTasks, self).__init__(transClient, logger) useCertificates = True if (bool(ownerDN) and bool(ownerGroup)) else False if not requestClient: self.requestClient = ReqClient(useCertificates=useCertificates, delegatedDN=ownerDN, delegatedGroup=ownerGroup) else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN='', bulkSubmissionFlag=False): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername(owner) if not res['OK']: return res ownerDN = res['Value'][0] try: transJson = json.loads(transBody) self._multiOperationsBody(transJson, taskDict, ownerDN, ownerGroup) except ValueError: # #json couldn't load self._singleOperationsBody(transBody, taskDict, ownerDN, ownerGroup) return S_OK(taskDict) def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup): """ deal with a Request that has multiple operations :param transJson: list of lists of string and dictionaries, e.g.: .. code :: python body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }), ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ), ] :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ failedTasks = [] for taskID, task in taskDict.items(): transID = task['TransformationID'] if not task.get('InputData'): self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID) taskDict.pop(taskID) continue files = [] oRequest = Request() if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') # create the operations from the json structure for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] for parameter, value in operationTuple[1].iteritems(): setattr(op, parameter, value) for lfn in files: opFile = File() opFile.LFN = lfn op.addFile(opFile) oRequest.addOperation(op) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID) def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.iteritems(): transID = task['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task['TargetSE'] # If there are input files if task.get('InputData'): if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID) def _assignRequestToTask(self, oRequest, taskDict, transID, taskID, ownerDN, ownerGroup): """set ownerDN and group to request, and add the request to taskDict if it is valid, otherwise remove the task from the taskDict :param oRequest: Request :param dict taskDict: dictionary of tasks, modified in this function :param int transID: Transformation ID :param int taskID: Task ID :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ oRequest.RequestName = self._transTaskName(transID, taskID) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate(oRequest) if not isValid['OK']: self._logError("Error creating request for task", "%s %s" % (taskID, isValid), transID=transID) return S_ERROR('Error creating request') taskDict[taskID]['TaskObject'] = oRequest return S_OK() def submitTransformationTasks(self, taskDict): """ Submit requests one by one """ submitted = 0 failed = 0 startTime = time.time() method = 'submitTransformationTasks' for task in taskDict.itervalues(): # transID is the same for all tasks, so pick it up every time here transID = task['TransformationID'] if not task['TaskObject']: task['Success'] = False failed += 1 continue res = self.submitTaskToExternal(task['TaskObject']) if res['OK']: task['ExternalID'] = res['Value'] task['Success'] = True submitted += 1 else: self._logError("Failed to submit task to RMS", res['Message'], transID=transID) task['Success'] = False failed += 1 if submitted: self._logInfo('Submitted %d tasks to RMS in %.1f seconds' % (submitted, time.time() - startTime), transID=transID, method=method) if failed: self._logWarn('Failed to submit %d tasks to RMS.' % (failed), transID=transID, method=method) return S_OK(taskDict) def submitTaskToExternal(self, oRequest): """ Submits a request to RMS """ if isinstance(oRequest, self.requestClass): return self.requestClient.putRequest(oRequest, useFailoverProxy=False, retryMainService=2) return S_ERROR("Request should be a Request object") def updateTransformationReservedTasks(self, taskDicts): requestNameIDs = {} noTasks = [] for taskDict in taskDicts: requestName = self._transTaskName(taskDict['TransformationID'], taskDict['TaskID']) reqID = taskDict['ExternalID'] if reqID: requestNameIDs[requestName] = reqID else: noTasks.append(requestName) return S_OK({'NoTasks': noTasks, 'TaskNameIDs': requestNameIDs}) def getSubmittedTaskStatus(self, taskDicts): """ Check if tasks changed status, and return a list of tasks per new status """ updateDict = {} badRequestID = 0 for taskDict in taskDicts: oldStatus = taskDict['ExternalStatus'] # ExternalID is normally a string if taskDict['ExternalID'] and int(taskDict['ExternalID']): newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID']) if not newStatus['OK']: log = self._logVerbose if 'not exist' in newStatus[ 'Message'] else self._logWarn log("getSubmittedTaskStatus: Failed to get requestID for request", newStatus['Message'], transID=taskDict['TransformationID']) else: newStatus = newStatus['Value'] # We don't care updating the tasks to Assigned while the request is being processed if newStatus != oldStatus and newStatus != 'Assigned': updateDict.setdefault(newStatus, []).append(taskDict['TaskID']) else: badRequestID += 1 if badRequestID: self._logWarn("%d requests have identifier 0" % badRequestID) return S_OK(updateDict) def getSubmittedFileStatus(self, fileDicts): """ Check if transformation files changed status, and return a list of taskIDs per new status """ # Don't try and get status of not submitted tasks! transID = None taskFiles = {} for fileDict in fileDicts: # There is only one transformation involved, get however the transID in the loop transID = fileDict['TransformationID'] taskID = int(fileDict['TaskID']) taskFiles.setdefault(taskID, []).append(fileDict['LFN']) # Should not happen, but just in case there are no files, return if transID is None: return S_OK({}) res = self.transClient.getTransformationTasks({ 'TransformationID': transID, 'TaskID': taskFiles.keys() }) if not res['OK']: return res requestFiles = {} for taskDict in res['Value']: taskID = taskDict['TaskID'] externalID = taskDict['ExternalID'] # Only consider tasks that are submitted, ExternalID is a string if taskDict['ExternalStatus'] != 'Created' and externalID and int( externalID): requestFiles[externalID] = taskFiles[taskID] updateDict = {} for requestID, lfnList in requestFiles.iteritems(): statusDict = self.requestClient.getRequestFileStatus( requestID, lfnList) if not statusDict['OK']: log = self._logVerbose if 'not exist' in statusDict[ 'Message'] else self._logWarn log("Failed to get files status for request", statusDict['Message'], transID=transID, method='getSubmittedFileStatus') else: for lfn, newStatus in statusDict['Value'].iteritems(): if newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': updateDict[lfn] = 'Problematic' return S_OK(updateDict)
class RequestTasks(TaskBase): def __init__(self, transClient=None, logger=None, requestClient=None, requestClass=None, requestValidator=None): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger('RequestTasks') super(RequestTasks, self).__init__(transClient, logger) if not requestClient: self.requestClient = ReqClient() else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN=''): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername(owner) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass for taskID in sorted(taskDict): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance(paramDict['InputData'], list): files = paramDict['InputData'] elif isinstance(paramDict['InputData'], basestring): files = paramDict['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) oRequest.RequestName = _requestName(transID, taskID) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate(oRequest) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK(taskDict) def submitTransformationTasks(self, taskDict): """ Submit requests one by one """ submitted = 0 failed = 0 startTime = time.time() for taskID in sorted(taskDict): if not taskDict[taskID]['TaskObject']: taskDict[taskID]['Success'] = False failed += 1 continue res = self.submitTaskToExternal(taskDict[taskID]['TaskObject']) if res['OK']: taskDict[taskID]['ExternalID'] = res['Value'] taskDict[taskID]['Success'] = True submitted += 1 else: self._logError("Failed to submit task to RMS", res['Message']) taskDict[taskID]['Success'] = False failed += 1 self._logInfo( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % (submitted, time.time() - startTime)) if failed: self._logWarn( 'submitTasks: But at the same time failed to submit %d tasks to RMS.' % (failed)) return S_OK(taskDict) def submitTaskToExternal(self, oRequest): """ Submits a request using ReqClient """ if isinstance(oRequest, self.requestClass): return self.requestClient.putRequest(oRequest) else: return S_ERROR("Request should be a Request object") def updateTransformationReservedTasks(self, taskDicts): requestNameIDs = {} noTasks = [] for taskDict in taskDicts: requestName = _requestName(taskDict['TransformationID'], taskDict['TaskID']) reqID = taskDict['ExternalID'] if reqID: requestNameIDs[requestName] = reqID else: noTasks.append(requestName) return S_OK({'NoTasks': noTasks, 'TaskNameIDs': requestNameIDs}) def getSubmittedTaskStatus(self, taskDicts): updateDict = {} for taskDict in taskDicts: oldStatus = taskDict['ExternalStatus'] newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID']) if not newStatus['OK']: log = self._logVerbose if 'not exist' in newStatus[ 'Message'] else self.log.warn log( "getSubmittedTaskStatus: Failed to get requestID for request", '%s' % newStatus['Message']) else: newStatus = newStatus['Value'] if newStatus != oldStatus: updateDict.setdefault(newStatus, []).append(taskDict['TaskID']) return S_OK(updateDict) def getSubmittedFileStatus(self, fileDicts): taskFiles = {} submittedTasks = {} externalIds = {} # Don't try and get status of not submitted tasks! for fileDict in fileDicts: submittedTasks.setdefault(fileDict['TransformationID'], set()).add(int(fileDict['TaskID'])) for transID in submittedTasks: res = self.transClient.getTransformationTasks({ 'TransformationID': transID, 'TaskID': list(submittedTasks[transID]) }) if not res['OK']: return res for taskDict in res['Value']: taskID = taskDict['TaskID'] externalIds[taskID] = taskDict['ExternalID'] if taskDict['ExternalStatus'] == 'Created': submittedTasks[transID].remove(taskID) for fileDict in fileDicts: transID = fileDict['TransformationID'] taskID = int(fileDict['TaskID']) if taskID in submittedTasks[transID]: requestID = externalIds[taskID] taskFiles.setdefault(requestID, {})[fileDict['LFN']] = fileDict['Status'] updateDict = {} for requestID in sorted(taskFiles): lfnDict = taskFiles[requestID] statusDict = self.requestClient.getRequestFileStatus( requestID, lfnDict.keys()) if not statusDict['OK']: log = self._logVerbose if 'not exist' in statusDict[ 'Message'] else self.log.warn log( "getSubmittedFileStatus: Failed to get files status for request", '%s' % statusDict['Message']) continue statusDict = statusDict['Value'] for lfn, newStatus in statusDict.items(): if newStatus == lfnDict[lfn]: pass elif newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': updateDict[lfn] = 'Problematic' return S_OK(updateDict)
class RequestTasks( TaskBase ): def __init__( self, transClient = None, logger = None, requestClient = None, requestClass = None, requestValidator = None ): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger( 'RequestTasks' ) super( RequestTasks, self ).__init__( transClient, logger ) if not requestClient: self.requestClient = ReqClient() else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if paramDict['InputData']: transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: return isValid taskDict[taskID]['TaskObject'] = oRequest return S_OK( taskDict ) def submitTransformationTasks( self, taskDict ): """ Submit requests one by one """ submitted = 0 failed = 0 startTime = time.time() for taskID in sorted( taskDict ): if not taskDict[taskID]['TaskObject']: taskDict[taskID]['Success'] = False failed += 1 continue res = self.submitTaskToExternal( taskDict[taskID]['TaskObject'] ) if res['OK']: taskDict[taskID]['ExternalID'] = res['Value'] taskDict[taskID]['Success'] = True submitted += 1 else: self._logError( "Failed to submit task to RMS", res['Message'] ) taskDict[taskID]['Success'] = False failed += 1 self._logInfo( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % ( submitted, time.time() - startTime ) ) if failed: self._logWarn( 'submitTasks: But at the same time failed to submit %d tasks to RMS.' % ( failed ) ) return S_OK( taskDict ) def submitTaskToExternal( self, oRequest ): """ Submits a request using ReqClient """ if isinstance( oRequest, self.requestClass ): return self.requestClient.putRequest( oRequest ) else: return S_ERROR( "Request should be a Request object" ) def updateTransformationReservedTasks( self, taskDicts ): requestNameIDs = {} noTasks = [] for taskDict in taskDicts: requestName = _requestName( taskDict['TransformationID'], taskDict['TaskID'] ) reqID = taskDict['ExternalID'] if reqID: requestNameIDs[requestName] = reqID else: noTasks.append( requestName ) return S_OK( {'NoTasks':noTasks, 'TaskNameIDs':requestNameIDs} ) def getSubmittedTaskStatus( self, taskDicts ): updateDict = {} for taskDict in taskDicts: oldStatus = taskDict['ExternalStatus'] newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID'] ) if not newStatus['OK']: log = self._logVerbose if 'not exist' in newStatus['Message'] else self.log.warn log( "getSubmittedTaskStatus: Failed to get requestID for request", '%s' % newStatus['Message'] ) else: newStatus = newStatus['Value'] if newStatus != oldStatus: updateDict.setdefault( newStatus, [] ).append( taskDict['TaskID'] ) return S_OK( updateDict ) def getSubmittedFileStatus( self, fileDicts ): taskFiles = {} submittedTasks = {} externalIds = {} # Don't try and get status of not submitted tasks! for fileDict in fileDicts: submittedTasks.setdefault( fileDict['TransformationID'], set() ).add( int( fileDict['TaskID'] ) ) for transID in submittedTasks: res = self.transClient.getTransformationTasks( { 'TransformationID':transID, 'TaskID': list( submittedTasks[transID] )} ) if not res['OK']: return res for taskDict in res['Value']: taskID = taskDict['TaskID'] externalIds[taskID] = taskDict['ExternalID'] if taskDict['ExternalStatus'] == 'Created': submittedTasks[transID].remove( taskID ) for fileDict in fileDicts: transID = fileDict['TransformationID'] taskID = int( fileDict['TaskID'] ) if taskID in submittedTasks[transID]: requestID = externalIds[taskID] taskFiles.setdefault( requestID, {} )[fileDict['LFN']] = fileDict['Status'] updateDict = {} for requestID in sorted( taskFiles ): lfnDict = taskFiles[requestID] statusDict = self.requestClient.getRequestFileStatus( requestID, lfnDict.keys() ) if not statusDict['OK']: log = self._logVerbose if 'not exist' in statusDict['Message'] else self.log.warn log( "getSubmittedFileStatus: Failed to get files status for request", '%s' % statusDict['Message'] ) continue statusDict = statusDict['Value'] for lfn, newStatus in statusDict.items(): if newStatus == lfnDict[lfn]: pass elif newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': updateDict[lfn] = 'Problematic' return S_OK( updateDict )
def testValidator(self): """ validator test """ ## create validator validator = RequestValidator() self.assertEqual(isinstance(validator, RequestValidator), True) ## RequestName not set ret = validator.validate(self.request) self.assertEqual(ret, {'Message': 'RequestName not set', 'OK': False}) self.request.RequestName = "test_request" # # no ownerDN ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': "Request 'test_request' is missing OwnerDN value", 'OK': False }) self.request.OwnerDN = "foo/bar=baz" # # no owner group ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': "Request 'test_request' is missing OwnerGroup value", 'OK': False }) self.request.OwnerGroup = "dirac_user" ## no operations ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': "Operations not present in request 'test_request'", 'OK': False }) self.request.addOperation(self.operation) ## type not set ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': "Operation #0 in request 'test_request' hasn't got Type set", 'OK': False }) self.operation.Type = "ReplicateAndRegister" ## files not present ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': "Operation #0 of type 'ReplicateAndRegister' hasn't got files to process.", 'OK': False }) self.operation.addFile(self.file) ## targetSE not set ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': "Operation #0 of type 'ReplicateAndRegister' is missing TargetSE attribute.", 'OK': False }) self.operation.TargetSE = "CERN-USER" ## missing LFN ret = validator.validate(self.request) self.assertEqual( ret, { "Message": "Operation #0 of type 'ReplicateAndRegister' is missing LFN attribute for file.", "OK": False }) self.file.LFN = "/a/b/c" ## Checksum set, ChecksumType not set self.file.Checksum = "abcdef" ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': 'File in operation #0 is missing Checksum (abcdef) or ChecksumType ()', 'OK': False }) ## ChecksumType set, Checksum not set self.file.Checksum = "" self.file.ChecksumType = "adler32" ret = validator.validate(self.request) self.assertEqual( ret, { 'Message': 'File in operation #0 is missing Checksum () or ChecksumType (ADLER32)', 'OK': False }) ## both set self.file.Checksum = "abcdef" self.file.ChecksumType = "adler32" ret = validator.validate(self.request) self.assertEqual(ret, {'OK': True, 'Value': ''}) ## both unset self.file.Checksum = "" self.file.ChecksumType = None ret = validator.validate(self.request) self.assertEqual(ret, {'OK': True, 'Value': ''}) ## all OK ret = validator.validate(self.request) self.assertEqual(ret, {'OK': True, 'Value': ''})
class ModuleBase(object): """ Base class for Modules - works only within DIRAC workflows """ ############################################################################# def __init__(self, loggerIn=None, operationsHelperIn=None, bkClientIn=None, dm=None): """ Initialization of module base. """ if loggerIn is None: self.log = gLogger.getSubLogger('ModuleBase') else: self.log = loggerIn if operationsHelperIn is None: self.opsH = Operations() else: self.opsH = operationsHelperIn if bkClientIn is None: self.bkClient = BookkeepingClient() else: self.bkClient = bkClientIn if dm is None: self.dataManager = DataManager() else: self.dataManager = dm self.requestValidator = RequestValidator() self.production_id = '' self.prod_job_id = '' self.jobID = 0 self.step_number = '' self.step_id = '' self.fileReport = None self.jobReport = None self.request = None self.workflowStatus = None self.stepStatus = None self.workflow_commons = None self.step_commons = None self.debugSE = 'CERN-DEBUG' self.executable = 'gaudirun.py' self.applicationName = 'Unknown' self.applicationVersion = 'Unknown' self.applicationLog = '' self.applicationType = None self.systemConfig = None self.extraPackages = None self.bkConfigName = None self.BKstepID = None self.condDBTag = None self.DDDBTag = None self.dqTag = None self.CPUe = None self.eventType = '' self.gaudiSteps = None self.InputData = '' self.inputDataList = [] self.inputDataType = None self.histoName = "Hist.root" self.optionsFile = None self.optionsFormat = None self.optionsLine = None self.extraOptionsLine = None self.jobType = None self.logFilePath = None self.onlineCondDBTag = None self.onlineDDDBTag = None self.outputSEs = {} self.outputDataFileMask = None self.numberOfEvents = -1 self.maxNumberOfEvents = None self.TCK = None self.mcTCK = None self.multicoreJob = None self.multicoreStep = None self.poolXMLCatName = 'pool_xml_catalog.xml' self.persistency = '' self.processingPass = None self.runNumber = 'Unknown' self.runTimeProjectName = None self.runTimeProjectVersion = None self.simDescription = '' self.siteName = None self.stepName = None self.stepInputData = None self.XMLSummary = '' # name of the file, not the object self.stepProcPass = None self.outputFilePrefix = '' ############################################################################# def execute(self, version=None, production_id=None, prod_job_id=None, wms_job_id=None, workflowStatus=None, stepStatus=None, wf_commons=None, step_commons=None, step_number=None, step_id=None): """ Function called by all super classes """ if version: self.log.info('===== Executing ' + version + ' ===== ') self.log.verbose("Executing directory for job is %s" % os.getcwd()) if production_id: self.production_id = production_id else: self.production_id = self.workflow_commons['PRODUCTION_ID'] # This is a string, like '00051753' if prod_job_id: self.prod_job_id = prod_job_id else: self.prod_job_id = self.workflow_commons['JOB_ID'] if 'JOBID' in os.environ: self.jobID = os.environ['JOBID'] if wms_job_id: self.jobID = wms_job_id if workflowStatus: self.workflowStatus = workflowStatus if stepStatus: self.stepStatus = stepStatus if wf_commons: self.workflow_commons = wf_commons if step_commons: self.step_commons = step_commons if step_number: self.step_number = step_number else: self.step_number = self.STEP_NUMBER # pylint: disable=no-member if step_id: self.step_id = step_id else: self.step_id = '%s_%s_%s' % (self.production_id, self.prod_job_id, self.step_number) self.siteName = siteName() ############################################################################# def finalize(self, version=None): """ Just finalizing """ self.log.flushAllMessages(0) if version: self.log.info('===== Terminating ' + version + ' ===== ') ############################################################################# def setApplicationStatus(self, status, sendFlag=True): """Wraps around setJobApplicationStatus of state update client """ if not self._WMSJob(): return 0 # e.g. running locally prior to submission if self._checkWFAndStepStatus(noPrint=True): # The application status won't be updated in case the workflow or the step is failed already if not isinstance(status, str): status = str(status) self.log.verbose('setJobApplicationStatus(%d, %s)' % (self.jobID, status)) jobStatus = self.jobReport.setApplicationStatus(status, sendFlag) if not jobStatus['OK']: self.log.warn(jobStatus['Message']) ############################################################################# def setJobParameter(self, name, value, sendFlag=True): """Wraps around setJobParameter of state update client """ if not self._WMSJob(): return 0 # e.g. running locally prior to submission self.log.verbose('setJobParameter(%d,%s,%s)' % (self.jobID, name, value)) jobParam = self.jobReport.setJobParameter(str(name), str(value), sendFlag) if not jobParam['OK']: self.log.warn(jobParam['Message']) ############################################################################# def _resolveInputVariables(self): """ By convention the module input parameters are resolved here. """ self.log.verbose("workflow_commons = ", self.workflow_commons) self.log.verbose("step_commons = ", self.step_commons) self.fileReport = self._getFileReporter() self.jobReport = self._getJobReporter() self.request = self._getRequestContainer() self.__resolveInputWorkflow() ############################################################################# def __resolveInputWorkflow(self): """ Resolve the input variables that are in the workflow_commons """ self.runNumber = self.workflow_commons.get('runNumber', self.runNumber) self.persistency = self.workflow_commons.get('persistency', self.persistency) self.jobType = self.workflow_commons.get('JobType', self.jobType) self.poolXMLCatName = self.workflow_commons.get('poolXMLCatName', self.poolXMLCatName) self.InputData = self.workflow_commons.get('InputData', self.InputData) if 'ParametricInputData' in self.workflow_commons: pID = copy.deepcopy(self.workflow_commons['ParametricInputData']) if pID: if isinstance(pID, list): pID = ';'.join(pID) # self.InputData += ';' + pID self.InputData = pID self.InputData = self.InputData.rstrip(';') if self.InputData == ';': self.InputData = '' self.inputDataList = [lfn.strip('LFN:') for lfn in self.InputData.split(';') if lfn] # only required until the stripping is the same for MC / data self.bkConfigName = self.workflow_commons.get('configName', self.bkConfigName) self.simDescription = self.workflow_commons.get('simDescription', self.simDescription) if 'runMetadata' in self.workflow_commons: runMetadataDict = eval(self.workflow_commons['runMetadata']) self.onlineDDDBTag = runMetadataDict['DDDB'] self.onlineCondDBTag = runMetadataDict['CondDb'] self.TCK = runMetadataDict['TCK'] if 'outputDataFileMask' in self.workflow_commons: self.outputDataFileMask = self.workflow_commons['outputDataFileMask'] if not isinstance(self.outputDataFileMask, list): self.outputDataFileMask = [i.lower().strip() for i in self.outputDataFileMask.split(';')] self.gaudiSteps = self.workflow_commons.get('gaudiSteps', self.gaudiSteps) if 'CPUe' in self.workflow_commons: self.CPUe = int(round(float(self.workflow_commons['CPUe']))) self.multicoreJob = self.workflow_commons.get('multicore', self.multicoreJob) self.processingPass = self.workflow_commons.get('processingPass', self.processingPass) self.logFilePath = self.workflow_commons.get('LogFilePath', self.logFilePath) if isinstance(self.logFilePath, list): self.logFilePath = self.logFilePath[0] else: if 'PRODUCTION_ID' and 'JOB_ID' and 'configVersion' and 'configName' in self.workflow_commons: self.log.info('LogFilePath parameter not found, creating on the fly') result = getLogPath(self.workflow_commons, self.bkClient) if not result['OK']: self.log.error('Could not create LogFilePath', result['Message']) raise RuntimeError(result['Message']) self.logFilePath = result['Value']['LogFilePath'][0] if 'maxNumberOfEvents' in self.workflow_commons: self.maxNumberOfEvents = int(self.workflow_commons['maxNumberOfEvents']) self.eventType = self.workflow_commons.get('eventType', self.eventType) self.numberOfEvents = int(self.workflow_commons.get('numberOfEvents', self.numberOfEvents)) if 'outputSEs' in self.workflow_commons: self.outputSEs = self.workflow_commons['outputSEs'] else: # this is here for backward compatibility histogramSE = self.opsH.getValue('Productions/HistogramSE', 'CERN-HIST') histoTypes = self.opsH.getValue('Productions/HistogramTypes', ['HIST', 'BRUNELHIST', 'DAVINCIHIST', 'GAUSSHIST']) self.outputSEs = dict((ht, histogramSE) for ht in histoTypes) # for older productions we construct it based on what should be found in the steps if 'listoutput' in self.step_commons: listOutputStep = self.step_commons['listoutput'] for lOutput in listOutputStep: try: for outputDataType in lOutput['outputDataType'].split(';'): if outputDataType: self.outputSEs.setdefault(outputDataType.upper(), lOutput['outputDataSE']) except KeyError: continue self.workflow_commons['outputSEs'] = self.outputSEs ############################################################################# def _resolveInputStep(self): """ Resolve the input variables for an application step """ prodID = self.workflow_commons.get('PRODUCTION_ID', '') jobID = self.workflow_commons.get('JOB_ID', '') stepInstanceNumber = self.step_commons.get('STEP_NUMBER', '') self.stepName = self.step_commons['STEP_INSTANCE_NAME'] self.executable = self.step_commons.get('executable', self.executable) self.applicationName = self.step_commons.get('applicationName', self.applicationName) self.applicationVersion = self.step_commons.get('applicationVersion', self.applicationVersion) self.BKstepID = self.step_commons.get('BKStepID', self.BKstepID) self.stepProcPass = self.step_commons.get('StepProcPass', self.stepProcPass) # this is only for production jobs and for application steps if prodID and jobID and stepInstanceNumber and 'listoutput' in self.step_commons: self.outputFilePrefix = "%s_%s_%s" % (prodID, jobID, stepInstanceNumber) self.applicationLog = self.applicationName + '_' + self.outputFilePrefix + '.log' self.XMLSummary = 'summary' + self.applicationName + '_' + self.outputFilePrefix + '.xml' self.histoName = self.applicationName + '_' + self.outputFilePrefix + '.Hist.root' for fileTypeDict in self.step_commons['listoutput']: # this is a dict like {'outputDataType': 'sim'} # for non histo-merging prods if 'hist' in fileTypeDict['outputDataType'].lower() and self.jobType.lower() != 'merge': # Watch out: this assumes that: # - 'hist' is always in the file type name # - merging jobs won't produce histograms # - the only merging jobs that produce output types with hist are histomerging productions if 'outputDataName' not in fileTypeDict: fileTypeDict['outputDataName'] = self.histoName else: fileTypeDict['outputDataName'] = self.outputFilePrefix + '.' + fileTypeDict['outputDataType'] else: self.applicationLog = self.step_commons.get('applicationLog', self.applicationLog) self.inputDataType = self.step_commons.get('inputDataType', self.inputDataType) self.applicationType = self.step_commons.get('applicationType', self.applicationType) self.optionsFile = self.step_commons.get('optionsFile', self.optionsFile) self.optionsLine = self.step_commons.get('optionsLine', self.optionsLine) self.extraOptionsLine = self.step_commons.get('extraOptionsLine', self.extraOptionsLine) if 'runTimeProjectName' in self.step_commons: self.runTimeProjectName = self.step_commons['runTimeProjectName'] self.runTimeProjectVersion = self.step_commons['runTimeProjectVersion'] if 'extraPackages' in self.step_commons: self.extraPackages = self.step_commons['extraPackages'] if self.extraPackages: if isinstance(self.extraPackages, basestring): eps = self.extraPackages.split(';') # pylint: disable=no-member epList = [] for ep in eps: epList.append(tuple(ep.split('.'))) self.extraPackages = epList stepInputData = [] if 'inputData' in self.step_commons: if self.step_commons['inputData']: stepInputData = self.step_commons['inputData'] elif self.InputData: stepInputData = copy.deepcopy(self.InputData) if stepInputData: stepInputData = self._determineStepInputData(stepInputData, ) self.stepInputData = [sid.strip('LFN:') for sid in stepInputData] self.optionsFormat = self.step_commons.get('optionsFormat', self.optionsFormat) self.multicoreStep = self.step_commons.get('multiCore', self.multicoreStep) self.systemConfig = self.step_commons.get('SystemConfig', self.systemConfig) self.mcTCK = self.step_commons.get('mcTCK', self.mcTCK) self.DDDBTag = self.step_commons.get('DDDBTag', self.DDDBTag) self.condDBTag = self.step_commons.get('CondDBTag', self.condDBTag) self.dqTag = self.step_commons.get('DQTag', self.dqTag) # This is resolved also in __resolveInputWorkflow but can be specialized per step (e.g. LHCbJob().setApplication()) self.numberOfEvents = int(self.step_commons.get('numberOfEvents', self.numberOfEvents)) ############################################################################# def _determineOutputs(self): """ Method that determines the correct outputs. For merging jobs the output has normally to be the same as the input, but there might be exceptions (like for the productions for the merging of histograms) For the others, we use what is in the step definition We always remove the 'HIST'(s), when present, from the list of output file types as these are treated differently. There is anyway also here the special case of histogram merging productions. """ histoTypes = self.opsH.getValue('Productions/HistogramTypes', ['HIST', 'BRUNELHIST', 'DAVINCIHIST', 'GAUSSHIST']) stepOutputs = self.step_commons['listoutput'] stepOutputsT = [x['outputDataType'] for x in stepOutputs] stepOutTypes = [] for fts in stepOutputsT: for ft in fts.split(';'): if ft and ft not in stepOutTypes: stepOutTypes.append(ft.lower()) # corrections for Merge productions if self.jobType.lower() in ('merge', 'histomerge'): if len(stepOutTypes) == 1 and stepOutTypes[0] in [hts.lower() for hts in histoTypes] + ['root']: # If it is root/histo/ntuple merging job, we treat it almost as a stripping job pass else: res = self.bkClient.getFileMetadata(self.stepInputData) if not res['OK']: raise RuntimeError(res['Message']) outputTypes = set() success = res['Value']['Successful'] for mdDict in success.values(): outputTypes.add(mdDict['FileType']) if len(success) != len(self.stepInputData): self.log.warn("Some inputs are not in BKK, trying to parse the file names") for sid in set(self.stepInputData) - set(success): # File types in the BK are upper case fType = '.'.join(os.path.basename(sid).split('.')[1:]).upper() outputTypes.add(fType) outputTypes = list(outputTypes) if len(outputTypes) > 1: raise ValueError("Not all input files have the same type (%s)" % ','.join(outputTypes)) outputType = outputTypes[0].lower() stepOutTypes = [outputType.lower()] stepOutputs = [{'outputDataName': self.step_id + '.' + outputType.lower(), 'outputDataType': outputType.lower(), 'outputBKType': outputType.upper()}] histogram = False # first here treating the special case of root/histo/ntuple merging jobs if self.jobType.lower() in ('merge', 'histomerge') \ and len(stepOutTypes) == 1 \ and stepOutTypes[0] in [hts.lower() for hts in histoTypes] + ['root']: pass else: # This is not a histogram merging production, so we remove the histograms from the step outputs for hist in histoTypes: try: stepOutTypes.remove(hist) histogram = True except ValueError: pass try: stepOutTypes.remove(hist.lower()) histogram = True except ValueError: pass return stepOutputs, stepOutTypes, histogram ############################################################################# def _getJobReporter(self): """ just return the job reporter (object, always defined by dirac-jobexec) """ if 'JobReport' in self.workflow_commons: return self.workflow_commons['JobReport'] jobReport = JobReport(self.jobID) self.workflow_commons['JobReport'] = jobReport return jobReport ############################################################################# def _getFileReporter(self): """ just return the file reporter (object) """ if 'FileReport' in self.workflow_commons: return self.workflow_commons['FileReport'] fileReport = FileReport() self.workflow_commons['FileReport'] = fileReport return fileReport ############################################################################# def _getRequestContainer(self): """ just return the Request reporter (object) """ if 'Request' in self.workflow_commons: return self.workflow_commons['Request'] request = Request() self.workflow_commons['Request'] = request return request ############################################################################# def getCandidateFiles(self, outputList, outputLFNs, fileMask='', stepMask=''): """ Returns list of candidate files to upload, check if some outputs are missing. :param list outputList: list of outputs with the following structure:: [{'outputDataType': '', 'outputDataName': ''} , {...}] :param list outputLFNs: output LFNs for the job :param str fileMask: the output file extensions to restrict the outputs to. Can also be a list of strings :param str stepMask: the step ID to restrict the outputs to. Can also be a list of strings. :returns: dictionary containing type, SE and LFN for files restricted by mask """ fileInfo = {} for outputFile in outputList: if 'outputDataType' in outputFile and 'outputDataName' in outputFile: fname = outputFile['outputDataName'] fileType = outputFile['outputDataType'] fileInfo[fname] = {'type': fileType} else: self.log.error('Ignoring mal-formed output data specification', str(outputFile)) for lfn in outputLFNs: if os.path.basename(lfn).lower() in list(fi.lower() for fi in fileInfo): try: fileInfo[os.path.basename(lfn)]['lfn'] = lfn self.log.verbose("Found LFN for file", "%s -> %s" % (lfn, os.path.basename(lfn))) except KeyError: fileInfo[os.path.basename(lfn).lower()]['lfn'] = lfn self.log.verbose("Found LFN for file", "%s -> %s" % (lfn, os.path.basename(lfn).lower())) elif os.path.basename(lfn).split('_')[-1].lower() in list(fi.lower() for fi in fileInfo): try: fileInfo[os.path.basename(lfn).split('_')[-1]]['lfn'] = lfn self.log.verbose("Found LFN for file", " %s -> %s" % (lfn, os.path.basename(lfn).split('_')[-1])) except KeyError: fileInfo[os.path.basename(lfn).split('_')[-1].lower()]['lfn'] = lfn self.log.verbose("Found LFN for file", " %s -> %s" % (lfn, os.path.basename(lfn).split('_')[-1].lower())) else: self.log.warn("LFN not recognized", "for LFN %s" % lfn) # check local existance fileList = self._checkLocalExistance(list(fileInfo.keys())) # really horrible stuff for updating the name with what's found on the disk # (because maybe the case is not the same as the expected) newFileInfo = {} for fi in fileInfo.iteritems(): for li in fileList: if fi[0].lower() == li.lower(): newFileInfo[li] = fi[1] # Select which files have to be uploaded: in principle all candidateFiles = self._applyMask(newFileInfo, fileMask, stepMask) # Sanity check all final candidate metadata keys are present self._checkSanity(candidateFiles) # Adding the SEs for candidateFile in candidateFiles: try: fType = candidateFiles[candidateFile]['type'] for fType in [fType, fType.lower(), fType.upper(), fType.capitalize()]: try: wfSE = self.outputSEs[fType] candidateFiles[candidateFile]['workflowSE'] = wfSE break except KeyError: continue except AttributeError: break return candidateFiles ############################################################################# def _checkLocalExistance(self, fileList): """ Check that the list of output files are present locally """ notPresentFiles = [] filesOnDisk = set(os.listdir('.')) diff = set(fileList).difference(filesOnDisk) if diff: self.log.warn("Not all files found", "set of what's on the disk: %s" % filesOnDisk) self.log.warn("Looking for files with different case") diffCI = set(fx.lower() for fx in fileList).difference(set(fod.lower() for fod in filesOnDisk)) if diffCI: self.log.error("Output data not found", "File list %s does not exist locally" % notPresentFiles) raise os.error("Output data not found") # now checking what's the actual filename case that's written self.log.warn("Found files with filename with different case, returning those") filesActuallyOnDisk = [] for fod in os.listdir('.'): if fod.lower() in [fl.lower() for fl in fileList]: filesActuallyOnDisk.append(fod) return filesActuallyOnDisk return fileList ############################################################################# def _applyMask(self, candidateFilesIn, fileMask, stepMask): """ Select which files have to be uploaded: in principle all :param dict candidateFilesIn: dictionary like {'00012345_00012345_4.dst': {'lfn': '/lhcb/MC/2010/DST/123/123_45_4.dst', type': 'dst'}, '00012345_00012345_2.digi': {'type': 'digi'}} :param str fileMask: the output file extensions to restrict the outputs to. Can also be a list of strings :param str stepMask: the step ID to restrict the outputs to. Can also be a list of strings. :returns: a dict like the one in candidateFilesIn """ candidateFiles = copy.deepcopy(candidateFilesIn) if fileMask and not isinstance(fileMask, list): fileMask = [fileMask] if isinstance(stepMask, int): stepMask = str(stepMask) if stepMask and not isinstance(stepMask, list): stepMask = [stepMask] if fileMask and fileMask != ['']: for fileName, metadata in list(candidateFiles.iteritems()): if metadata['type'].lower() not in [fm.lower() for fm in fileMask]: del candidateFiles[fileName] self.log.info('Output file %s was produced but will not be treated (fileMask is %s)' % (fileName, ', '.join(fileMask))) else: self.log.info('No outputDataFileMask provided, the files with all the extensions will be considered') if stepMask and stepMask != ['']: for fileName, metadata in list(candidateFiles.iteritems()): if fileName.lower().replace(metadata['type'].lower(), '').split('_')[-1].split('.')[0] not in stepMask: del candidateFiles[fileName] self.log.info('Output file %s was produced but will not be treated (stepMask is %s)' % (fileName, ', '.join(stepMask))) else: self.log.info('No outputDataStep provided, the files output of all the steps will be considered') return candidateFiles ############################################################################# def _checkSanity(self, candidateFiles): """ Sanity check all final candidate metadata keys are present :param dict candidateFiles: dictionary like {'00012345_00012345_4.dst': {'lfn': '/lhcb/MC/2010/DST/123/123_45_4.dst', type': 'dst'}, '00012345_00012345_2.digi': {'type': 'digi'}} :returns: None or raises ValueError """ notPresentKeys = [] mandatoryKeys = ['type', 'lfn'] # filedict is used for requests for fileName, metadata in candidateFiles.iteritems(): for key in mandatoryKeys: if key not in metadata: notPresentKeys.append((fileName, key)) if notPresentKeys: for fileName_keys in notPresentKeys: self.log.error("File %s has missing %s" % (fileName_keys[0], fileName_keys[1])) raise ValueError("Missing requested fileName keys") ############################################################################# def getFileMetadata(self, candidateFiles): """ Returns the candidate file dictionary with associated metadata. The input candidate files dictionary has the structure: {'foo_1.txt': {'lfn': '/lhcb/MC/2010/DST/00012345/0001/foo_1.txt', 'type': 'txt', 'workflowSE': SE1}, 'bar_2.py': {'lfn': '/lhcb/MC/2010/DST/00012345/0001/bar_2.py', 'type': 'py', 'workflowSE': 'SE2'}, } this also assumes the files are in the current working directory. """ # Retrieve the POOL File GUID(s) for any final output files self.log.info('Will search for POOL GUIDs for: %s' % (', '.join(candidateFiles.keys()))) pfnGUID = getGUID(candidateFiles.keys()) if not pfnGUID['OK']: self.log.error('''PoolXMLFile failed to determine POOL GUID(s) for output file list, these will be generated by the DataManager''', pfnGUID['Message']) for fileName in candidateFiles.keys(): candidateFiles[fileName]['guid'] = '' elif pfnGUID['generated']: self.log.warn('PoolXMLFile generated GUID(s) for the following files ', ', '.join(pfnGUID['generated'])) else: self.log.info('GUIDs found for all specified POOL files: %s' % (', '.join(candidateFiles.keys()))) for pfn, guid in pfnGUID['Value'].iteritems(): candidateFiles[pfn]['guid'] = guid # Get all additional metadata about the file necessary for requests final = {} for fileName, metadata in candidateFiles.iteritems(): fileDict = {} fileDict['LFN'] = metadata['lfn'] fileDict['Size'] = os.path.getsize(fileName) fileDict['Checksum'] = fileAdler(fileName) fileDict['ChecksumType'] = 'ADLER32' fileDict['GUID'] = metadata['guid'] fileDict['Status'] = 'Waiting' final[fileName] = metadata final[fileName]['filedict'] = fileDict final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName) # Sanity check all final candidate metadata keys are present (return S_ERROR if not) mandatoryKeys = ['guid', 'filedict'] # filedict is used for requests (this method adds guid and filedict) for fileName, metadata in final.iteritems(): for key in mandatoryKeys: if key not in metadata: raise RuntimeError("File %s has missing %s" % (fileName, key)) return final ############################################################################# def _determineStepInputData(self, inputData): """ determine the input data for the step """ if inputData == 'previousStep': stepIndex = self.gaudiSteps.index(self.stepName) previousStep = self.gaudiSteps[stepIndex - 1] stepInputData = [] for outputF in self.workflow_commons['outputList']: try: if outputF['stepName'] == previousStep and outputF['outputBKType'].lower() == self.inputDataType.lower(): stepInputData.append(outputF['outputDataName']) except KeyError: raise RuntimeError("Can't find output of step %s" % previousStep) return stepInputData return [x.strip('LFN:') for x in inputData.split(';')] ############################################################################# def _manageAppOutput(self, outputs): """ Calls self._findOutputs to find what's produced, then creates the LFNs outputs, as called here, is created starting from step_commons['listoutput'], but enriched with at least the outputDataName. example of outputs: [{'outputDataType': 'bhadron.dst', 'outputBKType': 'BHADRON.DST', 'outputDataName': '00012345_00012345_2.BHADRON.DST'}, {'outputDataType': 'calibration.dst','outputDataType': 'CALIBRATION.DST', 'outputDataName': '00012345_00012345_2.CALIBRATION.DST'}] :params list outputs: list of dicts of step output files descriptions """ if not outputs: self.log.warn('Step outputs are not defined (normal user jobs. Not normal in productions and SAM jobs)') return else: finalOutputs, _bkFileTypes = self._findOutputs(outputs) self.log.info('Final step outputs are: %s' % (finalOutputs)) self.step_commons['listoutput'] = finalOutputs if 'outputList' in self.workflow_commons: for outFile in finalOutputs: if outFile not in self.workflow_commons['outputList']: self.workflow_commons['outputList'].append(outFile) else: self.workflow_commons['outputList'] = finalOutputs if 'PRODUCTION_ID' and 'JOB_ID' and 'configVersion' and 'configName' in self.workflow_commons: self.log.info('Attempting to recreate the production output LFNs...') result = constructProductionLFNs(self.workflow_commons, self.bkClient) if not result['OK']: raise IOError("Could not create production LFNs: %s" % result['Message']) self.workflow_commons['BookkeepingLFNs'] = result['Value']['BookkeepingLFNs'] self.workflow_commons['LogFilePath'] = result['Value']['LogFilePath'] self.workflow_commons['ProductionOutputData'] = result['Value']['ProductionOutputData'] ############################################################################# def _findOutputs(self, stepOutput): """ Find which outputs of those in stepOutput (what are expected to be produced) are effectively produced. stepOutput, as called here, is created starting from step_commons['listoutput'] example of stepOutput: [{'outputDataType': 'bhadron.dst', 'outputBKType': 'BHADRON.DST', 'outputDataName': '00012345_00012345_2.BHADRON.DST'}, {'outputDataType': 'calibration.dst','outputDataType': 'CALIBRATION.DST', 'outputDataName': '00012345_00012345_2.CALIBRATION.DST'}] :params list stepOutput: list of dicts of step output files descriptions :returns: list, list """ bkFileTypes = [] # uppercase list of file types finalOutputs = [] # list of dicts of what's found on the local disk filesFound = [] for output in stepOutput: found = False fileOnDisk = None for fileOnDisk in os.listdir('.'): if output['outputDataName'].lower() == fileOnDisk.lower(): found = True break if found and fileOnDisk: self.log.info('Found output file (case is not considered)', '%s matching %s ' % (fileOnDisk, output['outputDataName'])) output['outputDataName'] = fileOnDisk filesFound.append(output) else: self.log.error('Output not found', output['outputDataName']) raise IOError("OutputData not found") for fileFound in filesFound: bkFileTypes.append(fileFound['outputDataType'].upper()) finalOutputs.append({'outputDataName': fileFound['outputDataName'], 'outputDataType': fileFound['outputDataType'].lower(), 'outputBKType': fileFound['outputDataType'].upper(), 'stepName': self.stepName}) return (finalOutputs, bkFileTypes) ############################################################################# def _WMSJob(self): """ Check if this job is running via WMS """ return True if self.jobID else False ############################################################################# def _enableModule(self): """ Enable module if it's running via WMS """ if not self._WMSJob(): self.log.info('No WMS JobID found, disabling module via control flag') return False self.log.verbose('Found WMS JobID', self.jobID) return True ############################################################################# def _checkWFAndStepStatus(self, noPrint=False): """ Check the WF and Step status """ if not self.workflowStatus['OK'] or not self.stepStatus['OK']: if not noPrint: self.log.info('Skip this module, failure detected in a previous step') self.log.info('Workflow status', self.workflowStatus) self.log.info('Step Status', self.stepStatus) return False else: return True ############################################################################# def _disableWatchdogCPUCheck(self): """ just writes a file to disable the watchdog """ self.log.info("Creating DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK in order to disable the Watchdog") with open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w') as fopen: fopen.write('%s' % time.asctime()) ############################################################################# def generateFailoverFile(self): """ Retrieve the accumulated reporting request, and produce a JSON file that is consumed by the JobWrapper """ reportRequest = None result = self.jobReport.generateForwardDISET() if not result['OK']: self.log.warn("Could not generate Operation for job report", "with result:\n%s" % (result)) else: reportRequest = result['Value'] if reportRequest: self.log.info("Populating request with job report information") self.request.addOperation(reportRequest) if len(self.request): try: optimized = self.request.optimize() except AttributeError: optimized = {'OK': True} if not optimized['OK']: self.log.error("Could not optimize", optimized['Message']) self.log.error("Not failing the job because of that, keep going") isValid = self.requestValidator.validate(self.request) if not isValid['OK']: raise RuntimeError("Failover request is not valid: %s" % isValid['Message']) else: requestJSON = self.request.toJSON() if requestJSON['OK']: self.log.info("Creating failover request for deferred operations", "for job %d" % self.jobID) request_string = str(requestJSON['Value']) self.log.debug(request_string) # Write out the request string fname = '%s_%s_request.json' % (self.production_id, self.prod_job_id) with open(fname, 'w') as jsonFile: jsonFile.write(request_string) self.log.info("Created file containing failover request", fname) result = self.request.getDigest() if result['OK']: self.log.info("Digest of the request", result['Value']) else: self.log.warn("No digest? That's not sooo important", "anyway: %s" % result['Message']) else: raise RuntimeError(requestJSON['Message']) accountingReport = None if 'AccountingReport' in self.workflow_commons: accountingReport = self.workflow_commons['AccountingReport'] if accountingReport: result = accountingReport.commit() if not result['OK']: self.log.error("!!! Both accounting and RequestDB are down? !!!") self.log.error("accountingReport result: %s" % result['Message']) self.log.error("Anyway, the job won't fail for this reason, because this is \"just\" the accounting report") ############################################################################# def setBKRegistrationRequest(self, lfn, error='', metaData={'Checksum': 'justSomething', 'ChecksumType': 'ADLER32', 'GUID': 'aGUID'}): """ Set a BK registration request for changing the replica flag. Uses the global request object (self.request). """ if error: self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error)) else: self.log.info('Setting BK registration request for %s' % (lfn)) regFile = Operation() regFile.Type = 'RegisterFile' regFile.Catalog = 'BookkeepingDB' bkFile = File() bkFile.LFN = lfn # this should NOT be needed... but RMS complains! bkFile.PFN = lfn bkFile.GUID = metaData['GUID'] bkFile.Checksum = metaData['Checksum'] bkFile.ChecksumType = metaData['ChecksumType'] regFile.addFile(bkFile) res = self.request.addOperation(regFile) if not res['OK']: raise RuntimeError(res['Message']) ############################################################################# def createProdConfFile(self, stepOutputTypes, histogram, runNumberGauss, firstEventNumberGauss): """ Utility that creates a ProdConf file, used mostly as input for gaudirun jobs """ # Creating ProdConf file prodConfFileName = 'prodConf_%s_%s_%s_%s.py' % (self.applicationName, self.production_id, self.prod_job_id, self.step_number) optionsDict = {} optionsDict['Application'] = self.applicationName optionsDict['AppVersion'] = self.applicationVersion if self.optionsFormat: optionsDict['OptionFormat'] = self.optionsFormat if self.stepInputData: optionsDict['InputFiles'] = ['LFN:' + sid for sid in self.stepInputData] else: if self.applicationName.lower() != "gauss": raise RuntimeError("No MC, but no input data") if self.outputFilePrefix: optionsDict['OutputFilePrefix'] = self.outputFilePrefix optionsDict['OutputFileTypes'] = stepOutputTypes optionsDict['XMLSummaryFile'] = self.XMLSummary optionsDict['XMLFileCatalog'] = self.poolXMLCatName if histogram: optionsDict['HistogramFile'] = self.histoName if self.DDDBTag: if self.DDDBTag.lower() == 'online': try: optionsDict['DDDBTag'] = self.onlineDDDBTag self.log.debug('Set the online DDDB tag') except NameError as e: self.log.error('Could not find online DDDb Tag', e) raise RuntimeError("Could not find online DDDb Tag") else: optionsDict['DDDBTag'] = self.DDDBTag if self.condDBTag: if self.condDBTag.lower() == 'online': optionsDict['CondDBTag'] = self.onlineCondDBTag self.log.debug('Set the online CondDB tag') else: optionsDict['CondDBTag'] = self.condDBTag if self.dqTag: optionsDict['DQTag'] = self.dqTag if self.applicationName.lower() == 'gauss': if self.CPUe and self.maxNumberOfEvents and self.numberOfEvents <= 0: # Here we set maxCPUTime to 24 hours, which seems reasonable eventsToProduce = getEventsToProduce(self.CPUe, maxNumberOfEvents=self.maxNumberOfEvents, jobMaxCPUTime=86400) else: eventsToProduce = self.numberOfEvents else: eventsToProduce = self.numberOfEvents optionsDict['NOfEvents'] = eventsToProduce if runNumberGauss: optionsDict['RunNumber'] = runNumberGauss if self.runNumber: if self.runNumber not in ('Unknown', 'Multiple'): optionsDict['RunNumber'] = self.runNumber if firstEventNumberGauss: optionsDict['FirstEventNumber'] = firstEventNumberGauss # TCK: can't have both set! if self.TCK and self.mcTCK: raise RuntimeError("%s step: TCK set in step, and should't be!" % self.applicationName) if self.TCK or self.mcTCK: optionsDict['TCK'] = self.TCK if self.TCK else self.mcTCK if self.processingPass: optionsDict['ProcessingPass'] = self.processingPass prodConfFile = ProdConf(prodConfFileName) self.log.debug(optionsDict) prodConfFile.putOptionsIn(optionsDict) return prodConfFileName ############################################################################# # properties def set_jobID(self, value): if isinstance(value, str): if value: value = int(value) else: value = 0 self._jobID = value def get_jobID(self): return self._jobID jobID = property(get_jobID, set_jobID)
class RequestTasks(TaskBase): """ Class for handling tasks for the RMS """ def __init__(self, transClient=None, logger=None, requestClient=None, requestClass=None, requestValidator=None, ownerDN=None, ownerGroup=None): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger('RequestTasks') super(RequestTasks, self).__init__(transClient, logger) useCertificates = True if (bool(ownerDN) and bool(ownerGroup)) else False if not requestClient: self.requestClient = ReqClient(useCertificates=useCertificates, delegatedDN=ownerDN, delegatedGroup=ownerGroup) else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN='', bulkSubmissionFlag=False): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if (not owner) or (not ownerGroup): res = getProxyInfo(False, False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername(owner) if not res['OK']: return res ownerDN = res['Value'][0] try: transJson = json.loads(transBody) self._multiOperationsBody(transJson, taskDict, ownerDN, ownerGroup) except ValueError: # #json couldn't load self._singleOperationsBody(transBody, taskDict, ownerDN, ownerGroup) return S_OK(taskDict) def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup): """ deal with a Request that has multiple operations :param transJson: list of lists of string and dictionaries, e.g.: .. code :: python body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }), ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ), ] :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ failedTasks = [] for taskID, task in taskDict.items(): transID = task['TransformationID'] if not task.get('InputData'): self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID) taskDict.pop(taskID) continue files = [] oRequest = Request() if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') # create the operations from the json structure for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] for parameter, value in operationTuple[1].iteritems(): setattr(op, parameter, value) for lfn in files: opFile = File() opFile.LFN = lfn op.addFile(opFile) oRequest.addOperation(op) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID) def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split(';') except AttributeError: pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop for taskID, task in taskDict.iteritems(): transID = task['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = task['TargetSE'] # If there are input files if task.get('InputData'): if isinstance(task['InputData'], list): files = task['InputData'] elif isinstance(task['InputData'], basestring): files = task['InputData'].split(';') for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile(trFile) oRequest.addOperation(transfer) result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup) if not result['OK']: failedTasks.append(taskID) # Remove failed tasks for taskID in failedTasks: taskDict.pop(taskID) def _assignRequestToTask(self, oRequest, taskDict, transID, taskID, ownerDN, ownerGroup): """set ownerDN and group to request, and add the request to taskDict if it is valid, otherwise remove the task from the taskDict :param oRequest: Request :param dict taskDict: dictionary of tasks, modified in this function :param int transID: Transformation ID :param int taskID: Task ID :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ oRequest.RequestName = self._transTaskName(transID, taskID) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate(oRequest) if not isValid['OK']: self._logError("Error creating request for task", "%s %s" % (taskID, isValid), transID=transID) return S_ERROR('Error creating request') taskDict[taskID]['TaskObject'] = oRequest return S_OK() def submitTransformationTasks(self, taskDict): """ Submit requests one by one """ submitted = 0 failed = 0 startTime = time.time() method = 'submitTransformationTasks' for task in taskDict.itervalues(): # transID is the same for all tasks, so pick it up every time here transID = task['TransformationID'] if not task['TaskObject']: task['Success'] = False failed += 1 continue res = self.submitTaskToExternal(task['TaskObject']) if res['OK']: task['ExternalID'] = res['Value'] task['Success'] = True submitted += 1 else: self._logError("Failed to submit task to RMS", res['Message'], transID=transID) task['Success'] = False failed += 1 if submitted: self._logInfo('Submitted %d tasks to RMS in %.1f seconds' % (submitted, time.time() - startTime), transID=transID, method=method) if failed: self._logWarn('Failed to submit %d tasks to RMS.' % (failed), transID=transID, method=method) return S_OK(taskDict) def submitTaskToExternal(self, oRequest): """ Submits a request to RMS """ if isinstance(oRequest, self.requestClass): return self.requestClient.putRequest(oRequest, useFailoverProxy=False, retryMainService=2) return S_ERROR("Request should be a Request object") def updateTransformationReservedTasks(self, taskDicts): requestNameIDs = {} noTasks = [] for taskDict in taskDicts: requestName = self._transTaskName(taskDict['TransformationID'], taskDict['TaskID']) reqID = taskDict['ExternalID'] if reqID: requestNameIDs[requestName] = reqID else: noTasks.append(requestName) return S_OK({'NoTasks': noTasks, 'TaskNameIDs': requestNameIDs}) def getSubmittedTaskStatus(self, taskDicts): """ Check if tasks changed status, and return a list of tasks per new status """ updateDict = {} badRequestID = 0 for taskDict in taskDicts: oldStatus = taskDict['ExternalStatus'] # ExternalID is normally a string if taskDict['ExternalID'] and int(taskDict['ExternalID']): newStatus = self.requestClient.getRequestStatus(taskDict['ExternalID']) if not newStatus['OK']: log = self._logVerbose if 'not exist' in newStatus['Message'] else self._logWarn log("getSubmittedTaskStatus: Failed to get requestID for request", newStatus['Message'], transID=taskDict['TransformationID']) else: newStatus = newStatus['Value'] # We don't care updating the tasks to Assigned while the request is being processed if newStatus != oldStatus and newStatus != 'Assigned': updateDict.setdefault(newStatus, []).append(taskDict['TaskID']) else: badRequestID += 1 if badRequestID: self._logWarn("%d requests have identifier 0" % badRequestID) return S_OK(updateDict) def getSubmittedFileStatus(self, fileDicts): """ Check if transformation files changed status, and return a list of taskIDs per new status """ # Don't try and get status of not submitted tasks! transID = None taskFiles = {} for fileDict in fileDicts: # There is only one transformation involved, get however the transID in the loop transID = fileDict['TransformationID'] taskID = int(fileDict['TaskID']) taskFiles.setdefault(taskID, []).append(fileDict['LFN']) # Should not happen, but just in case there are no files, return if transID is None: return S_OK({}) res = self.transClient.getTransformationTasks({'TransformationID': transID, 'TaskID': taskFiles.keys()}) if not res['OK']: return res requestFiles = {} for taskDict in res['Value']: taskID = taskDict['TaskID'] externalID = taskDict['ExternalID'] # Only consider tasks that are submitted, ExternalID is a string if taskDict['ExternalStatus'] != 'Created' and externalID and int(externalID): requestFiles[externalID] = taskFiles[taskID] updateDict = {} for requestID, lfnList in requestFiles.iteritems(): statusDict = self.requestClient.getRequestFileStatus(requestID, lfnList) if not statusDict['OK']: log = self._logVerbose if 'not exist' in statusDict['Message'] else self._logWarn log("Failed to get files status for request", statusDict['Message'], transID=transID, method='getSubmittedFileStatus') else: for lfn, newStatus in statusDict['Value'].iteritems(): if newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': updateDict[lfn] = 'Problematic' return S_OK(updateDict)
def testValidator(self): """validator test""" # create validator validator = RequestValidator() self.assertEqual(isinstance(validator, RequestValidator), True) # RequestName not set ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.request.RequestName = "test_request" # # no operations ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.request.addOperation(self.operation) # # type not set ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.operation.Type = "ReplicateAndRegister" # # files not present ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.operation.addFile(self.file) # # targetSE not set ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.operation.TargetSE = "CERN-USER" # # missing LFN ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.file.LFN = "/a/b/c" # # no ownerDN # force no owner DN because it takes the one of the current user self.request.OwnerDN = "" ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.request.OwnerDN = "foo/bar=baz" # # no owner group # same, force it self.request.OwnerGroup = "" ret = validator.validate(self.request) self.assertFalse(ret["OK"]) self.request.OwnerGroup = "dirac_user" # Checksum set, ChecksumType not set self.file.Checksum = "abcdef" ret = validator.validate(self.request) self.assertFalse(ret["OK"]) # ChecksumType set, Checksum not set self.file.Checksum = "" self.file.ChecksumType = "adler32" ret = validator.validate(self.request) self.assertFalse(ret["OK"]) # both set self.file.Checksum = "abcdef" self.file.ChecksumType = "adler32" ret = validator.validate(self.request) self.assertEqual(ret, {"OK": True, "Value": None}) # both unset self.file.Checksum = "" self.file.ChecksumType = None ret = validator.validate(self.request) self.assertEqual(ret, {"OK": True, "Value": None}) # all OK ret = validator.validate(self.request) self.assertEqual(ret, {"OK": True, "Value": None})
def testValidator( self ): """ validator test """ ## create validator validator = RequestValidator() self.assertEqual( isinstance( validator, RequestValidator ), True ) ## RequestName not set ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : 'RequestName not set', 'OK' : False } ) self.request.RequestName = "test_request" # # no operations ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : "Operations not present in request 'test_request'", 'OK': False} ) self.request.addOperation( self.operation ) # # type not set ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : "Operation #0 in request 'test_request' hasn't got Type set", 'OK' : False } ) self.operation.Type = "ReplicateAndRegister" # # files not present ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : "Operation #0 of type 'ReplicateAndRegister' hasn't got files to process.", 'OK' : False } ) self.operation.addFile( self.file ) # # targetSE not set ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : "Operation #0 of type 'ReplicateAndRegister' is missing TargetSE attribute.", 'OK': False } ) self.operation.TargetSE = "CERN-USER" # # missing LFN ret = validator.validate( self.request ) self.assertEqual( ret, { "Message" : "Operation #0 of type 'ReplicateAndRegister' is missing LFN attribute for file.", "OK": False } ) self.file.LFN = "/a/b/c" # # no ownerDN # force no owner DN because it takes the one of the current user self.request.OwnerDN = '' ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : "Request 'test_request' is missing OwnerDN value", 'OK': False} ) self.request.OwnerDN = "foo/bar=baz" # # no owner group # same, force it self.request.OwnerGroup = '' ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : "Request 'test_request' is missing OwnerGroup value", 'OK': False} ) self.request.OwnerGroup = "dirac_user" ## Checksum set, ChecksumType not set self.file.Checksum = "abcdef" ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : 'File in operation #0 is missing Checksum (abcdef) or ChecksumType ()', 'OK' : False } ) ## ChecksumType set, Checksum not set self.file.Checksum = "" self.file.ChecksumType = "adler32" ret = validator.validate( self.request ) self.assertEqual( ret, { 'Message' : 'File in operation #0 is missing Checksum () or ChecksumType (ADLER32)', 'OK' : False } ) ## both set self.file.Checksum = "abcdef" self.file.ChecksumType = "adler32" ret = validator.validate( self.request ) self.assertEqual( ret, {'OK': True, 'Value': None} ) ## both unset self.file.Checksum = "" self.file.ChecksumType = None ret = validator.validate( self.request ) self.assertEqual( ret, {'OK': True, 'Value': None} ) ## all OK ret = validator.validate( self.request ) self.assertEqual( ret, {'OK': True, 'Value': None} )
class RequestTasks( TaskBase ): def __init__( self, transClient = None, logger = None, requestClient = None, requestClass = None, requestValidator = None ): """ c'tor the requestClass is by default Request. If extensions want to use an extended type, they can pass it as a parameter. This is the same behavior as WorfkloTasks and jobClass """ if not logger: logger = gLogger.getSubLogger( 'RequestTasks' ) super( RequestTasks, self ).__init__( transClient, logger ) if not requestClient: self.requestClient = ReqClient() else: self.requestClient = requestClient if not requestClass: self.requestClass = Request else: self.requestClass = requestClass if not requestValidator: self.requestValidator = RequestValidator() else: self.requestValidator = requestValidator def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '', bulkSubmissionFlag = False): """ Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB """ if not taskDict: return S_OK({}) if ( not owner ) or ( not ownerGroup ): res = getProxyInfo( False, False ) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] if not ownerDN: res = getDNForUsername( owner ) if not res['OK']: return res ownerDN = res['Value'][0] try: transJson = json.loads(transBody) self._multiOperationsBody( transJson, taskDict, ownerDN, ownerGroup ) except ValueError: ##json couldn't load self._singleOperationsBody( transBody, taskDict, ownerDN, ownerGroup ) return S_OK( taskDict ) def _multiOperationsBody( self, transJson, taskDict, ownerDN, ownerGroup ): """ deal with a Request that has multiple operations :param transJson: list of lists of string and dictionaries, e.g.: .. code :: python body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }), ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ), ] :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ for taskID in sorted( taskDict ): paramDict = taskDict[taskID] if not paramDict.get('InputData'): self.log.error( "Error creating request for task", "%s, No input data" % taskID ) taskDict.pop( taskID ) continue files = [] transID = paramDict['TransformationID'] oRequest = Request() if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) # create the operations from the json structure for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] for parameter, value in operationTuple[1].iteritems(): setattr( op, parameter, value ) for lfn in files: opFile = File() opFile.LFN = lfn op.addFile( opFile ) oRequest.addOperation( op ) self._assignRequestToTask( oRequest, taskDict, transID, taskID, ownerDN, ownerGroup ) def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup ): """ deal with a Request that has just one operation, as it was sofar :param transBody: string, can be an empty string :param dict taskDict: dictionary of tasks, modified in this function :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ requestOperation = 'ReplicateAndRegister' if transBody: try: _requestType, requestOperation = transBody.split( ';' ) except AttributeError: pass # Do not remove sorted, we might pop elements in the loop for taskID in sorted( taskDict ): paramDict = taskDict[taskID] transID = paramDict['TransformationID'] oRequest = Request() transfer = Operation() transfer.Type = requestOperation transfer.TargetSE = paramDict['TargetSE'] # If there are input files if paramDict.get('InputData'): if isinstance( paramDict['InputData'], list ): files = paramDict['InputData'] elif isinstance( paramDict['InputData'], basestring ): files = paramDict['InputData'].split( ';' ) for lfn in files: trFile = File() trFile.LFN = lfn transfer.addFile( trFile ) oRequest.addOperation( transfer ) self._assignRequestToTask( oRequest, taskDict, transID, taskID, ownerDN, ownerGroup ) def _assignRequestToTask( self, oRequest, taskDict, transID, taskID, ownerDN, ownerGroup ): """set ownerDN and group to request, and add the request to taskDict if it is valid, otherwise remove the task from the taskDict :param oRequest: Request :param dict taskDict: dictionary of tasks, modified in this function :param int transID: Transformation ID :param int taskID: Task ID :param str ownerDN: certificate DN used for the requests :param str onwerGroup: dirac group used for the requests :returns: None """ oRequest.RequestName = _requestName( transID, taskID ) oRequest.OwnerDN = ownerDN oRequest.OwnerGroup = ownerGroup isValid = self.requestValidator.validate( oRequest ) if not isValid['OK']: self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) ) # This works because we loop over a copy of the keys ! taskDict.pop( taskID ) return taskDict[taskID]['TaskObject'] = oRequest return def submitTransformationTasks( self, taskDict ): """ Submit requests one by one """ submitted = 0 failed = 0 startTime = time.time() for taskID in sorted( taskDict ): if not taskDict[taskID]['TaskObject']: taskDict[taskID]['Success'] = False failed += 1 continue res = self.submitTaskToExternal( taskDict[taskID]['TaskObject'] ) if res['OK']: taskDict[taskID]['ExternalID'] = res['Value'] taskDict[taskID]['Success'] = True submitted += 1 else: self._logError( "Failed to submit task to RMS", res['Message'] ) taskDict[taskID]['Success'] = False failed += 1 self._logInfo( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % ( submitted, time.time() - startTime ) ) if failed: self._logWarn( 'submitTasks: But at the same time failed to submit %d tasks to RMS.' % ( failed ) ) return S_OK( taskDict ) def submitTaskToExternal( self, oRequest ): """ Submits a request using ReqClient """ if isinstance( oRequest, self.requestClass ): return self.requestClient.putRequest( oRequest, useFailoverProxy = False, retryMainService = 2 ) else: return S_ERROR( "Request should be a Request object" ) def updateTransformationReservedTasks( self, taskDicts ): requestNameIDs = {} noTasks = [] for taskDict in taskDicts: requestName = _requestName( taskDict['TransformationID'], taskDict['TaskID'] ) reqID = taskDict['ExternalID'] if reqID: requestNameIDs[requestName] = reqID else: noTasks.append( requestName ) return S_OK( {'NoTasks':noTasks, 'TaskNameIDs':requestNameIDs} ) def getSubmittedTaskStatus( self, taskDicts ): updateDict = {} for taskDict in taskDicts: oldStatus = taskDict['ExternalStatus'] newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID'] ) if not newStatus['OK']: log = self._logVerbose if 'not exist' in newStatus['Message'] else self.log.warn log( "getSubmittedTaskStatus: Failed to get requestID for request", '%s' % newStatus['Message'] ) else: newStatus = newStatus['Value'] if newStatus != oldStatus: updateDict.setdefault( newStatus, [] ).append( taskDict['TaskID'] ) return S_OK( updateDict ) def getSubmittedFileStatus( self, fileDicts ): taskFiles = {} submittedTasks = {} externalIds = {} # Don't try and get status of not submitted tasks! for fileDict in fileDicts: submittedTasks.setdefault( fileDict['TransformationID'], set() ).add( int( fileDict['TaskID'] ) ) for transID in submittedTasks: res = self.transClient.getTransformationTasks( { 'TransformationID':transID, 'TaskID': list( submittedTasks[transID] )} ) if not res['OK']: return res for taskDict in res['Value']: taskID = taskDict['TaskID'] externalIds[taskID] = taskDict['ExternalID'] if taskDict['ExternalStatus'] == 'Created': submittedTasks[transID].remove( taskID ) for fileDict in fileDicts: transID = fileDict['TransformationID'] taskID = int( fileDict['TaskID'] ) if taskID in submittedTasks[transID]: taskFiles.setdefault( externalIds[taskID], [] ).append( fileDict['LFN'] ) updateDict = {} for requestID in sorted( taskFiles ): lfnList = taskFiles[requestID] statusDict = self.requestClient.getRequestFileStatus( requestID, lfnList ) if not statusDict['OK']: log = self._logVerbose if 'not exist' in statusDict['Message'] else self.log.warn log( "getSubmittedFileStatus: Failed to get files status for request", '%s' % statusDict['Message'] ) continue for lfn, newStatus in statusDict['Value'].items(): if newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': updateDict[lfn] = 'Problematic' return S_OK( updateDict )