class RequestTasks(TaskBase):
"""
Class for handling tasks for the RMS
"""
def __init__(
self,
transClient=None,
logger=None,
requestClient=None,
requestClass=None,
requestValidator=None,
ownerDN=None,
ownerGroup=None,
):
"""c'tor
the requestClass is by default Request.
If extensions want to use an extended type, they can pass it as a parameter.
This is the same behavior as WorfkloTasks and jobClass
"""
if not logger:
logger = gLogger.getSubLogger(self.__class__.__name__)
super(RequestTasks, self).__init__(transClient, logger)
useCertificates = True if (bool(ownerDN)
and bool(ownerGroup)) else False
if not requestClient:
self.requestClient = ReqClient(useCertificates=useCertificates,
delegatedDN=ownerDN,
delegatedGroup=ownerGroup)
else:
self.requestClient = requestClient
if not requestClass:
self.requestClass = Request
else:
self.requestClass = requestClass
if not requestValidator:
self.requestValidator = RequestValidator()
else:
self.requestValidator = requestValidator
def prepareTransformationTasks(self,
transBody,
taskDict,
owner="",
ownerGroup="",
ownerDN="",
bulkSubmissionFlag=False):
"""Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB"""
if not taskDict:
return S_OK({})
if (not owner) or (not ownerGroup):
res = getProxyInfo(False, False)
if not res["OK"]:
return res
proxyInfo = res["Value"]
owner = proxyInfo["username"]
ownerGroup = proxyInfo["group"]
if not ownerDN:
res = getDNForUsername(owner)
if not res["OK"]:
return res
ownerDN = res["Value"][0]
try:
transJson, _decLen = decode(transBody)
if isinstance(transJson, BaseBody):
self._bodyPlugins(transJson, taskDict, ownerDN, ownerGroup)
else:
self._multiOperationsBody(transJson, taskDict, ownerDN,
ownerGroup)
except ValueError: # #json couldn't load
self._singleOperationsBody(transBody, taskDict, ownerDN,
ownerGroup)
return S_OK(taskDict)
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
"""Deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"TASK:TargetSE" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
If a value of an operation parameter in the body starts with ``TASK:``,
we take it from the taskDict.
For example ``TASK:TargetSE`` is replaced with ``task['TargetSE']``
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
for taskID, task in list(taskDict.items()):
try:
transID = task["TransformationID"]
if not task.get("InputData"):
raise StopTaskIteration("No input data")
files = []
oRequest = Request()
if isinstance(task["InputData"], list):
files = task["InputData"]
elif isinstance(task["InputData"], six.string_types):
files = task["InputData"].split(";")
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].items():
# Here we massage a bit the body to replace some parameters
# with what we have in the task.
try:
taskKey = value.split("TASK:")[1]
value = task[taskKey]
# Either the attribute is not a string (AttributeError)
# or it does not start with 'TASK:' (IndexError)
except (AttributeError, IndexError):
pass
# That happens when the requested substitution is not
# a key in the task, and that's a problem
except KeyError:
raise StopTaskIteration(
"Parameter %s does not exist in taskDict" %
taskKey)
setattr(op, parameter, value)
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile(opFile)
oRequest.addOperation(op)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result["OK"]:
raise StopTaskIteration(
"Could not assign request to task: %s" %
result["Message"])
except StopTaskIteration as e:
self._logError("Error creating request for task",
"%s, %s" % (taskID, e),
transID=transID)
taskDict.pop(taskID)
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
"""deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = "ReplicateAndRegister"
if transBody:
try:
_requestType, requestOperation = transBody.split(";")
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.items():
transID = task["TransformationID"]
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task["TargetSE"]
# If there are input files
if task.get("InputData"):
if isinstance(task["InputData"], list):
files = task["InputData"]
elif isinstance(task["InputData"], six.string_types):
files = task["InputData"].split(";")
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result["OK"]:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def _bodyPlugins(self, bodyObj, taskDict, ownerDN, ownerGroup):
"""Deal with complex body object"""
for taskID, task in list(taskDict.items()):
try:
transID = task["TransformationID"]
if not task.get("InputData"):
raise StopTaskIteration("No input data")
oRequest = bodyObj.taskToRequest(taskID, task, transID)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result["OK"]:
raise StopTaskIteration(
"Could not assign request to task: %s" %
result["Message"])
except StopTaskIteration as e:
self._logError("Error creating request for task",
"%s, %s" % (taskID, e),
transID=transID)
taskDict.pop(taskID)
def _assignRequestToTask(self, oRequest, taskDict, transID, taskID,
ownerDN, ownerGroup):
"""set ownerDN and group to request, and add the request to taskDict if it is
valid, otherwise remove the task from the taskDict
:param oRequest: Request
:param dict taskDict: dictionary of tasks, modified in this function
:param int transID: Transformation ID
:param int taskID: Task ID
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
oRequest.RequestName = self._transTaskName(transID, taskID)
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate(oRequest)
if not isValid["OK"]:
self._logError("Error creating request for task",
"%s %s" % (taskID, isValid),
transID=transID)
return S_ERROR("Error creating request")
taskDict[taskID]["TaskObject"] = oRequest
return S_OK()
def submitTransformationTasks(self, taskDict):
"""Submit requests one by one"""
submitted = 0
failed = 0
startTime = time.time()
method = "submitTransformationTasks"
for task in taskDict.values():
# transID is the same for all tasks, so pick it up every time here
transID = task["TransformationID"]
if not task["TaskObject"]:
task["Success"] = False
failed += 1
continue
res = self.submitTaskToExternal(task["TaskObject"])
if res["OK"]:
task["ExternalID"] = res["Value"]
task["Success"] = True
submitted += 1
else:
self._logError("Failed to submit task to RMS",
res["Message"],
transID=transID)
task["Success"] = False
failed += 1
if submitted:
self._logInfo(
"Submitted %d tasks to RMS in %.1f seconds" %
(submitted, time.time() - startTime),
transID=transID,
method=method,
)
if failed:
self._logWarn("Failed to submit %d tasks to RMS." % (failed),
transID=transID,
method=method)
return S_OK(taskDict)
def submitTaskToExternal(self, oRequest):
"""
Submits a request to RMS
"""
if isinstance(oRequest, self.requestClass):
return self.requestClient.putRequest(oRequest,
useFailoverProxy=False,
retryMainService=2)
return S_ERROR("Request should be a Request object")
def updateTransformationReservedTasks(self, taskDicts):
requestNameIDs = {}
noTasks = []
for taskDict in taskDicts:
requestName = self._transTaskName(taskDict["TransformationID"],
taskDict["TaskID"])
reqID = taskDict["ExternalID"]
if reqID and int(reqID):
requestNameIDs[requestName] = reqID
else:
noTasks.append(requestName)
return S_OK({"NoTasks": noTasks, "TaskNameIDs": requestNameIDs})
def getSubmittedTaskStatus(self, taskDicts):
"""
Check if tasks changed status, and return a list of tasks per new status
"""
updateDict = {}
badRequestID = 0
for taskDict in taskDicts:
oldStatus = taskDict["ExternalStatus"]
# ExternalID is normally a string
if taskDict["ExternalID"] and int(taskDict["ExternalID"]):
newStatus = self.requestClient.getRequestStatus(
taskDict["ExternalID"])
if not newStatus["OK"]:
log = self._logVerbose if "not exist" in newStatus[
"Message"] else self._logWarn
log(
"getSubmittedTaskStatus: Failed to get requestID for request",
newStatus["Message"],
transID=taskDict["TransformationID"],
)
else:
newStatus = newStatus["Value"]
# We don't care updating the tasks to Assigned while the request is being processed
if newStatus != oldStatus and newStatus != "Assigned":
updateDict.setdefault(newStatus,
[]).append(taskDict["TaskID"])
else:
badRequestID += 1
if badRequestID:
self._logWarn("%d requests have identifier 0" % badRequestID)
return S_OK(updateDict)
def getSubmittedFileStatus(self, fileDicts):
"""
Check if transformation files changed status, and return a list of taskIDs per new status
"""
# Don't try and get status of not submitted tasks!
transID = None
taskFiles = {}
for fileDict in fileDicts:
# There is only one transformation involved, get however the transID in the loop
transID = fileDict["TransformationID"]
taskID = int(fileDict["TaskID"])
taskFiles.setdefault(taskID, []).append(fileDict["LFN"])
# Should not happen, but just in case there are no files, return
if transID is None:
return S_OK({})
res = self.transClient.getTransformationTasks({
"TransformationID": transID,
"TaskID": list(taskFiles)
})
if not res["OK"]:
return res
requestFiles = {}
for taskDict in res["Value"]:
taskID = taskDict["TaskID"]
externalID = taskDict["ExternalID"]
# Only consider tasks that are submitted, ExternalID is a string
if taskDict["ExternalStatus"] != "Created" and externalID and int(
externalID):
requestFiles[externalID] = taskFiles[taskID]
updateDict = {}
for requestID, lfnList in requestFiles.items():
statusDict = self.requestClient.getRequestFileStatus(
requestID, lfnList)
if not statusDict["OK"]:
log = self._logVerbose if "not exist" in statusDict[
"Message"] else self._logWarn
log(
"Failed to get files status for request",
statusDict["Message"],
transID=transID,
method="getSubmittedFileStatus",
)
else:
for lfn, newStatus in statusDict["Value"].items():
if newStatus == "Done":
updateDict[lfn] = TransformationFilesStatus.PROCESSED
elif newStatus == "Failed":
updateDict[lfn] = TransformationFilesStatus.PROBLEMATIC
return S_OK(updateDict)
class RequestTasks(TaskBase):
"""
Class for handling tasks for the RMS
"""
def __init__(self,
transClient=None,
logger=None,
requestClient=None,
requestClass=None,
requestValidator=None,
ownerDN=None,
ownerGroup=None):
""" c'tor
the requestClass is by default Request.
If extensions want to use an extended type, they can pass it as a parameter.
This is the same behavior as WorfkloTasks and jobClass
"""
if not logger:
logger = gLogger.getSubLogger('RequestTasks')
super(RequestTasks, self).__init__(transClient, logger)
useCertificates = True if (bool(ownerDN)
and bool(ownerGroup)) else False
if not requestClient:
self.requestClient = ReqClient(useCertificates=useCertificates,
delegatedDN=ownerDN,
delegatedGroup=ownerGroup)
else:
self.requestClient = requestClient
if not requestClass:
self.requestClass = Request
else:
self.requestClass = requestClass
if not requestValidator:
self.requestValidator = RequestValidator()
else:
self.requestValidator = requestValidator
def prepareTransformationTasks(self,
transBody,
taskDict,
owner='',
ownerGroup='',
ownerDN='',
bulkSubmissionFlag=False):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if not taskDict:
return S_OK({})
if (not owner) or (not ownerGroup):
res = getProxyInfo(False, False)
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername(owner)
if not res['OK']:
return res
ownerDN = res['Value'][0]
try:
transJson = json.loads(transBody)
self._multiOperationsBody(transJson, taskDict, ownerDN, ownerGroup)
except ValueError: # #json couldn't load
self._singleOperationsBody(transBody, taskDict, ownerDN,
ownerGroup)
return S_OK(taskDict)
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
failedTasks = []
for taskID, task in taskDict.items():
transID = task['TransformationID']
if not task.get('InputData'):
self._logError("Error creating request for task",
"%s, No input data" % taskID,
transID=transID)
taskDict.pop(taskID)
continue
files = []
oRequest = Request()
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].iteritems():
setattr(op, parameter, value)
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile(opFile)
oRequest.addOperation(op)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split(';')
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.iteritems():
transID = task['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task['TargetSE']
# If there are input files
if task.get('InputData'):
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def _assignRequestToTask(self, oRequest, taskDict, transID, taskID,
ownerDN, ownerGroup):
"""set ownerDN and group to request, and add the request to taskDict if it is
valid, otherwise remove the task from the taskDict
:param oRequest: Request
:param dict taskDict: dictionary of tasks, modified in this function
:param int transID: Transformation ID
:param int taskID: Task ID
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
oRequest.RequestName = self._transTaskName(transID, taskID)
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate(oRequest)
if not isValid['OK']:
self._logError("Error creating request for task",
"%s %s" % (taskID, isValid),
transID=transID)
return S_ERROR('Error creating request')
taskDict[taskID]['TaskObject'] = oRequest
return S_OK()
def submitTransformationTasks(self, taskDict):
""" Submit requests one by one
"""
submitted = 0
failed = 0
startTime = time.time()
method = 'submitTransformationTasks'
for task in taskDict.itervalues():
# transID is the same for all tasks, so pick it up every time here
transID = task['TransformationID']
if not task['TaskObject']:
task['Success'] = False
failed += 1
continue
res = self.submitTaskToExternal(task['TaskObject'])
if res['OK']:
task['ExternalID'] = res['Value']
task['Success'] = True
submitted += 1
else:
self._logError("Failed to submit task to RMS",
res['Message'],
transID=transID)
task['Success'] = False
failed += 1
if submitted:
self._logInfo('Submitted %d tasks to RMS in %.1f seconds' %
(submitted, time.time() - startTime),
transID=transID,
method=method)
if failed:
self._logWarn('Failed to submit %d tasks to RMS.' % (failed),
transID=transID,
method=method)
return S_OK(taskDict)
def submitTaskToExternal(self, oRequest):
"""
Submits a request to RMS
"""
if isinstance(oRequest, self.requestClass):
return self.requestClient.putRequest(oRequest,
useFailoverProxy=False,
retryMainService=2)
return S_ERROR("Request should be a Request object")
def updateTransformationReservedTasks(self, taskDicts):
requestNameIDs = {}
noTasks = []
for taskDict in taskDicts:
requestName = self._transTaskName(taskDict['TransformationID'],
taskDict['TaskID'])
reqID = taskDict['ExternalID']
if reqID:
requestNameIDs[requestName] = reqID
else:
noTasks.append(requestName)
return S_OK({'NoTasks': noTasks, 'TaskNameIDs': requestNameIDs})
def getSubmittedTaskStatus(self, taskDicts):
"""
Check if tasks changed status, and return a list of tasks per new status
"""
updateDict = {}
badRequestID = 0
for taskDict in taskDicts:
oldStatus = taskDict['ExternalStatus']
# ExternalID is normally a string
if taskDict['ExternalID'] and int(taskDict['ExternalID']):
newStatus = self.requestClient.getRequestStatus(
taskDict['ExternalID'])
if not newStatus['OK']:
log = self._logVerbose if 'not exist' in newStatus[
'Message'] else self._logWarn
log("getSubmittedTaskStatus: Failed to get requestID for request",
newStatus['Message'],
transID=taskDict['TransformationID'])
else:
newStatus = newStatus['Value']
# We don't care updating the tasks to Assigned while the request is being processed
if newStatus != oldStatus and newStatus != 'Assigned':
updateDict.setdefault(newStatus,
[]).append(taskDict['TaskID'])
else:
badRequestID += 1
if badRequestID:
self._logWarn("%d requests have identifier 0" % badRequestID)
return S_OK(updateDict)
def getSubmittedFileStatus(self, fileDicts):
"""
Check if transformation files changed status, and return a list of taskIDs per new status
"""
# Don't try and get status of not submitted tasks!
transID = None
taskFiles = {}
for fileDict in fileDicts:
# There is only one transformation involved, get however the transID in the loop
transID = fileDict['TransformationID']
taskID = int(fileDict['TaskID'])
taskFiles.setdefault(taskID, []).append(fileDict['LFN'])
# Should not happen, but just in case there are no files, return
if transID is None:
return S_OK({})
res = self.transClient.getTransformationTasks({
'TransformationID':
transID,
'TaskID':
taskFiles.keys()
})
if not res['OK']:
return res
requestFiles = {}
for taskDict in res['Value']:
taskID = taskDict['TaskID']
externalID = taskDict['ExternalID']
# Only consider tasks that are submitted, ExternalID is a string
if taskDict['ExternalStatus'] != 'Created' and externalID and int(
externalID):
requestFiles[externalID] = taskFiles[taskID]
updateDict = {}
for requestID, lfnList in requestFiles.iteritems():
statusDict = self.requestClient.getRequestFileStatus(
requestID, lfnList)
if not statusDict['OK']:
log = self._logVerbose if 'not exist' in statusDict[
'Message'] else self._logWarn
log("Failed to get files status for request",
statusDict['Message'],
transID=transID,
method='getSubmittedFileStatus')
else:
for lfn, newStatus in statusDict['Value'].iteritems():
if newStatus == 'Done':
updateDict[lfn] = 'Processed'
elif newStatus == 'Failed':
updateDict[lfn] = 'Problematic'
return S_OK(updateDict)
class RequestTasks(TaskBase):
def __init__(self,
transClient=None,
logger=None,
requestClient=None,
requestClass=None,
requestValidator=None):
""" c'tor
the requestClass is by default Request.
If extensions want to use an extended type, they can pass it as a parameter.
This is the same behavior as WorfkloTasks and jobClass
"""
if not logger:
logger = gLogger.getSubLogger('RequestTasks')
super(RequestTasks, self).__init__(transClient, logger)
if not requestClient:
self.requestClient = ReqClient()
else:
self.requestClient = requestClient
if not requestClass:
self.requestClass = Request
else:
self.requestClass = requestClass
if not requestValidator:
self.requestValidator = RequestValidator()
else:
self.requestValidator = requestValidator
def prepareTransformationTasks(self,
transBody,
taskDict,
owner='',
ownerGroup='',
ownerDN=''):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if (not owner) or (not ownerGroup):
res = getProxyInfo(False, False)
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername(owner)
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split(';')
except AttributeError:
pass
for taskID in sorted(taskDict):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance(paramDict['InputData'], list):
files = paramDict['InputData']
elif isinstance(paramDict['InputData'], basestring):
files = paramDict['InputData'].split(';')
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
oRequest.RequestName = _requestName(transID, taskID)
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate(oRequest)
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK(taskDict)
def submitTransformationTasks(self, taskDict):
""" Submit requests one by one
"""
submitted = 0
failed = 0
startTime = time.time()
for taskID in sorted(taskDict):
if not taskDict[taskID]['TaskObject']:
taskDict[taskID]['Success'] = False
failed += 1
continue
res = self.submitTaskToExternal(taskDict[taskID]['TaskObject'])
if res['OK']:
taskDict[taskID]['ExternalID'] = res['Value']
taskDict[taskID]['Success'] = True
submitted += 1
else:
self._logError("Failed to submit task to RMS", res['Message'])
taskDict[taskID]['Success'] = False
failed += 1
self._logInfo(
'submitTasks: Submitted %d tasks to RMS in %.1f seconds' %
(submitted, time.time() - startTime))
if failed:
self._logWarn(
'submitTasks: But at the same time failed to submit %d tasks to RMS.'
% (failed))
return S_OK(taskDict)
def submitTaskToExternal(self, oRequest):
""" Submits a request using ReqClient
"""
if isinstance(oRequest, self.requestClass):
return self.requestClient.putRequest(oRequest)
else:
return S_ERROR("Request should be a Request object")
def updateTransformationReservedTasks(self, taskDicts):
requestNameIDs = {}
noTasks = []
for taskDict in taskDicts:
requestName = _requestName(taskDict['TransformationID'],
taskDict['TaskID'])
reqID = taskDict['ExternalID']
if reqID:
requestNameIDs[requestName] = reqID
else:
noTasks.append(requestName)
return S_OK({'NoTasks': noTasks, 'TaskNameIDs': requestNameIDs})
def getSubmittedTaskStatus(self, taskDicts):
updateDict = {}
for taskDict in taskDicts:
oldStatus = taskDict['ExternalStatus']
newStatus = self.requestClient.getRequestStatus(
taskDict['ExternalID'])
if not newStatus['OK']:
log = self._logVerbose if 'not exist' in newStatus[
'Message'] else self.log.warn
log(
"getSubmittedTaskStatus: Failed to get requestID for request",
'%s' % newStatus['Message'])
else:
newStatus = newStatus['Value']
if newStatus != oldStatus:
updateDict.setdefault(newStatus,
[]).append(taskDict['TaskID'])
return S_OK(updateDict)
def getSubmittedFileStatus(self, fileDicts):
taskFiles = {}
submittedTasks = {}
externalIds = {}
# Don't try and get status of not submitted tasks!
for fileDict in fileDicts:
submittedTasks.setdefault(fileDict['TransformationID'],
set()).add(int(fileDict['TaskID']))
for transID in submittedTasks:
res = self.transClient.getTransformationTasks({
'TransformationID':
transID,
'TaskID':
list(submittedTasks[transID])
})
if not res['OK']:
return res
for taskDict in res['Value']:
taskID = taskDict['TaskID']
externalIds[taskID] = taskDict['ExternalID']
if taskDict['ExternalStatus'] == 'Created':
submittedTasks[transID].remove(taskID)
for fileDict in fileDicts:
transID = fileDict['TransformationID']
taskID = int(fileDict['TaskID'])
if taskID in submittedTasks[transID]:
requestID = externalIds[taskID]
taskFiles.setdefault(requestID,
{})[fileDict['LFN']] = fileDict['Status']
updateDict = {}
for requestID in sorted(taskFiles):
lfnDict = taskFiles[requestID]
statusDict = self.requestClient.getRequestFileStatus(
requestID, lfnDict.keys())
if not statusDict['OK']:
log = self._logVerbose if 'not exist' in statusDict[
'Message'] else self.log.warn
log(
"getSubmittedFileStatus: Failed to get files status for request",
'%s' % statusDict['Message'])
continue
statusDict = statusDict['Value']
for lfn, newStatus in statusDict.items():
if newStatus == lfnDict[lfn]:
pass
elif newStatus == 'Done':
updateDict[lfn] = 'Processed'
elif newStatus == 'Failed':
updateDict[lfn] = 'Problematic'
return S_OK(updateDict)
class RequestTasks( TaskBase ):
def __init__( self, transClient = None, logger = None, requestClient = None,
requestClass = None, requestValidator = None ):
""" c'tor
the requestClass is by default Request.
If extensions want to use an extended type, they can pass it as a parameter.
This is the same behavior as WorfkloTasks and jobClass
"""
if not logger:
logger = gLogger.getSubLogger( 'RequestTasks' )
super( RequestTasks, self ).__init__( transClient, logger )
if not requestClient:
self.requestClient = ReqClient()
else:
self.requestClient = requestClient
if not requestClass:
self.requestClass = Request
else:
self.requestClass = requestClass
if not requestValidator:
self.requestValidator = RequestValidator()
else:
self.requestValidator = requestValidator
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def submitTransformationTasks( self, taskDict ):
""" Submit requests one by one
"""
submitted = 0
failed = 0
startTime = time.time()
for taskID in sorted( taskDict ):
if not taskDict[taskID]['TaskObject']:
taskDict[taskID]['Success'] = False
failed += 1
continue
res = self.submitTaskToExternal( taskDict[taskID]['TaskObject'] )
if res['OK']:
taskDict[taskID]['ExternalID'] = res['Value']
taskDict[taskID]['Success'] = True
submitted += 1
else:
self._logError( "Failed to submit task to RMS", res['Message'] )
taskDict[taskID]['Success'] = False
failed += 1
self._logInfo( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % ( submitted, time.time() - startTime ) )
if failed:
self._logWarn( 'submitTasks: But at the same time failed to submit %d tasks to RMS.' % ( failed ) )
return S_OK( taskDict )
def submitTaskToExternal( self, oRequest ):
""" Submits a request using ReqClient
"""
if isinstance( oRequest, self.requestClass ):
return self.requestClient.putRequest( oRequest )
else:
return S_ERROR( "Request should be a Request object" )
def updateTransformationReservedTasks( self, taskDicts ):
requestNameIDs = {}
noTasks = []
for taskDict in taskDicts:
requestName = _requestName( taskDict['TransformationID'], taskDict['TaskID'] )
reqID = taskDict['ExternalID']
if reqID:
requestNameIDs[requestName] = reqID
else:
noTasks.append( requestName )
return S_OK( {'NoTasks':noTasks, 'TaskNameIDs':requestNameIDs} )
def getSubmittedTaskStatus( self, taskDicts ):
updateDict = {}
for taskDict in taskDicts:
oldStatus = taskDict['ExternalStatus']
newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID'] )
if not newStatus['OK']:
log = self._logVerbose if 'not exist' in newStatus['Message'] else self.log.warn
log( "getSubmittedTaskStatus: Failed to get requestID for request", '%s' % newStatus['Message'] )
else:
newStatus = newStatus['Value']
if newStatus != oldStatus:
updateDict.setdefault( newStatus, [] ).append( taskDict['TaskID'] )
return S_OK( updateDict )
def getSubmittedFileStatus( self, fileDicts ):
taskFiles = {}
submittedTasks = {}
externalIds = {}
# Don't try and get status of not submitted tasks!
for fileDict in fileDicts:
submittedTasks.setdefault( fileDict['TransformationID'], set() ).add( int( fileDict['TaskID'] ) )
for transID in submittedTasks:
res = self.transClient.getTransformationTasks( { 'TransformationID':transID, 'TaskID': list( submittedTasks[transID] )} )
if not res['OK']:
return res
for taskDict in res['Value']:
taskID = taskDict['TaskID']
externalIds[taskID] = taskDict['ExternalID']
if taskDict['ExternalStatus'] == 'Created':
submittedTasks[transID].remove( taskID )
for fileDict in fileDicts:
transID = fileDict['TransformationID']
taskID = int( fileDict['TaskID'] )
if taskID in submittedTasks[transID]:
requestID = externalIds[taskID]
taskFiles.setdefault( requestID, {} )[fileDict['LFN']] = fileDict['Status']
updateDict = {}
for requestID in sorted( taskFiles ):
lfnDict = taskFiles[requestID]
statusDict = self.requestClient.getRequestFileStatus( requestID, lfnDict.keys() )
if not statusDict['OK']:
log = self._logVerbose if 'not exist' in statusDict['Message'] else self.log.warn
log( "getSubmittedFileStatus: Failed to get files status for request", '%s' % statusDict['Message'] )
continue
statusDict = statusDict['Value']
for lfn, newStatus in statusDict.items():
if newStatus == lfnDict[lfn]:
pass
elif newStatus == 'Done':
updateDict[lfn] = 'Processed'
elif newStatus == 'Failed':
updateDict[lfn] = 'Problematic'
return S_OK( updateDict )
def testValidator(self):
""" validator test """
## create validator
validator = RequestValidator()
self.assertEqual(isinstance(validator, RequestValidator), True)
## RequestName not set
ret = validator.validate(self.request)
self.assertEqual(ret, {'Message': 'RequestName not set', 'OK': False})
self.request.RequestName = "test_request"
# # no ownerDN
ret = validator.validate(self.request)
self.assertEqual(
ret, {
'Message': "Request 'test_request' is missing OwnerDN value",
'OK': False
})
self.request.OwnerDN = "foo/bar=baz"
# # no owner group
ret = validator.validate(self.request)
self.assertEqual(
ret, {
'Message':
"Request 'test_request' is missing OwnerGroup value",
'OK': False
})
self.request.OwnerGroup = "dirac_user"
## no operations
ret = validator.validate(self.request)
self.assertEqual(
ret, {
'Message': "Operations not present in request 'test_request'",
'OK': False
})
self.request.addOperation(self.operation)
## type not set
ret = validator.validate(self.request)
self.assertEqual(
ret, {
'Message':
"Operation #0 in request 'test_request' hasn't got Type set",
'OK': False
})
self.operation.Type = "ReplicateAndRegister"
## files not present
ret = validator.validate(self.request)
self.assertEqual(
ret, {
'Message':
"Operation #0 of type 'ReplicateAndRegister' hasn't got files to process.",
'OK': False
})
self.operation.addFile(self.file)
## targetSE not set
ret = validator.validate(self.request)
self.assertEqual(
ret, {
'Message':
"Operation #0 of type 'ReplicateAndRegister' is missing TargetSE attribute.",
'OK': False
})
self.operation.TargetSE = "CERN-USER"
## missing LFN
ret = validator.validate(self.request)
self.assertEqual(
ret, {
"Message":
"Operation #0 of type 'ReplicateAndRegister' is missing LFN attribute for file.",
"OK": False
})
self.file.LFN = "/a/b/c"
## Checksum set, ChecksumType not set
self.file.Checksum = "abcdef"
ret = validator.validate(self.request)
self.assertEqual(
ret, {
'Message':
'File in operation #0 is missing Checksum (abcdef) or ChecksumType ()',
'OK': False
})
## ChecksumType set, Checksum not set
self.file.Checksum = ""
self.file.ChecksumType = "adler32"
ret = validator.validate(self.request)
self.assertEqual(
ret, {
'Message':
'File in operation #0 is missing Checksum () or ChecksumType (ADLER32)',
'OK': False
})
## both set
self.file.Checksum = "abcdef"
self.file.ChecksumType = "adler32"
ret = validator.validate(self.request)
self.assertEqual(ret, {'OK': True, 'Value': ''})
## both unset
self.file.Checksum = ""
self.file.ChecksumType = None
ret = validator.validate(self.request)
self.assertEqual(ret, {'OK': True, 'Value': ''})
## all OK
ret = validator.validate(self.request)
self.assertEqual(ret, {'OK': True, 'Value': ''})
class ModuleBase(object):
""" Base class for Modules - works only within DIRAC workflows
"""
#############################################################################
def __init__(self, loggerIn=None, operationsHelperIn=None, bkClientIn=None, dm=None):
""" Initialization of module base.
"""
if loggerIn is None:
self.log = gLogger.getSubLogger('ModuleBase')
else:
self.log = loggerIn
if operationsHelperIn is None:
self.opsH = Operations()
else:
self.opsH = operationsHelperIn
if bkClientIn is None:
self.bkClient = BookkeepingClient()
else:
self.bkClient = bkClientIn
if dm is None:
self.dataManager = DataManager()
else:
self.dataManager = dm
self.requestValidator = RequestValidator()
self.production_id = ''
self.prod_job_id = ''
self.jobID = 0
self.step_number = ''
self.step_id = ''
self.fileReport = None
self.jobReport = None
self.request = None
self.workflowStatus = None
self.stepStatus = None
self.workflow_commons = None
self.step_commons = None
self.debugSE = 'CERN-DEBUG'
self.executable = 'gaudirun.py'
self.applicationName = 'Unknown'
self.applicationVersion = 'Unknown'
self.applicationLog = ''
self.applicationType = None
self.systemConfig = None
self.extraPackages = None
self.bkConfigName = None
self.BKstepID = None
self.condDBTag = None
self.DDDBTag = None
self.dqTag = None
self.CPUe = None
self.eventType = ''
self.gaudiSteps = None
self.InputData = ''
self.inputDataList = []
self.inputDataType = None
self.histoName = "Hist.root"
self.optionsFile = None
self.optionsFormat = None
self.optionsLine = None
self.extraOptionsLine = None
self.jobType = None
self.logFilePath = None
self.onlineCondDBTag = None
self.onlineDDDBTag = None
self.outputSEs = {}
self.outputDataFileMask = None
self.numberOfEvents = -1
self.maxNumberOfEvents = None
self.TCK = None
self.mcTCK = None
self.multicoreJob = None
self.multicoreStep = None
self.poolXMLCatName = 'pool_xml_catalog.xml'
self.persistency = ''
self.processingPass = None
self.runNumber = 'Unknown'
self.runTimeProjectName = None
self.runTimeProjectVersion = None
self.simDescription = ''
self.siteName = None
self.stepName = None
self.stepInputData = None
self.XMLSummary = '' # name of the file, not the object
self.stepProcPass = None
self.outputFilePrefix = ''
#############################################################################
def execute(self, version=None,
production_id=None, prod_job_id=None, wms_job_id=None,
workflowStatus=None, stepStatus=None,
wf_commons=None, step_commons=None,
step_number=None, step_id=None):
""" Function called by all super classes
"""
if version:
self.log.info('===== Executing ' + version + ' ===== ')
self.log.verbose("Executing directory for job is %s" % os.getcwd())
if production_id:
self.production_id = production_id
else:
self.production_id = self.workflow_commons['PRODUCTION_ID'] # This is a string, like '00051753'
if prod_job_id:
self.prod_job_id = prod_job_id
else:
self.prod_job_id = self.workflow_commons['JOB_ID']
if 'JOBID' in os.environ:
self.jobID = os.environ['JOBID']
if wms_job_id:
self.jobID = wms_job_id
if workflowStatus:
self.workflowStatus = workflowStatus
if stepStatus:
self.stepStatus = stepStatus
if wf_commons:
self.workflow_commons = wf_commons
if step_commons:
self.step_commons = step_commons
if step_number:
self.step_number = step_number
else:
self.step_number = self.STEP_NUMBER # pylint: disable=no-member
if step_id:
self.step_id = step_id
else:
self.step_id = '%s_%s_%s' % (self.production_id, self.prod_job_id, self.step_number)
self.siteName = siteName()
#############################################################################
def finalize(self, version=None):
""" Just finalizing
"""
self.log.flushAllMessages(0)
if version:
self.log.info('===== Terminating ' + version + ' ===== ')
#############################################################################
def setApplicationStatus(self, status, sendFlag=True):
"""Wraps around setJobApplicationStatus of state update client
"""
if not self._WMSJob():
return 0 # e.g. running locally prior to submission
if self._checkWFAndStepStatus(noPrint=True):
# The application status won't be updated in case the workflow or the step is failed already
if not isinstance(status, str):
status = str(status)
self.log.verbose('setJobApplicationStatus(%d, %s)' % (self.jobID, status))
jobStatus = self.jobReport.setApplicationStatus(status, sendFlag)
if not jobStatus['OK']:
self.log.warn(jobStatus['Message'])
#############################################################################
def setJobParameter(self, name, value, sendFlag=True):
"""Wraps around setJobParameter of state update client
"""
if not self._WMSJob():
return 0 # e.g. running locally prior to submission
self.log.verbose('setJobParameter(%d,%s,%s)' % (self.jobID, name, value))
jobParam = self.jobReport.setJobParameter(str(name), str(value), sendFlag)
if not jobParam['OK']:
self.log.warn(jobParam['Message'])
#############################################################################
def _resolveInputVariables(self):
""" By convention the module input parameters are resolved here.
"""
self.log.verbose("workflow_commons = ", self.workflow_commons)
self.log.verbose("step_commons = ", self.step_commons)
self.fileReport = self._getFileReporter()
self.jobReport = self._getJobReporter()
self.request = self._getRequestContainer()
self.__resolveInputWorkflow()
#############################################################################
def __resolveInputWorkflow(self):
""" Resolve the input variables that are in the workflow_commons
"""
self.runNumber = self.workflow_commons.get('runNumber', self.runNumber)
self.persistency = self.workflow_commons.get('persistency', self.persistency)
self.jobType = self.workflow_commons.get('JobType', self.jobType)
self.poolXMLCatName = self.workflow_commons.get('poolXMLCatName', self.poolXMLCatName)
self.InputData = self.workflow_commons.get('InputData', self.InputData)
if 'ParametricInputData' in self.workflow_commons:
pID = copy.deepcopy(self.workflow_commons['ParametricInputData'])
if pID:
if isinstance(pID, list):
pID = ';'.join(pID)
# self.InputData += ';' + pID
self.InputData = pID
self.InputData = self.InputData.rstrip(';')
if self.InputData == ';':
self.InputData = ''
self.inputDataList = [lfn.strip('LFN:') for lfn in self.InputData.split(';') if lfn]
# only required until the stripping is the same for MC / data
self.bkConfigName = self.workflow_commons.get('configName', self.bkConfigName)
self.simDescription = self.workflow_commons.get('simDescription', self.simDescription)
if 'runMetadata' in self.workflow_commons:
runMetadataDict = eval(self.workflow_commons['runMetadata'])
self.onlineDDDBTag = runMetadataDict['DDDB']
self.onlineCondDBTag = runMetadataDict['CondDb']
self.TCK = runMetadataDict['TCK']
if 'outputDataFileMask' in self.workflow_commons:
self.outputDataFileMask = self.workflow_commons['outputDataFileMask']
if not isinstance(self.outputDataFileMask, list):
self.outputDataFileMask = [i.lower().strip() for i in self.outputDataFileMask.split(';')]
self.gaudiSteps = self.workflow_commons.get('gaudiSteps', self.gaudiSteps)
if 'CPUe' in self.workflow_commons:
self.CPUe = int(round(float(self.workflow_commons['CPUe'])))
self.multicoreJob = self.workflow_commons.get('multicore', self.multicoreJob)
self.processingPass = self.workflow_commons.get('processingPass', self.processingPass)
self.logFilePath = self.workflow_commons.get('LogFilePath', self.logFilePath)
if isinstance(self.logFilePath, list):
self.logFilePath = self.logFilePath[0]
else:
if 'PRODUCTION_ID' and 'JOB_ID' and 'configVersion' and 'configName' in self.workflow_commons:
self.log.info('LogFilePath parameter not found, creating on the fly')
result = getLogPath(self.workflow_commons, self.bkClient)
if not result['OK']:
self.log.error('Could not create LogFilePath', result['Message'])
raise RuntimeError(result['Message'])
self.logFilePath = result['Value']['LogFilePath'][0]
if 'maxNumberOfEvents' in self.workflow_commons:
self.maxNumberOfEvents = int(self.workflow_commons['maxNumberOfEvents'])
self.eventType = self.workflow_commons.get('eventType', self.eventType)
self.numberOfEvents = int(self.workflow_commons.get('numberOfEvents', self.numberOfEvents))
if 'outputSEs' in self.workflow_commons:
self.outputSEs = self.workflow_commons['outputSEs']
else:
# this is here for backward compatibility
histogramSE = self.opsH.getValue('Productions/HistogramSE', 'CERN-HIST')
histoTypes = self.opsH.getValue('Productions/HistogramTypes', ['HIST', 'BRUNELHIST', 'DAVINCIHIST',
'GAUSSHIST'])
self.outputSEs = dict((ht, histogramSE) for ht in histoTypes)
# for older productions we construct it based on what should be found in the steps
if 'listoutput' in self.step_commons:
listOutputStep = self.step_commons['listoutput']
for lOutput in listOutputStep:
try:
for outputDataType in lOutput['outputDataType'].split(';'):
if outputDataType:
self.outputSEs.setdefault(outputDataType.upper(), lOutput['outputDataSE'])
except KeyError:
continue
self.workflow_commons['outputSEs'] = self.outputSEs
#############################################################################
def _resolveInputStep(self):
""" Resolve the input variables for an application step
"""
prodID = self.workflow_commons.get('PRODUCTION_ID', '')
jobID = self.workflow_commons.get('JOB_ID', '')
stepInstanceNumber = self.step_commons.get('STEP_NUMBER', '')
self.stepName = self.step_commons['STEP_INSTANCE_NAME']
self.executable = self.step_commons.get('executable', self.executable)
self.applicationName = self.step_commons.get('applicationName', self.applicationName)
self.applicationVersion = self.step_commons.get('applicationVersion', self.applicationVersion)
self.BKstepID = self.step_commons.get('BKStepID', self.BKstepID)
self.stepProcPass = self.step_commons.get('StepProcPass', self.stepProcPass)
# this is only for production jobs and for application steps
if prodID and jobID and stepInstanceNumber and 'listoutput' in self.step_commons:
self.outputFilePrefix = "%s_%s_%s" % (prodID, jobID, stepInstanceNumber)
self.applicationLog = self.applicationName + '_' + self.outputFilePrefix + '.log'
self.XMLSummary = 'summary' + self.applicationName + '_' + self.outputFilePrefix + '.xml'
self.histoName = self.applicationName + '_' + self.outputFilePrefix + '.Hist.root'
for fileTypeDict in self.step_commons['listoutput']: # this is a dict like {'outputDataType': 'sim'}
# for non histo-merging prods
if 'hist' in fileTypeDict['outputDataType'].lower() and self.jobType.lower() != 'merge':
# Watch out: this assumes that:
# - 'hist' is always in the file type name
# - merging jobs won't produce histograms
# - the only merging jobs that produce output types with hist are histomerging productions
if 'outputDataName' not in fileTypeDict:
fileTypeDict['outputDataName'] = self.histoName
else:
fileTypeDict['outputDataName'] = self.outputFilePrefix + '.' + fileTypeDict['outputDataType']
else:
self.applicationLog = self.step_commons.get('applicationLog', self.applicationLog)
self.inputDataType = self.step_commons.get('inputDataType', self.inputDataType)
self.applicationType = self.step_commons.get('applicationType', self.applicationType)
self.optionsFile = self.step_commons.get('optionsFile', self.optionsFile)
self.optionsLine = self.step_commons.get('optionsLine', self.optionsLine)
self.extraOptionsLine = self.step_commons.get('extraOptionsLine', self.extraOptionsLine)
if 'runTimeProjectName' in self.step_commons:
self.runTimeProjectName = self.step_commons['runTimeProjectName']
self.runTimeProjectVersion = self.step_commons['runTimeProjectVersion']
if 'extraPackages' in self.step_commons:
self.extraPackages = self.step_commons['extraPackages']
if self.extraPackages:
if isinstance(self.extraPackages, basestring):
eps = self.extraPackages.split(';') # pylint: disable=no-member
epList = []
for ep in eps:
epList.append(tuple(ep.split('.')))
self.extraPackages = epList
stepInputData = []
if 'inputData' in self.step_commons:
if self.step_commons['inputData']:
stepInputData = self.step_commons['inputData']
elif self.InputData:
stepInputData = copy.deepcopy(self.InputData)
if stepInputData:
stepInputData = self._determineStepInputData(stepInputData, )
self.stepInputData = [sid.strip('LFN:') for sid in stepInputData]
self.optionsFormat = self.step_commons.get('optionsFormat', self.optionsFormat)
self.multicoreStep = self.step_commons.get('multiCore', self.multicoreStep)
self.systemConfig = self.step_commons.get('SystemConfig', self.systemConfig)
self.mcTCK = self.step_commons.get('mcTCK', self.mcTCK)
self.DDDBTag = self.step_commons.get('DDDBTag', self.DDDBTag)
self.condDBTag = self.step_commons.get('CondDBTag', self.condDBTag)
self.dqTag = self.step_commons.get('DQTag', self.dqTag)
# This is resolved also in __resolveInputWorkflow but can be specialized per step (e.g. LHCbJob().setApplication())
self.numberOfEvents = int(self.step_commons.get('numberOfEvents', self.numberOfEvents))
#############################################################################
def _determineOutputs(self):
""" Method that determines the correct outputs.
For merging jobs the output has normally to be the same as the input, but there might be exceptions
(like for the productions for the merging of histograms)
For the others, we use what is in the step definition
We always remove the 'HIST'(s), when present, from the list of output file types
as these are treated differently.
There is anyway also here the special case of histogram merging productions.
"""
histoTypes = self.opsH.getValue('Productions/HistogramTypes', ['HIST', 'BRUNELHIST', 'DAVINCIHIST', 'GAUSSHIST'])
stepOutputs = self.step_commons['listoutput']
stepOutputsT = [x['outputDataType'] for x in stepOutputs]
stepOutTypes = []
for fts in stepOutputsT:
for ft in fts.split(';'):
if ft and ft not in stepOutTypes:
stepOutTypes.append(ft.lower())
# corrections for Merge productions
if self.jobType.lower() in ('merge', 'histomerge'):
if len(stepOutTypes) == 1 and stepOutTypes[0] in [hts.lower() for hts in histoTypes] + ['root']:
# If it is root/histo/ntuple merging job, we treat it almost as a stripping job
pass
else:
res = self.bkClient.getFileMetadata(self.stepInputData)
if not res['OK']:
raise RuntimeError(res['Message'])
outputTypes = set()
success = res['Value']['Successful']
for mdDict in success.values():
outputTypes.add(mdDict['FileType'])
if len(success) != len(self.stepInputData):
self.log.warn("Some inputs are not in BKK, trying to parse the file names")
for sid in set(self.stepInputData) - set(success):
# File types in the BK are upper case
fType = '.'.join(os.path.basename(sid).split('.')[1:]).upper()
outputTypes.add(fType)
outputTypes = list(outputTypes)
if len(outputTypes) > 1:
raise ValueError("Not all input files have the same type (%s)" % ','.join(outputTypes))
outputType = outputTypes[0].lower()
stepOutTypes = [outputType.lower()]
stepOutputs = [{'outputDataName': self.step_id + '.' + outputType.lower(),
'outputDataType': outputType.lower(),
'outputBKType': outputType.upper()}]
histogram = False
# first here treating the special case of root/histo/ntuple merging jobs
if self.jobType.lower() in ('merge', 'histomerge') \
and len(stepOutTypes) == 1 \
and stepOutTypes[0] in [hts.lower() for hts in histoTypes] + ['root']:
pass
else: # This is not a histogram merging production, so we remove the histograms from the step outputs
for hist in histoTypes:
try:
stepOutTypes.remove(hist)
histogram = True
except ValueError:
pass
try:
stepOutTypes.remove(hist.lower())
histogram = True
except ValueError:
pass
return stepOutputs, stepOutTypes, histogram
#############################################################################
def _getJobReporter(self):
""" just return the job reporter (object, always defined by dirac-jobexec)
"""
if 'JobReport' in self.workflow_commons:
return self.workflow_commons['JobReport']
jobReport = JobReport(self.jobID)
self.workflow_commons['JobReport'] = jobReport
return jobReport
#############################################################################
def _getFileReporter(self):
""" just return the file reporter (object)
"""
if 'FileReport' in self.workflow_commons:
return self.workflow_commons['FileReport']
fileReport = FileReport()
self.workflow_commons['FileReport'] = fileReport
return fileReport
#############################################################################
def _getRequestContainer(self):
""" just return the Request reporter (object)
"""
if 'Request' in self.workflow_commons:
return self.workflow_commons['Request']
request = Request()
self.workflow_commons['Request'] = request
return request
#############################################################################
def getCandidateFiles(self, outputList, outputLFNs, fileMask='', stepMask=''):
""" Returns list of candidate files to upload, check if some outputs are missing.
:param list outputList: list of outputs with the following structure::
[{'outputDataType': '', 'outputDataName': ''} , {...}]
:param list outputLFNs: output LFNs for the job
:param str fileMask: the output file extensions to restrict the outputs to. Can also be a list of strings
:param str stepMask: the step ID to restrict the outputs to. Can also be a list of strings.
:returns: dictionary containing type, SE and LFN for files restricted by mask
"""
fileInfo = {}
for outputFile in outputList:
if 'outputDataType' in outputFile and 'outputDataName' in outputFile:
fname = outputFile['outputDataName']
fileType = outputFile['outputDataType']
fileInfo[fname] = {'type': fileType}
else:
self.log.error('Ignoring mal-formed output data specification', str(outputFile))
for lfn in outputLFNs:
if os.path.basename(lfn).lower() in list(fi.lower() for fi in fileInfo):
try:
fileInfo[os.path.basename(lfn)]['lfn'] = lfn
self.log.verbose("Found LFN for file",
"%s -> %s" % (lfn, os.path.basename(lfn)))
except KeyError:
fileInfo[os.path.basename(lfn).lower()]['lfn'] = lfn
self.log.verbose("Found LFN for file",
"%s -> %s" % (lfn, os.path.basename(lfn).lower()))
elif os.path.basename(lfn).split('_')[-1].lower() in list(fi.lower() for fi in fileInfo):
try:
fileInfo[os.path.basename(lfn).split('_')[-1]]['lfn'] = lfn
self.log.verbose("Found LFN for file",
" %s -> %s" % (lfn, os.path.basename(lfn).split('_')[-1]))
except KeyError:
fileInfo[os.path.basename(lfn).split('_')[-1].lower()]['lfn'] = lfn
self.log.verbose("Found LFN for file",
" %s -> %s" % (lfn, os.path.basename(lfn).split('_')[-1].lower()))
else:
self.log.warn("LFN not recognized", "for LFN %s" % lfn)
# check local existance
fileList = self._checkLocalExistance(list(fileInfo.keys()))
# really horrible stuff for updating the name with what's found on the disk
# (because maybe the case is not the same as the expected)
newFileInfo = {}
for fi in fileInfo.iteritems():
for li in fileList:
if fi[0].lower() == li.lower():
newFileInfo[li] = fi[1]
# Select which files have to be uploaded: in principle all
candidateFiles = self._applyMask(newFileInfo, fileMask, stepMask)
# Sanity check all final candidate metadata keys are present
self._checkSanity(candidateFiles)
# Adding the SEs
for candidateFile in candidateFiles:
try:
fType = candidateFiles[candidateFile]['type']
for fType in [fType, fType.lower(), fType.upper(), fType.capitalize()]:
try:
wfSE = self.outputSEs[fType]
candidateFiles[candidateFile]['workflowSE'] = wfSE
break
except KeyError:
continue
except AttributeError:
break
return candidateFiles
#############################################################################
def _checkLocalExistance(self, fileList):
""" Check that the list of output files are present locally
"""
notPresentFiles = []
filesOnDisk = set(os.listdir('.'))
diff = set(fileList).difference(filesOnDisk)
if diff:
self.log.warn("Not all files found",
"set of what's on the disk: %s" % filesOnDisk)
self.log.warn("Looking for files with different case")
diffCI = set(fx.lower() for fx in fileList).difference(set(fod.lower() for fod in filesOnDisk))
if diffCI:
self.log.error("Output data not found",
"File list %s does not exist locally" % notPresentFiles)
raise os.error("Output data not found")
# now checking what's the actual filename case that's written
self.log.warn("Found files with filename with different case, returning those")
filesActuallyOnDisk = []
for fod in os.listdir('.'):
if fod.lower() in [fl.lower() for fl in fileList]:
filesActuallyOnDisk.append(fod)
return filesActuallyOnDisk
return fileList
#############################################################################
def _applyMask(self, candidateFilesIn, fileMask, stepMask):
""" Select which files have to be uploaded: in principle all
:param dict candidateFilesIn: dictionary like
{'00012345_00012345_4.dst': {'lfn': '/lhcb/MC/2010/DST/123/123_45_4.dst',
type': 'dst'},
'00012345_00012345_2.digi': {'type': 'digi'}}
:param str fileMask: the output file extensions to restrict the outputs to. Can also be a list of strings
:param str stepMask: the step ID to restrict the outputs to. Can also be a list of strings.
:returns: a dict like the one in candidateFilesIn
"""
candidateFiles = copy.deepcopy(candidateFilesIn)
if fileMask and not isinstance(fileMask, list):
fileMask = [fileMask]
if isinstance(stepMask, int):
stepMask = str(stepMask)
if stepMask and not isinstance(stepMask, list):
stepMask = [stepMask]
if fileMask and fileMask != ['']:
for fileName, metadata in list(candidateFiles.iteritems()):
if metadata['type'].lower() not in [fm.lower() for fm in fileMask]:
del candidateFiles[fileName]
self.log.info('Output file %s was produced but will not be treated (fileMask is %s)' % (fileName,
', '.join(fileMask)))
else:
self.log.info('No outputDataFileMask provided, the files with all the extensions will be considered')
if stepMask and stepMask != ['']:
for fileName, metadata in list(candidateFiles.iteritems()):
if fileName.lower().replace(metadata['type'].lower(), '').split('_')[-1].split('.')[0] not in stepMask:
del candidateFiles[fileName]
self.log.info('Output file %s was produced but will not be treated (stepMask is %s)' % (fileName,
', '.join(stepMask)))
else:
self.log.info('No outputDataStep provided, the files output of all the steps will be considered')
return candidateFiles
#############################################################################
def _checkSanity(self, candidateFiles):
""" Sanity check all final candidate metadata keys are present
:param dict candidateFiles: dictionary like
{'00012345_00012345_4.dst': {'lfn': '/lhcb/MC/2010/DST/123/123_45_4.dst',
type': 'dst'},
'00012345_00012345_2.digi': {'type': 'digi'}}
:returns: None or raises ValueError
"""
notPresentKeys = []
mandatoryKeys = ['type', 'lfn'] # filedict is used for requests
for fileName, metadata in candidateFiles.iteritems():
for key in mandatoryKeys:
if key not in metadata:
notPresentKeys.append((fileName, key))
if notPresentKeys:
for fileName_keys in notPresentKeys:
self.log.error("File %s has missing %s" % (fileName_keys[0], fileName_keys[1]))
raise ValueError("Missing requested fileName keys")
#############################################################################
def getFileMetadata(self, candidateFiles):
""" Returns the candidate file dictionary with associated metadata.
The input candidate files dictionary has the structure:
{'foo_1.txt': {'lfn': '/lhcb/MC/2010/DST/00012345/0001/foo_1.txt',
'type': 'txt',
'workflowSE': SE1},
'bar_2.py': {'lfn': '/lhcb/MC/2010/DST/00012345/0001/bar_2.py',
'type': 'py',
'workflowSE': 'SE2'},
}
this also assumes the files are in the current working directory.
"""
# Retrieve the POOL File GUID(s) for any final output files
self.log.info('Will search for POOL GUIDs for: %s' % (', '.join(candidateFiles.keys())))
pfnGUID = getGUID(candidateFiles.keys())
if not pfnGUID['OK']:
self.log.error('''PoolXMLFile failed to determine POOL GUID(s) for output file list,
these will be generated by the DataManager''', pfnGUID['Message'])
for fileName in candidateFiles.keys():
candidateFiles[fileName]['guid'] = ''
elif pfnGUID['generated']:
self.log.warn('PoolXMLFile generated GUID(s) for the following files ', ', '.join(pfnGUID['generated']))
else:
self.log.info('GUIDs found for all specified POOL files: %s' % (', '.join(candidateFiles.keys())))
for pfn, guid in pfnGUID['Value'].iteritems():
candidateFiles[pfn]['guid'] = guid
# Get all additional metadata about the file necessary for requests
final = {}
for fileName, metadata in candidateFiles.iteritems():
fileDict = {}
fileDict['LFN'] = metadata['lfn']
fileDict['Size'] = os.path.getsize(fileName)
fileDict['Checksum'] = fileAdler(fileName)
fileDict['ChecksumType'] = 'ADLER32'
fileDict['GUID'] = metadata['guid']
fileDict['Status'] = 'Waiting'
final[fileName] = metadata
final[fileName]['filedict'] = fileDict
final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName)
# Sanity check all final candidate metadata keys are present (return S_ERROR if not)
mandatoryKeys = ['guid', 'filedict'] # filedict is used for requests (this method adds guid and filedict)
for fileName, metadata in final.iteritems():
for key in mandatoryKeys:
if key not in metadata:
raise RuntimeError("File %s has missing %s" % (fileName, key))
return final
#############################################################################
def _determineStepInputData(self, inputData):
""" determine the input data for the step
"""
if inputData == 'previousStep':
stepIndex = self.gaudiSteps.index(self.stepName)
previousStep = self.gaudiSteps[stepIndex - 1]
stepInputData = []
for outputF in self.workflow_commons['outputList']:
try:
if outputF['stepName'] == previousStep and outputF['outputBKType'].lower() == self.inputDataType.lower():
stepInputData.append(outputF['outputDataName'])
except KeyError:
raise RuntimeError("Can't find output of step %s" % previousStep)
return stepInputData
return [x.strip('LFN:') for x in inputData.split(';')]
#############################################################################
def _manageAppOutput(self, outputs):
""" Calls self._findOutputs to find what's produced, then creates the LFNs
outputs, as called here, is created starting from step_commons['listoutput'],
but enriched with at least the outputDataName.
example of outputs:
[{'outputDataType': 'bhadron.dst', 'outputBKType': 'BHADRON.DST',
'outputDataName': '00012345_00012345_2.BHADRON.DST'},
{'outputDataType': 'calibration.dst','outputDataType': 'CALIBRATION.DST',
'outputDataName': '00012345_00012345_2.CALIBRATION.DST'}]
:params list outputs: list of dicts of step output files descriptions
"""
if not outputs:
self.log.warn('Step outputs are not defined (normal user jobs. Not normal in productions and SAM jobs)')
return
else:
finalOutputs, _bkFileTypes = self._findOutputs(outputs)
self.log.info('Final step outputs are: %s' % (finalOutputs))
self.step_commons['listoutput'] = finalOutputs
if 'outputList' in self.workflow_commons:
for outFile in finalOutputs:
if outFile not in self.workflow_commons['outputList']:
self.workflow_commons['outputList'].append(outFile)
else:
self.workflow_commons['outputList'] = finalOutputs
if 'PRODUCTION_ID' and 'JOB_ID' and 'configVersion' and 'configName' in self.workflow_commons:
self.log.info('Attempting to recreate the production output LFNs...')
result = constructProductionLFNs(self.workflow_commons, self.bkClient)
if not result['OK']:
raise IOError("Could not create production LFNs: %s" % result['Message'])
self.workflow_commons['BookkeepingLFNs'] = result['Value']['BookkeepingLFNs']
self.workflow_commons['LogFilePath'] = result['Value']['LogFilePath']
self.workflow_commons['ProductionOutputData'] = result['Value']['ProductionOutputData']
#############################################################################
def _findOutputs(self, stepOutput):
""" Find which outputs of those in stepOutput (what are expected to be produced) are effectively produced.
stepOutput, as called here, is created starting from step_commons['listoutput']
example of stepOutput:
[{'outputDataType': 'bhadron.dst', 'outputBKType': 'BHADRON.DST',
'outputDataName': '00012345_00012345_2.BHADRON.DST'},
{'outputDataType': 'calibration.dst','outputDataType': 'CALIBRATION.DST',
'outputDataName': '00012345_00012345_2.CALIBRATION.DST'}]
:params list stepOutput: list of dicts of step output files descriptions
:returns: list, list
"""
bkFileTypes = [] # uppercase list of file types
finalOutputs = [] # list of dicts of what's found on the local disk
filesFound = []
for output in stepOutput:
found = False
fileOnDisk = None
for fileOnDisk in os.listdir('.'):
if output['outputDataName'].lower() == fileOnDisk.lower():
found = True
break
if found and fileOnDisk:
self.log.info('Found output file (case is not considered)',
'%s matching %s ' % (fileOnDisk, output['outputDataName']))
output['outputDataName'] = fileOnDisk
filesFound.append(output)
else:
self.log.error('Output not found', output['outputDataName'])
raise IOError("OutputData not found")
for fileFound in filesFound:
bkFileTypes.append(fileFound['outputDataType'].upper())
finalOutputs.append({'outputDataName': fileFound['outputDataName'],
'outputDataType': fileFound['outputDataType'].lower(),
'outputBKType': fileFound['outputDataType'].upper(),
'stepName': self.stepName})
return (finalOutputs, bkFileTypes)
#############################################################################
def _WMSJob(self):
""" Check if this job is running via WMS
"""
return True if self.jobID else False
#############################################################################
def _enableModule(self):
""" Enable module if it's running via WMS
"""
if not self._WMSJob():
self.log.info('No WMS JobID found, disabling module via control flag')
return False
self.log.verbose('Found WMS JobID', self.jobID)
return True
#############################################################################
def _checkWFAndStepStatus(self, noPrint=False):
""" Check the WF and Step status
"""
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
if not noPrint:
self.log.info('Skip this module, failure detected in a previous step')
self.log.info('Workflow status', self.workflowStatus)
self.log.info('Step Status', self.stepStatus)
return False
else:
return True
#############################################################################
def _disableWatchdogCPUCheck(self):
""" just writes a file to disable the watchdog
"""
self.log.info("Creating DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK in order to disable the Watchdog")
with open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w') as fopen:
fopen.write('%s' % time.asctime())
#############################################################################
def generateFailoverFile(self):
""" Retrieve the accumulated reporting request, and produce a JSON file that is consumed by the JobWrapper
"""
reportRequest = None
result = self.jobReport.generateForwardDISET()
if not result['OK']:
self.log.warn("Could not generate Operation for job report",
"with result:\n%s" % (result))
else:
reportRequest = result['Value']
if reportRequest:
self.log.info("Populating request with job report information")
self.request.addOperation(reportRequest)
if len(self.request):
try:
optimized = self.request.optimize()
except AttributeError:
optimized = {'OK': True}
if not optimized['OK']:
self.log.error("Could not optimize", optimized['Message'])
self.log.error("Not failing the job because of that, keep going")
isValid = self.requestValidator.validate(self.request)
if not isValid['OK']:
raise RuntimeError("Failover request is not valid: %s" % isValid['Message'])
else:
requestJSON = self.request.toJSON()
if requestJSON['OK']:
self.log.info("Creating failover request for deferred operations",
"for job %d" % self.jobID)
request_string = str(requestJSON['Value'])
self.log.debug(request_string)
# Write out the request string
fname = '%s_%s_request.json' % (self.production_id, self.prod_job_id)
with open(fname, 'w') as jsonFile:
jsonFile.write(request_string)
self.log.info("Created file containing failover request", fname)
result = self.request.getDigest()
if result['OK']:
self.log.info("Digest of the request", result['Value'])
else:
self.log.warn("No digest? That's not sooo important",
"anyway: %s" % result['Message'])
else:
raise RuntimeError(requestJSON['Message'])
accountingReport = None
if 'AccountingReport' in self.workflow_commons:
accountingReport = self.workflow_commons['AccountingReport']
if accountingReport:
result = accountingReport.commit()
if not result['OK']:
self.log.error("!!! Both accounting and RequestDB are down? !!!")
self.log.error("accountingReport result: %s" % result['Message'])
self.log.error("Anyway, the job won't fail for this reason, because this is \"just\" the accounting report")
#############################################################################
def setBKRegistrationRequest(self, lfn, error='',
metaData={'Checksum': 'justSomething',
'ChecksumType': 'ADLER32',
'GUID': 'aGUID'}):
""" Set a BK registration request for changing the replica flag.
Uses the global request object (self.request).
"""
if error:
self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error))
else:
self.log.info('Setting BK registration request for %s' % (lfn))
regFile = Operation()
regFile.Type = 'RegisterFile'
regFile.Catalog = 'BookkeepingDB'
bkFile = File()
bkFile.LFN = lfn
# this should NOT be needed... but RMS complains!
bkFile.PFN = lfn
bkFile.GUID = metaData['GUID']
bkFile.Checksum = metaData['Checksum']
bkFile.ChecksumType = metaData['ChecksumType']
regFile.addFile(bkFile)
res = self.request.addOperation(regFile)
if not res['OK']:
raise RuntimeError(res['Message'])
#############################################################################
def createProdConfFile(self, stepOutputTypes, histogram, runNumberGauss, firstEventNumberGauss):
""" Utility that creates a ProdConf file, used mostly as input for gaudirun jobs
"""
# Creating ProdConf file
prodConfFileName = 'prodConf_%s_%s_%s_%s.py' % (self.applicationName,
self.production_id,
self.prod_job_id,
self.step_number)
optionsDict = {}
optionsDict['Application'] = self.applicationName
optionsDict['AppVersion'] = self.applicationVersion
if self.optionsFormat:
optionsDict['OptionFormat'] = self.optionsFormat
if self.stepInputData:
optionsDict['InputFiles'] = ['LFN:' + sid for sid in self.stepInputData]
else:
if self.applicationName.lower() != "gauss":
raise RuntimeError("No MC, but no input data")
if self.outputFilePrefix:
optionsDict['OutputFilePrefix'] = self.outputFilePrefix
optionsDict['OutputFileTypes'] = stepOutputTypes
optionsDict['XMLSummaryFile'] = self.XMLSummary
optionsDict['XMLFileCatalog'] = self.poolXMLCatName
if histogram:
optionsDict['HistogramFile'] = self.histoName
if self.DDDBTag:
if self.DDDBTag.lower() == 'online':
try:
optionsDict['DDDBTag'] = self.onlineDDDBTag
self.log.debug('Set the online DDDB tag')
except NameError as e:
self.log.error('Could not find online DDDb Tag', e)
raise RuntimeError("Could not find online DDDb Tag")
else:
optionsDict['DDDBTag'] = self.DDDBTag
if self.condDBTag:
if self.condDBTag.lower() == 'online':
optionsDict['CondDBTag'] = self.onlineCondDBTag
self.log.debug('Set the online CondDB tag')
else:
optionsDict['CondDBTag'] = self.condDBTag
if self.dqTag:
optionsDict['DQTag'] = self.dqTag
if self.applicationName.lower() == 'gauss':
if self.CPUe and self.maxNumberOfEvents and self.numberOfEvents <= 0:
# Here we set maxCPUTime to 24 hours, which seems reasonable
eventsToProduce = getEventsToProduce(self.CPUe,
maxNumberOfEvents=self.maxNumberOfEvents,
jobMaxCPUTime=86400)
else:
eventsToProduce = self.numberOfEvents
else:
eventsToProduce = self.numberOfEvents
optionsDict['NOfEvents'] = eventsToProduce
if runNumberGauss:
optionsDict['RunNumber'] = runNumberGauss
if self.runNumber:
if self.runNumber not in ('Unknown', 'Multiple'):
optionsDict['RunNumber'] = self.runNumber
if firstEventNumberGauss:
optionsDict['FirstEventNumber'] = firstEventNumberGauss
# TCK: can't have both set!
if self.TCK and self.mcTCK:
raise RuntimeError("%s step: TCK set in step, and should't be!" % self.applicationName)
if self.TCK or self.mcTCK:
optionsDict['TCK'] = self.TCK if self.TCK else self.mcTCK
if self.processingPass:
optionsDict['ProcessingPass'] = self.processingPass
prodConfFile = ProdConf(prodConfFileName)
self.log.debug(optionsDict)
prodConfFile.putOptionsIn(optionsDict)
return prodConfFileName
#############################################################################
# properties
def set_jobID(self, value):
if isinstance(value, str):
if value:
value = int(value)
else:
value = 0
self._jobID = value
def get_jobID(self):
return self._jobID
jobID = property(get_jobID, set_jobID)
class RequestTasks(TaskBase):
"""
Class for handling tasks for the RMS
"""
def __init__(self, transClient=None, logger=None, requestClient=None,
requestClass=None, requestValidator=None,
ownerDN=None, ownerGroup=None):
""" c'tor
the requestClass is by default Request.
If extensions want to use an extended type, they can pass it as a parameter.
This is the same behavior as WorfkloTasks and jobClass
"""
if not logger:
logger = gLogger.getSubLogger('RequestTasks')
super(RequestTasks, self).__init__(transClient, logger)
useCertificates = True if (bool(ownerDN) and bool(ownerGroup)) else False
if not requestClient:
self.requestClient = ReqClient(useCertificates=useCertificates,
delegatedDN=ownerDN,
delegatedGroup=ownerGroup)
else:
self.requestClient = requestClient
if not requestClass:
self.requestClass = Request
else:
self.requestClass = requestClass
if not requestValidator:
self.requestValidator = RequestValidator()
else:
self.requestValidator = requestValidator
def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup='', ownerDN='',
bulkSubmissionFlag=False):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if not taskDict:
return S_OK({})
if (not owner) or (not ownerGroup):
res = getProxyInfo(False, False)
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername(owner)
if not res['OK']:
return res
ownerDN = res['Value'][0]
try:
transJson = json.loads(transBody)
self._multiOperationsBody(transJson, taskDict, ownerDN, ownerGroup)
except ValueError: # #json couldn't load
self._singleOperationsBody(transBody, taskDict, ownerDN, ownerGroup)
return S_OK(taskDict)
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
failedTasks = []
for taskID, task in taskDict.items():
transID = task['TransformationID']
if not task.get('InputData'):
self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID)
taskDict.pop(taskID)
continue
files = []
oRequest = Request()
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].iteritems():
setattr(op, parameter, value)
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile(opFile)
oRequest.addOperation(op)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split(';')
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.iteritems():
transID = task['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task['TargetSE']
# If there are input files
if task.get('InputData'):
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def _assignRequestToTask(self, oRequest, taskDict, transID, taskID, ownerDN, ownerGroup):
"""set ownerDN and group to request, and add the request to taskDict if it is
valid, otherwise remove the task from the taskDict
:param oRequest: Request
:param dict taskDict: dictionary of tasks, modified in this function
:param int transID: Transformation ID
:param int taskID: Task ID
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
oRequest.RequestName = self._transTaskName(transID, taskID)
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate(oRequest)
if not isValid['OK']:
self._logError("Error creating request for task", "%s %s" % (taskID, isValid),
transID=transID)
return S_ERROR('Error creating request')
taskDict[taskID]['TaskObject'] = oRequest
return S_OK()
def submitTransformationTasks(self, taskDict):
""" Submit requests one by one
"""
submitted = 0
failed = 0
startTime = time.time()
method = 'submitTransformationTasks'
for task in taskDict.itervalues():
# transID is the same for all tasks, so pick it up every time here
transID = task['TransformationID']
if not task['TaskObject']:
task['Success'] = False
failed += 1
continue
res = self.submitTaskToExternal(task['TaskObject'])
if res['OK']:
task['ExternalID'] = res['Value']
task['Success'] = True
submitted += 1
else:
self._logError("Failed to submit task to RMS", res['Message'], transID=transID)
task['Success'] = False
failed += 1
if submitted:
self._logInfo('Submitted %d tasks to RMS in %.1f seconds' % (submitted, time.time() - startTime),
transID=transID, method=method)
if failed:
self._logWarn('Failed to submit %d tasks to RMS.' % (failed),
transID=transID, method=method)
return S_OK(taskDict)
def submitTaskToExternal(self, oRequest):
"""
Submits a request to RMS
"""
if isinstance(oRequest, self.requestClass):
return self.requestClient.putRequest(oRequest, useFailoverProxy=False, retryMainService=2)
return S_ERROR("Request should be a Request object")
def updateTransformationReservedTasks(self, taskDicts):
requestNameIDs = {}
noTasks = []
for taskDict in taskDicts:
requestName = self._transTaskName(taskDict['TransformationID'], taskDict['TaskID'])
reqID = taskDict['ExternalID']
if reqID:
requestNameIDs[requestName] = reqID
else:
noTasks.append(requestName)
return S_OK({'NoTasks': noTasks, 'TaskNameIDs': requestNameIDs})
def getSubmittedTaskStatus(self, taskDicts):
"""
Check if tasks changed status, and return a list of tasks per new status
"""
updateDict = {}
badRequestID = 0
for taskDict in taskDicts:
oldStatus = taskDict['ExternalStatus']
# ExternalID is normally a string
if taskDict['ExternalID'] and int(taskDict['ExternalID']):
newStatus = self.requestClient.getRequestStatus(taskDict['ExternalID'])
if not newStatus['OK']:
log = self._logVerbose if 'not exist' in newStatus['Message'] else self._logWarn
log("getSubmittedTaskStatus: Failed to get requestID for request", newStatus['Message'],
transID=taskDict['TransformationID'])
else:
newStatus = newStatus['Value']
# We don't care updating the tasks to Assigned while the request is being processed
if newStatus != oldStatus and newStatus != 'Assigned':
updateDict.setdefault(newStatus, []).append(taskDict['TaskID'])
else:
badRequestID += 1
if badRequestID:
self._logWarn("%d requests have identifier 0" % badRequestID)
return S_OK(updateDict)
def getSubmittedFileStatus(self, fileDicts):
"""
Check if transformation files changed status, and return a list of taskIDs per new status
"""
# Don't try and get status of not submitted tasks!
transID = None
taskFiles = {}
for fileDict in fileDicts:
# There is only one transformation involved, get however the transID in the loop
transID = fileDict['TransformationID']
taskID = int(fileDict['TaskID'])
taskFiles.setdefault(taskID, []).append(fileDict['LFN'])
# Should not happen, but just in case there are no files, return
if transID is None:
return S_OK({})
res = self.transClient.getTransformationTasks({'TransformationID': transID, 'TaskID': taskFiles.keys()})
if not res['OK']:
return res
requestFiles = {}
for taskDict in res['Value']:
taskID = taskDict['TaskID']
externalID = taskDict['ExternalID']
# Only consider tasks that are submitted, ExternalID is a string
if taskDict['ExternalStatus'] != 'Created' and externalID and int(externalID):
requestFiles[externalID] = taskFiles[taskID]
updateDict = {}
for requestID, lfnList in requestFiles.iteritems():
statusDict = self.requestClient.getRequestFileStatus(requestID, lfnList)
if not statusDict['OK']:
log = self._logVerbose if 'not exist' in statusDict['Message'] else self._logWarn
log("Failed to get files status for request", statusDict['Message'],
transID=transID, method='getSubmittedFileStatus')
else:
for lfn, newStatus in statusDict['Value'].iteritems():
if newStatus == 'Done':
updateDict[lfn] = 'Processed'
elif newStatus == 'Failed':
updateDict[lfn] = 'Problematic'
return S_OK(updateDict)
def testValidator(self):
"""validator test"""
# create validator
validator = RequestValidator()
self.assertEqual(isinstance(validator, RequestValidator), True)
# RequestName not set
ret = validator.validate(self.request)
self.assertFalse(ret["OK"])
self.request.RequestName = "test_request"
# # no operations
ret = validator.validate(self.request)
self.assertFalse(ret["OK"])
self.request.addOperation(self.operation)
# # type not set
ret = validator.validate(self.request)
self.assertFalse(ret["OK"])
self.operation.Type = "ReplicateAndRegister"
# # files not present
ret = validator.validate(self.request)
self.assertFalse(ret["OK"])
self.operation.addFile(self.file)
# # targetSE not set
ret = validator.validate(self.request)
self.assertFalse(ret["OK"])
self.operation.TargetSE = "CERN-USER"
# # missing LFN
ret = validator.validate(self.request)
self.assertFalse(ret["OK"])
self.file.LFN = "/a/b/c"
# # no ownerDN
# force no owner DN because it takes the one of the current user
self.request.OwnerDN = ""
ret = validator.validate(self.request)
self.assertFalse(ret["OK"])
self.request.OwnerDN = "foo/bar=baz"
# # no owner group
# same, force it
self.request.OwnerGroup = ""
ret = validator.validate(self.request)
self.assertFalse(ret["OK"])
self.request.OwnerGroup = "dirac_user"
# Checksum set, ChecksumType not set
self.file.Checksum = "abcdef"
ret = validator.validate(self.request)
self.assertFalse(ret["OK"])
# ChecksumType set, Checksum not set
self.file.Checksum = ""
self.file.ChecksumType = "adler32"
ret = validator.validate(self.request)
self.assertFalse(ret["OK"])
# both set
self.file.Checksum = "abcdef"
self.file.ChecksumType = "adler32"
ret = validator.validate(self.request)
self.assertEqual(ret, {"OK": True, "Value": None})
# both unset
self.file.Checksum = ""
self.file.ChecksumType = None
ret = validator.validate(self.request)
self.assertEqual(ret, {"OK": True, "Value": None})
# all OK
ret = validator.validate(self.request)
self.assertEqual(ret, {"OK": True, "Value": None})
def testValidator( self ):
""" validator test """
## create validator
validator = RequestValidator()
self.assertEqual( isinstance( validator, RequestValidator ), True )
## RequestName not set
ret = validator.validate( self.request )
self.assertEqual( ret, { 'Message' : 'RequestName not set',
'OK' : False } )
self.request.RequestName = "test_request"
# # no operations
ret = validator.validate( self.request )
self.assertEqual( ret, { 'Message' : "Operations not present in request 'test_request'",
'OK': False} )
self.request.addOperation( self.operation )
# # type not set
ret = validator.validate( self.request )
self.assertEqual( ret, { 'Message' : "Operation #0 in request 'test_request' hasn't got Type set",
'OK' : False } )
self.operation.Type = "ReplicateAndRegister"
# # files not present
ret = validator.validate( self.request )
self.assertEqual( ret, { 'Message' : "Operation #0 of type 'ReplicateAndRegister' hasn't got files to process.",
'OK' : False } )
self.operation.addFile( self.file )
# # targetSE not set
ret = validator.validate( self.request )
self.assertEqual( ret, { 'Message' : "Operation #0 of type 'ReplicateAndRegister' is missing TargetSE attribute.",
'OK': False } )
self.operation.TargetSE = "CERN-USER"
# # missing LFN
ret = validator.validate( self.request )
self.assertEqual( ret,
{ "Message" : "Operation #0 of type 'ReplicateAndRegister' is missing LFN attribute for file.",
"OK": False } )
self.file.LFN = "/a/b/c"
# # no ownerDN
# force no owner DN because it takes the one of the current user
self.request.OwnerDN = ''
ret = validator.validate( self.request )
self.assertEqual( ret, { 'Message' : "Request 'test_request' is missing OwnerDN value",
'OK': False} )
self.request.OwnerDN = "foo/bar=baz"
# # no owner group
# same, force it
self.request.OwnerGroup = ''
ret = validator.validate( self.request )
self.assertEqual( ret, { 'Message' : "Request 'test_request' is missing OwnerGroup value",
'OK': False} )
self.request.OwnerGroup = "dirac_user"
## Checksum set, ChecksumType not set
self.file.Checksum = "abcdef"
ret = validator.validate( self.request )
self.assertEqual( ret,
{ 'Message' : 'File in operation #0 is missing Checksum (abcdef) or ChecksumType ()',
'OK' : False } )
## ChecksumType set, Checksum not set
self.file.Checksum = ""
self.file.ChecksumType = "adler32"
ret = validator.validate( self.request )
self.assertEqual( ret,
{ 'Message' : 'File in operation #0 is missing Checksum () or ChecksumType (ADLER32)',
'OK' : False } )
## both set
self.file.Checksum = "abcdef"
self.file.ChecksumType = "adler32"
ret = validator.validate( self.request )
self.assertEqual( ret, {'OK': True, 'Value': None} )
## both unset
self.file.Checksum = ""
self.file.ChecksumType = None
ret = validator.validate( self.request )
self.assertEqual( ret, {'OK': True, 'Value': None} )
## all OK
ret = validator.validate( self.request )
self.assertEqual( ret, {'OK': True, 'Value': None} )
class RequestTasks( TaskBase ):
def __init__( self, transClient = None, logger = None, requestClient = None,
requestClass = None, requestValidator = None ):
""" c'tor
the requestClass is by default Request.
If extensions want to use an extended type, they can pass it as a parameter.
This is the same behavior as WorfkloTasks and jobClass
"""
if not logger:
logger = gLogger.getSubLogger( 'RequestTasks' )
super( RequestTasks, self ).__init__( transClient, logger )
if not requestClient:
self.requestClient = ReqClient()
else:
self.requestClient = requestClient
if not requestClass:
self.requestClass = Request
else:
self.requestClass = requestClass
if not requestValidator:
self.requestValidator = RequestValidator()
else:
self.requestValidator = requestValidator
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '',
bulkSubmissionFlag = False):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if not taskDict:
return S_OK({})
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
try:
transJson = json.loads(transBody)
self._multiOperationsBody( transJson, taskDict, ownerDN, ownerGroup )
except ValueError: ##json couldn't load
self._singleOperationsBody( transBody, taskDict, ownerDN, ownerGroup )
return S_OK( taskDict )
def _multiOperationsBody( self, transJson, taskDict, ownerDN, ownerGroup ):
""" deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if not paramDict.get('InputData'):
self.log.error( "Error creating request for task", "%s, No input data" % taskID )
taskDict.pop( taskID )
continue
files = []
transID = paramDict['TransformationID']
oRequest = Request()
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].iteritems():
setattr( op, parameter, value )
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile( opFile )
oRequest.addOperation( op )
self._assignRequestToTask( oRequest, taskDict, transID, taskID, ownerDN, ownerGroup )
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup ):
""" deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
# Do not remove sorted, we might pop elements in the loop
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
# If there are input files
if paramDict.get('InputData'):
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
self._assignRequestToTask( oRequest, taskDict, transID, taskID, ownerDN, ownerGroup )
def _assignRequestToTask( self, oRequest, taskDict, transID, taskID, ownerDN, ownerGroup ):
"""set ownerDN and group to request, and add the request to taskDict if it is
valid, otherwise remove the task from the taskDict
:param oRequest: Request
:param dict taskDict: dictionary of tasks, modified in this function
:param int transID: Transformation ID
:param int taskID: Task ID
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) )
# This works because we loop over a copy of the keys !
taskDict.pop( taskID )
return
taskDict[taskID]['TaskObject'] = oRequest
return
def submitTransformationTasks( self, taskDict ):
""" Submit requests one by one
"""
submitted = 0
failed = 0
startTime = time.time()
for taskID in sorted( taskDict ):
if not taskDict[taskID]['TaskObject']:
taskDict[taskID]['Success'] = False
failed += 1
continue
res = self.submitTaskToExternal( taskDict[taskID]['TaskObject'] )
if res['OK']:
taskDict[taskID]['ExternalID'] = res['Value']
taskDict[taskID]['Success'] = True
submitted += 1
else:
self._logError( "Failed to submit task to RMS", res['Message'] )
taskDict[taskID]['Success'] = False
failed += 1
self._logInfo( 'submitTasks: Submitted %d tasks to RMS in %.1f seconds' % ( submitted, time.time() - startTime ) )
if failed:
self._logWarn( 'submitTasks: But at the same time failed to submit %d tasks to RMS.' % ( failed ) )
return S_OK( taskDict )
def submitTaskToExternal( self, oRequest ):
""" Submits a request using ReqClient
"""
if isinstance( oRequest, self.requestClass ):
return self.requestClient.putRequest( oRequest, useFailoverProxy = False, retryMainService = 2 )
else:
return S_ERROR( "Request should be a Request object" )
def updateTransformationReservedTasks( self, taskDicts ):
requestNameIDs = {}
noTasks = []
for taskDict in taskDicts:
requestName = _requestName( taskDict['TransformationID'], taskDict['TaskID'] )
reqID = taskDict['ExternalID']
if reqID:
requestNameIDs[requestName] = reqID
else:
noTasks.append( requestName )
return S_OK( {'NoTasks':noTasks, 'TaskNameIDs':requestNameIDs} )
def getSubmittedTaskStatus( self, taskDicts ):
updateDict = {}
for taskDict in taskDicts:
oldStatus = taskDict['ExternalStatus']
newStatus = self.requestClient.getRequestStatus( taskDict['ExternalID'] )
if not newStatus['OK']:
log = self._logVerbose if 'not exist' in newStatus['Message'] else self.log.warn
log( "getSubmittedTaskStatus: Failed to get requestID for request", '%s' % newStatus['Message'] )
else:
newStatus = newStatus['Value']
if newStatus != oldStatus:
updateDict.setdefault( newStatus, [] ).append( taskDict['TaskID'] )
return S_OK( updateDict )
def getSubmittedFileStatus( self, fileDicts ):
taskFiles = {}
submittedTasks = {}
externalIds = {}
# Don't try and get status of not submitted tasks!
for fileDict in fileDicts:
submittedTasks.setdefault( fileDict['TransformationID'], set() ).add( int( fileDict['TaskID'] ) )
for transID in submittedTasks:
res = self.transClient.getTransformationTasks( { 'TransformationID':transID, 'TaskID': list( submittedTasks[transID] )} )
if not res['OK']:
return res
for taskDict in res['Value']:
taskID = taskDict['TaskID']
externalIds[taskID] = taskDict['ExternalID']
if taskDict['ExternalStatus'] == 'Created':
submittedTasks[transID].remove( taskID )
for fileDict in fileDicts:
transID = fileDict['TransformationID']
taskID = int( fileDict['TaskID'] )
if taskID in submittedTasks[transID]:
taskFiles.setdefault( externalIds[taskID], [] ).append( fileDict['LFN'] )
updateDict = {}
for requestID in sorted( taskFiles ):
lfnList = taskFiles[requestID]
statusDict = self.requestClient.getRequestFileStatus( requestID, lfnList )
if not statusDict['OK']:
log = self._logVerbose if 'not exist' in statusDict['Message'] else self.log.warn
log( "getSubmittedFileStatus: Failed to get files status for request", '%s' % statusDict['Message'] )
continue
for lfn, newStatus in statusDict['Value'].items():
if newStatus == 'Done':
updateDict[lfn] = 'Processed'
elif newStatus == 'Failed':
updateDict[lfn] = 'Problematic'
return S_OK( updateDict )