def __getRequestString(self, requestName):
""" Obtain the string for request (including all sub-requests)
:param self: self reference
:param str requestName: request name
"""
self.log.info("__getRequestString: Attempting to get string for %s." % requestName)
res = self.__locateRequest(requestName)
if not res["OK"]:
return res
subRequestPaths = res["Value"]
try:
oRequest = RequestContainer(init=False)
for subRequestPath in subRequestPaths:
res = self.__readSubRequestString(subRequestPath)
if not res["OK"]:
return res
subRequestString = res["Value"]
tempRequest = RequestContainer(subRequestString)
oRequest.setRequestAttributes(tempRequest.getRequestAttributes()["Value"])
oRequest.update(tempRequest)
requestString = oRequest.toXML()["Value"]
self.log.info("__getRequestString: Successfully obtained string for %s." % requestName)
result = S_OK(requestString)
result["Request"] = oRequest
return result
except Exception, error:
errStr = "__getRequestString: Exception while obtaining request string."
self.log.exception(errStr, requestName, lException=error)
return S_ERROR(errStr)
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
request = RequestContainer()
request.setRequestAttributes( { 'OwnerDN':ownerDN, 'OwnerGroup':ownerGroup } )
requestName = os.path.basename( lfn ).strip()+'_removal_request.xml'
request.setRequestName( requestName )
request.setSourceComponent( 'JobCleaningAgent' )
removalDict = {'Attributes':{ 'Operation':'removeFile',
'TargetSE':'',
'ExecutionOrder':0
}
}
result = request.addSubRequest( removalDict, 'removal' )
if not result['OK']:
return result
index = result['Value']
fileDict = { 'LFN':lfn, 'PFN':'', 'Status':'Waiting' }
request.setSubRequestFiles( index, 'removal', [fileDict] )
client = RequestClient()
result = request.toXML()
if not result['OK']:
return result
xmlRequest = result['Value']
result = client.setRequest( requestName, xmlRequest )
return result
def __getRequestString(self, requestName):
""" Obtain the string for request (including all sub-requests)
"""
gLogger.info(
"RequestDBFile.__getRequestString: Attempting to get string for %s."
% requestName)
res = self.__locateRequest(requestName)
if not res['OK']:
return res
subRequestPaths = res['Value']
try:
oRequest = RequestContainer(init=False)
for subRequestPath in subRequestPaths:
res = self.__readSubRequestString(subRequestPath)
if not res['OK']:
return res
subRequestString = res['Value']
tempRequest = RequestContainer(subRequestString) #,init=False)
oRequest.setRequestAttributes(
tempRequest.getRequestAttributes()['Value'])
oRequest.update(tempRequest)
requestString = oRequest.toXML()['Value']
gLogger.info(
"RequestDBFile.__getRequestString: Successfully obtained string for %s."
% requestName)
result = S_OK(requestString)
result['Request'] = oRequest
return result
except Exception, x:
errStr = "RequestDBFile.__getRequestString: Exception while obtaining request string."
gLogger.exception(errStr, requestName, lException=x)
return S_ERROR(errStr)
def __getRequestString( self, requestName ):
""" Obtain the string for request (including all sub-requests)
"""
gLogger.info( "RequestDBFile.__getRequestString: Attempting to get string for %s." % requestName )
res = self.__locateRequest( requestName )
if not res['OK']:
return res
subRequestPaths = res['Value']
try:
oRequest = RequestContainer( init = False )
for subRequestPath in subRequestPaths:
res = self.__readSubRequestString( subRequestPath )
if not res['OK']:
return res
subRequestString = res['Value']
tempRequest = RequestContainer( subRequestString )#,init=False)
oRequest.setRequestAttributes( tempRequest.getRequestAttributes()['Value'] )
oRequest.update( tempRequest )
requestString = oRequest.toXML()['Value']
gLogger.info( "RequestDBFile.__getRequestString: Successfully obtained string for %s." % requestName )
result = S_OK( requestString )
result['Request'] = oRequest
return result
except Exception, x:
errStr = "RequestDBFile.__getRequestString: Exception while obtaining request string."
gLogger.exception( errStr, requestName, lException = x )
return S_ERROR( errStr )
def prepareTransformationTasks(self,
transBody,
taskDict,
owner='',
ownerGroup=''):
requestType = 'transfer'
requestOperation = 'replicateAndRegister'
try:
requestType, requestOperation = transBody.split(';')
except:
pass
for taskID in sortList(taskDict.keys()):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = RequestContainer(init=False)
subRequestIndex = oRequest.initiateSubRequest(requestType)['Value']
attributeDict = {
'Operation': requestOperation,
'TargetSE': paramDict['TargetSE']
}
oRequest.setSubRequestAttributes(subRequestIndex, requestType,
attributeDict)
files = []
for lfn in paramDict['InputData'].split(';'):
files.append({'LFN': lfn})
oRequest.setSubRequestFiles(subRequestIndex, requestType, files)
requestName = str(transID).zfill(8) + '_' + str(taskID).zfill(8)
oRequest.setRequestAttributes({'RequestName': requestName})
taskDict[taskID]['TaskObject'] = oRequest.toXML()['Value']
return S_OK(taskDict)
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup):
""" Set removal request with the given credentials
"""
request = RequestContainer()
request.setRequestAttributes({
'OwnerDN': ownerDN,
'OwnerGroup': ownerGroup
})
requestName = os.path.basename(lfn).strip() + '_removal_request.xml'
request.setRequestName(requestName)
request.setSourceComponent('JobCleaningAgent')
removalDict = {
'Attributes': {
'Operation': 'removeFile',
'TargetSE': '',
'ExecutionOrder': 0
}
}
result = request.addSubRequest(removalDict, 'removal')
if not result['OK']:
return result
index = result['Value']
fileDict = {'LFN': lfn, 'PFN': '', 'Status': 'Waiting'}
request.setSubRequestFiles(index, 'removal', [fileDict])
client = RequestClient()
result = request.toXML()
if not result['OK']:
return result
xmlRequest = result['Value']
result = client.setRequest(requestName, xmlRequest)
return result
def __getRequestString(self, requestName):
""" Obtain the string for request (including all sub-requests)
:param self: self reference
:param str requestName: request name
"""
self.log.info("__getRequestString: Attempting to get string for %s." %
requestName)
res = self.__locateRequest(requestName)
if not res['OK']:
return res
subRequestPaths = res['Value']
try:
oRequest = RequestContainer(init=False)
for subRequestPath in subRequestPaths:
res = self.__readSubRequestString(subRequestPath)
if not res['OK']:
return res
subRequestString = res['Value']
tempRequest = RequestContainer(subRequestString)
oRequest.setRequestAttributes(
tempRequest.getRequestAttributes()['Value'])
oRequest.update(tempRequest)
requestString = oRequest.toXML()['Value']
self.log.info(
"__getRequestString: Successfully obtained string for %s." %
requestName)
result = S_OK(requestString)
result['Request'] = oRequest
return result
except Exception, error:
errStr = "__getRequestString: Exception while obtaining request string."
self.log.exception(errStr, requestName, lException=error)
return S_ERROR(errStr)
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
requestType = 'transfer'
requestOperation = 'replicateAndRegister'
try:
requestType, requestOperation = transBody.split( ';' )
except:
pass
for taskID in sortList( taskDict.keys() ):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = RequestContainer( init = False )
subRequestIndex = oRequest.initiateSubRequest( requestType )['Value']
attributeDict = {'Operation':requestOperation, 'TargetSE':paramDict['TargetSE']}
oRequest.setSubRequestAttributes( subRequestIndex, requestType, attributeDict )
files = []
for lfn in paramDict['InputData'].split( ';' ):
files.append( {'LFN':lfn} )
oRequest.setSubRequestFiles( subRequestIndex, requestType, files )
requestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
oRequest.setRequestAttributes( {'RequestName':requestName} )
taskDict[taskID]['TaskObject'] = oRequest.toXML()['Value']
return S_OK( taskDict )
requestOperation = 'removeFile'
for lfnList in breakListIntoChunks(lfns, 100):
oRequest = RequestContainer()
subRequestIndex = oRequest.initiateSubRequest(requestType)['Value']
attributeDict = {'Operation': requestOperation, 'TargetSE': targetSE}
oRequest.setSubRequestAttributes(subRequestIndex, requestType,
attributeDict)
files = []
for lfn in lfnList:
files.append({'LFN': lfn})
oRequest.setSubRequestFiles(subRequestIndex, requestType, files)
requestName = "%s_%s" % (md5(repr(time.time())).hexdigest()[:16],
md5(repr(time.time())).hexdigest()[:16])
oRequest.setRequestAttributes({'RequestName': requestName})
DIRAC.gLogger.info(oRequest.toXML()['Value'])
result = requestClient.setRequest(requestName, oRequest.toXML()['Value'])
if result['OK']:
print 'Submitted Request:', result['Value']
else:
print 'Failed to submit Request', result['Message']
if monitor:
requestID = result['Value']
while True:
result = requestClient.getRequestStatus(requestID)
if not result['OK']:
Script.gLogger.error(result['Message'])
break
def readRequestsForJobs( self, jobIDs ):
""" read and return Requests for jobs
:param mixed jobIDs: list with jobIDs or long JobIDs
"""
if type(jobIDs) != list:
return S_ERROR("RequestDB: wrong format for jobIDs argument, got %s, expecting a list" )
# make sure list is uniqe and has only longs
jobIDs = list( set( [ int(jobID) for jobID in jobIDs if int(jobID) != 0 ] ) )
reqCols = [ "RequestID", "RequestName", "JobID", "Status",
"OwnerDN", "OwnerGroup", "DIRACSetup", "SourceComponent",
"CreationTime", "SubmissionTime", "LastUpdate" ]
subCols = [ "SubRequestID", "Operation", "Arguments", "RequestType", "ExecutionOrder", "Error",
"SourceSE", "TargetSE", "Catalogue", "CreationTime", "SubmissionTime", "LastUpdate" ]
fileCols = [ "FileID", "LFN", "Size", "PFN", "GUID", "Md5", "Addler", "Attempt", "Status" , "Error" ]
requestNames = self.getRequestForJobs( jobIDs )
if not requestNames["OK"]:
return requestNames
requestNames = requestNames["Value"]
## this will be returned
retDict = { "Successful" : dict(), "Failed" : dict() }
for jobID in jobIDs:
## missing requests
if jobID not in requestNames:
retDict["Failed"][jobID] = "Request not found"
continue
requestName = requestNames[jobID]
## get request
queryStr = "SELECT %s FROM Requests WHERE RequestName = '%s';" % ( ",".join( reqCols ), requestName )
queryRes = self._query( queryStr )
if not queryRes["OK"]:
retDict["Failed"][jobID] = queryRes["Message"]
continue
queryRes = queryRes["Value"] if queryRes["Value"] else None
if not queryRes:
retDict["Failed"][jobID] = "Unable to read request attributes."
continue
requestObj = RequestContainer( init=False )
reqAttrs = dict( zip( reqCols, queryRes[0] ) )
requestObj.setRequestAttributes( reqAttrs )
queryStr = "SELECT %s FROM `SubRequests` WHERE `RequestID`=%s;" % ( ",".join(subCols), reqAttrs["RequestID"] )
queryRes = self._query( queryStr )
if not queryRes["OK"]:
retDict["Failed"][jobID] = queryRes["Message"]
continue
queryRes = queryRes["Value"] if queryRes["Value"] else None
if not queryRes:
retDict["Failed"][jobID] = "Unable to read subrequest attributes."
continue
## get sub-requests
for recTuple in queryRes:
subReqAttrs = dict( zip( subCols, recTuple ) )
subType = subReqAttrs["RequestType"]
subReqAttrs["ExecutionOrder"] = int( subReqAttrs["ExecutionOrder"] )
del subReqAttrs["RequestType"]
index = requestObj.initiateSubRequest( subType )
index = index["Value"]
requestObj.setSubRequestAttributes( index, subType, subReqAttrs )
## get files
subFiles = []
fileQuery = "SELECT %s FROM `Files` WHERE `SubRequestID` = %s ORDER BY `FileID`;" % ( ",".join(fileCols),
subReqAttrs["SubRequestID"] )
fileQueryRes = self._query( fileQuery )
if fileQueryRes["OK"] and fileQueryRes["Value"]:
for fileRec in fileQueryRes["Value"]:
subFiles.append( dict( zip(fileCols, fileRec) ) )
if subFiles:
requestObj.setSubRequestFiles( index, subType, subFiles )
retDict["Successful"][jobID] = requestObj.toXML()["Value"]
return S_OK( retDict )
reqClient = ReqClient()
requestType = 'transfer'
requestOperation = 'replicateAndRegister'
for lfnList in breakListIntoChunks( lfns, 100 ):
oRequest = RequestContainer()
subRequestIndex = oRequest.initiateSubRequest( requestType )['Value']
attributeDict = {'Operation':requestOperation, 'TargetSE':targetSE}
oRequest.setSubRequestAttributes( subRequestIndex, requestType, attributeDict )
files = []
for lfn in lfnList:
files.append( {'LFN':lfn} )
oRequest.setSubRequestFiles( subRequestIndex, requestType, files )
requestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] )
oRequest.setRequestAttributes( {'RequestName':requestName} )
DIRAC.gLogger.info( oRequest.toXML()['Value'] )
result = reqClient.setRequest( requestName, oRequest.toXML()['Value'] )
if result['OK']:
print 'Submitted Request:', result['Value']
else:
print 'Failed to submit Request', result['Message']
if monitor:
requestID = result['Value']
while True:
result = reqClient.getRequestStatus( requestID )
if not result['OK']:
Script.gLogger.error( result['Message'] )
break
print
Script.showHelp()
from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer
from DIRAC.RequestManagementSystem.Client.RequestClient import RequestClient
requestClient = RequestClient()
requestType = "transfer"
requestOperation = "replicateAndRegister"
for lfnList in breakListIntoChunks(lfns, 100):
oRequest = RequestContainer()
subRequestIndex = oRequest.initiateSubRequest(requestType)["Value"]
attributeDict = {"Operation": requestOperation, "TargetSE": targetSE}
oRequest.setSubRequestAttributes(subRequestIndex, requestType, attributeDict)
files = []
for lfn in lfnList:
files.append({"LFN": lfn})
oRequest.setSubRequestFiles(subRequestIndex, requestType, files)
requestName = "%s_%s" % (md5(repr(time.time())).hexdigest()[:16], md5(repr(time.time())).hexdigest()[:16])
oRequest.setRequestAttributes({"RequestName": requestName})
DIRAC.gLogger.info(oRequest.toXML()["Value"])
result = requestClient.setRequest(requestName, oRequest.toXML()["Value"])
if result["OK"]:
print "Submitted Request:", result["Value"]
else:
print "Failed to submit Request", result["Message"]
