def __getRequestString(self, requestName):
""" Obtain the string for request (including all sub-requests)
:param self: self reference
:param str requestName: request name
"""
self.log.info("__getRequestString: Attempting to get string for %s." %
requestName)
res = self.__locateRequest(requestName)
if not res['OK']:
return res
subRequestPaths = res['Value']
try:
oRequest = RequestContainer(init=False)
for subRequestPath in subRequestPaths:
res = self.__readSubRequestString(subRequestPath)
if not res['OK']:
return res
subRequestString = res['Value']
tempRequest = RequestContainer(subRequestString)
oRequest.setRequestAttributes(
tempRequest.getRequestAttributes()['Value'])
oRequest.update(tempRequest)
requestString = oRequest.toXML()['Value']
self.log.info(
"__getRequestString: Successfully obtained string for %s." %
requestName)
result = S_OK(requestString)
result['Request'] = oRequest
return result
except Exception, error:
errStr = "__getRequestString: Exception while obtaining request string."
self.log.exception(errStr, requestName, lException=error)
return S_ERROR(errStr)
def __getRequestString(self, requestName):
""" Obtain the string for request (including all sub-requests)
"""
gLogger.info(
"RequestDBFile.__getRequestString: Attempting to get string for %s."
% requestName)
res = self.__locateRequest(requestName)
if not res['OK']:
return res
subRequestPaths = res['Value']
try:
oRequest = RequestContainer(init=False)
for subRequestPath in subRequestPaths:
res = self.__readSubRequestString(subRequestPath)
if not res['OK']:
return res
subRequestString = res['Value']
tempRequest = RequestContainer(subRequestString) #,init=False)
oRequest.setRequestAttributes(
tempRequest.getRequestAttributes()['Value'])
oRequest.update(tempRequest)
requestString = oRequest.toXML()['Value']
gLogger.info(
"RequestDBFile.__getRequestString: Successfully obtained string for %s."
% requestName)
result = S_OK(requestString)
result['Request'] = oRequest
return result
except Exception, x:
errStr = "RequestDBFile.__getRequestString: Exception while obtaining request string."
gLogger.exception(errStr, requestName, lException=x)
return S_ERROR(errStr)
def generateRequest(self):
""" Generate failover requests for the operations in the internal cache
"""
request = RequestContainer()
result = self.sendStoredStatusInfo()
if not result['OK']:
if result.has_key('rpcStub'):
request.addSubRequest(
DISETSubRequest(result['rpcStub']).getDictionary(),
'diset')
else:
return S_ERROR('Could not create job state sub-request')
result = self.sendStoredJobParameters()
if not result['OK']:
if result.has_key('rpcStub'):
request.addSubRequest(
DISETSubRequest(result['rpcStub']).getDictionary(),
'diset')
else:
return S_ERROR('Could not create job parameters sub-request')
if request.isEmpty()['Value']:
request = None
return S_OK(request)
def jobexec(jobxml, wfParameters={}):
jobfile = os.path.abspath(jobxml)
if not os.path.exists(jobfile):
gLogger.warn('Path to specified workflow %s does not exist' %
(jobfile))
sys.exit(1)
workflow = fromXMLFile(jobfile)
gLogger.debug(workflow)
code = workflow.createCode()
gLogger.debug(code)
jobID = 0
if os.environ.has_key('JOBID'):
jobID = os.environ['JOBID']
gLogger.info('DIRAC JobID %s is running at site %s' %
(jobID, DIRAC.siteName()))
workflow.addTool('JobReport', JobReport(jobID))
workflow.addTool('AccountingReport', DataStoreClient())
workflow.addTool('Request', RequestContainer())
# Propagate the command line parameters to the workflow if any
for name, value in wfParameters.items():
workflow.setValue(name, value)
result = workflow.execute()
return result
def prepareTransformationTasks(self,
transBody,
taskDict,
owner='',
ownerGroup=''):
requestType = 'transfer'
requestOperation = 'replicateAndRegister'
try:
requestType, requestOperation = transBody.split(';')
except:
pass
for taskID in sortList(taskDict.keys()):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = RequestContainer(init=False)
subRequestIndex = oRequest.initiateSubRequest(requestType)['Value']
attributeDict = {
'Operation': requestOperation,
'TargetSE': paramDict['TargetSE']
}
oRequest.setSubRequestAttributes(subRequestIndex, requestType,
attributeDict)
files = []
for lfn in paramDict['InputData'].split(';'):
files.append({'LFN': lfn})
oRequest.setSubRequestFiles(subRequestIndex, requestType, files)
requestName = str(transID).zfill(8) + '_' + str(taskID).zfill(8)
oRequest.setRequestAttributes({'RequestName': requestName})
taskDict[taskID]['TaskObject'] = oRequest.toXML()['Value']
return S_OK(taskDict)
def test_toFile( self ):
lfn = '/lhcb/production/test/case.lfn'
# Add dummy transfer request
transferDic = {'Attributes': {'Status': 'Waiting', 'SubRequestID': '7F7C1D94-E452-CD50-204C-EE2E2F1816A9',
'Catalogue':'', 'TargetSE':'CERN-tape', 'Operation':'MoveAndRegister', 'SourceSE':'RAL-tape'},
'Files':[{'LFN':lfn, 'Status': 'Waiting', 'Attempt': 1, 'PFN': '', 'Size': 1231231,
'GUID': '7E9CED5A-295B-ED88-CE9A-CF41A62D2175', 'Addler': '', 'Md5': ''}],
'Datasets':[{'Dataset1':'DC06Stripping'}]
}
self.reqContainer.addSubRequest( transferDic, 'transfer' )
# Add dummy register request
registerDic = {'Attributes':{'Status': 'Waiting', 'SubRequestID': '7F7C1D94-E452-CD50-204C-EE2E2F1816A9',
'Catalogue':'', 'TargetSE':'CERN-tape', 'Operation':'RegisterFile'},
'Files':[{'LFN':lfn, 'Status': 'Waiting', 'Attempt': 1,
'PFN': 'srm://srm.cern.ch/castor/cern.ch/grid/lhcb/production/test/case.lfn',
'Size': 1231231, 'GUID': '7E9CED5A-295B-ED88-CE9A-CF41A62D2175', 'Addler': 'addler32', 'Md5': 'md5'}],
'Datasets':[{'Dataset1':'DC06Stripping'}]
}
self.reqContainer.addSubRequest( registerDic, 'register' )
# Add dummy removal request
removalDic = {'Attributes':{'Status': 'Waiting', 'SubRequestID': '7F7C1D94-E452-CD50-204C-EE2E2F1816A9',
'Catalogue':'', 'TargetSE':'CERN-tape', 'Operation':'RemoveReplica', 'Catalogue':'LFC'},
'Files':[{'LFN':lfn, 'Status': 'Waiting', 'Attempt': 1,
'PFN': 'srm://srm.cern.ch/castor/cern.ch/grid/lhcb/production/test/case.lfn',
'Size': 1231231, 'GUID': '7E9CED5A-295B-ED88-CE9A-CF41A62D2175', 'Addler': 'addler32', 'Md5': 'md5'}],
'Datasets':[{'Dataset1':'DC06Stripping'}]
}
self.reqContainer.addSubRequest( removalDic, 'removal' )
# Add dummy stage request
stageDic = {'Attributes':{'Status': 'Waiting', 'SubRequestID': '7F7C1D94-E452-CD50-204C-EE2E2F1816A9',
'Catalogue':'', 'TargetSE':'CERN-tape', 'Operation':'StageAndPin'},
'Files':[{'LFN':lfn, 'Status': 'Waiting', 'Attempt': 1,
'PFN': 'srm://srm.cern.ch/castor/cern.ch/grid/lhcb/production/test/case.lfn',
'Size': 1231231, 'GUID': '7E9CED5A-295B-ED88-CE9A-CF41A62D2175', 'Addler': 'addler32', 'Md5': 'md5'}],
'Datasets':[{'Dataset1':'DC06Stripping'}]
}
self.reqContainer.addSubRequest( stageDic, 'stage' )
# Get the XML string of the DM request
string = self.reqContainer.toXML()
fname = 'testRequest.xml'
# Write the reqContainer to a file
self.reqContainer.toFile( fname )
# Get the file contents
reqfile = open( fname, 'r' )
testString = reqfile.read()
reqfile.close()
# Check the file contents are what is expected
self.assertEqual( string['Value'], testString )
testReq = RequestContainer( string )
# Test that what is obtained when parsing the request is the same as what is given.
transferReqDouble = self.reqContainer.getSubRequest( 0, 'transfer' )
for key in transferReqDouble.keys():
if key == 'Files':
self.assertEqual( transferDic['Files'], transferReqDouble['Files'] )
elif key == 'Datasets':
self.assertEqual( transferDic[key], transferReqDouble[key] )
else:
for att in transferDic['Attributes'].keys():
self.assertEqual( transferDic['Attributes'][att], transferReqDouble['Value']['Attributes'][att] )
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup):
""" Set removal request with the given credentials
"""
request = RequestContainer()
request.setRequestAttributes({
'OwnerDN': ownerDN,
'OwnerGroup': ownerGroup
})
requestName = os.path.basename(lfn).strip() + '_removal_request.xml'
request.setRequestName(requestName)
request.setSourceComponent('JobCleaningAgent')
removalDict = {
'Attributes': {
'Operation': 'removeFile',
'TargetSE': '',
'ExecutionOrder': 0
}
}
result = request.addSubRequest(removalDict, 'removal')
if not result['OK']:
return result
index = result['Value']
fileDict = {'LFN': lfn, 'PFN': '', 'Status': 'Waiting'}
request.setSubRequestFiles(index, 'removal', [fileDict])
client = RequestClient()
result = request.toXML()
if not result['OK']:
return result
xmlRequest = result['Value']
result = client.setRequest(requestName, xmlRequest)
return result
def setRequest(self, requestName, requestString, desiredStatus=None):
""" Set request to the database (including all sub-requests)
"""
gLogger.info("RequestDBFile._setRequest: Attempting to set %s." %
requestName)
request = RequestContainer(requestString)
requestTypes = request.getSubRequestTypes()['Value']
try:
for requestType in requestTypes:
subRequestString = request.toXML(
desiredType=requestType)['Value']
if subRequestString:
if desiredStatus:
status = desiredStatus
elif not request.isRequestTypeEmpty(requestType)['Value']:
status = 'ToDo'
else:
status = 'Done'
subRequestDir = os.path.join(self.root, requestType,
status)
if not os.path.exists(subRequestDir):
os.makedirs(subRequestDir)
subRequestPath = os.path.join(subRequestDir, requestName)
subRequestFile = open(subRequestPath, 'w')
subRequestFile.write(subRequestString)
subRequestFile.close()
gLogger.info("RequestDBFile._setRequest: Successfully set %s." %
requestName)
return S_OK()
except Exception, x:
errStr = "RequestDBFile._setRequest: Exception while setting request."
gLogger.exception(errStr, requestName, lException=x)
self.deleteRequest(requestName)
return S_ERROR(errStr)
def setRequest( self, requestName, requestString, desiredStatus = None ):
""" Set request to the database (including all sub-requests)
:param self: self reference
:param str requestName: request name
:param str requestString: serilised request
:param mixed desiredState: optional request status, defult = None
"""
self.log.info( "setRequest: Attempting to set %s." % requestName )
request = RequestContainer( requestString )
requestTypes = request.getSubRequestTypes()['Value']
try:
for requestType in requestTypes:
subRequestString = request.toXML( desiredType = requestType )['Value']
if subRequestString:
if desiredStatus:
status = desiredStatus
elif not request.isRequestTypeEmpty( requestType )['Value']:
status = 'Waiting'
else:
status = 'Done'
subRequestDir = os.path.join( self.root, requestType, status )
if not os.path.exists( subRequestDir ):
os.makedirs( subRequestDir )
subRequestPath = os.path.join( subRequestDir, requestName )
subRequestFile = open( subRequestPath, 'w' )
subRequestFile.write( subRequestString )
subRequestFile.close()
self.log.info( "setRequest: Successfully set %s." % requestName )
return S_OK()
except Exception, error:
errStr = "setRequest: Exception while setting request."
self.log.exception( errStr, requestName, lException = error )
self.deleteRequest( requestName )
return S_ERROR( errStr )
def getRequest():
"""
helper fcn to build requestContainer
"""
requestContainer = RequestContainer(init=False)
## get request
requestContainer.setRequestName("00009423_00000118")
requestContainer.setJobID(0)
requestContainer.setOwnerDN("")
requestContainer.setOwnerGroup("")
requestContainer.setDIRACSetup("")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-02-19 04:57:02")
requestContainer.setStatus("Waiting")
## get subrequest
requestContainer.initiateSubRequest("transfer")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2259916,
"Operation": "replicateAndRegister",
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "GRIDKA_MC-DST,GRIDKA_MC_M-DST",
"Catalogue": None,
"CreationTime": "2011-02-19 04:57:02",
"SubmissionTime": "2011-02-19 04:57:02",
"LastUpdate": "2011-08-18 20:14:22"
}
requestContainer.setSubRequestAttributes(0, "transfer", subRequestDict)
## get subrequest files
files = [{
"FileID": 1610538,
"LFN":
"/lhcb/MC/MC10/ALLSTREAMS.DST/00009422/0000/00009422_00000171_1.allstreams.dst",
"Size": None,
"PFN": None,
"GUID": None,
"Md5": None,
"Addler": None,
"Attempt": 1,
"Status": "Scheduled"
}]
requestContainer.setSubRequestFiles(0, "transfer", files)
return {
"OK": True,
"Value": {
"RequestName": "00009423_00000118",
"RequestString": requestContainer.toXML()["Value"],
"JobID": 0,
"RequestContainer": requestContainer
}
}
def _sendToFailover( rpcStub ):
requestClient = RequestClient()
request = RequestContainer()
request.setDISETRequest( rpcStub )
requestStub = request.toXML()['Value']
return requestClient.setRequest( "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ),
requestStub )
def getRegisterRequest():
""" helper fcn to build request
"""
requestContainer = RequestContainer(init=False)
requestContainer.setJobID(11889410)
#requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" )
requestContainer.setOwnerGroup("lhcb_user")
requestContainer.setDIRACSetup("LHCb-Production")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-02-19 04:57:02")
requestContainer.setStatus("Waiting")
requestContainer.initiateSubRequest("register")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2259916,
"Operation": "registerFile",
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "RAL-USER",
"Catalogue": "LcgFileCatalogCombined",
"CreationTime": "2011-02-19 04:57:02",
"SubmissionTime": "2011-02-19 04:57:02",
"LastUpdate": "2011-08-18 20:14:22"
}
requestContainer.setSubRequestAttributes(0, "register", subRequestDict)
files = [{
"FileID": 1610538,
"LFN": "/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL",
"Size": 153961749,
"PFN":
"srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL",
"GUID": "5911A19C-7CDF-7F2A-36ED-089CD410F98A",
"Md5": None,
"Addler": "92b85e26",
"Attempt": 1,
"Status": "Waiting"
}]
requestContainer.setSubRequestFiles(0, "register", files)
return {
"requestName":
"11889410.xml",
"requestString":
requestContainer.toXML()["Value"],
"jobID":
11889410,
"executionOrder":
0,
"sourceServer":
"foobarserver",
"configPath":
"/Systems/DataManagement/Development/Agents/RegistrationAgent"
}
def test_01_getRequestStatus(self):
self.assertEqual(
self.requestDB.getRequestStatus("testRequest"), {
'OK': True,
'Value': {
'SubRequestStatus': 'Waiting',
'RequestStatus': 'Waiting'
}
})
## get request
getRemoval = self.requestDB.getRequest("removal")
oRequest = RequestContainer(getRemoval["Value"]["RequestString"])
self.assertEqual(
self.requestDB.getRequestStatus("testRequest"), {
'OK': True,
'Value': {
'SubRequestStatus': 'Assigned',
'RequestStatus': 'Waiting'
}
})
## make removal Done
oRequest.subRequests["removal"][0]["Attributes"]["Status"] = "Done"
oRequest.subRequests["removal"][0]["Files"][0]["Status"] = "Done"
update = self.requestDB.updateRequest(
getRemoval["Value"]["RequestName"],
oRequest.toXML()["Value"])
## get status
self.assertEqual(
self.requestDB.getRequestStatus("testRequest"), {
'OK': True,
'Value': {
'SubRequestStatus': 'Waiting',
'RequestStatus': u'Waiting'
}
})
## make transfer Done
oRequest.subRequests["transfer"][0]["Attributes"]["Status"] = "Done"
oRequest.subRequests["transfer"][0]["Files"][0]["Status"] = "Done"
update = self.requestDB.updateRequest(
getRemoval["Value"]["RequestName"],
oRequest.toXML()["Value"])
## get status
self.assertEqual(
self.requestDB.getRequestStatus("testRequest"), {
'OK': True,
'Value': {
'SubRequestStatus': 'Done',
'RequestStatus': 'Done'
}
})
def submitTaskToExternal(self, request):
if type(request) in types.StringTypes:
oRequest = RequestContainer(request)
name = oRequest.getRequestName()['Value']
elif type(request) == types.InstanceType:
name = request.getRequestName()['Value']
request = request.toXML()['Value']
else:
return S_ERROR("Request should be string or request object")
return self.requestClient.setRequest(name, request)
def updateRequest(self, requestName, requestString):
request = RequestContainer(request=requestString)
requestTypes = [
'transfer', 'register', 'removal', 'stage', 'diset', 'logupload'
]
requestID = request.getRequestID()['Value']
updateRequestFailed = False
for requestType in requestTypes:
res = request.getNumSubRequests(requestType)
if res['OK']:
numRequests = res['Value']
for ind in range(numRequests):
res = request.getSubRequestAttributes(ind, requestType)
if res['OK']:
subRequestDict = res['Value']
if 'SubRequestID' in subRequestDict:
subRequestID = res['Value']['SubRequestID']
res = self.__updateSubRequestFiles(
ind, requestType, subRequestID, request)
if res['OK']:
if request.isSubRequestDone(
ind, requestType)['Value']:
res = self._setSubRequestAttribute(
requestID, subRequestID, 'Status',
'Done')
else:
res = self._setSubRequestAttribute(
requestID, subRequestID, 'Status',
'Waiting')
if not res['OK']:
updateRequestFailed = True
else:
updateRequestFailed = True
if "Error" in subRequestDict:
result = self._setSubRequestAttribute(
requestID, subRequestID, 'Error',
subRequestDict['Error'])
if not result['OK']:
updateRequestFailed = True
else:
updateRequestFailed = True
else:
updateRequestFailed = True
else:
updateRequestFailed = True
if updateRequestFailed:
errStr = 'Failed to update request %s.' % requestID
return S_ERROR(errStr)
else:
if request.isRequestDone()['Value']:
res = self._setRequestAttribute(requestID, 'Status', 'Done')
if not res['OK']:
errStr = 'Failed to update request status of %s to Done.' % requestID
return S_ERROR(errStr)
return S_OK()
def getRequest(operation):
""" fake requestDict
:param str operation: sub-request operation attribute
"""
requestContainer = RequestContainer(init=False)
requestContainer.setJobID(1)
#requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" )
requestContainer.setOwnerGroup("lhcb_user")
requestContainer.setDIRACSetup("LHCb-Production")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-12-01 04:57:02")
requestContainer.setStatus("Waiting")
requestContainer.setAttribute("RequestID", 123456789)
requestContainer.initiateSubRequest("transfer")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2222222,
"Operation": operation,
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "CERN-USER",
"Catalogue": "LcgFileCatalogCombined",
"CreationTime": "2011-12-01 04:57:02",
"SubmissionTime": "2011-12-01 04:57:02",
"LastUpdate": "2011-12-01 20:14:22"
}
requestContainer.setSubRequestAttributes(0, "transfer", subRequestDict)
files = [{
"FileID": 3333333,
"LFN": "/lhcb/user/c/cibak/11889/11889410/test.zzz",
"Size": 44444444,
"PFN":
"srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz",
"GUID": "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R",
"Md5": None,
"Addler": "92b85e26",
"Attempt": 1,
"Status": "Waiting"
}]
requestContainer.setSubRequestFiles(0, "transfer", files)
return {
"OK": True,
"Value": {
"requestName": "%s.xml" % operation,
"requestString": requestContainer.toXML_new()["Value"],
"requestObj": requestContainer,
"jobID": 1,
"executionOrder": 0,
"sourceServer": "foobarserver"
}
}
def generateRequest(self):
''' Commit the accumulated records and generate request eventually '''
result = self.commit()
request = None
if not result['OK']:
# Generate Request
request = RequestContainer()
if result.has_key('FailedResults'):
for res in result['FailedResults']:
if res.has_key('rpcStub'):
request.setDISETRequest(res['rpcStub'])
return S_OK(request)
def __init__(self, requestObject=False):
""" Constructor function, can specify request object to instantiate
FailoverTransfer or a new request object is created.
"""
self.log = gLogger.getSubLogger("FailoverTransfer")
self.rm = ReplicaManager()
self.request = requestObject
if not self.request:
self.request = RequestContainer()
self.request.setRequestName('default_request.xml')
self.request.setSourceComponent('FailoverTransfer')
def sweeper(cls):
""" move cached request to the central request manager
:param self: self reference
"""
cacheDir = cls.cacheDir()
## cache dir empty?
if not os.listdir(cacheDir):
gLogger.always("sweeper: CacheDir %s is empty, nothing to do" %
cacheDir)
return S_OK()
else:
## read 10 cache dir files, the oldest first
cachedRequests = [
os.path.abspath(requestFile)
for requestFile in sorted(filter(os.path.isfile, [
os.path.join(cacheDir, requestName)
for requestName in os.listdir(cacheDir)
]),
key=os.path.getctime)
][:30]
## set cached requests to the central RequestManager
for cachedFile in cachedRequests:
try:
requestString = "".join(open(cachedFile, "r").readlines())
cachedRequest = RequestContainer(requestString)
requestName = cachedRequest.getAttribute(
"RequestName")["Value"]
## cibak: hack for DISET requests
if requestName == "Unknown":
cachedRequest.setAttribute("RequestName", makeGuid())
requestName = cachedRequest.getAttribute(
"RequestName")["Value"]
setRequest = cls.requestManager().setRequest(
requestName, requestString)
if not setRequest["OK"]:
gLogger.error(
"sweeper: unable to set request '%s' @ RequestManager: %s"
% (requestName, setRequest["Message"]))
continue
gLogger.info(
"sweeper: successfully set request '%s' @ RequestManager"
% requestName)
os.unlink(cachedFile)
except Exception, error:
gLogger.exception("sweeper: hit by exception %s" %
str(error))
return S_ERROR("sweeper: hit by exception: %s" %
str(error))
return S_OK()
def getKwargsRemoveFile():
""" helper fcn to build request """
requestContainer = RequestContainer(init=False)
requestContainer.setJobID(11111111)
#requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" )
requestContainer.setOwnerGroup("lhcb_user")
requestContainer.setDIRACSetup("LHCb-Production")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-12-01 04:57:02")
requestContainer.setStatus("Waiting")
requestContainer.initiateSubRequest("removal")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2222222,
"Operation": "removeFile",
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "RAL-USER",
"Catalogue": "LcgFileCatalogCombined",
"CreationTime": "2011-12-01 04:57:02",
"SubmissionTime": "2011-12-01 04:57:02",
"LastUpdate": "2011-12-01 20:14:22"
}
requestContainer.setSubRequestAttributes(0, "removal", subRequestDict)
files = [{
"FileID": 3333333,
"LFN": "/lhcb/user/c/cibak/11889/11889410/test.zzz",
"Size": 44444444,
"PFN":
"srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz",
"GUID": "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R",
"Md5": None,
"Addler": "92b85e26",
"Attempt": 1,
"Status": "Waiting"
}]
requestContainer.setSubRequestFiles(0, "removal", files)
return {
"requestName": "00000001.xml",
"requestString": requestContainer.toXML()["Value"],
"jobID": 1,
"executionOrder": 0,
"sourceServer": "foobarserver",
"configPath": "/Systems/DataManagement/Development/Agents/RemovalAgent"
}
def getRequestDict(cls, requestType):
""" retrive Request of type requestType from RequestDB
:param cls: class reference
:param str requestType: type of request
:return: S_ERROR on error
:return: S_OK with request dictionary::
requestDict = {
"requestString" : str,
"requestName" : str,
"sourceServer" : str,
"executionOrder" : list,
"requestObj" : RequestContainer,
"jobId" : int }
"""
## prepare requestDict
requestDict = {
"requestString": None,
"requestName": None,
"sourceServer": None,
"executionOrder": None,
"requestObj": None,
"jobId": None
}
## get request out of DB
res = cls.requestClient().getRequest(requestType)
if not res["OK"]:
gLogger.error(res["Message"])
return res
elif not res["Value"]:
msg = "Request of type '%s' not found in RequestDB." % requestType
gLogger.info(msg)
return S_OK()
## store values
requestDict["requestName"] = res["Value"]["RequestName"]
requestDict["requestString"] = res["Value"]["RequestString"]
requestDict["sourceServer"] = res["Value"]["Server"]
requestDict["requestObj"] = RequestContainer(
request=requestDict["requestString"])
## get JobID
try:
requestDict["jobId"] = int(res["JobID"])
except ValueError, exc:
gLogger.warn(
"Cannot read JobID for request %s, setting it to 0: %s" %
(requestDict["requestName"], str(exc)))
requestDict["jobId"] = 0
def __forwardable(requestString):
""" check if request if forwardable
The sub-request of type transfer:putAndRegister, removal:physicalRemoval and removal:reTransfer are
definitely not, they should be executed locally, as they are using local fs.
:param str requestString: XML-serialised request
"""
request = RequestContainer(requestString)
subRequests = request.getSubRequests(
"transfer")["Value"] + request.getSubRequests("removal")["Value"]
for subRequest in subRequests:
if subRequest["Attributes"]["Operation"] in ("putAndRegister",
"physicalRemoval",
"reTransfer"):
return S_ERROR(
"found operation '%s' that cannot be forwarded" %
subRequest["Attributes"]["Operation"])
return S_OK()
def readRequestsForJobs( self, jobIDs ):
""" read requests for jobs
:param list jobIDs: list with jobIDs
:return: S_OK( { "Successful" : { jobID1 : RequestContainer, ... },
"Failed" : { jobIDn : "Fail reason" } } )
"""
readReqsForJobs = self.requestManager().readRequestsForJobs( jobIDs )
if not readReqsForJobs["OK"]:
return readReqsForJobs
ret = readReqsForJobs["Value"] if readReqsForJobs["Value"] else None
if not ret:
return S_ERROR( "No values returned" )
# # create RequestContainers out of xml strings for successful reads
if "Successful" in ret:
for jobID, xmlStr in ret["Successful"].items():
req = RequestContainer( init = False )
req.parseRequest( request = xmlStr )
ret["Successful"][jobID] = req
return S_OK( ret )
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
request = RequestContainer()
result = request.addSubRequest(
{
'Attributes': {
'Operation': 'removePhysicalFile',
'TargetSE': SEName,
'ExecutionOrder': 1
}
}, 'removal')
index = result['Value']
fileDict = {'PFN': SEPFN, 'Status': 'Waiting'}
request.setSubRequestFiles(index, 'removal', [fileDict])
return RequestClient().setRequest(
"RemoteSBDeletion:%s|%s:%s" % (SEName, SEPFN, time.time()),
request.toXML()['Value'])
except Exception, e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR("Cannot set deletion request: %s" % str(e))
def getRequest(self, requestType):
""" Obtain a request from the database of a certain type
"""
gLogger.info(
"RequestDBFile._getRequest: Attempting to get %s type request." %
requestType)
try:
# Determine the request name to be obtained
candidateRequests = []
reqDir = os.path.join(self.root, requestType, "ToDo")
self.getIdLock.acquire()
if os.path.exists(reqDir):
candidateRequests = [
os.path.basename(requestFile)
for requestFile in sorted(filter(os.path.isfile, [
os.path.join(reqDir, requestName)
for requestName in os.listdir(reqDir)
]),
key=os.path.getctime)
]
if not len(candidateRequests) > 0:
self.getIdLock.release()
gLogger.info(
"RequestDBFile._getRequest: No request of type %s found." %
requestType)
return S_OK()
# Select a request
if not self.lastRequest.has_key(requestType):
self.lastRequest[requestType] = ('', 0)
lastRequest, lastRequestIndex = self.lastRequest[requestType]
res = self.__selectRequestCursor(candidateRequests, lastRequest,
lastRequestIndex)
if not res['OK']:
self.getIdLock.release()
errStr = "RequestDBFile._getRequest: Failed to get request cursor."
gLogger.error(errStr, res['Message'])
return S_ERROR(errStr)
selectedRequestName, selectedRequestIndex = res['Value']
# Obtain the string for the selected request
res = self.__getRequestString(selectedRequestName)
if not res['OK']:
self.getIdLock.release()
errStr = "RequestDBFile._getRequest: Failed to get request string for %s." % selectedRequestName
gLogger.error(errStr, res['Message'])
return S_ERROR(errStr)
selectedRequestString = res['Value']
# Set the request status to assigned
res = self.setRequestStatus(selectedRequestName, 'Assigned')
if not res['OK']:
self.getIdLock.release()
errStr = "RequestDBFile._getRequest: Failed to set %s status to 'Assigned'." % selectedRequestName
gLogger.error(errStr, res['Message'])
return S_ERROR(errStr)
# Update the request cursor and return the selected request
self.lastRequest[requestType] = (selectedRequestName,
selectedRequestIndex)
self.getIdLock.release()
gLogger.info(
"RequestDBFile._getRequest: Successfully obtained %s request."
% selectedRequestName)
jobID = 'Unknown'
try:
oRequest = RequestContainer(request=selectedRequestString)
jobID = oRequest.getJobID()['Value']
except:
gLogger.exception('Could not get JobID from Request')
resDict = {
'RequestString': selectedRequestString,
'RequestName': selectedRequestName,
'JobID': jobID
}
return S_OK(resDict)
except Exception, x:
errStr = "RequestDBFile._getRequest: Exception while getting request."
gLogger.exception(errStr, requestType, lException=x)
return S_ERROR(errStr)
def setUp( self ): self.reqContainer = RequestContainer()
def test_addSubRequest( self ):
rc_o = RequestContainer()
op1_Index = rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Operation': 'op1'}},
'someType' )
op1_Index = op1_Index['Value']
subRequestExpected = {'someType': [{'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op1'},
'Datasets': []}]}
self.assertEqual( rc_o.subRequests, subRequestExpected )
op2_index = rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Operation': 'op2'}},
'someType' )
op2_index = op2_index['Value']
subRequestExpected = {
'someType':
[
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op1'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op2'},
'Datasets': []
}
]
}
self.assertEqual( rc_o.subRequests, subRequestExpected )
rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'ExecutionOrder': 'last'}},
'someType' )
subRequestExpected = {
'someType':
[
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op1'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op2'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
}
]
}
self.assertEqual( rc_o.subRequests, subRequestExpected )
rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'ExecutionOrder': 'last'}},
'someOtherType' )
subRequestExpected = {
'someType':
[
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op1'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op2'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
}
],
'someOtherType':
[
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 2,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
},
]
}
self.assertEqual( rc_o.subRequests, subRequestExpected )
fileDict = {'LFN':'foo', 'Status':'Waiting'}
rc_o.setSubRequestFiles( op1_Index, 'someType', [fileDict] )
subRequestExpected = {
'someType':
[
{
'Files': [{'LFN':'foo', 'Status':'Waiting'}],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op1'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op2'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
}
],
'someOtherType':
[
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 2,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
},
]
}
self.assertEqual( rc_o.subRequests, subRequestExpected )
fileLastOp = rc_o._getLastOrder( 'foo' )
rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'ExecutionOrder': fileLastOp + 1}},
'someOtherType' )
subRequestExpected = {
'someType':
[
{
'Files': [{'LFN':'foo', 'Status':'Waiting'}],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op1'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op2'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
}
],
'someOtherType':
[
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 2,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
},
]
}
self.assertEqual( rc_o.subRequests, subRequestExpected )
def execute(self):
res = self.RequestDBClient.getRequest('integrity')
if not res['OK']:
gLogger.info(
"LFCvsSEAgent.execute: Failed to get request from database.")
return S_OK()
elif not res['Value']:
gLogger.info(
"LFCvsSEAgent.execute: No requests to be executed found.")
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
gLogger.info("LFCvsSEAgent.execute: Obtained request %s" % requestName)
oRequest = RequestContainer(request=requestString)
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('integrity')
if not res['OK']:
errStr = "LFCvsSEAgent.execute: Failed to obtain number of integrity subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info("LFCvsSEAgent.execute: Found %s sub requests." %
res['Value'])
################################################
# For all the sub-requests in the request
for ind in range(res['Value']):
gLogger.info("LFCvsSEAgent.execute: Processing sub-request %s." %
ind)
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'integrity')['Value']
if subRequestAttributes['Status'] == 'Waiting':
subRequestFiles = oRequest.getSubRequestFiles(
ind, 'integrity')['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a lfcvsse operation
if operation == 'LFCvsSE':
gLogger.info(
"LFCvsSEAgent.execute: Attempting to execute %s sub-request."
% operation)
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile['LFN']
oNamespaceBrowser = NamespaceBrowser(lfn)
# Loop over all the directories and sub-directories
while (oNamespaceBrowser.isActive()):
currentDir = oNamespaceBrowser.getActiveDir()
gLogger.info(
"LFCvsSEAgent.execute: Attempting to get contents of %s."
% currentDir)
res = self.ReplicaManager.getCatalogDirectoryContents(
currentDir)
if not res['OK']:
subDirs = [currentDir]
elif res['Value']['Failed'].has_key(
currentDir):
subDirs = [currentDir]
else:
subDirs = res['Value']['Successful'][
currentDir]['SubDirs']
files = res['Value']['Successful'][
currentDir]['Files']
lfnSizeDict = {}
pfnLfnDict = {}
pfnStatusDict = {}
sePfnDict = {}
for lfn, lfnDict in files.items():
lfnSizeDict[lfn] = lfnDict['MetaData'][
'Size']
for se in lfnDict['Replicas'].keys():
pfn = lfnDict['Replicas'][se][
'PFN']
status = lfnDict['Replicas'][se][
'Status']
pfnStatusDict[pfn] = status
pfnLfnDict[pfn] = lfn
if not sePfnDict.has_key(se):
sePfnDict[se] = []
sePfnDict[se].append(pfn)
for storageElementName, physicalFiles in sePfnDict.items(
):
gLogger.info(
"LFCvsSEAgent.execute: Attempting to get metadata for files on %s."
% storageElementName)
res = self.ReplicaManager.getStorageFileMetadata(
physicalFiles, storageElementName)
if not res['OK']:
gLogger.error(
"LFCvsSEAgent.execute: Completely failed to get physical file metadata.",
res['Message'])
else:
for pfn in res['Value'][
'Failed'].keys():
gLogger.error(
"LFCvsSEAgent.execute: Failed to get metadata.",
"%s %s" %
(pfn, res['Value']
['Failed'][pfn]))
lfn = pfnLfnDict[pfn]
fileMetadata = {
'Prognosis':
'MissingSEPfn',
'LFN': lfn,
'PFN': pfn,
'StorageElement':
storageElementName,
'Size': lfnSizeDict[lfn]
}
IntegrityDB = RPCClient(
'DataManagement/DataIntegrity'
)
resInsert = IntegrityDB.insertProblematic(
AGENT_NAME, fileMetadata)
if resInsert['OK']:
gLogger.info(
"LFCvsSEAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
for pfn, pfnDict in res['Value'][
'Successful'].items():
lfn = pfnLfnDict[pfn]
catalogSize = int(
lfnSizeDict[lfn])
storageSize = int(
pfnDict['Size'])
if int(catalogSize) == int(
storageSize):
gLogger.info(
"LFCvsSEAgent.execute: Catalog and storage sizes match.",
"%s %s" %
(pfn,
storageElementName))
gLogger.info(
"Change the status in the LFC"
)
else:
gLogger.error(
"LFCvsSEAgent.execute: Catalog and storage size mis-match.",
"%s %s" %
(pfn,
storageElementName))
fileMetadata = {
'Prognosis':
'PfnSizeMismatch',
'LFN':
lfn,
'PFN':
pfn,
'StorageElement':
storageElementName
}
IntegrityDB = RPCClient(
'DataManagement/DataIntegrity'
)
resInsert = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if resInsert['OK']:
gLogger.info(
"LFCvsSEAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
oNamespaceBrowser.updateDirs(subDirs)
oRequest.setSubRequestFileAttributeValue(
ind, 'integrity', lfn, 'Status', 'Done')
################################################
# If the sub-request is none of the above types
else:
gLogger.info(
"LFCvsSEAgent.execute: Operation not supported.",
operation)
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty(ind, 'integrity')['Value']:
oRequest.setSubRequestStatus(ind, 'integrity', 'Done')
################################################
# If the sub-request is already in terminal state
else:
gLogger.info(
"LFCvsSEAgent.execute: Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest(requestName, requestString,
sourceServer)
return S_OK()
print se.errorReason
print
Script.showHelp()
from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer
from DIRAC.RequestManagementSystem.Client.RequestClient import RequestClient
requestClient = RequestClient()
requestType = 'removal'
requestOperation = 'replicaRemoval'
if targetSE == 'All':
requestOperation = 'removeFile'
for lfnList in breakListIntoChunks(lfns, 100):
oRequest = RequestContainer()
subRequestIndex = oRequest.initiateSubRequest(requestType)['Value']
attributeDict = {'Operation': requestOperation, 'TargetSE': targetSE}
oRequest.setSubRequestAttributes(subRequestIndex, requestType,
attributeDict)
files = []
for lfn in lfnList:
files.append({'LFN': lfn})
oRequest.setSubRequestFiles(subRequestIndex, requestType, files)
requestName = "%s_%s" % (md5(repr(time.time())).hexdigest()[:16],
md5(repr(time.time())).hexdigest()[:16])
oRequest.setRequestAttributes({'RequestName': requestName})
DIRAC.gLogger.info(oRequest.toXML()['Value'])
result = requestClient.setRequest(requestName, oRequest.toXML()['Value'])
def executeRequest(self):
"""
Do the actual work in the Thread
"""
################################################
# Get a request from request DB
gMonitor.addMark("Iteration", 1)
res = self.requestDBClient.getRequest('removal')
if not res['OK']:
gLogger.info(
"RemovalAgent.execute: Failed to get request from database.")
return S_OK()
elif not res['Value']:
gLogger.info(
"RemovalAgent.execute: No requests to be executed found.")
self.pendingRequests = False
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
jobID = 0
try:
jobID = int(res['Value']['JobID'])
except:
gLogger.warn(
"RemovalAgent.execute: JobID not present or malformed in request '%s', will use 0 instead."
% requestName)
gLogger.info("RemovalAgent.execute: Obtained request %s" % requestName)
try:
result = self.requestDBClient.getCurrentExecutionOrder(
requestName, sourceServer)
if result['OK']:
currentOrder = result['Value']
else:
gLogger.error('Can not get the request execution order')
self.requestDBClient.updateRequest(requestName, requestString,
sourceServer)
return S_OK('Can not get the request execution order')
oRequest = RequestContainer(request=requestString)
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('removal')
if not res['OK']:
errStr = "RemovalAgent.execute: Failed to obtain number of removal subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info("RemovalAgent.execute: Found %s sub requests." %
res['Value'])
################################################
# For all the sub-requests in the request
modified = False
for ind in range(res['Value']):
gMonitor.addMark("Execute", 1)
gLogger.info(
"RemovalAgent.execute: Processing sub-request %s." % ind)
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'removal')['Value']
subExecutionOrder = int(subRequestAttributes['ExecutionOrder'])
subStatus = subRequestAttributes['Status']
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
subRequestFiles = oRequest.getSubRequestFiles(
ind, 'removal')['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a physical removal operation
if operation == 'physicalRemoval':
gLogger.info(
"RemovalAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSEs = subRequestAttributes['TargetSE'].split(',')
physicalFiles = []
pfnToLfn = {}
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
pfn = str(subRequestFile['PFN'])
lfn = str(subRequestFile['LFN'])
pfnToLfn[pfn] = lfn
physicalFiles.append(pfn)
gMonitor.addMark('PhysicalRemovalAtt',
len(physicalFiles))
failed = {}
errMsg = {}
for diracSE in diracSEs:
res = self.replicaManager.removeStorageFile(
physicalFiles, diracSE)
if res['OK']:
for pfn in res['Value']['Failed'].keys():
if not failed.has_key(pfn):
failed[pfn] = {}
failed[pfn][diracSE] = res['Value'][
'Failed'][pfn]
else:
errMsg[diracSE] = res['Message']
for pfn in physicalFiles:
if not failed.has_key(pfn):
failed[pfn] = {}
failed[pfn][diracSE] = 'Completely'
# Now analyse the results
failedPFNs = failed.keys()
pfnsOK = [
pfn for pfn in physicalFiles
if not pfn in failedPFNs
]
gMonitor.addMark('PhysicalRemovalDone', len(pfnsOK))
for pfn in pfnsOK:
gLogger.info(
"RemovalAgent.execute: Successfully removed %s at %s"
% (pfn, str(diracSEs)))
res = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', pfnToLfn[pfn], 'Status',
'Done')
if not res['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', pfnToLfn[pfn]))
modified = True
if failed:
gMonitor.addMark('PhysicalRemovalFail',
len(failedPFNs))
for pfn in failedPFNs:
for diracSE in failed[pfn].keys():
if type(failed[pfn]
[diracSE]) in StringTypes:
if re.search(
'no such file or directory',
failed[pfn][diracSE].lower()):
gLogger.info(
"RemovalAgent.execute: File did not exist.",
pfn)
res = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', pfnToLfn[pfn],
'Status', 'Done')
if not res['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', pfnToLfn[pfn]))
modified = True
else:
gLogger.info(
"RemovalAgent.execute: Failed to remove file.",
"%s at %s - %s" %
(pfn, diracSE,
failed[pfn][diracSE]))
if errMsg:
for diracSE in errMsg.keys():
errStr = "RemovalAgent.execute: Completely failed to remove replicas. At %s", diracSE
gLogger.error(errStr, errMsg[diracSE])
################################################
# If the sub-request is a physical removal operation
elif operation == 'removeFile':
gLogger.info(
"RemovalAgent.execute: Attempting to execute %s sub-request."
% operation)
lfns = []
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = str(subRequestFile['LFN'])
lfns.append(lfn)
gMonitor.addMark('RemoveFileAtt', len(lfns))
res = self.replicaManager.removeFile(lfns)
if res['OK']:
gMonitor.addMark(
'RemoveFileDone',
len(res['Value']['Successful'].keys()))
for lfn in res['Value']['Successful'].keys():
gLogger.info(
"RemovalAgent.execute: Successfully removed %s."
% lfn)
result = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', lfn, 'Status', 'Done')
if not result['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', lfn))
modified = True
gMonitor.addMark(
'RemoveFileFail',
len(res['Value']['Failed'].keys()))
for lfn in res['Value']['Failed'].keys():
if type(res['Value']['Failed']
[lfn]) in StringTypes:
if re.search(
'no such file or directory',
res['Value']['Failed']
[lfn].lower()):
gLogger.info(
"RemovalAgent.execute: File did not exist.",
lfn)
result = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', lfn, 'Status',
'Done')
if not result['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', lfn))
modified = True
else:
gLogger.info(
"RemovalAgent.execute: Failed to remove file:",
"%s %s" %
(lfn, res['Value']['Failed'][lfn]))
else:
gMonitor.addMark('RemoveFileFail', len(lfns))
errStr = "RemovalAgent.execute: Completely failed to remove files files."
gLogger.error(errStr, res['Message'])
################################################
# If the sub-request is a physical removal operation
elif operation == 'replicaRemoval':
gLogger.info(
"RemovalAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSEs = subRequestAttributes['TargetSE'].split(',')
lfns = []
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = str(subRequestFile['LFN'])
lfns.append(lfn)
gMonitor.addMark('ReplicaRemovalAtt', len(lfns))
failed = {}
errMsg = {}
for diracSE in diracSEs:
res = self.replicaManager.removeReplica(
diracSE, lfns)
if res['OK']:
for lfn in res['Value']['Failed'].keys():
errorMessage = str(
res['Value']['Failed'][lfn])
if errorMessage.find(
'Write access not permitted for this credential.'
) != -1:
if self.__getProxyAndRemoveReplica(
diracSE, lfn):
continue
if errorMessage.find(
'seconds timeout for "__gfal_wrapper" call'
) != -1:
self.timeOutCounter += 1
if not failed.has_key(lfn):
failed[lfn] = {}
failed[lfn][diracSE] = res['Value'][
'Failed'][lfn]
else:
errMsg[diracSE] = res['Message']
for lfn in lfns:
if not failed.has_key(lfn):
failed[lfn] = {}
failed[lfn][diracSE] = 'Completely'
# Now analyse the results
failedLFNs = failed.keys()
lfnsOK = [lfn for lfn in lfns if not lfn in failedLFNs]
gMonitor.addMark('ReplicaRemovalDone', len(lfnsOK))
for lfn in lfnsOK:
gLogger.info(
"RemovalAgent.execute: Successfully removed %s at %s"
% (lfn, str(diracSEs)))
res = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', lfn, 'Status', 'Done')
if not res['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', lfn))
modified = True
if failed:
gMonitor.addMark('PhysicalRemovalFail',
len(failedLFNs))
for lfn in failedLFNs:
for diracSE in failed[lfn].keys():
if type(failed[lfn]
[diracSE]) in StringTypes:
if re.search(
'no such file or directory',
failed[lfn][diracSE].lower()):
gLogger.info(
"RemovalAgent.execute: File did not exist.",
lfn)
res = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', lfn, 'Status',
'Done')
if not res['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', lfn))
modified = True
else:
gLogger.info(
"RemovalAgent.execute: Failed to remove file.",
"%s at %s - %s" %
(lfn, diracSE,
failed[lfn][diracSE]))
if errMsg:
for diracSE in errMsg.keys():
errStr = "RemovalAgent.execute: Completely failed to remove replicas. At %s", diracSE
gLogger.error(errStr, errMsg[diracSE])
################################################
# If the sub-request is a request to the online system to retransfer
elif operation == 'reTransfer':
gLogger.info(
"RemovalAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
pfn = str(subRequestFile['PFN'])
lfn = str(subRequestFile['LFN'])
res = self.replicaManager.onlineRetransfer(
diracSE, pfn)
if res['OK']:
if res['Value']['Successful'].has_key(pfn):
gLogger.info(
"RemovalAgent.execute: Successfully requested retransfer of %s."
% pfn)
result = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', lfn, 'Status',
'Done')
if not result['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', lfn))
modified = True
else:
errStr = "RemovalAgent.execute: Failed to request retransfer."
gLogger.error(
errStr, "%s %s %s" %
(pfn, diracSE,
res['Value']['Failed'][pfn]))
else:
errStr = "RemovalAgent.execute: Completely failed to request retransfer."
gLogger.error(errStr, res['Message'])
else:
gLogger.info(
"RemovalAgent.execute: File already completed."
)
################################################
# If the sub-request is none of the above types
else:
gLogger.error(
"RemovalAgent.execute: Operation not supported.",
operation)
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty(ind, 'removal')['Value']:
oRequest.setSubRequestStatus(ind, 'removal', 'Done')
gMonitor.addMark("Done", 1)
################################################
# If the sub-request is already in terminal state
else:
gLogger.info(
"RemovalAgent.execute:",
"Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
newrequestString = oRequest.toXML()['Value']
except:
# if something fails return the original request back to the server
res = self.requestDBClient.updateRequest(requestName,
requestString,
sourceServer)
return S_OK()
res = self.requestDBClient.updateRequest(requestName, newrequestString,
sourceServer)
if modified and jobID:
result = self.finalizeRequest(requestName, jobID, sourceServer)
return S_OK()
