def test_01_getRequestStatus( self ):
self.assertEqual( self.requestDB.getRequestStatus( "testRequest" ),
{'OK': True, 'Value': {'SubRequestStatus': 'Waiting', 'RequestStatus': 'Waiting'}})
## get request
getRemoval = self.requestDB.getRequest( "removal" )
oRequest = RequestContainer( getRemoval["Value"]["RequestString"] )
self.assertEqual( self.requestDB.getRequestStatus( "testRequest" ),
{'OK': True, 'Value': {'SubRequestStatus': 'Assigned', 'RequestStatus': 'Waiting'}} )
## make removal Done
oRequest.subRequests["removal"][0]["Attributes"]["Status"] = "Done"
oRequest.subRequests["removal"][0]["Files"][0]["Status"] = "Done"
update = self.requestDB.updateRequest( getRemoval["Value"]["RequestName"],
oRequest.toXML()["Value"] )
## get status
self.assertEqual( self.requestDB.getRequestStatus( "testRequest" ),
{'OK': True, 'Value': {'SubRequestStatus': 'Waiting', 'RequestStatus': u'Waiting'}})
## make transfer Done
oRequest.subRequests["transfer"][0]["Attributes"]["Status"] = "Done"
oRequest.subRequests["transfer"][0]["Files"][0]["Status"] = "Done"
update = self.requestDB.updateRequest( getRemoval["Value"]["RequestName"],
oRequest.toXML()["Value"] )
## get status
self.assertEqual( self.requestDB.getRequestStatus( "testRequest" ),
{'OK': True, 'Value': {'SubRequestStatus': 'Done', 'RequestStatus': 'Done'}} )
def test_01_getRequestStatus(self):
self.assertEqual(
self.requestDB.getRequestStatus("testRequest"), {
'OK': True,
'Value': {
'SubRequestStatus': 'Waiting',
'RequestStatus': 'Waiting'
}
})
## get request
getRemoval = self.requestDB.getRequest("removal")
oRequest = RequestContainer(getRemoval["Value"]["RequestString"])
self.assertEqual(
self.requestDB.getRequestStatus("testRequest"), {
'OK': True,
'Value': {
'SubRequestStatus': 'Assigned',
'RequestStatus': 'Waiting'
}
})
## make removal Done
oRequest.subRequests["removal"][0]["Attributes"]["Status"] = "Done"
oRequest.subRequests["removal"][0]["Files"][0]["Status"] = "Done"
update = self.requestDB.updateRequest(
getRemoval["Value"]["RequestName"],
oRequest.toXML()["Value"])
## get status
self.assertEqual(
self.requestDB.getRequestStatus("testRequest"), {
'OK': True,
'Value': {
'SubRequestStatus': 'Waiting',
'RequestStatus': u'Waiting'
}
})
## make transfer Done
oRequest.subRequests["transfer"][0]["Attributes"]["Status"] = "Done"
oRequest.subRequests["transfer"][0]["Files"][0]["Status"] = "Done"
update = self.requestDB.updateRequest(
getRemoval["Value"]["RequestName"],
oRequest.toXML()["Value"])
## get status
self.assertEqual(
self.requestDB.getRequestStatus("testRequest"), {
'OK': True,
'Value': {
'SubRequestStatus': 'Done',
'RequestStatus': 'Done'
}
})
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
request = RequestContainer()
request.setRequestAttributes( { 'OwnerDN':ownerDN, 'OwnerGroup':ownerGroup } )
requestName = os.path.basename( lfn ).strip()+'_removal_request.xml'
request.setRequestName( requestName )
request.setSourceComponent( 'JobCleaningAgent' )
removalDict = {'Attributes':{ 'Operation':'removeFile',
'TargetSE':'',
'ExecutionOrder':0
}
}
result = request.addSubRequest( removalDict, 'removal' )
if not result['OK']:
return result
index = result['Value']
fileDict = { 'LFN':lfn, 'PFN':'', 'Status':'Waiting' }
request.setSubRequestFiles( index, 'removal', [fileDict] )
client = RequestClient()
result = request.toXML()
if not result['OK']:
return result
xmlRequest = result['Value']
result = client.setRequest( requestName, xmlRequest )
return result
def __getRequestString(self, requestName):
""" Obtain the string for request (including all sub-requests)
"""
gLogger.info(
"RequestDBFile.__getRequestString: Attempting to get string for %s."
% requestName)
res = self.__locateRequest(requestName)
if not res['OK']:
return res
subRequestPaths = res['Value']
try:
oRequest = RequestContainer(init=False)
for subRequestPath in subRequestPaths:
res = self.__readSubRequestString(subRequestPath)
if not res['OK']:
return res
subRequestString = res['Value']
tempRequest = RequestContainer(subRequestString) #,init=False)
oRequest.setRequestAttributes(
tempRequest.getRequestAttributes()['Value'])
oRequest.update(tempRequest)
requestString = oRequest.toXML()['Value']
gLogger.info(
"RequestDBFile.__getRequestString: Successfully obtained string for %s."
% requestName)
result = S_OK(requestString)
result['Request'] = oRequest
return result
except Exception, x:
errStr = "RequestDBFile.__getRequestString: Exception while obtaining request string."
gLogger.exception(errStr, requestName, lException=x)
return S_ERROR(errStr)
def setRequest(self, requestName, requestString, desiredStatus=None):
""" Set request to the database (including all sub-requests)
"""
gLogger.info("RequestDBFile._setRequest: Attempting to set %s." %
requestName)
request = RequestContainer(requestString)
requestTypes = request.getSubRequestTypes()['Value']
try:
for requestType in requestTypes:
subRequestString = request.toXML(
desiredType=requestType)['Value']
if subRequestString:
if desiredStatus:
status = desiredStatus
elif not request.isRequestTypeEmpty(requestType)['Value']:
status = 'ToDo'
else:
status = 'Done'
subRequestDir = os.path.join(self.root, requestType,
status)
if not os.path.exists(subRequestDir):
os.makedirs(subRequestDir)
subRequestPath = os.path.join(subRequestDir, requestName)
subRequestFile = open(subRequestPath, 'w')
subRequestFile.write(subRequestString)
subRequestFile.close()
gLogger.info("RequestDBFile._setRequest: Successfully set %s." %
requestName)
return S_OK()
except Exception, x:
errStr = "RequestDBFile._setRequest: Exception while setting request."
gLogger.exception(errStr, requestName, lException=x)
self.deleteRequest(requestName)
return S_ERROR(errStr)
def setRequest( self, requestName, requestString, desiredStatus = None ):
""" Set request to the database (including all sub-requests)
:param self: self reference
:param str requestName: request name
:param str requestString: serilised request
:param mixed desiredState: optional request status, defult = None
"""
self.log.info( "setRequest: Attempting to set %s." % requestName )
request = RequestContainer( requestString )
requestTypes = request.getSubRequestTypes()['Value']
try:
for requestType in requestTypes:
subRequestString = request.toXML( desiredType = requestType )['Value']
if subRequestString:
if desiredStatus:
status = desiredStatus
elif not request.isRequestTypeEmpty( requestType )['Value']:
status = 'Waiting'
else:
status = 'Done'
subRequestDir = os.path.join( self.root, requestType, status )
if not os.path.exists( subRequestDir ):
os.makedirs( subRequestDir )
subRequestPath = os.path.join( subRequestDir, requestName )
subRequestFile = open( subRequestPath, 'w' )
subRequestFile.write( subRequestString )
subRequestFile.close()
self.log.info( "setRequest: Successfully set %s." % requestName )
return S_OK()
except Exception, error:
errStr = "setRequest: Exception while setting request."
self.log.exception( errStr, requestName, lException = error )
self.deleteRequest( requestName )
return S_ERROR( errStr )
def __getRequestString(self, requestName):
""" Obtain the string for request (including all sub-requests)
:param self: self reference
:param str requestName: request name
"""
self.log.info("__getRequestString: Attempting to get string for %s." % requestName)
res = self.__locateRequest(requestName)
if not res["OK"]:
return res
subRequestPaths = res["Value"]
try:
oRequest = RequestContainer(init=False)
for subRequestPath in subRequestPaths:
res = self.__readSubRequestString(subRequestPath)
if not res["OK"]:
return res
subRequestString = res["Value"]
tempRequest = RequestContainer(subRequestString)
oRequest.setRequestAttributes(tempRequest.getRequestAttributes()["Value"])
oRequest.update(tempRequest)
requestString = oRequest.toXML()["Value"]
self.log.info("__getRequestString: Successfully obtained string for %s." % requestName)
result = S_OK(requestString)
result["Request"] = oRequest
return result
except Exception, error:
errStr = "__getRequestString: Exception while obtaining request string."
self.log.exception(errStr, requestName, lException=error)
return S_ERROR(errStr)
def prepareTransformationTasks(self,
transBody,
taskDict,
owner='',
ownerGroup=''):
requestType = 'transfer'
requestOperation = 'replicateAndRegister'
try:
requestType, requestOperation = transBody.split(';')
except:
pass
for taskID in sortList(taskDict.keys()):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = RequestContainer(init=False)
subRequestIndex = oRequest.initiateSubRequest(requestType)['Value']
attributeDict = {
'Operation': requestOperation,
'TargetSE': paramDict['TargetSE']
}
oRequest.setSubRequestAttributes(subRequestIndex, requestType,
attributeDict)
files = []
for lfn in paramDict['InputData'].split(';'):
files.append({'LFN': lfn})
oRequest.setSubRequestFiles(subRequestIndex, requestType, files)
requestName = str(transID).zfill(8) + '_' + str(taskID).zfill(8)
oRequest.setRequestAttributes({'RequestName': requestName})
taskDict[taskID]['TaskObject'] = oRequest.toXML()['Value']
return S_OK(taskDict)
def __getRequestString( self, requestName ):
""" Obtain the string for request (including all sub-requests)
"""
gLogger.info( "RequestDBFile.__getRequestString: Attempting to get string for %s." % requestName )
res = self.__locateRequest( requestName )
if not res['OK']:
return res
subRequestPaths = res['Value']
try:
oRequest = RequestContainer( init = False )
for subRequestPath in subRequestPaths:
res = self.__readSubRequestString( subRequestPath )
if not res['OK']:
return res
subRequestString = res['Value']
tempRequest = RequestContainer( subRequestString )#,init=False)
oRequest.setRequestAttributes( tempRequest.getRequestAttributes()['Value'] )
oRequest.update( tempRequest )
requestString = oRequest.toXML()['Value']
gLogger.info( "RequestDBFile.__getRequestString: Successfully obtained string for %s." % requestName )
result = S_OK( requestString )
result['Request'] = oRequest
return result
except Exception, x:
errStr = "RequestDBFile.__getRequestString: Exception while obtaining request string."
gLogger.exception( errStr, requestName, lException = x )
return S_ERROR( errStr )
def setRequest(self, requestName, requestString, desiredStatus=None):
""" Set request to the database (including all sub-requests)
:param self: self reference
:param str requestName: request name
:param str requestString: serilised request
:param mixed desiredState: optional request status, defult = None
"""
self.log.info("setRequest: Attempting to set %s." % requestName)
request = RequestContainer(requestString)
requestTypes = request.getSubRequestTypes()["Value"]
try:
for requestType in requestTypes:
subRequestString = request.toXML(desiredType=requestType)["Value"]
if subRequestString:
if desiredStatus:
status = desiredStatus
elif not request.isRequestTypeEmpty(requestType)["Value"]:
status = "Waiting"
else:
status = "Done"
subRequestDir = os.path.join(self.root, requestType, status)
if not os.path.exists(subRequestDir):
os.makedirs(subRequestDir)
subRequestPath = os.path.join(subRequestDir, requestName)
subRequestFile = open(subRequestPath, "w")
subRequestFile.write(subRequestString)
subRequestFile.close()
self.log.info("setRequest: Successfully set %s." % requestName)
return S_OK()
except Exception, error:
errStr = "setRequest: Exception while setting request."
self.log.exception(errStr, requestName, lException=error)
self.deleteRequest(requestName)
return S_ERROR(errStr)
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup):
""" Set removal request with the given credentials
"""
request = RequestContainer()
request.setRequestAttributes({
'OwnerDN': ownerDN,
'OwnerGroup': ownerGroup
})
requestName = os.path.basename(lfn).strip() + '_removal_request.xml'
request.setRequestName(requestName)
request.setSourceComponent('JobCleaningAgent')
removalDict = {
'Attributes': {
'Operation': 'removeFile',
'TargetSE': '',
'ExecutionOrder': 0
}
}
result = request.addSubRequest(removalDict, 'removal')
if not result['OK']:
return result
index = result['Value']
fileDict = {'LFN': lfn, 'PFN': '', 'Status': 'Waiting'}
request.setSubRequestFiles(index, 'removal', [fileDict])
client = RequestClient()
result = request.toXML()
if not result['OK']:
return result
xmlRequest = result['Value']
result = client.setRequest(requestName, xmlRequest)
return result
def __getRequestString(self, requestName):
""" Obtain the string for request (including all sub-requests)
:param self: self reference
:param str requestName: request name
"""
self.log.info("__getRequestString: Attempting to get string for %s." %
requestName)
res = self.__locateRequest(requestName)
if not res['OK']:
return res
subRequestPaths = res['Value']
try:
oRequest = RequestContainer(init=False)
for subRequestPath in subRequestPaths:
res = self.__readSubRequestString(subRequestPath)
if not res['OK']:
return res
subRequestString = res['Value']
tempRequest = RequestContainer(subRequestString)
oRequest.setRequestAttributes(
tempRequest.getRequestAttributes()['Value'])
oRequest.update(tempRequest)
requestString = oRequest.toXML()['Value']
self.log.info(
"__getRequestString: Successfully obtained string for %s." %
requestName)
result = S_OK(requestString)
result['Request'] = oRequest
return result
except Exception, error:
errStr = "__getRequestString: Exception while obtaining request string."
self.log.exception(errStr, requestName, lException=error)
return S_ERROR(errStr)
def setRequest( self, requestName, requestString, desiredStatus = None ):
""" Set request to the database (including all sub-requests)
"""
gLogger.info( "RequestDBFile._setRequest: Attempting to set %s." % requestName )
request = RequestContainer( requestString )
requestTypes = request.getSubRequestTypes()['Value']
try:
for requestType in requestTypes:
subRequestString = request.toXML( desiredType = requestType )['Value']
if subRequestString:
if desiredStatus:
status = desiredStatus
elif not request.isRequestTypeEmpty( requestType )['Value']:
status = 'ToDo'
else:
status = 'Done'
subRequestDir = '%s/%s/%s' % ( self.root, requestType, status )
if not os.path.exists( subRequestDir ):
os.makedirs( subRequestDir )
subRequestPath = '%s/%s' % ( subRequestDir, requestName )
subRequestFile = open( subRequestPath, 'w' )
subRequestFile.write( subRequestString )
subRequestFile.close()
gLogger.info( "RequestDBFile._setRequest: Successfully set %s." % requestName )
return S_OK()
except Exception, x:
errStr = "RequestDBFile._setRequest: Exception while setting request."
gLogger.exception( errStr, requestName, lException = x )
self.deleteRequest( requestName )
return S_ERROR( errStr )
def getRequest():
"""
helper fcn to build requestContainer
"""
requestContainer = RequestContainer(init=False)
## get request
requestContainer.setRequestName("00009423_00000118")
requestContainer.setJobID(0)
requestContainer.setOwnerDN("")
requestContainer.setOwnerGroup("")
requestContainer.setDIRACSetup("")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-02-19 04:57:02")
requestContainer.setStatus("Waiting")
## get subrequest
requestContainer.initiateSubRequest("transfer")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2259916,
"Operation": "replicateAndRegister",
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "GRIDKA_MC-DST,GRIDKA_MC_M-DST",
"Catalogue": None,
"CreationTime": "2011-02-19 04:57:02",
"SubmissionTime": "2011-02-19 04:57:02",
"LastUpdate": "2011-08-18 20:14:22"
}
requestContainer.setSubRequestAttributes(0, "transfer", subRequestDict)
## get subrequest files
files = [{
"FileID": 1610538,
"LFN":
"/lhcb/MC/MC10/ALLSTREAMS.DST/00009422/0000/00009422_00000171_1.allstreams.dst",
"Size": None,
"PFN": None,
"GUID": None,
"Md5": None,
"Addler": None,
"Attempt": 1,
"Status": "Scheduled"
}]
requestContainer.setSubRequestFiles(0, "transfer", files)
return {
"OK": True,
"Value": {
"RequestName": "00009423_00000118",
"RequestString": requestContainer.toXML()["Value"],
"JobID": 0,
"RequestContainer": requestContainer
}
}
def _sendToFailover( rpcStub ):
requestClient = RequestClient()
request = RequestContainer()
request.setDISETRequest( rpcStub )
requestStub = request.toXML()['Value']
return requestClient.setRequest( "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ),
requestStub )
def _sendToFailover( rpcStub ):
requestClient = RequestClient()
request = RequestContainer()
request.setDISETRequest( rpcStub )
requestStub = request.toXML()['Value']
return requestClient.setRequest( "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ),
requestStub )
def getRegisterRequest():
""" helper fcn to build request
"""
requestContainer = RequestContainer(init=False)
requestContainer.setJobID(11889410)
#requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" )
requestContainer.setOwnerGroup("lhcb_user")
requestContainer.setDIRACSetup("LHCb-Production")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-02-19 04:57:02")
requestContainer.setStatus("Waiting")
requestContainer.initiateSubRequest("register")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2259916,
"Operation": "registerFile",
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "RAL-USER",
"Catalogue": "LcgFileCatalogCombined",
"CreationTime": "2011-02-19 04:57:02",
"SubmissionTime": "2011-02-19 04:57:02",
"LastUpdate": "2011-08-18 20:14:22"
}
requestContainer.setSubRequestAttributes(0, "register", subRequestDict)
files = [{
"FileID": 1610538,
"LFN": "/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL",
"Size": 153961749,
"PFN":
"srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL",
"GUID": "5911A19C-7CDF-7F2A-36ED-089CD410F98A",
"Md5": None,
"Addler": "92b85e26",
"Attempt": 1,
"Status": "Waiting"
}]
requestContainer.setSubRequestFiles(0, "register", files)
return {
"requestName":
"11889410.xml",
"requestString":
requestContainer.toXML()["Value"],
"jobID":
11889410,
"executionOrder":
0,
"sourceServer":
"foobarserver",
"configPath":
"/Systems/DataManagement/Development/Agents/RegistrationAgent"
}
def getRegisterRequest():
""" helper fcn to build request
"""
requestContainer = RequestContainer(init=False)
requestContainer.setJobID(11889410)
# requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" )
requestContainer.setOwnerGroup("lhcb_user")
requestContainer.setDIRACSetup("LHCb-Production")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-02-19 04:57:02")
requestContainer.setStatus("Waiting")
requestContainer.initiateSubRequest("register")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2259916,
"Operation": "registerFile",
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "RAL-USER",
"Catalogue": "LcgFileCatalogCombined",
"CreationTime": "2011-02-19 04:57:02",
"SubmissionTime": "2011-02-19 04:57:02",
"LastUpdate": "2011-08-18 20:14:22",
}
requestContainer.setSubRequestAttributes(0, "register", subRequestDict)
files = [
{
"FileID": 1610538,
"LFN": "/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL",
"Size": 153961749,
"PFN": "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cblanks/11889/11889410/LDSB.rsQrRL",
"GUID": "5911A19C-7CDF-7F2A-36ED-089CD410F98A",
"Md5": None,
"Addler": "92b85e26",
"Attempt": 1,
"Status": "Waiting",
}
]
requestContainer.setSubRequestFiles(0, "register", files)
return {
"requestName": "11889410.xml",
"requestString": requestContainer.toXML()["Value"],
"jobID": 11889410,
"executionOrder": 0,
"sourceServer": "foobarserver",
"configPath": "/Systems/DataManagement/Development/Agents/RegistrationAgent",
}
def getRequest():
"""
helper fcn to build requestContainer
"""
requestContainer = RequestContainer( init = False )
## get request
requestContainer.setRequestName( "00009423_00000118" )
requestContainer.setJobID( 0 )
requestContainer.setOwnerDN( "" )
requestContainer.setOwnerGroup( "" )
requestContainer.setDIRACSetup( "" )
requestContainer.setSourceComponent( None )
requestContainer.setCreationTime( "0000-00-00 00:00:00" )
requestContainer.setLastUpdate( "2011-02-19 04:57:02" )
requestContainer.setStatus( "Waiting" )
## get subrequest
requestContainer.initiateSubRequest( "transfer" )
subRequestDict = { "Status" : "Waiting",
"SubRequestID" : 2259916,
"Operation" : "replicateAndRegister",
"Arguments" : None,
"ExecutionOrder" : 0,
"SourceSE" : None,
"TargetSE" : "GRIDKA_MC-DST,GRIDKA_MC_M-DST",
"Catalogue" : None,
"CreationTime" : "2011-02-19 04:57:02",
"SubmissionTime" : "2011-02-19 04:57:02",
"LastUpdate" : "2011-08-18 20:14:22" }
requestContainer.setSubRequestAttributes( 0, "transfer", subRequestDict )
## get subrequest files
files = [ { "FileID" : 1610538,
"LFN" : "/lhcb/MC/MC10/ALLSTREAMS.DST/00009422/0000/00009422_00000171_1.allstreams.dst",
"Size" : None,
"PFN" : None,
"GUID" : None,
"Md5" : None,
"Addler" : None,
"Attempt" : 1,
"Status" : "Scheduled" } ]
requestContainer.setSubRequestFiles( 0, "transfer", files )
return { "OK" : True,
"Value" : { "RequestName" : "00009423_00000118",
"RequestString" : requestContainer.toXML()["Value"],
"JobID" : 0,
"RequestContainer" : requestContainer } }
def getKwargsRemoveFile():
""" helper fcn to build request """
requestContainer = RequestContainer(init=False)
requestContainer.setJobID(11111111)
#requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" )
requestContainer.setOwnerGroup("lhcb_user")
requestContainer.setDIRACSetup("LHCb-Production")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-12-01 04:57:02")
requestContainer.setStatus("Waiting")
requestContainer.initiateSubRequest("removal")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2222222,
"Operation": "removeFile",
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "RAL-USER",
"Catalogue": "LcgFileCatalogCombined",
"CreationTime": "2011-12-01 04:57:02",
"SubmissionTime": "2011-12-01 04:57:02",
"LastUpdate": "2011-12-01 20:14:22"
}
requestContainer.setSubRequestAttributes(0, "removal", subRequestDict)
files = [{
"FileID": 3333333,
"LFN": "/lhcb/user/c/cibak/11889/11889410/test.zzz",
"Size": 44444444,
"PFN":
"srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz",
"GUID": "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R",
"Md5": None,
"Addler": "92b85e26",
"Attempt": 1,
"Status": "Waiting"
}]
requestContainer.setSubRequestFiles(0, "removal", files)
return {
"requestName": "00000001.xml",
"requestString": requestContainer.toXML()["Value"],
"jobID": 1,
"executionOrder": 0,
"sourceServer": "foobarserver",
"configPath": "/Systems/DataManagement/Development/Agents/RemovalAgent"
}
def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ):
if self.getCSOption( "DelayedExternalDeletion", True ):
gLogger.info( "Setting deletion request" )
try:
request = RequestContainer()
result = request.addSubRequest( { 'Attributes' : { 'Operation' : 'removePhysicalFile',
'TargetSE' : SEName,
'ExecutionOrder' : 1
} },
'removal' )
index = result['Value']
fileDict = { 'PFN' : SEPFN, 'Status' : 'Waiting' }
request.setSubRequestFiles( index, 'removal', [ fileDict ] )
return RequestClient().setRequest( "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() ),
request.toXML()[ 'Value' ] )
except Exception, e:
gLogger.exception( "Exception while setting deletion request" )
return S_ERROR( "Cannot set deletion request: %s" % str( e ) )
def getKwargsRemoveFile():
""" helper fcn to build request """
requestContainer = RequestContainer( init = False )
requestContainer.setJobID( 11111111 )
#requestContainer.setOwnerDN( "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba" )
requestContainer.setOwnerGroup( "lhcb_user" )
requestContainer.setDIRACSetup( "LHCb-Production" )
requestContainer.setSourceComponent( None )
requestContainer.setCreationTime( "0000-00-00 00:00:00" )
requestContainer.setLastUpdate( "2011-12-01 04:57:02" )
requestContainer.setStatus( "Waiting" )
requestContainer.initiateSubRequest( "removal" )
subRequestDict = { "Status" : "Waiting",
"SubRequestID" : 2222222,
"Operation" : "removeFile",
"Arguments" : None,
"ExecutionOrder" : 0,
"SourceSE" : None,
"TargetSE" : "RAL-USER",
"Catalogue" : "LcgFileCatalogCombined",
"CreationTime" : "2011-12-01 04:57:02",
"SubmissionTime" : "2011-12-01 04:57:02",
"LastUpdate" : "2011-12-01 20:14:22" }
requestContainer.setSubRequestAttributes( 0, "removal", subRequestDict )
files = [ { "FileID" : 3333333,
"LFN" : "/lhcb/user/c/cibak/11889/11889410/test.zzz",
"Size" : 44444444,
"PFN" : "srm://srm-lhcb.gridpp.rl.ac.uk/castor/ads.rl.ac.uk/prod/lhcb/user/c/cibak/11889/11889410/test.zzz",
"GUID" : "5P13RD4L-4J5L-3D21-U5P1-3RD4L4J5P13R",
"Md5" : None,
"Addler" : "92b85e26",
"Attempt" : 1,
"Status" : "Waiting" } ]
requestContainer.setSubRequestFiles( 0, "removal", files )
return { "requestName" : "00000001.xml",
"requestString" : requestContainer.toXML()["Value"],
"jobID" : 1,
"executionOrder" : 0,
"sourceServer" : "foobarserver",
"configPath" : "/Systems/DataManagement/Development/Agents/RemovalAgent" }
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
requestType = 'transfer'
requestOperation = 'replicateAndRegister'
try:
requestType, requestOperation = transBody.split( ';' )
except:
pass
for taskID in sortList( taskDict.keys() ):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = RequestContainer( init = False )
subRequestIndex = oRequest.initiateSubRequest( requestType )['Value']
attributeDict = {'Operation':requestOperation, 'TargetSE':paramDict['TargetSE']}
oRequest.setSubRequestAttributes( subRequestIndex, requestType, attributeDict )
files = []
for lfn in paramDict['InputData'].split( ';' ):
files.append( {'LFN':lfn} )
oRequest.setSubRequestFiles( subRequestIndex, requestType, files )
requestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
oRequest.setRequestAttributes( {'RequestName':requestName} )
taskDict[taskID]['TaskObject'] = oRequest.toXML()['Value']
return S_OK( taskDict )
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
request = RequestContainer()
result = request.addSubRequest(
{
'Attributes': {
'Operation': 'removePhysicalFile',
'TargetSE': SEName,
'ExecutionOrder': 1
}
}, 'removal')
index = result['Value']
fileDict = {'PFN': SEPFN, 'Status': 'Waiting'}
request.setSubRequestFiles(index, 'removal', [fileDict])
return RequestClient().setRequest(
"RemoteSBDeletion:%s|%s:%s" % (SEName, SEPFN, time.time()),
request.toXML()['Value'])
except Exception, e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR("Cannot set deletion request: %s" % str(e))
else:
lfnpath = "%s%s" % (path, os.path.basename(appTar))
res = rm.putAndRegister(lfnpath, appTar, ops.getValue('Software/BaseStorageElement',"CERN-SRM"))
if not res['OK']:
return res
request = RequestContainer()
request.setCreationTime()
requestClient = RequestClient()
request.setRequestName('copy_%s' % os.path.basename(appTar).replace(".tgz","").replace(".tar.gz",""))
request.setSourceComponent('ReplicateILCSoft')
copies_at = ops.getValue('Software/CopiesAt',[])
index_copy = 0
for copies in copies_at:
res = request.addSubRequest({'Attributes':{'Operation' : 'replicateAndRegister',
'TargetSE' : copies,
'ExecutionOrder' : index_copy},
'Files':[{'LFN':lfnpath}]},
'transfer')
#res = rm.replicateAndRegister("%s%s"%(path,appTar),"IN2P3-SRM")
if not res['OK']:
return res
index_copy += 1
requestxml = request.toXML()['Value']
if copies_at:
res = requestClient.setRequest(request.getRequestName()['Value'], requestxml)
if not res['OK']:
gLogger.error('Could not set replication request %s' % res['Message'])
return S_OK('Application uploaded')
return S_OK()
def executeRequest( self ):
################################################
# Get a request from request DB
res = self.RequestDBClient.getRequest( 'register' )
if not res['OK']:
gLogger.info( "RegistrationAgent.execute: Failed to get request from database." )
return S_OK()
elif not res['Value']:
gLogger.info( "RegistrationAgent.execute: No requests to be executed found." )
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
try:
jobID = int( res['Value']['JobID'] )
except:
jobID = 0
gLogger.info( "RegistrationAgent.execute: Obtained request %s" % requestName )
result = self.RequestDBClient.getCurrentExecutionOrder( requestName, sourceServer )
if result['OK']:
currentOrder = result['Value']
else:
return S_OK( 'Can not get the request execution order' )
oRequest = RequestContainer( request = requestString )
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests( 'register' )
if not res['OK']:
errStr = "RegistrationAgent.execute: Failed to obtain number of transfer subrequests."
gLogger.error( errStr, res['Message'] )
return S_OK()
gLogger.info( "RegistrationAgent.execute: Found %s sub requests." % res['Value'] )
################################################
# For all the sub-requests in the request
modified = False
for ind in range( res['Value'] ):
gLogger.info( "RegistrationAgent.execute: Processing sub-request %s." % ind )
subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'register' )['Value']
subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
subStatus = subRequestAttributes['Status']
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
subRequestFiles = oRequest.getSubRequestFiles( ind, 'register' )['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a register file operation
if operation == 'registerFile':
gLogger.info( "RegistrationAgent.execute: Attempting to execute %s sub-request." % operation )
diracSE = str( subRequestAttributes['TargetSE'] )
if diracSE == 'SE':
# We do not care about SE, put any there
diracSE = "CERN-FAILOVER"
catalog = subRequestAttributes['Catalogue']
if catalog == "None":
catalog = ''
subrequest_done = True
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile.get( 'LFN', '' )
if lfn: lfn = str( lfn )
physicalFile = subRequestFile.get( 'PFN', '' )
if physicalFile: physicalFile = str( physicalFile )
fileSize = subRequestFile.get( 'Size', 0 )
if fileSize: fileSize = int( fileSize )
fileGuid = subRequestFile.get( 'GUID', '' )
if fileGuid: fileGuid = str( fileGuid )
checksum = subRequestFile.get( 'Addler', '' )
if checksum: checksum = str( checksum )
if catalog == 'BookkeepingDB':
diracSE = 'CERN-HIST'
fileTuple = ( lfn, physicalFile, fileSize, diracSE, fileGuid, checksum )
res = self.ReplicaManager.registerFile( fileTuple, catalog )
print res
if not res['OK']:
self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'RegistrationAgent' )
errStr = "RegistrationAgent.execute: Completely failed to register file."
gLogger.error( errStr, res['Message'] )
subrequest_done = False
elif lfn in res['Value']['Failed'].keys():
self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'RegistrationAgent' )
errStr = "RegistrationAgent.execute: Completely failed to register file."
gLogger.error( errStr, res['Value']['Failed'][lfn] )
subrequest_done = False
else:
self.DataLog.addFileRecord( lfn, 'Register', diracSE, '', 'TransferAgent' )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gLogger.info( "RegistrationAgent.execute: File already completed." )
if subrequest_done:
oRequest.setSubRequestStatus( ind, 'register', 'Done' )
################################################
# If the sub-request is none of the above types
else:
gLogger.error( "RegistrationAgent.execute: Operation not supported.", operation )
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty( ind, 'register' )['Value']:
oRequest.setSubRequestStatus( ind, 'register', 'Done' )
################################################
# If the sub-request is already in terminal state
else:
gLogger.info( "RegistrationAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )
if modified and jobID:
result = self.finalizeRequest( requestName, jobID, sourceServer )
return S_OK()
class RequestTask( object ):
"""
.. class:: RequestTask
Base class for DMS 'transfer', 'removal' and 'register' Requests processing.
This class is meant to be executed as a ProcessTask inside ProcessPool.
The most important and common global DIRAC objects are created in RequestTask constructor.
This includes gLogger, gConfig, gProxyManager, S_OK and S_ERROR. The constructor also
imports a set of common modules: os, sys, re, time and everything from types module.
All other DIRAC tools and clients (i.e. RequestManager) are private in RequestTask class and will
be imported and instantiated on demand during first usage. They are accessible using proxied public methods, i.e.
if you are going to use ReplicaManager just call::
self.replicaManager().someMethod()
All currently proxied tools are::
DataLoggingClient -- self.dataLoggingClient()
ReplicaManager -- self.replicaManager()
RequestClient -- self.requestClient()
StorageFactory -- self.storageFactory()
SubLogger message handles for all levels are also proxied, so you can directly use them in your code, i.e.::
self.info("An info message")
self.debug("This will be shown only in debug")
For handling sub-request one has to register their actions handlers using :self.addOperationAction:
method. This method checks if handler is defined as a method of inherited class and then puts its
definition into internal operation dispatcher dictionary with a key of sub-request's operation name.
Each operation handler should have the signature::
def operationName( self, index, requestObj, subRequestAttrs, subRequestFiles )
where index is a sub-request counter, requestObj is a RequestContainer instance,
subRequestAttrs is a dict with sub-request attributes and subRequestFiles is a dict with
files attached to the sub-request.
Handlers shoudl always return S_OK with value of (modified or not) requestObj, S_ERROR with some
error message otherwise.
Processing of request is done automatically in self.__call__, one doesn't have to worry about changing
credentials, looping over subrequests or request finalizing -- only sub-request processing matters in
the all inherited classes.
Concerning :MonitringClient: (or better known its global instance :gMonitor:), if someone wants to send
some metric over there, she has to put in agent's code registration of activity and then in a particular
task use :RequestTask.addMark: to save monitoring data. All monitored activities are held in
:RequestTask.__monitor: dict which at the end of processing is returned from :RequestTask.__call__:.
The values are then processed and pushed to the gMonitor instance in the default callback function.
"""
## reference to ReplicaManager
__replicaManager = None
## reference to DataLoggingClient
__dataLoggingClient = None
## reference to RequestClient
__requestClient = None
## reference to StotageFactory
__storageFactory = None
## subLogger
__log = None
## request type
__requestType = None
## placeholder for request owner DB
requestOwnerDN = None
## placeholder for Request owner group
requestOwnerGroup = None
## operation dispatcher for SubRequests,
## a dictonary
## "operation" => methodToRun
##
__operationDispatcher = {}
## holder for DataManager proxy file
__dataManagerProxy = None
## monitoring dict
__monitor = {}
def __init__( self, requestString, requestName, executionOrder, jobID, configPath ):
""" c'tor
:param self: self reference
:param str requestString: XML serialised RequestContainer
:param str requestName: request name
:param list executionOrder: request execution order
:param int jobID: jobID
:param str sourceServer: request's source server
:param str configPath: path in CS for parent agent
"""
## fixtures
## python fixtures
import os, os.path, sys, time, re, types
self.makeGlobal( "os", os )
self.makeGlobal( "os.path", os.path )
self.makeGlobal( "sys", sys )
self.makeGlobal( "time", time )
self.makeGlobal( "re", re )
## export all Types from types
[ self.makeGlobal( item, getattr( types, item ) ) for item in dir(types) if "Type" in item ]
## DIRAC fixtures
from DIRAC.FrameworkSystem.Client.Logger import gLogger
self.__log = gLogger.getSubLogger( "%s/%s" % ( self.__class__.__name__, str(requestName) ) )
self.always = self.__log.always
self.notice = self.__log.notice
self.info = self.__log.info
self.debug = self.__log.debug
self.warn = self.__log.warn
self.error = self.__log.error
self.exception = self.__log.exception
self.fatal = self.__log.fatal
from DIRAC import S_OK, S_ERROR
from DIRAC.ConfigurationSystem.Client.Config import gConfig
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getGroupsWithVOMSAttribute
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
## export DIRAC global tools and functions
self.makeGlobal( "S_OK", S_OK )
self.makeGlobal( "S_ERROR", S_ERROR )
self.makeGlobal( "gLogger", gLogger )
self.makeGlobal( "gConfig", gConfig )
self.makeGlobal( "gProxyManager", gProxyManager )
self.makeGlobal( "getGroupsWithVOMSAttribute", getGroupsWithVOMSAttribute )
self.makeGlobal( "gConfigurationData", gConfigurationData )
## save request string
self.requestString = requestString
## build request object
from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer
self.requestObj = RequestContainer( init = False )
self.requestObj.parseRequest( request = self.requestString )
## save request name
self.requestName = requestName
## .. and jobID
self.jobID = jobID
## .. and execution order
self.executionOrder = executionOrder
## save config path
self.__configPath = configPath
## set requestType
self.setRequestType( gConfig.getValue( os.path.join( configPath, "RequestType" ), "" ) )
## get log level
self.__log.setLevel( gConfig.getValue( os.path.join( configPath, self.__class__.__name__, "LogLevel" ), "INFO" ) )
## clear monitoring
self.__monitor = {}
## save DataManager proxy
if "X509_USER_PROXY" in os.environ:
self.info("saving path to current proxy file")
self.__dataManagerProxy = os.environ["X509_USER_PROXY"]
else:
self.error("'X509_USER_PROXY' environment variable not set")
def dataManagerProxy( self ):
""" get dataManagerProxy file
:param self: self reference
"""
return self.__dataManagerProxy
def addMark( self, name, value = 1 ):
""" add mark to __monitor dict
:param self: self reference
:param name: mark name
:param value: value to be
"""
if name not in self.__monitor:
self.__monitor.setdefault( name, 0 )
self.__monitor[name] += value
def monitor( self ):
""" get monitoring dict
:param cls: class reference
"""
return self.__monitor
def makeGlobal( self, objName, objDef ):
""" export :objDef: to global name space using :objName: name
:param self: self reference
:param str objName: symbol name
:param mixed objDef: symbol definition
:throws: NameError if symbol of that name is already in
"""
if objName not in __builtins__:
if type( __builtins__) == type( {} ):
__builtins__[objName] = objDef
else:
setattr( __builtins__, objName, objDef )
return True
def requestType( self ):
""" get request type
:params self: self reference
"""
return self.__requestType
def setRequestType( self, requestType ):
""" set request type
:param self: self reference
"""
self.debug( "Setting requestType to %s" % str(requestType) )
self.__requestType = requestType
@classmethod
def replicaManager( cls ):
""" ReplicaManager getter
:param cls: class reference
"""
if not cls.__replicaManager:
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
cls.__replicaManager = ReplicaManager()
return cls.__replicaManager
@classmethod
def dataLoggingClient( cls ):
""" DataLoggingClient getter
:param cls: class reference
"""
if not cls.__dataLoggingClient:
from DIRAC.DataManagementSystem.Client.DataLoggingClient import DataLoggingClient
cls.__dataLoggingClient = DataLoggingClient()
return cls.__dataLoggingClient
@classmethod
def requestClient( cls ):
""" RequestClient getter
:param cls: class reference
"""
if not cls.__requestClient:
from DIRAC.RequestManagementSystem.Client.RequestClient import RequestClient
cls.__requestClient = RequestClient()
return cls.__requestClient
@classmethod
def storageFactory( cls ):
""" StorageFactory getter
:param cls: class reference
"""
if not cls.__storageFactory:
from DIRAC.Resources.Storage.StorageFactory import StorageFactory
cls.__storageFactory = StorageFactory()
return cls.__storageFactory
def changeProxy( self, ownerDN, ownerGroup ):
""" get proxy from gProxyManager, save it to file
:param self: self reference
:param str ownerDN: request owner DN
:param str ownerGroup: request owner group
:return: S_OK with name of newly created owner proxy file
"""
ownerProxy = gProxyManager.downloadVOMSProxy( str(ownerDN), str(ownerGroup) )
if not ownerProxy["OK"] or not ownerProxy["Value"]:
reason = ownerProxy["Message"] if "Message" in ownerProxy else "No valid proxy found in ProxyManager."
return S_ERROR( "Change proxy error for '%s'@'%s': %s" % ( ownerDN, ownerGroup, reason ) )
ownerProxyFile = ownerProxy["Value"].dumpAllToFile()
if not ownerProxyFile["OK"]:
return S_ERROR( ownerProxyFile["Message"] )
ownerProxyFile = ownerProxyFile["Value"]
os.environ["X509_USER_PROXY"] = ownerProxyFile
return S_OK( ownerProxyFile )
######################################################################
# operationDispatcher
@classmethod
def operationDispatcher( cls ):
""" operation dispatcher getter
:param cls: class reference
"""
return cls.__operationDispatcher
@classmethod
def addOperationAction( cls, operation, methodToRun, overwrite = True ):
""" register handler :methodToRun: for SubRequest operation :operation:
:warn: all handlers should have the same signature
:param self: self reference
:param str operation: SubRequest operation name
:param MethodType methodToRun: handler to be executed for SubRequest
:param bool overwrite: flag to overwrite handler, if already present
:return: S_OK/S_ERROR
Every action handler should return S_OK with of a structure::
{ "OK" : True,
"Value" : requestObj # that has been sent to operation handler
}
otherwise S_ERROR.
"""
if operation in cls.__operationDispatcher and not overwrite:
return S_ERROR("addOperationAction: operation for '%s' is already registered" % operation )
if type(methodToRun) is not MethodType:
return S_ERROR("addOperationAction: wrong type (%s = types.MethodType) for '%s' operation" % \
( str(type(methodToRun)), operation ) )
cls.__operationDispatcher[operation] = methodToRun
return S_OK()
def __call__( self ):
""" generic function to process one Request of a type requestType
This method could be run in a thread.
:param self: self reference
:param str requestType: request type
:return: S_OK/S_ERROR
"""
self.always("executing request %s" % self.requestName )
################################################################
## get ownerDN and ownerGroup
ownerDN = self.requestObj.getAttribute( "OwnerDN" )
if not ownerDN["OK"]:
return ownerDN
ownerDN = ownerDN["Value"]
ownerGroup = self.requestObj.getAttribute( "OwnerGroup" )
if not ownerGroup["OK"]:
return ownerGroup
ownerGroup = ownerGroup["Value"]
## save request owner
self.requestOwnerDN = ownerDN if ownerDN else ""
self.requestOwnerGroup = ownerGroup if ownerGroup else ""
#################################################################
## change proxy
ownerProxyFile = None
if ownerDN and ownerGroup:
ownerProxyFile = self.changeProxy( ownerDN, ownerGroup )
if not ownerProxyFile["OK"]:
self.error( "handleReuqest: unable to get proxy for '%s'@'%s': %s" % ( ownerDN,
ownerGroup,
ownerProxyFile["Message"] ) )
update = self.putBackRequest( self.requestName, self.requestString )
if not update["OK"]:
self.error( "handleRequest: error when updating request: %s" % update["Message"] )
return update
return ownerProxyFile
ownerProxyFile = ownerProxyFile["Value"]
#self.ownerProxyFile = ownerProxyFile
self.info( "Will execute request for '%s'@'%s' using proxy file %s" % ( ownerDN, ownerGroup, ownerProxyFile ) )
else:
self.info( "Will execute request for DataManager using her/his proxy")
#################################################################
## execute handlers
ret = { "OK" : False, "Message" : "" }
useServerCert = gConfig.useServerCertificate()
try:
# Execute task with the owner proxy even for contacting DIRAC services
if useServerCert:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','false')
ret = self.handleRequest()
finally:
if useServerCert:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','true')
## delete owner proxy
if self.__dataManagerProxy:
os.environ["X509_USER_PROXY"] = self.__dataManagerProxy
if ownerProxyFile and os.path.exists( ownerProxyFile ):
os.unlink( ownerProxyFile )
if not ret["OK"]:
self.error( "handleRequest: error during request processing: %s" % ret["Message"] )
self.error( "handleRequest: will put original request back" )
update = self.putBackRequest( self.requestName, self.requestString )
if not update["OK"]:
self.error( "handleRequest: error when putting back request: %s" % update["Message"] )
## return at least
return ret
def handleRequest( self ):
""" read SubRequests and ExecutionOrder, fire registered handlers upon SubRequests operations
:param self: self reference
:param dict requestDict: request dictionary as read from self.readRequest
"""
##############################################################
# here comes the processing
##############################################################
res = self.requestObj.getNumSubRequests( self.__requestType )
if not res["OK"]:
errMsg = "handleRequest: failed to obtain number of '%s' subrequests." % self.__requestType
self.error( errMsg, res["Message"] )
return S_ERROR( res["Message"] )
## for gMonitor
self.addMark( "Execute", 1 )
## process sub requests
for index in range( res["Value"] ):
self.info( "handleRequest: processing subrequest %s." % str(index) )
subRequestAttrs = self.requestObj.getSubRequestAttributes( index, self.__requestType )["Value"]
if subRequestAttrs["ExecutionOrder"]:
subExecutionOrder = int( subRequestAttrs["ExecutionOrder"] )
else:
subExecutionOrder = 0
subRequestStatus = subRequestAttrs["Status"]
if subRequestStatus != "Waiting":
self.info( "handleRequest: subrequest %s has status '%s' and is not to be executed." % ( str(index),
subRequestStatus ) )
continue
if subExecutionOrder <= self.executionOrder:
operation = subRequestAttrs["Operation"]
if operation not in self.operationDispatcher():
self.error( "handleRequest: '%s' operation not supported" % operation )
else:
self.info( "handleRequest: will execute %s '%s' subrequest" % ( str(index), operation ) )
## get files
subRequestFiles = self.requestObj.getSubRequestFiles( index, self.__requestType )["Value"]
## execute operation action
ret = self.operationDispatcher()[operation].__call__( index,
self.requestObj,
subRequestAttrs,
subRequestFiles )
################################################
## error in operation action?
if not ret["OK"]:
self.error( "handleRequest: error when handling subrequest %s: %s" % ( str(index), ret["Message"] ) )
self.requestObj.setSubRequestAttributeValue( index, self.__requestType, "Error", ret["Message"] )
else:
## update ref to requestObj
self.requestObj = ret["Value"]
## check if subrequest status == Done, disable finalisation if not
subRequestDone = self.requestObj.isSubRequestDone( index, self.__requestType )
if not subRequestDone["OK"]:
self.error( "handleRequest: unable to determine subrequest status: %s" % subRequestDone["Message"] )
else:
if not subRequestDone["Value"]:
self.warn("handleRequest: subrequest %s is not done yet" % str(index) )
################################################
# Generate the new request string after operation
newRequestString = self.requestObj.toXML()['Value']
update = self.putBackRequest( self.requestName, newRequestString )
if not update["OK"]:
self.error( "handleRequest: error when updating request: %s" % update["Message"] )
return update
## get request status
if self.jobID:
requestStatus = self.requestClient().getRequestStatus( self.requestName )
if not requestStatus["OK"]:
return requestStatus
requestStatus = requestStatus["Value"]
## finalize request if jobID is present and request status = 'Done'
self.info("handleRequest: request status is %s" % requestStatus )
if ( requestStatus["RequestStatus"] == "Done" ) and ( requestStatus["SubRequestStatus"] not in ( "Waiting", "Assigned" ) ):
self.debug("handleRequest: request is going to be finalised")
finalize = self.requestClient().finalizeRequest( self.requestName, self.jobID )
if not finalize["OK"]:
self.error("handleRequest: error in request finalization: %s" % finalize["Message"] )
return finalize
self.info("handleRequest: request is finalised")
## for gMonitor
self.addMark( "Done", 1 )
## should return S_OK with monitor dict
return S_OK( { "monitor" : self.monitor() } )
def putBackRequest( self, requestName, requestString ):
""" put request back
:param self: self reference
:param str requestName: request name
:param str requestString: XML-serilised request
:param str sourceServer: request server URL
"""
update = self.requestClient().updateRequest( requestName, requestString )
if not update["OK"]:
self.error( "putBackRequest: error when updating request: %s" % update["Message"] )
return update
return S_OK()
def execute(self):
""" The main agent execution method """
# This allows dynamic changing of the throughput timescale
self.throughputTimescale = self.am_getOption('ThroughputTimescale',
3600)
self.throughputTimescale = 60 * 60 * 1
#print 'ThroughputTimescale:',self.throughputTimescale
######################################################################################
#
# Obtain information on the current state of the channel queues
#
res = self.TransferDB.getChannelQueues()
if not res['OK']:
errStr = "ReplicationScheduler._execute: Failed to get channel queues from TransferDB."
gLogger.error(errStr, res['Message'])
return S_OK()
if not res['Value']:
gLogger.info(
"ReplicationScheduler._execute: No active channels found for replication."
)
return S_OK()
channels = res['Value']
res = self.TransferDB.getChannelObservedThroughput(
self.throughputTimescale)
if not res['OK']:
errStr = "ReplicationScheduler._execute: Failed to get observed throughput from TransferDB."
gLogger.error(errStr, res['Message'])
return S_OK()
if not res['Value']:
gLogger.info(
"ReplicationScheduler._execute: No active channels found for replication."
)
return S_OK()
bandwidths = res['Value']
self.strategyHandler = StrategyHandler(bandwidths, channels,
self.section)
processedRequests = []
requestsPresent = True
while requestsPresent:
######################################################################################
#
# The first step is to obtain a transfer request from the RequestDB which should be scheduled.
#
gLogger.info(
"ReplicationScheduler._execute: Contacting RequestDB for suitable requests."
)
res = self.RequestDB.getRequest('transfer')
if not res['OK']:
gLogger.error(
"ReplicationScheduler._execute: Failed to get a request list from RequestDB.",
res['Message'])
continue
if not res['Value']:
gLogger.info(
"ReplicationScheduler._execute: No requests found in RequestDB."
)
requestsPresent = False
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
gLogger.info(
"ReplicationScheduler._execute: Obtained Request %s from RequestDB."
% (requestName))
######################################################################################
#
# The request must then be parsed to obtain the sub-requests, their attributes and files.
#
logStr = 'ReplicationScheduler._execute: Parsing Request %s.' % (
requestName)
gLogger.info(logStr)
oRequest = RequestContainer(requestString)
res = oRequest.getAttribute('RequestID')
if not res['OK']:
gLogger.error(
'ReplicationScheduler._execute: Failed to get requestID.',
res['Message'])
return S_ERROR(
'ReplicationScheduler._execute: Failed to get number of sub-requests.'
)
requestID = res['Value']
if requestID in processedRequests:
# Break the loop once we have iterated once over all requests
res = self.RequestDB.updateRequest(requestName, requestString)
if not res['OK']:
gLogger.error("Failed to update request",
"%s %s" % (requestName, res['Message']))
return S_OK()
processedRequests.append(requestID)
res = oRequest.getNumSubRequests('transfer')
if not res['OK']:
gLogger.error(
'ReplicationScheduler._execute: Failed to get number of sub-requests.',
res['Message'])
return S_ERROR(
'ReplicationScheduler._execute: Failed to get number of sub-requests.'
)
numberRequests = res['Value']
gLogger.info(
"ReplicationScheduler._execute: '%s' found with %s sub-requests."
% (requestName, numberRequests))
######################################################################################
#
# The important request attributes are the source and target SEs.
#
for ind in range(numberRequests):
gLogger.info(
"ReplicationScheduler._execute: Treating sub-request %s from '%s'."
% (ind, requestName))
attributes = oRequest.getSubRequestAttributes(
ind, 'transfer')['Value']
if attributes['Status'] != 'Waiting':
# If the sub-request is already in terminal state
gLogger.info(
"ReplicationScheduler._execute: Sub-request %s is status '%s' and not to be executed."
% (ind, attributes['Status']))
continue
sourceSE = attributes['SourceSE']
targetSE = attributes['TargetSE']
""" This section should go in the transfer request class """
if type(targetSE) in types.StringTypes:
if re.search(',', targetSE):
targetSEs = targetSE.split(',')
else:
targetSEs = [targetSE]
"""----------------------------------------------------- """
operation = attributes['Operation']
reqRepStrategy = None
if operation in self.strategyHandler.getSupportedStrategies():
reqRepStrategy = operation
######################################################################################
#
# Then obtain the file attribute of interest are the LFN and FileID
#
res = oRequest.getSubRequestFiles(ind, 'transfer')
if not res['OK']:
gLogger.error(
'ReplicationScheduler._execute: Failed to obtain sub-request files.',
res['Message'])
continue
files = res['Value']
gLogger.info(
"ReplicationScheduler._execute: Sub-request %s found with %s files."
% (ind, len(files)))
filesDict = {}
for file in files:
lfn = file['LFN']
if file['Status'] != 'Waiting':
gLogger.debug(
"ReplicationScheduler._execute: %s will not be scheduled because it is %s."
% (lfn, file['Status']))
else:
fileID = file['FileID']
filesDict[lfn] = fileID
if not filesDict:
gLogger.info(
"ReplicationScheduler._execute: No Waiting files found for request"
)
continue
notSched = len(files) - len(filesDict)
if notSched:
gLogger.info(
"ReplicationScheduler._execute: %d files found not Waiting"
% notSched)
######################################################################################
#
# Now obtain replica information for the files associated to the sub-request.
#
lfns = filesDict.keys()
gLogger.info(
"ReplicationScheduler._execute: Obtaining replica information for %d sub-request files."
% len(lfns))
res = self.rm.getCatalogReplicas(lfns)
if not res['OK']:
gLogger.error(
"ReplicationScheduler._execute: Failed to get replica information.",
res['Message'])
continue
for lfn, failure in res['Value']['Failed'].items():
gLogger.error(
"ReplicationScheduler._execute: Failed to get replicas.",
'%s: %s' % (lfn, failure))
replicas = res['Value']['Successful']
if not replicas.keys():
gLogger.error(
"ReplicationScheduler._execute: Failed to get replica information for all files."
)
continue
######################################################################################
#
# Now obtain the file sizes for the files associated to the sub-request.
#
lfns = replicas.keys()
gLogger.info(
"ReplicationScheduler._execute: Obtaining file sizes for %d sub-request files."
% len(lfns))
res = self.rm.getCatalogFileMetadata(lfns)
if not res['OK']:
gLogger.error(
"ReplicationScheduler._execute: Failed to get file size information.",
res['Message'])
continue
for lfn, failure in res['Value']['Failed'].items():
gLogger.error(
'ReplicationScheduler._execute: Failed to get file size.',
'%s: %s' % (lfn, failure))
metadata = res['Value']['Successful']
if not metadata.keys():
gLogger.error(
"ReplicationScheduler._execute: Failed to get metadata for all files."
)
continue
######################################################################################
#
# For each LFN determine the replication tree
#
for lfn in sortList(metadata.keys()):
fileSize = metadata[lfn]['Size']
lfnReps = replicas[lfn]
fileID = filesDict[lfn]
targets = []
for targetSE in targetSEs:
if targetSE in lfnReps.keys():
gLogger.debug(
"ReplicationScheduler.execute: %s already present at %s."
% (lfn, targetSE))
else:
targets.append(targetSE)
if not targets:
gLogger.info(
"ReplicationScheduler.execute: %s present at all targets."
% lfn)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Done')
continue
if not lfnReps:
gLogger.error(
"ReplicationScheduler.execute: The file has no replicas.",
lfn)
continue
res = self.strategyHandler.determineReplicationTree(
sourceSE,
targets,
lfnReps,
fileSize,
strategy=reqRepStrategy)
if not res['OK']:
gLogger.error(
"ReplicationScheduler.execute: Failed to determine replication tree.",
res['Message'])
continue
tree = res['Value']
######################################################################################
#
# For each item in the replication tree obtain the source and target SURLS
#
for channelID, dict in tree.items():
gLogger.info(
"ReplicationScheduler.execute: processing for channel %d %s"
% (channelID, str(dict)))
hopSourceSE = dict['SourceSE']
hopDestSE = dict['DestSE']
hopAncestor = dict['Ancestor']
# Get the sourceSURL
if hopAncestor:
status = 'Waiting%s' % (hopAncestor)
res = self.obtainLFNSURL(hopSourceSE, lfn)
if not res['OK']:
errStr = res['Message']
gLogger.error(errStr)
return S_ERROR(errStr)
sourceSURL = res['Value']
else:
status = 'Waiting'
res = self.resolvePFNSURL(hopSourceSE,
lfnReps[hopSourceSE])
if not res['OK']:
sourceSURL = lfnReps[hopSourceSE]
else:
sourceSURL = res['Value']
# Get the targetSURL
res = self.obtainLFNSURL(hopDestSE, lfn)
if not res['OK']:
errStr = res['Message']
gLogger.error(errStr)
return S_ERROR(errStr)
targetSURL = res['Value']
######################################################################################
#
# For each item in the replication tree add the file to the channel
#
res = self.TransferDB.addFileToChannel(
channelID,
fileID,
hopSourceSE,
sourceSURL,
hopDestSE,
targetSURL,
fileSize,
fileStatus=status)
if not res['OK']:
errStr = res['Message']
gLogger.error(
"ReplicationScheduler._execute: Failed to add File to Channel.",
"%s %s" % (fileID, channelID))
return S_ERROR(errStr)
res = self.TransferDB.addFileRegistration(
channelID, fileID, lfn, targetSURL, hopDestSE)
if not res['OK']:
errStr = res['Message']
gLogger.error(
"ReplicationScheduler._execute: Failed to add File registration.",
"%s %s" % (fileID, channelID))
result = self.TransferDB.removeFileFromChannel(
channelID, fileID)
if not result['OK']:
errStr += result['Message']
gLogger.error(
"ReplicationScheduler._execute: Failed to remove File.",
"%s %s" % (fileID, channelID))
return S_ERROR(errStr)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Scheduled')
res = self.TransferDB.addReplicationTree(fileID, tree)
if oRequest.isSubRequestEmpty(ind, 'transfer')['Value']:
oRequest.setSubRequestStatus(ind, 'transfer', 'Scheduled')
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDB.updateRequest(requestName, requestString)
if not res['OK']:
gLogger.error(
"ReplicationScheduler._execute: Failed to update request",
"%s %s" % (requestName, res['Message']))
class FailoverRequest(ModuleBase):
""" Handle the failover requests issued by previous steps. Used in production.
"""
#############################################################################
def __init__(self):
"""Module initialization.
"""
super(FailoverRequest, self).__init__()
self.version = __RCSID__
self.log = gLogger.getSubLogger( "FailoverRequest" )
#Internal parameters
self.enable = True
self.jobID = ''
self.productionID = None
self.prodJobID = None
#Workflow parameters
self.jobReport = None
self.fileReport = None
self.request = None
#############################################################################
def applicationSpecificInputs(self):
""" By convention the module input parameters are resolved here.
"""
self.log.debug(self.workflow_commons)
self.log.debug(self.step_commons)
if os.environ.has_key('JOBID'):
self.jobID = os.environ['JOBID']
self.log.verbose('Found WMS JobID = %s' %self.jobID)
else:
self.log.info('No WMS JobID found, disabling module via control flag')
self.enable = False
if self.step_commons.has_key('Enable'):
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
self.enable = False
#Earlier modules will have populated the report objects
if self.workflow_commons.has_key('JobReport'):
self.jobReport = self.workflow_commons['JobReport']
if self.workflow_commons.has_key('FileReport'):
self.fileReport = self.workflow_commons['FileReport']
if self.InputData:
if type(self.InputData) != type([]):
self.InputData = self.InputData.split(';')
self.InputData = [x.replace('LFN:','') for x in self.InputData]
if self.workflow_commons.has_key('Request'):
self.request = self.workflow_commons['Request']
if not self.request:
self.request = RequestContainer()
self.request.setRequestName('job_%s_request.xml' % self.jobID)
self.request.setJobID(self.jobID)
self.request.setSourceComponent("Job_%s" % self.jobID)
if self.workflow_commons.has_key('PRODUCTION_ID'):
self.productionID = self.workflow_commons['PRODUCTION_ID']
if self.workflow_commons.has_key('JOB_ID'):
self.prodJobID = self.workflow_commons['JOB_ID']
return S_OK('Parameters resolved')
#############################################################################
def execute(self):
""" Main execution function.
"""
self.log.info('Initializing %s' % self.version)
result = self.resolveInputVariables()
if not result['OK']:
self.log.error(result['Message'])
return result
if not self.fileReport:
self.fileReport = FileReport('Transformation/TransformationManager')
if self.InputData:
inputFiles = self.fileReport.getFiles()
for lfn in self.InputData:
if not lfn in inputFiles:
self.log.verbose('No status populated for input data %s, setting to "Unused"' % lfn)
result = self.fileReport.setFileStatus(int(self.productionID), lfn, 'Unused')
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.info('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
inputFiles = self.fileReport.getFiles()
for lfn in inputFiles:
if inputFiles[lfn] != 'ApplicationCrash':
self.log.info('Forcing status to "Unused" due to workflow failure for: %s' % (lfn))
self.fileReport.setFileStatus(int(self.productionID), lfn, 'Unused')
else:
inputFiles = self.fileReport.getFiles()
if inputFiles:
self.log.info('Workflow status OK, setting input file status to Processed')
for lfn in inputFiles:
self.log.info('Setting status to "Processed" for: %s' % (lfn))
self.fileReport.setFileStatus(int(self.productionID), lfn, 'Processed')
result = self.fileReport.commit()
if not result['OK']:
self.log.error('Failed to report file status to ProductionDB, request will be generated', result['Message'])
else:
self.log.info('Status of files have been properly updated in the ProcessingDB')
# Must ensure that the local job report instance is used to report the final status
# in case of failure and a subsequent failover operation
if self.workflowStatus['OK'] and self.stepStatus['OK']:
if not self.jobReport:
self.jobReport = JobReport(int(self.jobID))
jobStatus = self.jobReport.setApplicationStatus('Job Finished Successfully')
if not jobStatus['OK']:
self.log.warn(jobStatus['Message'])
# Retrieve the accumulated reporting request
reportRequest = None
if self.jobReport:
result = self.jobReport.generateRequest()
if not result['OK']:
self.log.warn('Could not generate request for job report with result:\n%s' % (result))
else:
reportRequest = result['Value']
if reportRequest:
self.log.info('Populating request with job report information')
self.request.update(reportRequest)
fileReportRequest = None
if self.fileReport:
result = self.fileReport.generateRequest()
if not result['OK']:
self.log.warn('Could not generate request for file report with result:\n%s' % (result))
else:
fileReportRequest = result['Value']
if fileReportRequest:
self.log.info('Populating request with file report information')
result = self.request.update(fileReportRequest)
accountingReport = None
if self.workflow_commons.has_key('AccountingReport'):
accountingReport = self.workflow_commons['AccountingReport']
if accountingReport:
result = accountingReport.commit()
if not result['OK']:
self.log.info('Populating request with accounting report information')
self.request.setDISETRequest(result['rpcStub'])
if self.request.isEmpty()['Value']:
self.log.info('Request is empty, nothing to do.')
return self.finalize()
request_string = self.request.toXML()['Value']
self.log.debug(request_string)
# Write out the request string
fname = '%s_%s_request.xml' % (self.productionID, self.prodJobID)
xmlfile = open(fname, 'w')
xmlfile.write(request_string)
xmlfile.close()
self.log.info('Creating failover request for deferred operations for job %s:' % self.jobID)
result = self.request.getDigest()
if result['OK']:
digest = result['Value']
self.log.info(digest)
if not self.enable:
self.log.info('Module is disabled by control flag')
return S_OK('Module is disabled by control flag')
return self.finalize()
#############################################################################
def finalize(self):
""" Finalize and report correct status for the workflow based on the workflow
or step status.
"""
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.warn('Workflow status is not ok, will not overwrite status')
self.log.info('Workflow failed, end of FailoverRequest module execution.')
return S_ERROR('Workflow failed, FailoverRequest module completed')
self.log.info('Workflow successful, end of FailoverRequest module execution.')
return S_OK('FailoverRequest module completed')
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#
def execute( self ):
IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
res = self.RequestDBClient.getRequest( 'integrity' )
if not res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Failed to get request from database." )
return S_OK()
elif not res['Value']:
gLogger.info( "SEvsLFCAgent.execute: No requests to be executed found." )
return S_OK()
requestString = res['Value']['requestString']
requestName = res['Value']['requestName']
sourceServer = res['Value']['Server']
gLogger.info( "SEvsLFCAgent.execute: Obtained request %s" % requestName )
oRequest = RequestContainer( request = requestString )
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests( 'integrity' )
if not res['OK']:
errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests."
gLogger.error( errStr, res['Message'] )
return S_OK()
gLogger.info( "SEvsLFCAgent.execute: Found %s sub requests." % res['Value'] )
################################################
# For all the sub-requests in the request
for ind in range( res['Value'] ):
gLogger.info( "SEvsLFCAgent.execute: Processing sub-request %s." % ind )
subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value']
if subRequestAttributes['Status'] == 'Waiting':
subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a lfcvsse operation
if operation == 'SEvsLFC':
gLogger.info( "SEvsLFCAgent.execute: Attempting to execute %s sub-request." % operation )
storageElementName = subRequestAttributes['StorageElement']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile['LFN']
storageElement = StorageElement( storageElementName )
res = storageElement.isValid()
if not res['OK']:
errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement."
gLogger.error( errStr, storageElement )
else:
res = storageElement.getPfnForLfn( lfn )
if not res['OK']:
gLogger.info( 'shit bugger do something.' )
else:
oNamespaceBrowser = NamespaceBrowser( res['Value'] )
# Loop over all the directories and sub-directories
while ( oNamespaceBrowser.isActive() ):
currentDir = oNamespaceBrowser.getActiveDir()
gLogger.info( "SEvsLFCAgent.execute: Attempting to list the contents of %s." % currentDir )
res = storageElement.listDirectory( currentDir )
if not res['Value']['Successful'].has_key( currentDir ):
gLogger.error( "SEvsLFCAgent.execute: Failed to list the directory contents.", "%s %s" % ( currentDir, res['Value']['Successful']['Failed'][currentDir] ) )
subDirs = [currentDir]
else:
subDirs = []
files = {}
for surl, surlDict in res['Value']['Successful'][currentDir]['Files'].items():
pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False )
surl = pfnRes['Value']
files[surl] = surlDict
for surl, surlDict in res['Value']['Successful'][currentDir]['SubDirs'].items():
pfnRes = storageElement.getPfnForProtocol( surl, 'SRM2', withPort = False )
surl = pfnRes['Value']
subDirs.append( surl )
#subDirs = res['Value']['Successful'][currentDir]['SubDirs']
gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s sub-directories." % len( subDirs ) )
#files = res['Value']['Successful'][currentDir]['Files']
gLogger.info( "SEvsLFCAgent.execute: Successfully obtained %s files." % len( files ) )
selectedLfns = []
lfnPfnDict = {}
pfnSize = {}
for pfn, pfnDict in files.items():
res = storageElement.getPfnPath( pfn )
if not res['OK']:
gLogger.error( "SEvsLFCAgent.execute: Failed to get determine LFN from pfn.", "%s %s" % ( pfn, res['Message'] ) )
fileMetadata = {'Prognosis':'NonConventionPfn', 'LFN':'', 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnDict['Size']}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
else:
lfn = res['Value']
selectedLfns.append( lfn )
lfnPfnDict[lfn] = pfn
pfnSize[pfn] = pfnDict['Size']
res = self.ReplicaManager.getCatalogFileMetadata( selectedLfns )
if not res['OK']:
subDirs = [currentDir]
else:
for lfn in res['Value']['Failed'].keys():
gLogger.error( "SEvsLFCAgent.execute: Failed to get metadata.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) )
pfn = lfnPfnDict[lfn]
fileMetadata = {'Prognosis':'SEPfnNoLfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
for lfn, lfnDict in res['Value']['Successful'].items():
pfn = lfnPfnDict[lfn]
storageSize = pfnSize[pfn]
catalogSize = lfnDict['Size']
if int( catalogSize ) == int( storageSize ):
gLogger.info( "SEvsLFCAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) )
gLogger.info( "Change the status in the LFC" )
elif int( storageSize ) == 0:
gLogger.error( "SEvsLFCAgent.execute: Physical file size is 0.", "%s %s" % ( pfn, storageElementName ) )
fileMetadata = {'Prognosis':'ZeroSizePfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
else:
gLogger.error( "SEvsLFCAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) )
fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
res = self.ReplicaManager.getCatalogReplicas( selectedLfns )
if not res['OK']:
subDirs = [currentDir]
else:
for lfn in res['Value']['Failed'].keys():
gLogger.error( "SEvsLFCAgent.execute: Failed to get replica information.", "%s %s" % ( lfn, res['Value']['Failed'][lfn] ) )
pfn = lfnPfnDict[lfn]
fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':pfnSize[pfn]}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
for lfn, repDict in res['Value']['Successful'].items():
pfn = lfnPfnDict[lfn]
registeredPfns = repDict.values()
if not pfn in registeredPfns:
gLogger.error( "SEvsLFCAgent.execute: SE PFN not registered.", "%s %s" % ( lfn, pfn ) )
fileMetadata = {'Prognosis':'PfnNoReplica', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
res = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if res['OK']:
gLogger.info( "SEvsLFCAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
else:
gLogger.info( "SEvsLFCAgent.execute: SE Pfn verified.", pfn )
oNamespaceBrowser.updateDirs( subDirs )
oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done' )
################################################
# If the sub-request is none of the above types
else:
gLogger.info( "SEvsLFCAgent.execute: Operation not supported.", operation )
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty( ind, 'integrity' )['Value']:
oRequest.setSubRequestStatus( ind, 'integrity', 'Done' )
################################################
# If the sub-request is already in terminal state
else:
gLogger.info( "SEvsLFCAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )
return S_OK()
def executeRequest( self ):
"""
Do the actual work in the Thread
"""
################################################
# Get a request from request DB
gMonitor.addMark( "Iteration", 1 )
res = self.requestDBClient.getRequest( 'removal' )
if not res['OK']:
gLogger.info( "RemovalAgent.execute: Failed to get request from database." )
return S_OK()
elif not res['Value']:
gLogger.info( "RemovalAgent.execute: No requests to be executed found." )
self.pendingRequests = False
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
try:
jobID = int( res['Value']['JobID'] )
except ValueError:
jobID = 0
gLogger.info( "RemovalAgent.execute: Obtained request %s" % requestName )
try:
result = self.requestDBClient.getCurrentExecutionOrder( requestName, sourceServer )
if result['OK']:
currentOrder = result['Value']
else:
gLogger.error( 'Can not get the request execution order' )
self.requestDBClient.updateRequest( requestName, requestString, sourceServer )
return S_OK( 'Can not get the request execution order' )
oRequest = RequestContainer( request = requestString )
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests( 'removal' )
if not res['OK']:
errStr = "RemovalAgent.execute: Failed to obtain number of removal subrequests."
gLogger.error( errStr, res['Message'] )
return S_OK()
gLogger.info( "RemovalAgent.execute: Found %s sub requests." % res['Value'] )
################################################
# For all the sub-requests in the request
modified = False
for ind in range( res['Value'] ):
gMonitor.addMark( "Execute", 1 )
gLogger.info( "RemovalAgent.execute: Processing sub-request %s." % ind )
subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'removal' )['Value']
subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
subStatus = subRequestAttributes['Status']
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
subRequestFiles = oRequest.getSubRequestFiles( ind, 'removal' )['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a physical removal operation
if operation == 'physicalRemoval':
gLogger.info( "RemovalAgent.execute: Attempting to execute %s sub-request." % operation )
diracSEs = subRequestAttributes['TargetSE'].split( ',' )
physicalFiles = []
pfnToLfn = {}
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
pfn = str( subRequestFile['PFN'] )
lfn = str( subRequestFile['LFN'] )
pfnToLfn[pfn] = lfn
physicalFiles.append( pfn )
gMonitor.addMark( 'PhysicalRemovalAtt', len( physicalFiles ) )
failed = {}
errMsg = {}
for diracSE in diracSEs:
res = self.replicaManager.removeStorageFile( physicalFiles, diracSE )
if res['OK']:
for pfn in res['Value']['Failed'].keys():
if not failed.has_key( pfn ):
failed[pfn] = {}
failed[pfn][diracSE] = res['Value']['Failed'][pfn]
else:
errMsg[diracSE] = res['Message']
for pfn in physicalFiles:
if not failed.has_key( pfn ):
failed[pfn] = {}
failed[pfn][diracSE] = 'Completely'
# Now analyse the results
failedPFNs = failed.keys()
pfnsOK = [pfn for pfn in physicalFiles if not pfn in failedPFNs]
gMonitor.addMark( 'PhysicalRemovalDone', len( pfnsOK ) )
for pfn in pfnsOK:
gLogger.info( "RemovalAgent.execute: Successfully removed %s at %s" % ( pfn, str( diracSEs ) ) )
res = oRequest.setSubRequestFileAttributeValue( ind, 'removal', pfnToLfn[pfn], 'Status', 'Done' )
if not res['OK']:
gLogger.error( "RemovalAgent.execute: Error setting status to %s for %s" % ( 'Done', pfnToLfn[pfn] ) )
modified = True
if failed:
gMonitor.addMark( 'PhysicalRemovalFail', len( failedPFNs ) )
for pfn in failedPFNs:
for diracSE in failed[pfn].keys():
if type( failed[pfn][diracSE] ) in StringTypes:
if re.search( 'no such file or directory', failed[pfn][diracSE].lower() ):
gLogger.info( "RemovalAgent.execute: File did not exist.", pfn )
res = oRequest.setSubRequestFileAttributeValue( ind, 'removal', pfnToLfn[pfn], 'Status', 'Done' )
if not res['OK']:
gLogger.error( "RemovalAgent.execute: Error setting status to %s for %s" % ( 'Done', pfnToLfn[pfn] ) )
modified = True
else:
gLogger.info( "RemovalAgent.execute: Failed to remove file.", "%s at %s - %s" % ( pfn, diracSE, failed[pfn][diracSE] ) )
if errMsg:
for diracSE in errMsg.keys():
errStr = "RemovalAgent.execute: Completely failed to remove replicas. At %s", diracSE
gLogger.error( errStr, errMsg[diracSE] )
################################################
# If the sub-request is a physical removal operation
elif operation == 'removeFile':
gLogger.info( "RemovalAgent.execute: Attempting to execute %s sub-request." % operation )
lfns = []
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = str( subRequestFile['LFN'] )
lfns.append( lfn )
gMonitor.addMark( 'RemoveFileAtt', len( lfns ) )
res = self.replicaManager.removeFile( lfns )
if res['OK']:
gMonitor.addMark( 'RemoveFileDone', len( res['Value']['Successful'].keys() ) )
for lfn in res['Value']['Successful'].keys():
gLogger.info( "RemovalAgent.execute: Successfully removed %s." % lfn )
result = oRequest.setSubRequestFileAttributeValue( ind, 'removal', lfn, 'Status', 'Done' )
if not result['OK']:
gLogger.error( "RemovalAgent.execute: Error setting status to %s for %s" % ( 'Done', lfn ) )
modified = True
gMonitor.addMark( 'RemoveFileFail', len( res['Value']['Failed'].keys() ) )
for lfn in res['Value']['Failed'].keys():
if type( res['Value']['Failed'][lfn] ) in StringTypes:
if re.search( 'no such file or directory', res['Value']['Failed'][lfn].lower() ):
gLogger.info( "RemovalAgent.execute: File did not exist.", lfn )
result = oRequest.setSubRequestFileAttributeValue( ind, 'removal', lfn, 'Status', 'Done' )
if not result['OK']:
gLogger.error( "RemovalAgent.execute: Error setting status to %s for %s" % ( 'Done', lfn ) )
modified = True
else:
gLogger.info( "RemovalAgent.execute: Failed to remove file:",
"%s %s" % ( lfn, res['Value']['Failed'][lfn] ) )
else:
gMonitor.addMark( 'RemoveFileFail', len( lfns ) )
errStr = "RemovalAgent.execute: Completely failed to remove files files."
gLogger.error( errStr, res['Message'] )
################################################
# If the sub-request is a physical removal operation
elif operation == 'replicaRemoval':
gLogger.info( "RemovalAgent.execute: Attempting to execute %s sub-request." % operation )
diracSEs = subRequestAttributes['TargetSE'].split( ',' )
lfns = []
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = str( subRequestFile['LFN'] )
lfns.append( lfn )
gMonitor.addMark( 'ReplicaRemovalAtt', len( lfns ) )
failed = {}
errMsg = {}
for diracSE in diracSEs:
res = self.replicaManager.removeReplica( diracSE, lfns )
if res['OK']:
for lfn in res['Value']['Failed'].keys():
errorMessage = str( res['Value']['Failed'][lfn] )
if errorMessage.find( 'Write access not permitted for this credential.' ) != -1:
if self.__getProxyAndRemoveReplica( diracSE, lfn ):
continue
if errorMessage.find( 'seconds timeout for "__gfal_wrapper" call' ) != -1:
self.timeOutCounter += 1
if not failed.has_key( lfn ):
failed[lfn] = {}
failed[lfn][diracSE] = res['Value']['Failed'][lfn]
else:
errMsg[diracSE] = res['Message']
for lfn in lfns:
if not failed.has_key( lfn ):
failed[lfn] = {}
failed[lfn][diracSE] = 'Completely'
# Now analyse the results
failedLFNs = failed.keys()
lfnsOK = [lfn for lfn in lfns if not lfn in failedLFNs]
gMonitor.addMark( 'ReplicaRemovalDone', len( lfnsOK ) )
for lfn in lfnsOK:
gLogger.info( "RemovalAgent.execute: Successfully removed %s at %s" % ( lfn, str( diracSEs ) ) )
res = oRequest.setSubRequestFileAttributeValue( ind, 'removal', lfn, 'Status', 'Done' )
if not res['OK']:
gLogger.error( "RemovalAgent.execute: Error setting status to %s for %s" % ( 'Done', lfn ) )
modified = True
if failed:
gMonitor.addMark( 'PhysicalRemovalFail', len( failedLFNs ) )
for lfn in failedLFNs:
for diracSE in failed[lfn].keys():
if type( failed[lfn][diracSE] ) in StringTypes:
if re.search( 'no such file or directory', failed[lfn][diracSE].lower() ):
gLogger.info( "RemovalAgent.execute: File did not exist.", lfn )
res = oRequest.setSubRequestFileAttributeValue( ind, 'removal', lfn, 'Status', 'Done' )
if not res['OK']:
gLogger.error( "RemovalAgent.execute: Error setting status to %s for %s" % ( 'Done', lfn ) )
modified = True
else:
gLogger.info( "RemovalAgent.execute: Failed to remove file.", "%s at %s - %s" % ( lfn, diracSE, failed[lfn][diracSE] ) )
if errMsg:
for diracSE in errMsg.keys():
errStr = "RemovalAgent.execute: Completely failed to remove replicas. At %s", diracSE
gLogger.error( errStr, errMsg[diracSE] )
################################################
# If the sub-request is a request to the online system to retransfer
elif operation == 'reTransfer':
gLogger.info( "RemovalAgent.execute: Attempting to execute %s sub-request." % operation )
diracSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
pfn = str( subRequestFile['PFN'] )
lfn = str( subRequestFile['LFN'] )
res = self.replicaManager.onlineRetransfer( diracSE, pfn )
if res['OK']:
if res['Value']['Successful'].has_key( pfn ):
gLogger.info( "RemovalAgent.execute: Successfully requested retransfer of %s." % pfn )
result = oRequest.setSubRequestFileAttributeValue( ind, 'removal', lfn, 'Status', 'Done' )
if not result['OK']:
gLogger.error( "RemovalAgent.execute: Error setting status to %s for %s" % ( 'Done', lfn ) )
modified = True
else:
errStr = "RemovalAgent.execute: Failed to request retransfer."
gLogger.error( errStr, "%s %s %s" % ( pfn, diracSE, res['Value']['Failed'][pfn] ) )
else:
errStr = "RemovalAgent.execute: Completely failed to request retransfer."
gLogger.error( errStr, res['Message'] )
else:
gLogger.info( "RemovalAgent.execute: File already completed." )
################################################
# If the sub-request is none of the above types
else:
gLogger.error( "RemovalAgent.execute: Operation not supported.", operation )
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty( ind, 'removal' )['Value']:
oRequest.setSubRequestStatus( ind, 'removal', 'Done' )
gMonitor.addMark( "Done", 1 )
################################################
# If the sub-request is already in terminal state
else:
gLogger.info( "RemovalAgent.execute:",
"Sub-request %s is status '%s' and not to be executed." %
( ind, subRequestAttributes['Status'] ) )
################################################
# Generate the new request string after operation
newrequestString = oRequest.toXML()['Value']
except:
# if something fails return the original request back to the server
res = self.requestDBClient.updateRequest( requestName, requestString, sourceServer )
return S_OK()
res = self.requestDBClient.updateRequest( requestName, newrequestString, sourceServer )
if modified and jobID:
result = self.finalizeRequest( requestName, jobID, sourceServer )
return S_OK()
def getRequest( self, requestType ):
""" Get a request of a given type eligible for execution
"""
# RG: What if requestType is not given?
# the first query will return nothing.
# KC: maybe returning S_ERROR would be enough?
# alternatively we should check if requestType is known (in 'transfer', 'removal', 'register' and 'diset')
if not requestType or type( requestType ) not in types.StringTypes:
return S_ERROR( "Request type not given." )
myRequestType = self._escapeString( requestType )
if not myRequestType:
return myRequestType
myRequestType = myRequestType['Value']
start = time.time()
dmRequest = RequestContainer( init = False )
requestID = 0
subIDList = []
fields = ['RequestID', 'SubRequestID', 'Operation', 'Arguments',
'ExecutionOrder', 'SourceSE', 'TargetSE', 'Catalogue',
'CreationTime', 'SubmissionTime', 'LastUpdate', 'Status', 'RequestType']
# get the pending SubRequest sorted by ExecutionOrder and LastUpdate
req = "SELECT `RequestID`,`ExecutionOrder`,`Status`,`RequestType`,`LastUpdate` FROM `SubRequests` "\
"WHERE `Status` IN ( 'Waiting', 'Assigned' ) ORDER BY `ExecutionOrder`,`LastUpdate`"
# now get sorted list of RequestID (according to the above)
req = "SELECT * FROM ( %s ) as T1 GROUP BY `RequestID`" % req
# and get the 100 oldest ones of Type requestType
req = "SELECT `RequestID`,`ExecutionOrder` FROM ( %s ) as T2 WHERE `RequestType`=%s "\
"ORDER BY `LastUpdate` LIMIT 100" % ( req, myRequestType )
# and now get all waiting SubRequest for the selected RequestID and ExecutionOrder
req = "SELECT A.%s FROM SubRequests AS A, ( %s ) AS B WHERE " % ( ', A.'.join( fields ), req )
req = "%s A.RequestID=B.RequestID AND A.ExecutionOrder=B.ExecutionOrder" % ( req )
result = self._query( req )
if not result['OK']:
err = 'RequestDB._getRequest: Failed to retrieve Requests'
return S_ERROR( '%s\n%s' % ( err, result['Message'] ) )
if not result['Value']:
return S_OK()
# We get up to 10 Request candidates, to add some randomness
reqDict = {}
for row in result['Value']:
if ('"%s"' % row[-1]) != myRequestType:
continue
if row[-2] != 'Waiting':
continue
reqDict.setdefault( row[0], [] )
reqDict[row[0]].append( row[1:-2] )
reqIDList = reqDict.keys()
random.shuffle( reqIDList )
for reqID in reqIDList:
sidList = [ x[0] for x in reqDict[reqID] ]
for subID in sidList:
req = "UPDATE SubRequests SET Status='Assigned' WHERE RequestID=%s AND SubRequestID=%s;" % ( reqID, subID )
resAssigned = self._update( req )
if not resAssigned['OK']:
if subIDList:
self.__releaseSubRequests( reqID, subIDList )
return S_ERROR( 'Failed to assign subrequests: %s' % resAssigned['Message'] )
if resAssigned['Value'] == 0:
# Somebody has assigned this request
gLogger.warn( 'Already assigned subrequest %d of request %d' % ( subID, reqID ) )
else:
subIDList.append( subID )
if subIDList:
# We managed to get some requests, can continue now
requestID = reqID
break
# Haven't succeeded to get any request
if not requestID:
return S_OK()
dmRequest.setRequestID( requestID )
fields = ['FileID', 'LFN', 'Size', 'PFN', 'GUID', 'Md5', 'Addler', 'Attempt', 'Status' ]
for subRequestID, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, \
creationTime, submissionTime, lastUpdate in reqDict[requestID]:
if not subRequestID in subIDList: continue
res = dmRequest.initiateSubRequest( requestType )
ind = res['Value']
subRequestDict = {
'Status' : 'Waiting',
'SubRequestID' : subRequestID,
'Operation' : operation,
'Arguments' : arguments,
'ExecutionOrder': int( executionOrder ),
'SourceSE' : sourceSE,
'TargetSE' : targetSE,
'Catalogue' : catalogue,
'CreationTime' : creationTime,
'SubmissionTime': submissionTime,
'LastUpdate' : lastUpdate
}
res = dmRequest.setSubRequestAttributes( ind, requestType, subRequestDict )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set subRequest attributes for RequestID %s' % requestID
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT %s FROM `Files` WHERE `SubRequestID`=%s ORDER BY `FileID`;" % ( ', '.join( fields ),
subRequestID )
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get File attributes for RequestID %s.%s' % ( requestID,
subRequestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res['Value']:
fileDict = {'FileID':fileID, 'LFN':lfn, 'Size':size, 'PFN':pfn, 'GUID':guid,
'Md5':md5, 'Addler':addler, 'Attempt':attempt, 'Status':status}
files.append( fileDict )
res = dmRequest.setSubRequestFiles( ind, requestType, files )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set files into Request for RequestID %s.%s' % ( requestID,
subRequestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT Dataset,Status FROM Datasets WHERE SubRequestID = %s;" % subRequestID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get Datasets for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
datasets = []
for dataset, status in res['Value']:
datasets.append( dataset )
res = dmRequest.setSubRequestDatasets( ind, requestType, datasets )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set datasets into Request for RequestID %s.%s' % ( requestID,
subRequestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
fields = ['RequestName', 'JobID', 'OwnerDN', 'OwnerGroup',
'DIRACSetup', 'SourceComponent', 'CreationTime',
'SubmissionTime', 'LastUpdate']
req = "SELECT %s FROM `Requests` WHERE `RequestID`=%s;" % ( ', '.join( fields ), requestID )
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve max RequestID'
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, \
creationTime, submissionTime, lastUpdate = res['Value'][0]
dmRequest.setRequestName( requestName )
dmRequest.setJobID( jobID )
dmRequest.setOwnerDN( ownerDN )
dmRequest.setOwnerGroup( ownerGroup )
dmRequest.setDIRACSetup( diracSetup )
dmRequest.setSourceComponent( sourceComponent )
dmRequest.setCreationTime( str( creationTime ) )
dmRequest.setLastUpdate( str( lastUpdate ) )
res = dmRequest.toXML()
if not res['OK']:
err = 'RequestDB._getRequest: Failed to create XML for RequestID %s' % ( requestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
requestString = res['Value']
#still have to manage the status of the dataset properly
resultDict = {}
resultDict['RequestName'] = requestName
resultDict['RequestString'] = requestString
resultDict['JobID'] = jobID
return S_OK( resultDict )
class UserJobFinalization(ModuleBase):
""" User Job finalization: takes care of uploading the output data to the specified storage elements
If it does not work, it will upload to a Failover SE, then register the request to replicate and remove.
"""
#############################################################################
def __init__(self):
"""Module initialization.
"""
super(UserJobFinalization, self).__init__()
self.version = __RCSID__
self.log = gLogger.getSubLogger( "UserJobFinalization" )
self.enable = True
self.failoverTest = False #flag to put file to failover SE by default
self.defaultOutputSE = gConfig.getValue( '/Resources/StorageElementGroups/Tier1-USER', [])
self.failoverSEs = gConfig.getValue('/Resources/StorageElementGroups/Tier1-Failover', [])
#List all parameters here
self.userFileCatalog = self.ops.getValue('/UserJobs/Catalogs', ['FileCatalog'] )
self.request = None
self.lastStep = False
#Always allow any files specified by users
self.outputDataFileMask = ''
self.userOutputData = ''
self.userOutputSE = ''
self.userOutputPath = ''
self.jobReport = None
#############################################################################
def applicationSpecificInputs(self):
""" By convention the module parameters are resolved here.
"""
self.log.verbose(self.workflow_commons)
self.log.verbose(self.step_commons)
#Earlier modules may have populated the report objects
if self.workflow_commons.has_key('JobReport'):
self.jobReport = self.workflow_commons['JobReport']
if self.step_commons.has_key('Enable'):
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
self.log.warn('Enable flag set to non-boolean value %s, setting to False' %self.enable)
self.enable = False
if self.step_commons.has_key('TestFailover'):
self.enable = self.step_commons['TestFailover']
if not type(self.failoverTest) == type(True):
self.log.warn('Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest)
self.failoverTest = False
if os.environ.has_key('JOBID'):
self.jobID = os.environ['JOBID']
self.log.verbose('Found WMS JobID = %s' % self.jobID)
else:
self.log.info('No WMS JobID found, disabling module via control flag')
self.enable = False
if self.workflow_commons.has_key('Request'):
self.request = self.workflow_commons['Request']
else:
self.request = RequestContainer()
self.request.setRequestName('job_%s_request.xml' % self.jobID)
self.request.setJobID(self.jobID)
self.request.setSourceComponent("Job_%s" % self.jobID)
#Use LHCb utility for local running via dirac-jobexec
if self.workflow_commons.has_key('UserOutputData'):
self.userOutputData = self.workflow_commons['UserOutputData']
if not type(self.userOutputData) == type([]):
self.userOutputData = [i.strip() for i in self.userOutputData.split(';')]
if self.workflow_commons.has_key('UserOutputSE'):
specifiedSE = self.workflow_commons['UserOutputSE']
if not type(specifiedSE) == type([]):
self.userOutputSE = [i.strip() for i in specifiedSE.split(';')]
else:
self.log.verbose('No UserOutputSE specified, using default value: %s' % (string.join(self.defaultOutputSE,
', ')))
self.userOutputSE = self.defaultOutputSE
if self.workflow_commons.has_key('UserOutputPath'):
self.userOutputPath = self.workflow_commons['UserOutputPath']
return S_OK('Parameters resolved')
#############################################################################
def execute(self):
""" Main execution function.
"""
#Have to work out if the module is part of the last step i.e.
#user jobs can have any number of steps and we only want
#to run the finalization once.
currentStep = int(self.step_commons['STEP_NUMBER'])
totalSteps = int(self.workflow_commons['TotalSteps'])
if currentStep == totalSteps:
self.lastStep = True
else:
self.log.verbose('Current step = %s, total steps of workflow = %s, UserJobFinalization will enable itself only \
at the last workflow step.' % (currentStep, totalSteps))
if not self.lastStep:
return S_OK()
result = self.resolveInputVariables()
if not result['OK']:
self.log.error(result['Message'])
return result
self.log.info('Initializing %s' % self.version)
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'],
self.stepStatus['OK']))
return S_OK('No output data upload attempted')
if not self.userOutputData:
self.log.info('No user output data is specified for this job, nothing to do')
return S_OK('No output data to upload')
#Determine the final list of possible output files for the
#workflow and all the parameters needed to upload them.
outputList = []
for i in self.userOutputData:
outputList.append({'outputPath' : string.upper(string.split(i, '.')[-1]),
'outputDataSE' : self.userOutputSE,
'outputFile' : os.path.basename(i)})
userOutputLFNs = []
if self.userOutputData:
self.log.info('Constructing user output LFN(s) for %s' % (string.join(self.userOutputData, ', ')))
if not self.jobID:
self.jobID = 12345
owner = ''
if self.workflow_commons.has_key('Owner'):
owner = self.workflow_commons['Owner']
else:
res = self.getCurrentOwner()
if not res['OK']:
return S_ERROR('Could not obtain owner from proxy')
owner = res['Value']
vo = ''
if self.workflow_commons.has_key('VO'):
vo = self.workflow_commons['VO']
else:
res = self.getCurrentVO()
if not res['OK']:
return S_ERROR('Could not obtain VO from proxy')
vo = res['Value']
result = constructUserLFNs(int(self.jobID), vo, owner, self.userOutputData, self.userOutputPath)
if not result['OK']:
self.log.error('Could not create user LFNs', result['Message'])
return result
userOutputLFNs = result['Value']
self.log.verbose('Calling getCandidateFiles( %s, %s, %s)' % (outputList, userOutputLFNs, self.outputDataFileMask))
result = self.getCandidateFiles(outputList, userOutputLFNs, self.outputDataFileMask)
if not result['OK']:
if not self.ignoreapperrors:
self.setApplicationStatus(result['Message'])
return S_OK()
fileDict = result['Value']
result = self.getFileMetadata(fileDict)
if not result['OK']:
if not self.ignoreapperrors:
self.setApplicationStatus(result['Message'])
return S_OK()
if not result['Value']:
if not self.ignoreapperrors:
self.log.info('No output data files were determined to be uploaded for this workflow')
self.setApplicationStatus('No Output Data Files To Upload')
return S_OK()
fileMetadata = result['Value']
#First get the local (or assigned) SE to try first for upload and others in random fashion
result = getDestinationSEList('Tier1-USER', DIRAC.siteName(), outputmode='local')
if not result['OK']:
self.log.error('Could not resolve output data SE', result['Message'])
self.setApplicationStatus('Failed To Resolve OutputSE')
return result
localSE = result['Value']
self.log.verbose('Site Local SE for user outputs is: %s' % (localSE))
orderedSEs = self.defaultOutputSE
for se in localSE:
if se in orderedSEs:
orderedSEs.remove(se)
for se in self.userOutputSE:
if se in orderedSEs:
orderedSEs.remove(se)
orderedSEs = localSE + List.randomize(orderedSEs)
if self.userOutputSE:
prependSEs = []
for userSE in self.userOutputSE:
if not userSE in orderedSEs:
prependSEs.append(userSE)
orderedSEs = prependSEs + orderedSEs
self.log.info('Ordered list of output SEs is: %s' % (string.join(orderedSEs, ', ')))
final = {}
for fileName, metadata in fileMetadata.items():
final[fileName] = metadata
final[fileName]['resolvedSE'] = orderedSEs
#At this point can exit and see exactly what the module will upload
if not self.enable:
self.log.info('Module is disabled by control flag, would have attempted \
to upload the following files %s' % string.join(final.keys(), ', '))
for fileName, metadata in final.items():
self.log.info('--------%s--------' % fileName)
for n, v in metadata.items():
self.log.info('%s = %s' %(n, v))
return S_OK('Module is disabled by control flag')
#Instantiate the failover transfer client with the global request object
failoverTransfer = FailoverTransfer(self.request)
#One by one upload the files with failover if necessary
replication = {}
failover = {}
uploaded = []
if not self.failoverTest:
for fileName, metadata in final.items():
self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName,
string.join(metadata['resolvedSE'],
', ')))
result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'], metadata['lfn'],
metadata['resolvedSE'], fileGUID = metadata['guid'],
fileCatalog = self.userFileCatalog)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
failover[fileName] = metadata
else:
#Only attempt replication after successful upload
lfn = metadata['lfn']
uploaded.append(lfn)
seList = metadata['resolvedSE']
replicateSE = ''
if result['Value'].has_key('uploadedSE'):
uploadedSE = result['Value']['uploadedSE']
for se in seList:
if not se == uploadedSE:
replicateSE = se
break
if replicateSE and lfn:
self.log.info('Will attempt to replicate %s to %s' % (lfn, replicateSE))
replication[lfn] = replicateSE
else:
failover = final
cleanUp = False
for fileName, metadata in failover.items():
random.shuffle(self.failoverSEs)
targetSE = metadata['resolvedSE'][0]
metadata['resolvedSE'] = self.failoverSEs
result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'], metadata['lfn'],
targetSE, metadata['resolvedSE'],
fileGUID = metadata['guid'],
fileCatalog = self.userFileCatalog)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
cleanUp = True
continue #for users can continue even if one completely fails
else:
lfn = metadata['lfn']
uploaded.append(lfn)
#For files correctly uploaded must report LFNs to job parameters
if uploaded:
report = string.join( uploaded, ', ' )
self.jobReport.setJobParameter( 'UploadedOutputData', report )
#Now after all operations, retrieve potentially modified request object
result = failoverTransfer.getRequestObject()
if not result['OK']:
self.log.error(result)
return S_ERROR('Could Not Retrieve Modified Request')
self.request = result['Value']
#If some or all of the files failed to be saved to failover
if cleanUp:
self.workflow_commons['Request'] = self.request
#Leave any uploaded files just in case it is useful for the user
#do not try to replicate any files.
return S_ERROR('Failed To Upload Output Data')
#If there is now at least one replica for uploaded files can trigger replication
rm = ReplicaManager()
self.log.info('Sleeping for 10 seconds before attempting replication of recently uploaded files')
time.sleep(10)
for lfn, repSE in replication.items():
result = rm.replicateAndRegister(lfn, repSE, catalog = self.userFileCatalog)
if not result['OK']:
self.log.info('Replication failed with below error but file already exists in Grid storage with \
at least one replica:\n%s' % (result))
self.workflow_commons['Request'] = self.request
#Now must ensure if any pending requests are generated that these are propagated to the job wrapper
reportRequest = None
if self.jobReport:
result = self.jobReport.generateRequest()
if not result['OK']:
self.log.warn('Could not generate request for job report with result:\n%s' % (result))
else:
reportRequest = result['Value']
if reportRequest:
self.log.info('Populating request with job report information')
self.request.update(reportRequest)
if not self.request.isEmpty()['Value']:
request_string = self.request.toXML()['Value']
# Write out the request string
fname = 'user_job_%s_request.xml' % (self.jobID)
xmlfile = open(fname, 'w')
xmlfile.write(request_string)
xmlfile.close()
self.log.info('Creating failover request for deferred operations for job %s:' % self.jobID)
result = self.request.getDigest()
if result['OK']:
digest = result['Value']
self.log.info(digest)
self.setApplicationStatus('Job Finished Successfully')
return S_OK('Output data uploaded')
#############################################################################
def getCurrentOwner(self):
"""Simple function to return current DIRAC username.
"""
result = getProxyInfo()
if not result['OK']:
return S_ERROR('Could not obtain proxy information')
if not result['Value'].has_key('username'):
return S_ERROR('Could not get username from proxy')
username = result['Value']['username']
return S_OK(username)
#############################################################################
def getCurrentVO(self):
"""Simple function to return current DIRAC username.
"""
result = getProxyInfo()
if not result['OK']:
return S_ERROR('Could not obtain proxy information')
if not result['Value'].has_key('group'):
return S_ERROR('Could not get group from proxy')
group = result['Value']['group']
vo = group.split("_")[0]
return S_OK(vo)
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#
def execute( self ):
""" The main agent execution method """
# This allows dynamic changing of the throughput timescale
self.throughputTimescale = self.am_getOption( 'ThroughputTimescale', 3600 )
self.throughputTimescale = 60 * 60 * 1
#print 'ThroughputTimescale:',self.throughputTimescale
######################################################################################
#
# Obtain information on the current state of the channel queues
#
res = self.TransferDB.getChannelQueues()
if not res['OK']:
errStr = "ReplicationScheduler._execute: Failed to get channel queues from TransferDB."
gLogger.error( errStr, res['Message'] )
return S_OK()
if not res['Value']:
gLogger.info( "ReplicationScheduler._execute: No active channels found for replication." )
return S_OK()
channels = res['Value']
res = self.TransferDB.getChannelObservedThroughput( self.throughputTimescale )
if not res['OK']:
errStr = "ReplicationScheduler._execute: Failed to get observed throughput from TransferDB."
gLogger.error( errStr, res['Message'] )
return S_OK()
if not res['Value']:
gLogger.info( "ReplicationScheduler._execute: No active channels found for replication." )
return S_OK()
bandwidths = res['Value']
self.strategyHandler = StrategyHandler( bandwidths, channels, self.section )
processedRequests = []
requestsPresent = True
while requestsPresent:
######################################################################################
#
# The first step is to obtain a transfer request from the RequestDB which should be scheduled.
#
gLogger.info( "ReplicationScheduler._execute: Contacting RequestDB for suitable requests." )
res = self.RequestDB.getRequest( 'transfer' )
if not res['OK']:
gLogger.error( "ReplicationScheduler._execute: Failed to get a request list from RequestDB.", res['Message'] )
continue
if not res['Value']:
gLogger.info( "ReplicationScheduler._execute: No requests found in RequestDB." )
requestsPresent = False
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
gLogger.info( "ReplicationScheduler._execute: Obtained Request %s from RequestDB." % ( requestName ) )
######################################################################################
#
# The request must then be parsed to obtain the sub-requests, their attributes and files.
#
logStr = 'ReplicationScheduler._execute: Parsing Request %s.' % ( requestName )
gLogger.info( logStr )
oRequest = RequestContainer( requestString )
res = oRequest.getAttribute( 'RequestID' )
if not res['OK']:
gLogger.error( 'ReplicationScheduler._execute: Failed to get requestID.', res['Message'] )
return S_ERROR( 'ReplicationScheduler._execute: Failed to get number of sub-requests.' )
requestID = res['Value']
if requestID in processedRequests:
# Break the loop once we have iterated once over all requests
res = self.RequestDB.updateRequest( requestName, requestString )
if not res['OK']:
gLogger.error( "Failed to update request", "%s %s" % ( requestName, res['Message'] ) )
return S_OK()
processedRequests.append( requestID )
res = oRequest.getNumSubRequests( 'transfer' )
if not res['OK']:
gLogger.error( 'ReplicationScheduler._execute: Failed to get number of sub-requests.', res['Message'] )
return S_ERROR( 'ReplicationScheduler._execute: Failed to get number of sub-requests.' )
numberRequests = res['Value']
gLogger.info( "ReplicationScheduler._execute: '%s' found with %s sub-requests." % ( requestName, numberRequests ) )
######################################################################################
#
# The important request attributes are the source and target SEs.
#
for ind in range( numberRequests ):
gLogger.info( "ReplicationScheduler._execute: Treating sub-request %s from '%s'." % ( ind, requestName ) )
attributes = oRequest.getSubRequestAttributes( ind, 'transfer' )['Value']
if attributes['Status'] != 'Waiting':
# If the sub-request is already in terminal state
gLogger.info( "ReplicationScheduler._execute: Sub-request %s is status '%s' and not to be executed." % ( ind, attributes['Status'] ) )
continue
sourceSE = attributes['SourceSE']
targetSE = attributes['TargetSE']
""" This section should go in the transfer request class """
if type( targetSE ) in types.StringTypes:
if re.search( ',', targetSE ):
targetSEs = targetSE.split( ',' )
else:
targetSEs = [targetSE]
"""----------------------------------------------------- """
operation = attributes['Operation']
reqRepStrategy = None
if operation in self.strategyHandler.getSupportedStrategies():
reqRepStrategy = operation
######################################################################################
#
# Then obtain the file attribute of interest are the LFN and FileID
#
res = oRequest.getSubRequestFiles( ind, 'transfer' )
if not res['OK']:
gLogger.error( 'ReplicationScheduler._execute: Failed to obtain sub-request files.' , res['Message'] )
continue
files = res['Value']
gLogger.info( "ReplicationScheduler._execute: Sub-request %s found with %s files." % ( ind, len( files ) ) )
filesDict = {}
for file in files:
lfn = file['LFN']
if file['Status'] != 'Waiting':
gLogger.debug( "ReplicationScheduler._execute: %s will not be scheduled because it is %s." % ( lfn, file['Status'] ) )
else:
fileID = file['FileID']
filesDict[lfn] = fileID
if not filesDict:
gLogger.info( "ReplicationScheduler._execute: No Waiting files found for request" )
continue
notSched = len( files ) - len( filesDict )
if notSched:
gLogger.info( "ReplicationScheduler._execute: %d files found not Waiting" % notSched )
######################################################################################
#
# Now obtain replica information for the files associated to the sub-request.
#
lfns = filesDict.keys()
gLogger.info( "ReplicationScheduler._execute: Obtaining replica information for %d sub-request files." % len( lfns ) )
res = self.rm.getCatalogReplicas( lfns )
if not res['OK']:
gLogger.error( "ReplicationScheduler._execute: Failed to get replica information.", res['Message'] )
continue
for lfn, failure in res['Value']['Failed'].items():
gLogger.error( "ReplicationScheduler._execute: Failed to get replicas.", '%s: %s' % ( lfn, failure ) )
replicas = res['Value']['Successful']
if not replicas.keys():
gLogger.error( "ReplicationScheduler._execute: Failed to get replica information for all files." )
continue
######################################################################################
#
# Now obtain the file sizes for the files associated to the sub-request.
#
lfns = replicas.keys()
gLogger.info( "ReplicationScheduler._execute: Obtaining file sizes for %d sub-request files." % len( lfns ) )
res = self.rm.getCatalogFileMetadata( lfns )
if not res['OK']:
gLogger.error( "ReplicationScheduler._execute: Failed to get file size information.", res['Message'] )
continue
for lfn, failure in res['Value']['Failed'].items():
gLogger.error( 'ReplicationScheduler._execute: Failed to get file size.', '%s: %s' % ( lfn, failure ) )
metadata = res['Value']['Successful']
if not metadata.keys():
gLogger.error( "ReplicationScheduler._execute: Failed to get metadata for all files." )
continue
######################################################################################
#
# For each LFN determine the replication tree
#
for lfn in sortList( metadata.keys() ):
fileSize = metadata[lfn]['Size']
lfnReps = replicas[lfn]
fileID = filesDict[lfn]
targets = []
for targetSE in targetSEs:
if targetSE in lfnReps.keys():
gLogger.debug( "ReplicationScheduler.execute: %s already present at %s." % ( lfn, targetSE ) )
else:
targets.append( targetSE )
if not targets:
gLogger.info( "ReplicationScheduler.execute: %s present at all targets." % lfn )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
continue
if not lfnReps:
gLogger.error( "ReplicationScheduler.execute: The file has no replicas.", lfn )
continue
res = self.strategyHandler.determineReplicationTree( sourceSE, targets, lfnReps, fileSize, strategy = reqRepStrategy )
if not res['OK']:
gLogger.error( "ReplicationScheduler.execute: Failed to determine replication tree.", res['Message'] )
continue
tree = res['Value']
######################################################################################
#
# For each item in the replication tree obtain the source and target SURLS
#
for channelID, dict in tree.items():
gLogger.info( "ReplicationScheduler.execute: processing for channel %d %s" % ( channelID, str( dict ) ) )
hopSourceSE = dict['SourceSE']
hopDestSE = dict['DestSE']
hopAncestor = dict['Ancestor']
# Get the sourceSURL
if hopAncestor:
status = 'Waiting%s' % ( hopAncestor )
res = self.obtainLFNSURL( hopSourceSE, lfn )
if not res['OK']:
errStr = res['Message']
gLogger.error( errStr )
return S_ERROR( errStr )
sourceSURL = res['Value']
else:
status = 'Waiting'
res = self.resolvePFNSURL( hopSourceSE, lfnReps[hopSourceSE] )
if not res['OK']:
sourceSURL = lfnReps[hopSourceSE]
else:
sourceSURL = res['Value']
# Get the targetSURL
res = self.obtainLFNSURL( hopDestSE, lfn )
if not res['OK']:
errStr = res['Message']
gLogger.error( errStr )
return S_ERROR( errStr )
targetSURL = res['Value']
######################################################################################
#
# For each item in the replication tree add the file to the channel
#
res = self.TransferDB.addFileToChannel( channelID, fileID, hopSourceSE, sourceSURL, hopDestSE, targetSURL, fileSize, fileStatus = status )
if not res['OK']:
errStr = res['Message']
gLogger.error( "ReplicationScheduler._execute: Failed to add File to Channel." , "%s %s" % ( fileID, channelID ) )
return S_ERROR( errStr )
res = self.TransferDB.addFileRegistration( channelID, fileID, lfn, targetSURL, hopDestSE )
if not res['OK']:
errStr = res['Message']
gLogger.error( "ReplicationScheduler._execute: Failed to add File registration." , "%s %s" % ( fileID, channelID ) )
result = self.TransferDB.removeFileFromChannel( channelID, fileID )
if not result['OK']:
errStr += result['Message']
gLogger.error( "ReplicationScheduler._execute: Failed to remove File." , "%s %s" % ( fileID, channelID ) )
return S_ERROR( errStr )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Scheduled' )
res = self.TransferDB.addReplicationTree( fileID, tree )
if oRequest.isSubRequestEmpty( ind, 'transfer' )['Value']:
oRequest.setSubRequestStatus( ind, 'transfer', 'Scheduled' )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDB.updateRequest( requestName, requestString )
if not res['OK']:
gLogger.error( "ReplicationScheduler._execute: Failed to update request", "%s %s" % ( requestName, res['Message'] ) )
def execute_request(self):
""" Takes one DISET request and forward it to the destination service
"""
gMonitor.addMark("Iteration", 1)
if self.RequestDB:
res = self.RequestDB.getRequest('diset')
else:
res = self.RequestDBClient.getRequest('diset', url=self.local)
if not res['OK']:
gLogger.error(
"DISETForwardingAgent.execute: Failed to get request from database.",
self.local)
return S_OK()
elif not res['Value']:
gLogger.info(
"DISETForwardingAgent.execute: No requests to be executed found."
)
return S_OK()
gMonitor.addMark("Attempted", 1)
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
try:
jobID = int(res['Value']['JobID'])
except:
jobID = 0
gLogger.info("DISETForwardingAgent.execute: Obtained request %s" %
requestName)
if self.RequestDB:
result = self.RequestDB._getRequestAttribute(
'RequestID', requestName=requestName)
if not result['OK']:
return S_OK('Can not get the request execution order')
requestID = result['Value']
result = self.RequestDB.getCurrentExecutionOrder(requestID)
else:
result = self.RequestDBClient.getCurrentExecutionOrder(
requestName, self.local)
if result['OK']:
currentOrder = result['Value']
else:
return S_OK('Can not get the request execution order')
oRequest = RequestContainer(request=requestString)
requestAttributes = oRequest.getRequestAttributes()['Value']
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('diset')
if not res['OK']:
errStr = "DISETForwardingAgent.execute: Failed to obtain number of diset subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info(
"DISETForwardingAgent.execute: Found %s sub requests for job %s" %
(res['Value'], jobID))
################################################
# For all the sub-requests in the request
modified = False
for ind in range(res['Value']):
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'diset')['Value']
subExecutionOrder = int(subRequestAttributes['ExecutionOrder'])
subStatus = subRequestAttributes['Status']
gLogger.info(
"DISETForwardingAgent.execute: Processing sub-request %s with execution order %d"
% (ind, subExecutionOrder))
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
operation = subRequestAttributes['Operation']
gLogger.info(
"DISETForwardingAgent.execute: Attempting to forward %s type."
% operation)
rpcStubString = subRequestAttributes['Arguments']
rpcStub, length = DEncode.decode(rpcStubString)
res = executeRPCStub(rpcStub)
if res['OK']:
gLogger.info(
"DISETForwardingAgent.execute: Successfully forwarded."
)
oRequest.setSubRequestStatus(ind, 'diset', 'Done')
gMonitor.addMark("Successful", 1)
modified = True
elif res['Message'] == 'No Matching Job':
gLogger.warn(
"DISETForwardingAgent.execute: No corresponding job found. Setting to done."
)
oRequest.setSubRequestStatus(ind, 'diset', 'Done')
else:
gLogger.error(
"DISETForwardingAgent.execute: Failed to forward request.",
res['Message'])
else:
gLogger.info(
"DISETForwardingAgent.execute: Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
if self.RequestDB:
res = self.RequestDB.updateRequest(requestName, requestString)
else:
res = self.RequestDBClient.updateRequest(requestName,
requestString, self.local)
if res['OK']:
gLogger.info(
"DISETForwardingAgent.execute: Successfully updated request.")
else:
gLogger.error(
"DISETForwardingAgent.execute: Failed to update request to",
self.local)
if modified and jobID:
result = self.RequestDBClient.finalizeRequest(
requestName, jobID, self.local)
return S_OK()
requestOperation = 'replicateAndRegister'
for lfnList in breakListIntoChunks( lfns, 100 ):
oRequest = RequestContainer()
subRequestIndex = oRequest.initiateSubRequest( requestType )['Value']
attributeDict = {'Operation':requestOperation, 'TargetSE':targetSE}
oRequest.setSubRequestAttributes( subRequestIndex, requestType, attributeDict )
files = []
for lfn in lfnList:
files.append( {'LFN':lfn} )
oRequest.setSubRequestFiles( subRequestIndex, requestType, files )
requestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] )
oRequest.setRequestAttributes( {'RequestName':requestName} )
DIRAC.gLogger.info( oRequest.toXML()['Value'] )
result = reqClient.setRequest( requestName, oRequest.toXML()['Value'] )
if result['OK']:
print 'Submitted Request:', result['Value']
else:
print 'Failed to submit Request', result['Message']
if monitor:
requestID = result['Value']
while True:
result = reqClient.getRequestStatus( requestID )
if not result['OK']:
Script.gLogger.error( result['Message'] )
break
Script.gLogger.notice( result['Value']['RequestStatus'] )
if result['Value']['RequestStatus'] == 'Done':
def getRequestForSubRequest(self, itself, subRequestID ):
"""
Select Request given SubRequestID.
:param self: plugin reference
:param itself: patient reference for injection
:param int subRequestID: SubRequests.SubRequestID
:warn: function has to be injected to RequestDBMySQL instance
"""
## get RequestID
requestID = "SELECT RequestID FROM SubRequests WHERE SubRequestID = %s;" % str(subRequestID)
requestID = self._query( requestID )
if not requestID["OK"]:
return requestID
requestID = requestID["Value"][0]
## create RequestContainer
requestContainer = RequestContainer( init = False )
requestContainer.setRequestID( requestID )
## put some basic infos in
requestInfo = "SELECT RequestName, JobID, OwnerDN, OwnerGroup, DIRACSetup, SourceComponent, CreationTime, SubmissionTime, LastUpdate, Status "
requestInfo += "FROM Requests WHERE RequestID = %d;" % requestID
requestInfo = self._query( requestInfo )
if not requestInfo["OK"]:
return requestInfo
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate, status = requestInfo['Value'][0]
requestContainer.setRequestName( requestName )
requestContainer.setJobID( jobID )
requestContainer.setOwnerDN( ownerDN )
requestContainer.setOwnerGroup( ownerGroup )
requestContainer.setDIRACSetup( diracSetup )
requestContainer.setSourceComponent( sourceComponent )
requestContainer.setCreationTime( str( creationTime ) )
requestContainer.setLastUpdate( str( lastUpdate ) )
requestContainer.setStatus( status )
## get sub-requests
subRequests = "SELECT SubRequestID, Status, RequestType, Operation, Arguments, ExecutionOrder, SourceSE, "
subRequests += "TargetSE, Catalogue, CreationTime, SubmissionTime, LastUpdate FROM SubRequests WHERE RequestID=%s;" % requestID
subRequests = self._query( subRequests )
if not subRequests["OK"]:
return subRequests
## loop over sub requests
for subRequestID, status, requestType, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in subRequests["Value"]:
res = requestContainer.initiateSubRequest( requestType )
ind = res["Value"]
subRequestDict = { "Status" : status, "SubRequestID" : subRequestID, "Operation" : operation, "Arguments" : arguments,
"ExecutionOrder" : int( executionOrder ), "SourceSE" : sourceSE, "TargetSE" : targetSE,
"Catalogue" : catalogue, "CreationTime" : creationTime, "SubmissionTime" : submissionTime,
"LastUpdate" : lastUpdate }
res = requestContainer.setSubRequestAttributes( ind, requestType, subRequestDict )
if not res["OK"]:
return res
## get files for this subrequest
req = "SELECT FileID, LFN, Size, PFN, GUID, Md5, Addler, Attempt, Status FROM Files WHERE SubRequestID = %s ORDER BY FileID;" % str(subRequestID)
res = self._query( req )
if not res["OK"]:
return res
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res["Value"]:
fileDict = { "FileID" : fileID, "LFN" : lfn, "Size" : size,
"PFN" : pfn, "GUID" : guid, "Md5" : md5,
"Addler" : addler, "Attempt" : attempt,
"Status" : status }
files.append( fileDict )
res = requestContainer.setSubRequestFiles( ind, requestType, files )
if not res["OK"]:
return res
## dump request to XML
res = requestContainer.toXML()
if not res["OK"]:
return res
requestString = res["Value"]
## return dictonary with all info in at least
return S_OK( {
"RequestName" : requestName,
"RequestString" : requestString,
"JobID" : jobID,
"RequestContainer" : requestContainer
} )
class RequestTask( object ):
"""
.. class:: RequestTask
Base class for DMS 'transfer', 'removal' and 'register' Requests processing.
This class is meant to be executed as a ProcessTask inside ProcessPool.
The most important and common global DIRAC objects are created in RequestTask constructor.
This includes gLogger, gConfig, gProxyManager, S_OK and S_ERROR. The constructor also
imports a set of common modules: os, sys, re, time and everything from types module.
All other DIRAC tools and clients (i.e. RequestManager) are private in RequestTask class and will
be imported and instantiated on demand during first usage. They are accessible using proxied public methods, i.e.
if you are going to use ReplicaManager just call::
self.replicaManager().someMethod()
All currently proxied tools are::
DataLoggingClient -- self.dataLoggingClient()
ReplicaManager -- self.replicaManager()
RequestClient -- self.requestClient()
StorageFactory -- self.storageFactory()
SubLogger message handles for all levels are also proxied, so you can directly use them in your code, i.e.::
self.info("An info message")
self.debug("This will be shown only in debug")
For handling sub-request one has to register their actions handlers using :self.addOperationAction:
method. This method checks if handler is defined as a method of inherited class and then puts its
definition into internal operation dispatcher dictionary with a key of sub-request's operation name.
Each operation handler should have the signature::
def operationName( self, index, requestObj, subRequestAttrs, subRequestFiles )
where index is a sub-request counter, requestObj is a RequestContainer instance,
subRequestAttrs is a dict with sub-request attributes and subRequestFiles is a dict with
files attached to the sub-request.
Handlers shoudl always return S_OK with value of (modified or not) requestObj, S_ERROR with some
error message otherwise.
Processing of request is done automatically in self.__call__, one doesn't have to worry about changing
credentials, looping over subrequests or request finalizing -- only sub-request processing matters in
the all inherited classes.
Concerning :MonitringClient: (or better known its global instance :gMonitor:), if someone wants to send
some metric over there, she has to put in agent's code registration of activity and then in a particular
task use :RequestTask.addMark: to save monitoring data. All monitored activities are held in
:RequestTask.__monitor: dict which at the end of processing is returned from :RequestTask.__call__:.
The values are then processed and pushed to the gMonitor instance in the default callback function.
"""
## reference to ReplicaManager
__replicaManager = None
## reference to DataLoggingClient
__dataLoggingClient = None
## reference to RequestClient
__requestClient = None
## reference to StotageFactory
__storageFactory = None
## subLogger
__log = None
## request type
__requestType = None
## placeholder for request owner DB
requestOwnerDN = None
## placeholder for Request owner group
requestOwnerGroup = None
## operation dispatcher for SubRequests,
## a dictonary
## "operation" => methodToRun
##
__operationDispatcher = {}
## holder for DataManager proxy file
__dataManagerProxy = None
## monitoring dict
__monitor = {}
def __init__( self, requestString, requestName, executionOrder, jobID, configPath ):
""" c'tor
:param self: self reference
:param str requestString: XML serialised RequestContainer
:param str requestName: request name
:param list executionOrder: request execution order
:param int jobID: jobID
:param str sourceServer: request's source server
:param str configPath: path in CS for parent agent
"""
## fixtures
## python fixtures
import os, os.path, sys, time, re, types
self.makeGlobal( "os", os )
self.makeGlobal( "os.path", os.path )
self.makeGlobal( "sys", sys )
self.makeGlobal( "time", time )
self.makeGlobal( "re", re )
## export all Types from types
[ self.makeGlobal( item, getattr( types, item ) ) for item in dir(types) if "Type" in item ]
## DIRAC fixtures
from DIRAC.FrameworkSystem.Client.Logger import gLogger
self.__log = gLogger.getSubLogger( "%s/%s" % ( self.__class__.__name__, str(requestName) ) )
self.always = self.__log.always
self.notice = self.__log.notice
self.info = self.__log.info
self.debug = self.__log.debug
self.warn = self.__log.warn
self.error = self.__log.error
self.exception = self.__log.exception
self.fatal = self.__log.fatal
from DIRAC import S_OK, S_ERROR
from DIRAC.ConfigurationSystem.Client.Config import gConfig
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getGroupsWithVOMSAttribute
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
## export DIRAC global tools and functions
self.makeGlobal( "S_OK", S_OK )
self.makeGlobal( "S_ERROR", S_ERROR )
self.makeGlobal( "gLogger", gLogger )
self.makeGlobal( "gConfig", gConfig )
self.makeGlobal( "gProxyManager", gProxyManager )
self.makeGlobal( "getGroupsWithVOMSAttribute", getGroupsWithVOMSAttribute )
self.makeGlobal( "gConfigurationData", gConfigurationData )
## save request string
self.requestString = requestString
## build request object
from DIRAC.RequestManagementSystem.Client.RequestContainer import RequestContainer
self.requestObj = RequestContainer( init = False )
self.requestObj.parseRequest( request = self.requestString )
## save request name
self.requestName = requestName
## .. and jobID
self.jobID = jobID
## .. and execution order
self.executionOrder = executionOrder
## save config path
self.__configPath = configPath
## set requestType
self.setRequestType( gConfig.getValue( os.path.join( configPath, "RequestType" ), "" ) )
## get log level
self.__log.setLevel( gConfig.getValue( os.path.join( configPath, self.__class__.__name__, "LogLevel" ), "INFO" ) )
## clear monitoring
self.__monitor = {}
## save DataManager proxy
if "X509_USER_PROXY" in os.environ:
self.info("saving path to current proxy file")
self.__dataManagerProxy = os.environ["X509_USER_PROXY"]
else:
self.error("'X509_USER_PROXY' environment variable not set")
def dataManagerProxy( self ):
""" get dataManagerProxy file
:param self: self reference
"""
return self.__dataManagerProxy
def addMark( self, name, value = 1 ):
""" add mark to __monitor dict
:param self: self reference
:param name: mark name
:param value: value to be
"""
if name not in self.__monitor:
self.__monitor.setdefault( name, 0 )
self.__monitor[name] += value
def monitor( self ):
""" get monitoring dict
:param cls: class reference
"""
return self.__monitor
def makeGlobal( self, objName, objDef ):
""" export :objDef: to global name space using :objName: name
:param self: self reference
:param str objName: symbol name
:param mixed objDef: symbol definition
:throws: NameError if symbol of that name is already in
"""
if objName not in __builtins__:
if type( __builtins__) == type( {} ):
__builtins__[objName] = objDef
else:
setattr( __builtins__, objName, objDef )
return True
def requestType( self ):
""" get request type
:params self: self reference
"""
return self.__requestType
def setRequestType( self, requestType ):
""" set request type
:param self: self reference
"""
self.debug( "Setting requestType to %s" % str(requestType) )
self.__requestType = requestType
@classmethod
def replicaManager( cls ):
""" ReplicaManager getter
:param cls: class reference
"""
if not cls.__replicaManager:
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
cls.__replicaManager = ReplicaManager()
return cls.__replicaManager
@classmethod
def dataLoggingClient( cls ):
""" DataLoggingClient getter
:param cls: class reference
"""
if not cls.__dataLoggingClient:
from DIRAC.DataManagementSystem.Client.DataLoggingClient import DataLoggingClient
cls.__dataLoggingClient = DataLoggingClient()
return cls.__dataLoggingClient
@classmethod
def requestClient( cls ):
""" RequestClient getter
:param cls: class reference
"""
if not cls.__requestClient:
from DIRAC.Core.DISET.RPCClient import RPCClient
from DIRAC.RequestManagementSystem.Client.RequestClient import RequestClient
cls.__requestClient = RequestClient()
return cls.__requestClient
@classmethod
def storageFactory( cls ):
""" StorageFactory getter
:param cls: class reference
"""
if not cls.__storageFactory:
from DIRAC.Resources.Storage.StorageFactory import StorageFactory
cls.__storageFactory = StorageFactory()
return cls.__storageFactory
def changeProxy( self, ownerDN, ownerGroup ):
""" get proxy from gProxyManager, save it to file
:param self: self reference
:param str ownerDN: request owner DN
:param str ownerGroup: request owner group
:return: S_OK with name of newly created owner proxy file
"""
ownerProxy = gProxyManager.downloadVOMSProxy( str(ownerDN), str(ownerGroup) )
if not ownerProxy["OK"] or not ownerProxy["Value"]:
reason = ownerProxy["Message"] if "Message" in ownerProxy else "No valid proxy found in ProxyManager."
return S_ERROR( "Change proxy error for '%s'@'%s': %s" % ( ownerDN, ownerGroup, reason ) )
ownerProxyFile = ownerProxy["Value"].dumpAllToFile()
if not ownerProxyFile["OK"]:
return S_ERROR( ownerProxyFile["Message"] )
ownerProxyFile = ownerProxyFile["Value"]
os.environ["X509_USER_PROXY"] = ownerProxyFile
return S_OK( ownerProxyFile )
######################################################################
# operationDispatcher
@classmethod
def operationDispatcher( cls ):
""" operation dispatcher getter
:param cls: class reference
"""
return cls.__operationDispatcher
@classmethod
def addOperationAction( cls, operation, methodToRun, overwrite = True ):
""" register handler :methodToRun: for SubRequest operation :operation:
:warn: all handlers should have the same signature
:param self: self reference
:param str operation: SubRequest operation name
:param MethodType methodToRun: handler to be executed for SubRequest
:param bool overwrite: flag to overwrite handler, if already present
:return: S_OK/S_ERROR
Every action handler should return S_OK with of a structure::
{ "OK" : True,
"Value" : requestObj # that has been sent to operation handler
}
otherwise S_ERROR.
"""
if operation in cls.__operationDispatcher and not overwrite:
return S_ERROR("addOperationAction: operation for '%s' is already registered" % operation )
if type(methodToRun) is not MethodType:
return S_ERROR("addOperationAction: wrong type (%s = types.MethodType) for '%s' operation" % \
( str(type(methodToRun)), operation ) )
cls.__operationDispatcher[operation] = methodToRun
return S_OK()
def __call__( self ):
""" generic function to process one Request of a type requestType
This method could be run in a thread.
:param self: self reference
:param str requestType: request type
:return: S_OK/S_ERROR
"""
self.always("executing request %s" % self.requestName )
################################################################
## get ownerDN and ownerGroup
ownerDN = self.requestObj.getAttribute( "OwnerDN" )
if not ownerDN["OK"]:
return ownerDN
ownerDN = ownerDN["Value"]
ownerGroup = self.requestObj.getAttribute( "OwnerGroup" )
if not ownerGroup["OK"]:
return ownerGroup
ownerGroup = ownerGroup["Value"]
## save request owner
self.requestOwnerDN = ownerDN if ownerDN else ""
self.requestOwnerGroup = ownerGroup if ownerGroup else ""
#################################################################
## change proxy
ownerProxyFile = None
if ownerDN and ownerGroup:
ownerProxyFile = self.changeProxy( ownerDN, ownerGroup )
if not ownerProxyFile["OK"]:
self.error( "handleReuqest: unable to get proxy for '%s'@'%s': %s" % ( ownerDN,
ownerGroup,
ownerProxyFile["Message"] ) )
update = self.putBackRequest( self.requestName, self.requestString )
if not update["OK"]:
self.error( "handleRequest: error when updating request: %s" % update["Message"] )
return update
return ownerProxyFile
ownerProxyFile = ownerProxyFile["Value"]
#self.ownerProxyFile = ownerProxyFile
self.info( "Will execute request for '%s'@'%s' using proxy file %s" % ( ownerDN, ownerGroup, ownerProxyFile ) )
else:
self.info( "Will execute request for DataManager using her/his proxy")
#################################################################
## execute handlers
ret = { "OK" : False, "Message" : "" }
useServerCert = gConfig.useServerCertificate()
try:
# Execute task with the owner proxy even for contacting DIRAC services
if useServerCert:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','false')
ret = self.handleRequest()
finally:
if useServerCert:
gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate','true')
## delete owner proxy
if self.__dataManagerProxy:
os.environ["X509_USER_PROXY"] = self.__dataManagerProxy
if ownerProxyFile and os.path.exists( ownerProxyFile ):
os.unlink( ownerProxyFile )
if not ret["OK"]:
self.error( "handleRequest: error during request processing: %s" % ret["Message"] )
self.error( "handleRequest: will put original request back" )
update = self.putBackRequest( self.requestName, self.requestString )
if not update["OK"]:
self.error( "handleRequest: error when putting back request: %s" % update["Message"] )
## return at least
return ret
def handleRequest( self ):
""" read SubRequests and ExecutionOrder, fire registered handlers upon SubRequests operations
:param self: self reference
:param dict requestDict: request dictionary as read from self.readRequest
"""
##############################################################
# here comes the processing
##############################################################
res = self.requestObj.getNumSubRequests( self.__requestType )
if not res["OK"]:
errMsg = "handleRequest: failed to obtain number of '%s' subrequests." % self.__requestType
self.error( errMsg, res["Message"] )
return S_ERROR( res["Message"] )
## for gMonitor
self.addMark( "Execute", 1 )
## process sub requests
for index in range( res["Value"] ):
self.info( "handleRequest: processing subrequest %s." % str(index) )
subRequestAttrs = self.requestObj.getSubRequestAttributes( index, self.__requestType )["Value"]
if subRequestAttrs["ExecutionOrder"]:
subExecutionOrder = int( subRequestAttrs["ExecutionOrder"] )
else:
subExecutionOrder = 0
subRequestStatus = subRequestAttrs["Status"]
if subRequestStatus != "Waiting":
self.info( "handleRequest: subrequest %s has status '%s' and is not to be executed." % ( str(index),
subRequestStatus ) )
continue
if subExecutionOrder <= self.executionOrder:
operation = subRequestAttrs["Operation"]
if operation not in self.operationDispatcher():
self.error( "handleRequest: '%s' operation not supported" % operation )
else:
self.info( "handleRequest: will execute %s '%s' subrequest" % ( str(index), operation ) )
## get files
subRequestFiles = self.requestObj.getSubRequestFiles( index, self.__requestType )["Value"]
## execute operation action
ret = self.operationDispatcher()[operation].__call__( index,
self.requestObj,
subRequestAttrs,
subRequestFiles )
################################################
## error in operation action?
if not ret["OK"]:
self.error( "handleRequest: error when handling subrequest %s: %s" % ( str(index), ret["Message"] ) )
self.requestObj.setSubRequestAttributeValue( index, self.__requestType, "Error", ret["Message"] )
else:
## update ref to requestObj
self.requestObj = ret["Value"]
## check if subrequest status == Done, disable finalisation if not
subRequestDone = self.requestObj.isSubRequestDone( index, self.__requestType )
if not subRequestDone["OK"]:
self.error( "handleRequest: unable to determine subrequest status: %s" % subRequestDone["Message"] )
else:
if not subRequestDone["Value"]:
self.warn("handleRequest: subrequest %s is not done yet" % str(index) )
################################################
# Generate the new request string after operation
newRequestString = self.requestObj.toXML()['Value']
update = self.putBackRequest( self.requestName, newRequestString )
if not update["OK"]:
self.error( "handleRequest: error when updating request: %s" % update["Message"] )
return update
## get request status
if self.jobID:
requestStatus = self.requestClient().getRequestStatus( self.requestName )
if not requestStatus["OK"]:
return requestStatus
requestStatus = requestStatus["Value"]
## finalize request if jobID is present and request status = 'Done'
self.info("handleRequest: request status is %s" % requestStatus )
if ( requestStatus["RequestStatus"] == "Done" ) and ( requestStatus["SubRequestStatus"] not in ( "Waiting", "Assigned" ) ):
self.debug("handleRequest: request is going to be finalised")
finalize = self.requestClient().finalizeRequest( self.requestName, self.jobID )
if not finalize["OK"]:
self.error("handleRequest: error in request finalization: %s" % finalize["Message"] )
return finalize
self.info("handleRequest: request is finalised")
## for gMonitor
self.addMark( "Done", 1 )
## should return S_OK with monitor dict
return S_OK( { "monitor" : self.monitor() } )
def putBackRequest( self, requestName, requestString ):
""" put request back
:param self: self reference
:param str requestName: request name
:param str requestString: XML-serilised request
:param str sourceServer: request server URL
"""
update = self.requestClient().updateRequest( requestName, requestString )
if not update["OK"]:
self.error( "putBackRequest: error when updating request: %s" % update["Message"] )
return update
return S_OK()
def getRequest( self, requestType = '' ):
""" Get a request of a given type
"""
# RG: What if requestType is not given?
# the first query will return nothing.
start = time.time()
dmRequest = RequestContainer( init = False )
requestID = 0
req = "SELECT RequestID,SubRequestID FROM SubRequests WHERE Status = 'Waiting' AND RequestType = '%s' ORDER BY LastUpdate ASC LIMIT 50;" % requestType
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve max RequestID'
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
if not res['Value']:
return S_OK()
reqIDList = [ x[0] for x in res['Value'] ]
random.shuffle( reqIDList )
count = 0
for reqID in reqIDList:
count += 1
if requestType:
req = "SELECT SubRequestID,Operation,Arguments,ExecutionOrder,SourceSE,TargetSE,Catalogue,CreationTime,SubmissionTime,LastUpdate \
from SubRequests WHERE RequestID=%s AND RequestType='%s' AND Status='%s'" % ( reqID, requestType, 'Waiting' )
else:
# RG: What if requestType is not given?
# we should never get there, and it misses the "AND Status='Waiting'"
req = "SELECT SubRequestID,Operation,Arguments,ExecutionOrder,SourceSE,TargetSE,Catalogue,CreationTime,SubmissionTime,LastUpdate \
from SubRequests WHERE RequestID=%s" % reqID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve SubRequests for RequestID %s' % reqID
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
subIDList = []
for tuple in res['Value']:
subID = tuple[0]
# RG: We should set the condition "AND Status='Waiting'"
# if the subrequest has got assigned it will failed
req = "UPDATE SubRequests SET Status='Assigned' WHERE RequestID=%s AND SubRequestID=%s;" % ( reqID, subID )
resAssigned = self._update( req )
if not resAssigned['OK']:
if subIDList:
self.__releaseSubRequests( reqID, subIDList )
return S_ERROR( 'Failed to assign subrequests: %s' % resAssigned['Message'] )
if resAssigned['Value'] == 0:
# Somebody has assigned this request
gLogger.warn( 'Already assigned subrequest %d of request %d' % ( subID, reqID ) )
else:
subIDList.append( subID )
# RG: We need to check that all subRequest with smaller ExecutionOrder are "Done"
if subIDList:
# We managed to get some requests, can continue now
requestID = reqID
break
# Haven't succeeded to get any request
if not requestID:
return S_OK()
dmRequest.setRequestID( requestID )
# RG: We have this list in subIDList, can different queries get part of the subrequets of the same type?
subRequestIDs = []
for subRequestID, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in res['Value']:
if not subRequestID in subIDList: continue
subRequestIDs.append( subRequestID )
# RG: res['Value'] is the range of the loop and it gets redefined here !!!!!!
res = dmRequest.initiateSubRequest( requestType )
ind = res['Value']
subRequestDict = {
'Status' : 'Waiting',
'SubRequestID' : subRequestID,
'Operation' : operation,
'Arguments' : arguments,
'ExecutionOrder': int( executionOrder ),
'SourceSE' : sourceSE,
'TargetSE' : targetSE,
'Catalogue' : catalogue,
'CreationTime' : creationTime,
'SubmissionTime': submissionTime,
'LastUpdate' : lastUpdate
}
res = dmRequest.setSubRequestAttributes( ind, requestType, subRequestDict )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set subRequest attributes for RequestID %s' % requestID
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT FileID,LFN,Size,PFN,GUID,Md5,Addler,Attempt,Status \
from Files WHERE SubRequestID = %s ORDER BY FileID;" % subRequestID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get File attributes for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res['Value']:
fileDict = {'FileID':fileID, 'LFN':lfn, 'Size':size, 'PFN':pfn, 'GUID':guid, 'Md5':md5, 'Addler':addler, 'Attempt':attempt, 'Status':status}
files.append( fileDict )
res = dmRequest.setSubRequestFiles( ind, requestType, files )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set files into Request for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT Dataset,Status FROM Datasets WHERE SubRequestID = %s;" % subRequestID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get Datasets for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
datasets = []
for dataset, status in res['Value']:
datasets.append( dataset )
res = dmRequest.setSubRequestDatasets( ind, requestType, datasets )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set datasets into Request for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT RequestName,JobID,OwnerDN,OwnerGroup,DIRACSetup,SourceComponent,CreationTime,SubmissionTime,LastUpdate from Requests WHERE RequestID = %s;" % requestID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve max RequestID'
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate = res['Value'][0]
dmRequest.setRequestName( requestName )
dmRequest.setJobID( jobID )
dmRequest.setOwnerDN( ownerDN )
dmRequest.setOwnerGroup( ownerGroup )
dmRequest.setDIRACSetup( diracSetup )
dmRequest.setSourceComponent( sourceComponent )
dmRequest.setCreationTime( str( creationTime ) )
dmRequest.setLastUpdate( str( lastUpdate ) )
res = dmRequest.toXML()
if not res['OK']:
err = 'RequestDB._getRequest: Failed to create XML for RequestID %s' % ( requestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
requestString = res['Value']
#still have to manage the status of the dataset properly
resultDict = {}
resultDict['RequestName'] = requestName
resultDict['RequestString'] = requestString
resultDict['JobID'] = jobID
return S_OK( resultDict )
def execute(self):
res = self.RequestDBClient.getRequest('integrity')
if not res['OK']:
gLogger.info(
"LFCvsSEAgent.execute: Failed to get request from database.")
return S_OK()
elif not res['Value']:
gLogger.info(
"LFCvsSEAgent.execute: No requests to be executed found.")
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
gLogger.info("LFCvsSEAgent.execute: Obtained request %s" % requestName)
oRequest = RequestContainer(request=requestString)
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('integrity')
if not res['OK']:
errStr = "LFCvsSEAgent.execute: Failed to obtain number of integrity subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info("LFCvsSEAgent.execute: Found %s sub requests." %
res['Value'])
################################################
# For all the sub-requests in the request
for ind in range(res['Value']):
gLogger.info("LFCvsSEAgent.execute: Processing sub-request %s." %
ind)
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'integrity')['Value']
if subRequestAttributes['Status'] == 'Waiting':
subRequestFiles = oRequest.getSubRequestFiles(
ind, 'integrity')['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a lfcvsse operation
if operation == 'LFCvsSE':
gLogger.info(
"LFCvsSEAgent.execute: Attempting to execute %s sub-request."
% operation)
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile['LFN']
oNamespaceBrowser = NamespaceBrowser(lfn)
# Loop over all the directories and sub-directories
while (oNamespaceBrowser.isActive()):
currentDir = oNamespaceBrowser.getActiveDir()
gLogger.info(
"LFCvsSEAgent.execute: Attempting to get contents of %s."
% currentDir)
res = self.ReplicaManager.getCatalogDirectoryContents(
currentDir)
if not res['OK']:
subDirs = [currentDir]
elif res['Value']['Failed'].has_key(
currentDir):
subDirs = [currentDir]
else:
subDirs = res['Value']['Successful'][
currentDir]['SubDirs']
files = res['Value']['Successful'][
currentDir]['Files']
lfnSizeDict = {}
pfnLfnDict = {}
pfnStatusDict = {}
sePfnDict = {}
for lfn, lfnDict in files.items():
lfnSizeDict[lfn] = lfnDict['MetaData'][
'Size']
for se in lfnDict['Replicas'].keys():
pfn = lfnDict['Replicas'][se][
'PFN']
status = lfnDict['Replicas'][se][
'Status']
pfnStatusDict[pfn] = status
pfnLfnDict[pfn] = lfn
if not sePfnDict.has_key(se):
sePfnDict[se] = []
sePfnDict[se].append(pfn)
for storageElementName, physicalFiles in sePfnDict.items(
):
gLogger.info(
"LFCvsSEAgent.execute: Attempting to get metadata for files on %s."
% storageElementName)
res = self.ReplicaManager.getStorageFileMetadata(
physicalFiles, storageElementName)
if not res['OK']:
gLogger.error(
"LFCvsSEAgent.execute: Completely failed to get physical file metadata.",
res['Message'])
else:
for pfn in res['Value'][
'Failed'].keys():
gLogger.error(
"LFCvsSEAgent.execute: Failed to get metadata.",
"%s %s" %
(pfn, res['Value']
['Failed'][pfn]))
lfn = pfnLfnDict[pfn]
fileMetadata = {
'Prognosis':
'MissingSEPfn',
'LFN': lfn,
'PFN': pfn,
'StorageElement':
storageElementName,
'Size': lfnSizeDict[lfn]
}
IntegrityDB = RPCClient(
'DataManagement/DataIntegrity'
)
resInsert = IntegrityDB.insertProblematic(
AGENT_NAME, fileMetadata)
if resInsert['OK']:
gLogger.info(
"LFCvsSEAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
for pfn, pfnDict in res['Value'][
'Successful'].items():
lfn = pfnLfnDict[pfn]
catalogSize = int(
lfnSizeDict[lfn])
storageSize = int(
pfnDict['Size'])
if int(catalogSize) == int(
storageSize):
gLogger.info(
"LFCvsSEAgent.execute: Catalog and storage sizes match.",
"%s %s" %
(pfn,
storageElementName))
gLogger.info(
"Change the status in the LFC"
)
else:
gLogger.error(
"LFCvsSEAgent.execute: Catalog and storage size mis-match.",
"%s %s" %
(pfn,
storageElementName))
fileMetadata = {
'Prognosis':
'PfnSizeMismatch',
'LFN':
lfn,
'PFN':
pfn,
'StorageElement':
storageElementName
}
IntegrityDB = RPCClient(
'DataManagement/DataIntegrity'
)
resInsert = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if resInsert['OK']:
gLogger.info(
"LFCvsSEAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
oNamespaceBrowser.updateDirs(subDirs)
oRequest.setSubRequestFileAttributeValue(
ind, 'integrity', lfn, 'Status', 'Done')
################################################
# If the sub-request is none of the above types
else:
gLogger.info(
"LFCvsSEAgent.execute: Operation not supported.",
operation)
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty(ind, 'integrity')['Value']:
oRequest.setSubRequestStatus(ind, 'integrity', 'Done')
################################################
# If the sub-request is already in terminal state
else:
gLogger.info(
"LFCvsSEAgent.execute: Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest(requestName, requestString,
sourceServer)
return S_OK()
def getRequestForSubRequest(self, itself, subRequestID):
"""
Select Request given SubRequestID.
:param self: plugin reference
:param itself: patient reference for injection
:param int subRequestID: SubRequests.SubRequestID
:warn: function has to be injected to RequestDBMySQL instance
"""
## get RequestID
requestID = "SELECT RequestID FROM SubRequests WHERE SubRequestID = %s;" % str(
subRequestID)
requestID = self._query(requestID)
if not requestID["OK"]:
return requestID
requestID = requestID["Value"][0]
## create RequestContainer
requestContainer = RequestContainer(init=False)
requestContainer.setRequestID(requestID)
## put some basic infos in
requestInfo = "SELECT RequestName, JobID, OwnerDN, OwnerGroup, DIRACSetup, SourceComponent, CreationTime, SubmissionTime, LastUpdate, Status "
requestInfo += "FROM Requests WHERE RequestID = %d;" % requestID
requestInfo = self._query(requestInfo)
if not requestInfo["OK"]:
return requestInfo
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate, status = requestInfo[
'Value'][0]
requestContainer.setRequestName(requestName)
requestContainer.setJobID(jobID)
requestContainer.setOwnerDN(ownerDN)
requestContainer.setOwnerGroup(ownerGroup)
requestContainer.setDIRACSetup(diracSetup)
requestContainer.setSourceComponent(sourceComponent)
requestContainer.setCreationTime(str(creationTime))
requestContainer.setLastUpdate(str(lastUpdate))
requestContainer.setStatus(status)
## get sub-requests
subRequests = "SELECT SubRequestID, Status, RequestType, Operation, Arguments, ExecutionOrder, SourceSE, "
subRequests += "TargetSE, Catalogue, CreationTime, SubmissionTime, LastUpdate FROM SubRequests WHERE RequestID=%s;" % requestID
subRequests = self._query(subRequests)
if not subRequests["OK"]:
return subRequests
## loop over sub requests
for subRequestID, status, requestType, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in subRequests[
"Value"]:
res = requestContainer.initiateSubRequest(requestType)
ind = res["Value"]
subRequestDict = {
"Status": status,
"SubRequestID": subRequestID,
"Operation": operation,
"Arguments": arguments,
"ExecutionOrder": int(executionOrder),
"SourceSE": sourceSE,
"TargetSE": targetSE,
"Catalogue": catalogue,
"CreationTime": creationTime,
"SubmissionTime": submissionTime,
"LastUpdate": lastUpdate
}
res = requestContainer.setSubRequestAttributes(
ind, requestType, subRequestDict)
if not res["OK"]:
return res
## get files for this subrequest
req = "SELECT FileID, LFN, Size, PFN, GUID, Md5, Addler, Attempt, Status FROM Files WHERE SubRequestID = %s ORDER BY FileID;" % str(
subRequestID)
res = self._query(req)
if not res["OK"]:
return res
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res[
"Value"]:
fileDict = {
"FileID": fileID,
"LFN": lfn,
"Size": size,
"PFN": pfn,
"GUID": guid,
"Md5": md5,
"Addler": addler,
"Attempt": attempt,
"Status": status
}
files.append(fileDict)
res = requestContainer.setSubRequestFiles(ind, requestType, files)
if not res["OK"]:
return res
## dump request to XML
res = requestContainer.toXML()
if not res["OK"]:
return res
requestString = res["Value"]
## return dictonary with all info in at least
return S_OK({
"RequestName": requestName,
"RequestString": requestString,
"JobID": jobID,
"RequestContainer": requestContainer
})
def executeRequest(self):
################################################
# Get a request from request DB
res = self.RequestDBClient.getRequest('register')
if not res['OK']:
gLogger.info(
"RegistrationAgent.execute: Failed to get request from database."
)
return S_OK()
elif not res['Value']:
gLogger.info(
"RegistrationAgent.execute: No requests to be executed found.")
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
try:
jobID = int(res['Value']['JobID'])
except:
jobID = 0
gLogger.info("RegistrationAgent.execute: Obtained request %s" %
requestName)
result = self.RequestDBClient.getCurrentExecutionOrder(
requestName, sourceServer)
if result['OK']:
currentOrder = result['Value']
else:
return S_OK('Can not get the request execution order')
oRequest = RequestContainer(request=requestString)
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('register')
if not res['OK']:
errStr = "RegistrationAgent.execute: Failed to obtain number of transfer subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info("RegistrationAgent.execute: Found %s sub requests." %
res['Value'])
################################################
# For all the sub-requests in the request
modified = False
for ind in range(res['Value']):
gLogger.info(
"RegistrationAgent.execute: Processing sub-request %s." % ind)
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'register')['Value']
subExecutionOrder = int(subRequestAttributes['ExecutionOrder'])
subStatus = subRequestAttributes['Status']
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
subRequestFiles = oRequest.getSubRequestFiles(
ind, 'register')['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a register file operation
if operation == 'registerFile':
gLogger.info(
"RegistrationAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSE = str(subRequestAttributes['TargetSE'])
if diracSE == 'SE':
# We do not care about SE, put any there
diracSE = "CERN-FAILOVER"
catalog = subRequestAttributes['Catalogue']
if catalog == "None":
catalog = ''
subrequest_done = True
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile.get('LFN', '')
if lfn: lfn = str(lfn)
physicalFile = subRequestFile.get('PFN', '')
if physicalFile: physicalFile = str(physicalFile)
fileSize = subRequestFile.get('Size', 0)
if fileSize: fileSize = int(fileSize)
fileGuid = subRequestFile.get('GUID', '')
if fileGuid: fileGuid = str(fileGuid)
checksum = subRequestFile.get('Addler', '')
if checksum: checksum = str(checksum)
if catalog == 'BookkeepingDB':
diracSE = 'CERN-HIST'
fileTuple = (lfn, physicalFile, fileSize, diracSE,
fileGuid, checksum)
res = self.ReplicaManager.registerFile(
fileTuple, catalog)
print res
if not res['OK']:
self.DataLog.addFileRecord(
lfn, 'RegisterFail', diracSE, '',
'RegistrationAgent')
errStr = "RegistrationAgent.execute: Completely failed to register file."
gLogger.error(errStr, res['Message'])
subrequest_done = False
elif lfn in res['Value']['Failed'].keys():
self.DataLog.addFileRecord(
lfn, 'RegisterFail', diracSE, '',
'RegistrationAgent')
errStr = "RegistrationAgent.execute: Completely failed to register file."
gLogger.error(errStr,
res['Value']['Failed'][lfn])
subrequest_done = False
else:
self.DataLog.addFileRecord(
lfn, 'Register', diracSE, '',
'TransferAgent')
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Done')
modified = True
else:
gLogger.info(
"RegistrationAgent.execute: File already completed."
)
if subrequest_done:
oRequest.setSubRequestStatus(ind, 'register', 'Done')
################################################
# If the sub-request is none of the above types
else:
gLogger.error(
"RegistrationAgent.execute: Operation not supported.",
operation)
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty(ind, 'register')['Value']:
oRequest.setSubRequestStatus(ind, 'register', 'Done')
################################################
# If the sub-request is already in terminal state
else:
gLogger.info(
"RegistrationAgent.execute: Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest(requestName, requestString,
sourceServer)
if modified and jobID:
result = self.finalizeRequest(requestName, jobID, sourceServer)
return S_OK()
def getRequest(self, requestType):
""" Get a request of a given type eligible for execution
"""
# RG: What if requestType is not given?
# the first query will return nothing.
# KC: maybe returning S_ERROR would be enough?
# alternatively we should check if requestType is known (in 'transfer', 'removal', 'register' and 'diset')
if not requestType or type(requestType) not in types.StringTypes:
return S_ERROR("Request type not given.")
myRequestType = self._escapeString(requestType)
if not myRequestType:
return myRequestType
myRequestType = myRequestType['Value']
start = time.time()
dmRequest = RequestContainer(init=False)
requestID = 0
subIDList = []
fields = [
'RequestID', 'SubRequestID', 'Operation', 'Arguments',
'ExecutionOrder', 'SourceSE', 'TargetSE', 'Catalogue',
'CreationTime', 'SubmissionTime', 'LastUpdate'
]
# get the pending SubRequest sorted by ExecutionOrder and LastUpdate
req = "SELECT RequestID, ExecutionOrder, Status, RequestType, LastUpdate from SubRequests WHERE Status IN ( 'Waiting', 'Assigned' ) ORDER BY ExecutionOrder, LastUpdate"
# now get sorted list of RequestID (according to the above)
req = "SELECT * from ( %s ) as T1 GROUP BY RequestID" % req
# and get the 100 oldest ones of Type requestType
req = "SELECT RequestID, ExecutionOrder FROM ( %s ) as T2 WHERE RequestType = %s ORDER BY LastUpdate limit 100" % (
req, myRequestType)
# and now get all waiting SubRequest for the selected RequestID and ExecutionOrder
req = "SELECT A.%s FROM SubRequests AS A, ( %s ) AS B WHERE " % (
', A.'.join(fields), req)
req = "%s A.RequestID = B.RequestID AND A.ExecutionOrder = B.ExecutionOrder AND A.Status = 'Waiting' AND A.RequestType = %s;" % (
req, myRequestType)
result = self._query(req)
if not result['OK']:
err = 'RequestDB._getRequest: Failed to retrieve Requests'
return S_ERROR('%s\n%s' % (err, result['Message']))
if not result['Value']:
return S_OK()
# We get up to 10 Request candidates, to add some randomness
reqDict = {}
for row in result['Value']:
reqDict.setdefault(row[0], [])
reqDict[row[0]].append(row[1:])
reqIDList = reqDict.keys()
random.shuffle(reqIDList)
for reqID in reqIDList:
sidList = [x[0] for x in reqDict[reqID]]
for subID in sidList:
req = "UPDATE SubRequests SET Status='Assigned' WHERE RequestID=%s AND SubRequestID=%s;" % (
reqID, subID)
resAssigned = self._update(req)
if not resAssigned['OK']:
if subIDList:
self.__releaseSubRequests(reqID, subIDList)
return S_ERROR('Failed to assign subrequests: %s' %
resAssigned['Message'])
if resAssigned['Value'] == 0:
# Somebody has assigned this request
gLogger.warn(
'Already assigned subrequest %d of request %d' %
(subID, reqID))
else:
subIDList.append(subID)
if subIDList:
# We managed to get some requests, can continue now
requestID = reqID
break
# Haven't succeeded to get any request
if not requestID:
return S_OK()
dmRequest.setRequestID(requestID)
fields = [
'FileID', 'LFN', 'Size', 'PFN', 'GUID', 'Md5', 'Addler', 'Attempt',
'Status'
]
for subRequestID, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in reqDict[
requestID]:
if not subRequestID in subIDList: continue
res = dmRequest.initiateSubRequest(requestType)
ind = res['Value']
subRequestDict = {
'Status': 'Waiting',
'SubRequestID': subRequestID,
'Operation': operation,
'Arguments': arguments,
'ExecutionOrder': int(executionOrder),
'SourceSE': sourceSE,
'TargetSE': targetSE,
'Catalogue': catalogue,
'CreationTime': creationTime,
'SubmissionTime': submissionTime,
'LastUpdate': lastUpdate
}
res = dmRequest.setSubRequestAttributes(ind, requestType,
subRequestDict)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set subRequest attributes for RequestID %s' % requestID
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
req = "SELECT %s FROM Files WHERE SubRequestID = %s ORDER BY FileID;" % (
', '.join(fields), subRequestID)
res = self._query(req)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get File attributes for RequestID %s.%s' % (
requestID, subRequestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res[
'Value']:
fileDict = {
'FileID': fileID,
'LFN': lfn,
'Size': size,
'PFN': pfn,
'GUID': guid,
'Md5': md5,
'Addler': addler,
'Attempt': attempt,
'Status': status
}
files.append(fileDict)
res = dmRequest.setSubRequestFiles(ind, requestType, files)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set files into Request for RequestID %s.%s' % (
requestID, subRequestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
req = "SELECT Dataset,Status FROM Datasets WHERE SubRequestID = %s;" % subRequestID
res = self._query(req)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get Datasets for RequestID %s.%s' % (
requestID, subRequestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
datasets = []
for dataset, status in res['Value']:
datasets.append(dataset)
res = dmRequest.setSubRequestDatasets(ind, requestType, datasets)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set datasets into Request for RequestID %s.%s' % (
requestID, subRequestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
fields = [
'RequestName', 'JobID', 'OwnerDN', 'OwnerGroup', 'DIRACSetup',
'SourceComponent', 'CreationTime', 'SubmissionTime', 'LastUpdate'
]
req = "SELECT %s from Requests WHERE RequestID = %s;" % (
', '.join(fields), requestID)
res = self._query(req)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve max RequestID'
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate = res[
'Value'][0]
dmRequest.setRequestName(requestName)
dmRequest.setJobID(jobID)
dmRequest.setOwnerDN(ownerDN)
dmRequest.setOwnerGroup(ownerGroup)
dmRequest.setDIRACSetup(diracSetup)
dmRequest.setSourceComponent(sourceComponent)
dmRequest.setCreationTime(str(creationTime))
dmRequest.setLastUpdate(str(lastUpdate))
res = dmRequest.toXML()
if not res['OK']:
err = 'RequestDB._getRequest: Failed to create XML for RequestID %s' % (
requestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
requestString = res['Value']
#still have to manage the status of the dataset properly
resultDict = {}
resultDict['RequestName'] = requestName
resultDict['RequestString'] = requestString
resultDict['JobID'] = jobID
return S_OK(resultDict)
def readRequestsForJobs( self, jobIDs ):
""" read and return Requests for jobs
:param mixed jobIDs: list with jobIDs or long JobIDs
"""
if type(jobIDs) != list:
return S_ERROR("RequestDB: wrong format for jobIDs argument, got %s, expecting a list" )
# make sure list is uniqe and has only longs
jobIDs = list( set( [ int(jobID) for jobID in jobIDs if int(jobID) != 0 ] ) )
reqCols = [ "RequestID", "RequestName", "JobID", "Status",
"OwnerDN", "OwnerGroup", "DIRACSetup", "SourceComponent",
"CreationTime", "SubmissionTime", "LastUpdate" ]
subCols = [ "SubRequestID", "Operation", "Arguments", "RequestType", "ExecutionOrder", "Error",
"SourceSE", "TargetSE", "Catalogue", "CreationTime", "SubmissionTime", "LastUpdate" ]
fileCols = [ "FileID", "LFN", "Size", "PFN", "GUID", "Md5", "Addler", "Attempt", "Status" , "Error" ]
requestNames = self.getRequestForJobs( jobIDs )
if not requestNames["OK"]:
return requestNames
requestNames = requestNames["Value"]
## this will be returned
retDict = { "Successful" : dict(), "Failed" : dict() }
for jobID in jobIDs:
## missing requests
if jobID not in requestNames:
retDict["Failed"][jobID] = "Request not found"
continue
requestName = requestNames[jobID]
## get request
queryStr = "SELECT %s FROM Requests WHERE RequestName = '%s';" % ( ",".join( reqCols ), requestName )
queryRes = self._query( queryStr )
if not queryRes["OK"]:
retDict["Failed"][jobID] = queryRes["Message"]
continue
queryRes = queryRes["Value"] if queryRes["Value"] else None
if not queryRes:
retDict["Failed"][jobID] = "Unable to read request attributes."
continue
requestObj = RequestContainer( init=False )
reqAttrs = dict( zip( reqCols, queryRes[0] ) )
requestObj.setRequestAttributes( reqAttrs )
queryStr = "SELECT %s FROM `SubRequests` WHERE `RequestID`=%s;" % ( ",".join(subCols), reqAttrs["RequestID"] )
queryRes = self._query( queryStr )
if not queryRes["OK"]:
retDict["Failed"][jobID] = queryRes["Message"]
continue
queryRes = queryRes["Value"] if queryRes["Value"] else None
if not queryRes:
retDict["Failed"][jobID] = "Unable to read subrequest attributes."
continue
## get sub-requests
for recTuple in queryRes:
subReqAttrs = dict( zip( subCols, recTuple ) )
subType = subReqAttrs["RequestType"]
subReqAttrs["ExecutionOrder"] = int( subReqAttrs["ExecutionOrder"] )
del subReqAttrs["RequestType"]
index = requestObj.initiateSubRequest( subType )
index = index["Value"]
requestObj.setSubRequestAttributes( index, subType, subReqAttrs )
## get files
subFiles = []
fileQuery = "SELECT %s FROM `Files` WHERE `SubRequestID` = %s ORDER BY `FileID`;" % ( ",".join(fileCols),
subReqAttrs["SubRequestID"] )
fileQueryRes = self._query( fileQuery )
if fileQueryRes["OK"] and fileQueryRes["Value"]:
for fileRec in fileQueryRes["Value"]:
subFiles.append( dict( zip(fileCols, fileRec) ) )
if subFiles:
requestObj.setSubRequestFiles( index, subType, subFiles )
retDict["Successful"][jobID] = requestObj.toXML()["Value"]
return S_OK( retDict )
def execute_request( self ):
""" Takes one DISET request and forward it to the destination service
"""
gMonitor.addMark( "Iteration", 1 )
if self.RequestDB:
res = self.RequestDB.getRequest( 'diset' )
else:
res = self.RequestDBClient.getRequest( 'diset' )
if not res['OK']:
gLogger.error( "DISETForwardingAgent.execute: Failed to get request from database." )
return S_OK()
elif not res['Value']:
gLogger.info( "DISETForwardingAgent.execute: No requests to be executed found." )
return S_OK()
gMonitor.addMark( "Attempted", 1 )
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
try:
jobID = int( res['Value']['JobID'] )
except:
jobID = 0
gLogger.info( "DISETForwardingAgent.execute: Obtained request %s" % requestName )
if self.RequestDB:
result = self.RequestDB._getRequestAttribute( 'RequestID', requestName = requestName )
if not result['OK']:
return S_OK( 'Can not get the request execution order' )
requestID = result['Value']
result = self.RequestDB.getCurrentExecutionOrder( requestID )
else:
result = self.RequestDBClient.getCurrentExecutionOrder( requestName )
if result['OK']:
currentOrder = result['Value']
else:
return S_OK( 'Can not get the request execution order' )
oRequest = RequestContainer( request = requestString )
requestAttributes = oRequest.getRequestAttributes()['Value']
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests( 'diset' )
if not res['OK']:
errStr = "DISETForwardingAgent.execute: Failed to obtain number of diset subrequests."
gLogger.error( errStr, res['Message'] )
return S_OK()
gLogger.info( "DISETForwardingAgent.execute: Found %s sub requests for job %s" % ( res['Value'], jobID ) )
################################################
# For all the sub-requests in the request
modified = False
for ind in range( res['Value'] ):
subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'diset' )['Value']
subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
subStatus = subRequestAttributes['Status']
gLogger.info( "DISETForwardingAgent.execute: Processing sub-request %s with execution order %d" % ( ind, subExecutionOrder ) )
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
operation = subRequestAttributes['Operation']
gLogger.info( "DISETForwardingAgent.execute: Attempting to forward %s type." % operation )
rpcStubString = subRequestAttributes['Arguments']
rpcStub, length = DEncode.decode( rpcStubString )
res = executeRPCStub( rpcStub )
if res['OK']:
gLogger.info( "DISETForwardingAgent.execute: Successfully forwarded." )
oRequest.setSubRequestStatus( ind, 'diset', 'Done' )
gMonitor.addMark( "Successful", 1 )
modified = True
elif res['Message'] == 'No Matching Job':
gLogger.warn( "DISETForwardingAgent.execute: No corresponding job found. Setting to done." )
oRequest.setSubRequestStatus( ind, 'diset', 'Done' )
else:
gLogger.error( "DISETForwardingAgent.execute: Failed to forward request.", res['Message'] )
else:
gLogger.info( "DISETForwardingAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
if self.RequestDB:
res = self.RequestDB.updateRequest( requestName, requestString )
else:
res = self.RequestDBClient.updateRequest( requestName, requestString )
if res['OK']:
gLogger.info( "DISETForwardingAgent.execute: Successfully updated request." )
else:
gLogger.error( "DISETForwardingAgent.execute: Failed to update request" )
if modified and jobID:
result = self.RequestDBClient.finalizeRequest( requestName, jobID )
return S_OK()
for lfnList in breakListIntoChunks(lfns, 100):
oRequest = RequestContainer()
subRequestIndex = oRequest.initiateSubRequest(requestType)['Value']
attributeDict = {'Operation': requestOperation, 'TargetSE': targetSE}
oRequest.setSubRequestAttributes(subRequestIndex, requestType,
attributeDict)
files = []
for lfn in lfnList:
files.append({'LFN': lfn})
oRequest.setSubRequestFiles(subRequestIndex, requestType, files)
requestName = "%s_%s" % (md5(repr(time.time())).hexdigest()[:16],
md5(repr(time.time())).hexdigest()[:16])
oRequest.setRequestAttributes({'RequestName': requestName})
DIRAC.gLogger.info(oRequest.toXML()['Value'])
result = requestClient.setRequest(requestName, oRequest.toXML()['Value'])
if result['OK']:
print 'Submitted Request:', result['Value']
else:
print 'Failed to submit Request', result['Message']
if monitor:
requestID = result['Value']
while True:
result = requestClient.getRequestStatus(requestID)
if not result['OK']:
Script.gLogger.error(result['Message'])
break
Script.gLogger.notice(result['Value']['RequestStatus'])
if result['Value']['RequestStatus'] == 'Done':
def execute(self):
IntegrityDB = RPCClient('DataManagement/DataIntegrity')
res = self.RequestDBClient.getRequest('integrity')
if not res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Failed to get request from database.")
return S_OK()
elif not res['Value']:
gLogger.info(
"SEvsLFCAgent.execute: No requests to be executed found.")
return S_OK()
requestString = res['Value']['requestString']
requestName = res['Value']['requestName']
sourceServer = res['Value']['Server']
gLogger.info("SEvsLFCAgent.execute: Obtained request %s" % requestName)
oRequest = RequestContainer(request=requestString)
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('integrity')
if not res['OK']:
errStr = "SEvsLFCAgent.execute: Failed to obtain number of integrity subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info("SEvsLFCAgent.execute: Found %s sub requests." %
res['Value'])
################################################
# For all the sub-requests in the request
for ind in range(res['Value']):
gLogger.info("SEvsLFCAgent.execute: Processing sub-request %s." %
ind)
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'integrity')['Value']
if subRequestAttributes['Status'] == 'Waiting':
subRequestFiles = oRequest.getSubRequestFiles(
ind, 'integrity')['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a lfcvsse operation
if operation == 'SEvsLFC':
gLogger.info(
"SEvsLFCAgent.execute: Attempting to execute %s sub-request."
% operation)
storageElementName = subRequestAttributes['StorageElement']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile['LFN']
storageElement = StorageElement(storageElementName)
res = storageElement.isValid()
if not res['OK']:
errStr = "SEvsLFCAgent.execute: Failed to instantiate destination StorageElement."
gLogger.error(errStr, storageElement)
else:
res = storageElement.getPfnForLfn(lfn)
if not res['OK']:
gLogger.info('shit bugger do something.')
else:
oNamespaceBrowser = NamespaceBrowser(
res['Value'])
# Loop over all the directories and sub-directories
while (oNamespaceBrowser.isActive()):
currentDir = oNamespaceBrowser.getActiveDir(
)
gLogger.info(
"SEvsLFCAgent.execute: Attempting to list the contents of %s."
% currentDir)
res = storageElement.listDirectory(
currentDir)
if not res['Value'][
'Successful'].has_key(
currentDir):
gLogger.error(
"SEvsLFCAgent.execute: Failed to list the directory contents.",
"%s %s" %
(currentDir,
res['Value']['Successful']
['Failed'][currentDir]))
subDirs = [currentDir]
else:
subDirs = []
files = {}
for surl, surlDict in res['Value'][
'Successful'][currentDir][
'Files'].items():
pfnRes = storageElement.getPfnForProtocol(
surl,
'SRM2',
withPort=False)
surl = pfnRes['Value']
files[surl] = surlDict
for surl, surlDict in res['Value'][
'Successful'][currentDir][
'SubDirs'].items():
pfnRes = storageElement.getPfnForProtocol(
surl,
'SRM2',
withPort=False)
surl = pfnRes['Value']
subDirs.append(surl)
#subDirs = res['Value']['Successful'][currentDir]['SubDirs']
gLogger.info(
"SEvsLFCAgent.execute: Successfully obtained %s sub-directories."
% len(subDirs))
#files = res['Value']['Successful'][currentDir]['Files']
gLogger.info(
"SEvsLFCAgent.execute: Successfully obtained %s files."
% len(files))
selectedLfns = []
lfnPfnDict = {}
pfnSize = {}
for pfn, pfnDict in files.items():
res = storageElement.getPfnPath(
pfn)
if not res['OK']:
gLogger.error(
"SEvsLFCAgent.execute: Failed to get determine LFN from pfn.",
"%s %s" %
(pfn, res['Message']))
fileMetadata = {
'Prognosis':
'NonConventionPfn',
'LFN': '',
'PFN': pfn,
'StorageElement':
storageElementName,
'Size': pfnDict['Size']
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
else:
lfn = res['Value']
selectedLfns.append(lfn)
lfnPfnDict[lfn] = pfn
pfnSize[pfn] = pfnDict[
'Size']
res = self.ReplicaManager.getCatalogFileMetadata(
selectedLfns)
if not res['OK']:
subDirs = [currentDir]
else:
for lfn in res['Value'][
'Failed'].keys():
gLogger.error(
"SEvsLFCAgent.execute: Failed to get metadata.",
"%s %s" %
(lfn, res['Value']
['Failed'][lfn]))
pfn = lfnPfnDict[lfn]
fileMetadata = {
'Prognosis':
'SEPfnNoLfn',
'LFN': lfn,
'PFN': pfn,
'StorageElement':
storageElementName,
'Size': pfnSize[pfn]
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
for lfn, lfnDict in res[
'Value'][
'Successful'].items(
):
pfn = lfnPfnDict[lfn]
storageSize = pfnSize[pfn]
catalogSize = lfnDict[
'Size']
if int(catalogSize) == int(
storageSize):
gLogger.info(
"SEvsLFCAgent.execute: Catalog and storage sizes match.",
"%s %s" %
(pfn,
storageElementName
))
gLogger.info(
"Change the status in the LFC"
)
elif int(storageSize) == 0:
gLogger.error(
"SEvsLFCAgent.execute: Physical file size is 0.",
"%s %s" %
(pfn,
storageElementName
))
fileMetadata = {
'Prognosis':
'ZeroSizePfn',
'LFN':
lfn,
'PFN':
pfn,
'StorageElement':
storageElementName
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
else:
gLogger.error(
"SEvsLFCAgent.execute: Catalog and storage size mis-match.",
"%s %s" %
(pfn,
storageElementName
))
fileMetadata = {
'Prognosis':
'PfnSizeMismatch',
'LFN':
lfn,
'PFN':
pfn,
'StorageElement':
storageElementName
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
res = self.ReplicaManager.getCatalogReplicas(
selectedLfns)
if not res['OK']:
subDirs = [currentDir]
else:
for lfn in res['Value'][
'Failed'].keys():
gLogger.error(
"SEvsLFCAgent.execute: Failed to get replica information.",
"%s %s" %
(lfn, res['Value']
['Failed'][lfn]))
pfn = lfnPfnDict[lfn]
fileMetadata = {
'Prognosis':
'PfnNoReplica',
'LFN': lfn,
'PFN': pfn,
'StorageElement':
storageElementName,
'Size':
pfnSize[pfn]
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
for lfn, repDict in res[
'Value'][
'Successful'].items(
):
pfn = lfnPfnDict[lfn]
registeredPfns = repDict.values(
)
if not pfn in registeredPfns:
gLogger.error(
"SEvsLFCAgent.execute: SE PFN not registered.",
"%s %s" %
(lfn, pfn))
fileMetadata = {
'Prognosis':
'PfnNoReplica',
'LFN':
lfn,
'PFN':
pfn,
'StorageElement':
storageElementName
}
res = IntegrityDB.insertProblematic(
AGENT_NAME,
fileMetadata)
if res['OK']:
gLogger.info(
"SEvsLFCAgent.execute: Successfully added to IntegrityDB."
)
gLogger.error(
"Change the status in the LFC,ProcDB...."
)
else:
gLogger.error(
"Shit, f**k, bugger. Add the failover."
)
else:
gLogger.info(
"SEvsLFCAgent.execute: SE Pfn verified.",
pfn)
oNamespaceBrowser.updateDirs(subDirs)
oRequest.setSubRequestFileAttributeValue(
ind, 'integrity', lfn, 'Status',
'Done')
################################################
# If the sub-request is none of the above types
else:
gLogger.info(
"SEvsLFCAgent.execute: Operation not supported.",
operation)
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty(ind, 'integrity')['Value']:
oRequest.setSubRequestStatus(ind, 'integrity', 'Done')
################################################
# If the sub-request is already in terminal state
else:
gLogger.info(
"SEvsLFCAgent.execute: Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest(requestName, requestString,
sourceServer)
return S_OK()
def execute( self ):
res = self.RequestDBClient.getRequest( 'integrity' )
if not res['OK']:
gLogger.info( "LFCvsSEAgent.execute: Failed to get request from database." )
return S_OK()
elif not res['Value']:
gLogger.info( "LFCvsSEAgent.execute: No requests to be executed found." )
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
gLogger.info( "LFCvsSEAgent.execute: Obtained request %s" % requestName )
oRequest = RequestContainer( request = requestString )
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests( 'integrity' )
if not res['OK']:
errStr = "LFCvsSEAgent.execute: Failed to obtain number of integrity subrequests."
gLogger.error( errStr, res['Message'] )
return S_OK()
gLogger.info( "LFCvsSEAgent.execute: Found %s sub requests." % res['Value'] )
################################################
# For all the sub-requests in the request
for ind in range( res['Value'] ):
gLogger.info( "LFCvsSEAgent.execute: Processing sub-request %s." % ind )
subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'integrity' )['Value']
if subRequestAttributes['Status'] == 'Waiting':
subRequestFiles = oRequest.getSubRequestFiles( ind, 'integrity' )['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a lfcvsse operation
if operation == 'LFCvsSE':
gLogger.info( "LFCvsSEAgent.execute: Attempting to execute %s sub-request." % operation )
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = subRequestFile['LFN']
oNamespaceBrowser = NamespaceBrowser( lfn )
# Loop over all the directories and sub-directories
while ( oNamespaceBrowser.isActive() ):
currentDir = oNamespaceBrowser.getActiveDir()
gLogger.info( "LFCvsSEAgent.execute: Attempting to get contents of %s." % currentDir )
res = self.ReplicaManager.getCatalogDirectoryContents( currentDir )
if not res['OK']:
subDirs = [currentDir]
elif res['Value']['Failed'].has_key( currentDir ):
subDirs = [currentDir]
else:
subDirs = res['Value']['Successful'][currentDir]['SubDirs']
files = res['Value']['Successful'][currentDir]['Files']
lfnSizeDict = {}
pfnLfnDict = {}
pfnStatusDict = {}
sePfnDict = {}
for lfn, lfnDict in files.items():
lfnSizeDict[lfn] = lfnDict['MetaData']['Size']
for se in lfnDict['Replicas'].keys():
pfn = lfnDict['Replicas'][se]['PFN']
status = lfnDict['Replicas'][se]['Status']
pfnStatusDict[pfn] = status
pfnLfnDict[pfn] = lfn
if not sePfnDict.has_key( se ):
sePfnDict[se] = []
sePfnDict[se].append( pfn )
for storageElementName, physicalFiles in sePfnDict.items():
gLogger.info( "LFCvsSEAgent.execute: Attempting to get metadata for files on %s." % storageElementName )
res = self.ReplicaManager.getStorageFileMetadata( physicalFiles, storageElementName )
if not res['OK']:
gLogger.error( "LFCvsSEAgent.execute: Completely failed to get physical file metadata.", res['Message'] )
else:
for pfn in res['Value']['Failed'].keys():
gLogger.error( "LFCvsSEAgent.execute: Failed to get metadata.", "%s %s" % ( pfn, res['Value']['Failed'][pfn] ) )
lfn = pfnLfnDict[pfn]
fileMetadata = {'Prognosis':'MissingSEPfn', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName, 'Size':lfnSizeDict[lfn]}
IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if resInsert['OK']:
gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
for pfn, pfnDict in res['Value']['Successful'].items():
lfn = pfnLfnDict[pfn]
catalogSize = int( lfnSizeDict[lfn] )
storageSize = int( pfnDict['Size'] )
if int( catalogSize ) == int( storageSize ):
gLogger.info( "LFCvsSEAgent.execute: Catalog and storage sizes match.", "%s %s" % ( pfn, storageElementName ) )
gLogger.info( "Change the status in the LFC" )
else:
gLogger.error( "LFCvsSEAgent.execute: Catalog and storage size mis-match.", "%s %s" % ( pfn, storageElementName ) )
fileMetadata = {'Prognosis':'PfnSizeMismatch', 'LFN':lfn, 'PFN':pfn, 'StorageElement':storageElementName}
IntegrityDB = RPCClient( 'DataManagement/DataIntegrity' )
resInsert = IntegrityDB.insertProblematic( AGENT_NAME, fileMetadata )
if resInsert['OK']:
gLogger.info( "LFCvsSEAgent.execute: Successfully added to IntegrityDB." )
gLogger.error( "Change the status in the LFC,ProcDB...." )
else:
gLogger.error( "Shit, f**k, bugger. Add the failover." )
oNamespaceBrowser.updateDirs( subDirs )
oRequest.setSubRequestFileAttributeValue( ind, 'integrity', lfn, 'Status', 'Done' )
################################################
# If the sub-request is none of the above types
else:
gLogger.info( "LFCvsSEAgent.execute: Operation not supported.", operation )
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty( ind, 'integrity' )['Value']:
oRequest.setSubRequestStatus( ind, 'integrity', 'Done' )
################################################
# If the sub-request is already in terminal state
else:
gLogger.info( "LFCvsSEAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )
return S_OK()
def executeRequest(self):
"""
Do the actual work in the Thread
"""
################################################
# Get a request from request DB
gMonitor.addMark("Iteration", 1)
res = self.requestDBClient.getRequest('removal')
if not res['OK']:
gLogger.info(
"RemovalAgent.execute: Failed to get request from database.")
return S_OK()
elif not res['Value']:
gLogger.info(
"RemovalAgent.execute: No requests to be executed found.")
self.pendingRequests = False
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
jobID = 0
try:
jobID = int(res['Value']['JobID'])
except:
gLogger.warn(
"RemovalAgent.execute: JobID not present or malformed in request '%s', will use 0 instead."
% requestName)
gLogger.info("RemovalAgent.execute: Obtained request %s" % requestName)
try:
result = self.requestDBClient.getCurrentExecutionOrder(
requestName, sourceServer)
if result['OK']:
currentOrder = result['Value']
else:
gLogger.error('Can not get the request execution order')
self.requestDBClient.updateRequest(requestName, requestString,
sourceServer)
return S_OK('Can not get the request execution order')
oRequest = RequestContainer(request=requestString)
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('removal')
if not res['OK']:
errStr = "RemovalAgent.execute: Failed to obtain number of removal subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info("RemovalAgent.execute: Found %s sub requests." %
res['Value'])
################################################
# For all the sub-requests in the request
modified = False
for ind in range(res['Value']):
gMonitor.addMark("Execute", 1)
gLogger.info(
"RemovalAgent.execute: Processing sub-request %s." % ind)
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'removal')['Value']
subExecutionOrder = int(subRequestAttributes['ExecutionOrder'])
subStatus = subRequestAttributes['Status']
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
subRequestFiles = oRequest.getSubRequestFiles(
ind, 'removal')['Value']
operation = subRequestAttributes['Operation']
################################################
# If the sub-request is a physical removal operation
if operation == 'physicalRemoval':
gLogger.info(
"RemovalAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSEs = subRequestAttributes['TargetSE'].split(',')
physicalFiles = []
pfnToLfn = {}
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
pfn = str(subRequestFile['PFN'])
lfn = str(subRequestFile['LFN'])
pfnToLfn[pfn] = lfn
physicalFiles.append(pfn)
gMonitor.addMark('PhysicalRemovalAtt',
len(physicalFiles))
failed = {}
errMsg = {}
for diracSE in diracSEs:
res = self.replicaManager.removeStorageFile(
physicalFiles, diracSE)
if res['OK']:
for pfn in res['Value']['Failed'].keys():
if not failed.has_key(pfn):
failed[pfn] = {}
failed[pfn][diracSE] = res['Value'][
'Failed'][pfn]
else:
errMsg[diracSE] = res['Message']
for pfn in physicalFiles:
if not failed.has_key(pfn):
failed[pfn] = {}
failed[pfn][diracSE] = 'Completely'
# Now analyse the results
failedPFNs = failed.keys()
pfnsOK = [
pfn for pfn in physicalFiles
if not pfn in failedPFNs
]
gMonitor.addMark('PhysicalRemovalDone', len(pfnsOK))
for pfn in pfnsOK:
gLogger.info(
"RemovalAgent.execute: Successfully removed %s at %s"
% (pfn, str(diracSEs)))
res = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', pfnToLfn[pfn], 'Status',
'Done')
if not res['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', pfnToLfn[pfn]))
modified = True
if failed:
gMonitor.addMark('PhysicalRemovalFail',
len(failedPFNs))
for pfn in failedPFNs:
for diracSE in failed[pfn].keys():
if type(failed[pfn]
[diracSE]) in StringTypes:
if re.search(
'no such file or directory',
failed[pfn][diracSE].lower()):
gLogger.info(
"RemovalAgent.execute: File did not exist.",
pfn)
res = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', pfnToLfn[pfn],
'Status', 'Done')
if not res['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', pfnToLfn[pfn]))
modified = True
else:
gLogger.info(
"RemovalAgent.execute: Failed to remove file.",
"%s at %s - %s" %
(pfn, diracSE,
failed[pfn][diracSE]))
if errMsg:
for diracSE in errMsg.keys():
errStr = "RemovalAgent.execute: Completely failed to remove replicas. At %s", diracSE
gLogger.error(errStr, errMsg[diracSE])
################################################
# If the sub-request is a physical removal operation
elif operation == 'removeFile':
gLogger.info(
"RemovalAgent.execute: Attempting to execute %s sub-request."
% operation)
lfns = []
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = str(subRequestFile['LFN'])
lfns.append(lfn)
gMonitor.addMark('RemoveFileAtt', len(lfns))
res = self.replicaManager.removeFile(lfns)
if res['OK']:
gMonitor.addMark(
'RemoveFileDone',
len(res['Value']['Successful'].keys()))
for lfn in res['Value']['Successful'].keys():
gLogger.info(
"RemovalAgent.execute: Successfully removed %s."
% lfn)
result = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', lfn, 'Status', 'Done')
if not result['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', lfn))
modified = True
gMonitor.addMark(
'RemoveFileFail',
len(res['Value']['Failed'].keys()))
for lfn in res['Value']['Failed'].keys():
if type(res['Value']['Failed']
[lfn]) in StringTypes:
if re.search(
'no such file or directory',
res['Value']['Failed']
[lfn].lower()):
gLogger.info(
"RemovalAgent.execute: File did not exist.",
lfn)
result = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', lfn, 'Status',
'Done')
if not result['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', lfn))
modified = True
else:
gLogger.info(
"RemovalAgent.execute: Failed to remove file:",
"%s %s" %
(lfn, res['Value']['Failed'][lfn]))
else:
gMonitor.addMark('RemoveFileFail', len(lfns))
errStr = "RemovalAgent.execute: Completely failed to remove files files."
gLogger.error(errStr, res['Message'])
################################################
# If the sub-request is a physical removal operation
elif operation == 'replicaRemoval':
gLogger.info(
"RemovalAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSEs = subRequestAttributes['TargetSE'].split(',')
lfns = []
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = str(subRequestFile['LFN'])
lfns.append(lfn)
gMonitor.addMark('ReplicaRemovalAtt', len(lfns))
failed = {}
errMsg = {}
for diracSE in diracSEs:
res = self.replicaManager.removeReplica(
diracSE, lfns)
if res['OK']:
for lfn in res['Value']['Failed'].keys():
errorMessage = str(
res['Value']['Failed'][lfn])
if errorMessage.find(
'Write access not permitted for this credential.'
) != -1:
if self.__getProxyAndRemoveReplica(
diracSE, lfn):
continue
if errorMessage.find(
'seconds timeout for "__gfal_wrapper" call'
) != -1:
self.timeOutCounter += 1
if not failed.has_key(lfn):
failed[lfn] = {}
failed[lfn][diracSE] = res['Value'][
'Failed'][lfn]
else:
errMsg[diracSE] = res['Message']
for lfn in lfns:
if not failed.has_key(lfn):
failed[lfn] = {}
failed[lfn][diracSE] = 'Completely'
# Now analyse the results
failedLFNs = failed.keys()
lfnsOK = [lfn for lfn in lfns if not lfn in failedLFNs]
gMonitor.addMark('ReplicaRemovalDone', len(lfnsOK))
for lfn in lfnsOK:
gLogger.info(
"RemovalAgent.execute: Successfully removed %s at %s"
% (lfn, str(diracSEs)))
res = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', lfn, 'Status', 'Done')
if not res['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', lfn))
modified = True
if failed:
gMonitor.addMark('PhysicalRemovalFail',
len(failedLFNs))
for lfn in failedLFNs:
for diracSE in failed[lfn].keys():
if type(failed[lfn]
[diracSE]) in StringTypes:
if re.search(
'no such file or directory',
failed[lfn][diracSE].lower()):
gLogger.info(
"RemovalAgent.execute: File did not exist.",
lfn)
res = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', lfn, 'Status',
'Done')
if not res['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', lfn))
modified = True
else:
gLogger.info(
"RemovalAgent.execute: Failed to remove file.",
"%s at %s - %s" %
(lfn, diracSE,
failed[lfn][diracSE]))
if errMsg:
for diracSE in errMsg.keys():
errStr = "RemovalAgent.execute: Completely failed to remove replicas. At %s", diracSE
gLogger.error(errStr, errMsg[diracSE])
################################################
# If the sub-request is a request to the online system to retransfer
elif operation == 'reTransfer':
gLogger.info(
"RemovalAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
pfn = str(subRequestFile['PFN'])
lfn = str(subRequestFile['LFN'])
res = self.replicaManager.onlineRetransfer(
diracSE, pfn)
if res['OK']:
if res['Value']['Successful'].has_key(pfn):
gLogger.info(
"RemovalAgent.execute: Successfully requested retransfer of %s."
% pfn)
result = oRequest.setSubRequestFileAttributeValue(
ind, 'removal', lfn, 'Status',
'Done')
if not result['OK']:
gLogger.error(
"RemovalAgent.execute: Error setting status to %s for %s"
% ('Done', lfn))
modified = True
else:
errStr = "RemovalAgent.execute: Failed to request retransfer."
gLogger.error(
errStr, "%s %s %s" %
(pfn, diracSE,
res['Value']['Failed'][pfn]))
else:
errStr = "RemovalAgent.execute: Completely failed to request retransfer."
gLogger.error(errStr, res['Message'])
else:
gLogger.info(
"RemovalAgent.execute: File already completed."
)
################################################
# If the sub-request is none of the above types
else:
gLogger.error(
"RemovalAgent.execute: Operation not supported.",
operation)
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty(ind, 'removal')['Value']:
oRequest.setSubRequestStatus(ind, 'removal', 'Done')
gMonitor.addMark("Done", 1)
################################################
# If the sub-request is already in terminal state
else:
gLogger.info(
"RemovalAgent.execute:",
"Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
newrequestString = oRequest.toXML()['Value']
except:
# if something fails return the original request back to the server
res = self.requestDBClient.updateRequest(requestName,
requestString,
sourceServer)
return S_OK()
res = self.requestDBClient.updateRequest(requestName, newrequestString,
sourceServer)
if modified and jobID:
result = self.finalizeRequest(requestName, jobID, sourceServer)
return S_OK()
def executeRequest( self ):
################################################
# Get a request from request DB
gMonitor.addMark( "Iteration", 1 )
res = self.RequestDBClient.getRequest( 'transfer' )
if not res['OK']:
gLogger.info( "TransferAgent.execute: Failed to get request from database." )
return S_OK()
elif not res['Value']:
gLogger.info( "TransferAgent.execute: No requests to be executed found." )
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
try:
jobID = int( res['Value']['JobID'] )
except:
jobID = 0
gLogger.info( "TransferAgent.execute: Obtained request %s" % requestName )
result = self.RequestDBClient.getCurrentExecutionOrder( requestName, sourceServer )
if result['OK']:
currentOrder = result['Value']
else:
return S_OK( 'Can not get the request execution order' )
oRequest = RequestContainer( request = requestString )
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests( 'transfer' )
if not res['OK']:
errStr = "TransferAgent.execute: Failed to obtain number of transfer subrequests."
gLogger.error( errStr, res['Message'] )
return S_OK()
gLogger.info( "TransferAgent.execute: Found %s sub requests." % res['Value'] )
################################################
# For all the sub-requests in the request
modified = False
for ind in range( res['Value'] ):
gMonitor.addMark( "Execute", 1 )
gLogger.info( "TransferAgent.execute: Processing sub-request %s." % ind )
subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'transfer' )['Value']
if subRequestAttributes['ExecutionOrder']:
subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
else:
subExecutionOrder = 0
subStatus = subRequestAttributes['Status']
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
subRequestFiles = oRequest.getSubRequestFiles( ind, 'transfer' )['Value']
operation = subRequestAttributes['Operation']
subRequestError = ''
################################################
# If the sub-request is a put and register operation
if operation == 'putAndRegister' or operation == 'putAndRegisterAndRemove':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
diracSE = str( subRequestAttributes['TargetSE'] )
catalog = ''
if subRequestAttributes.has_key( 'Catalogue' ):
catalog = subRequestAttributes['Catalogue']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark( "Put and register", 1 )
lfn = str( subRequestFile['LFN'] )
file = subRequestFile['PFN']
guid = subRequestFile['GUID']
addler = subRequestFile['Addler']
res = self.ReplicaManager.putAndRegister( lfn, file, diracSE, guid = guid, checksum = addler, catalog = catalog )
if res['OK']:
if res['Value']['Successful'].has_key( lfn ):
if not res['Value']['Successful'][lfn].has_key( 'put' ):
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
gLogger.info( "TransferAgent.execute: Failed to put %s to %s." % ( lfn, diracSE ) )
subRequestError = "Put operation failed for %s to %s" % ( lfn, diracSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' )
elif not res['Value']['Successful'][lfn].has_key( 'register' ):
gMonitor.addMark( "Put successful", 1 )
gMonitor.addMark( "File registration failed", 1 )
self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' )
self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'TransferAgent' )
gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['put'] ) )
gLogger.info( "TransferAgent.execute: Failed to register %s to %s." % ( lfn, diracSE ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Registration failed' )
subRequestError = "Registration failed for %s to %s" % ( lfn, diracSE )
fileDict = res['Value']['Failed'][lfn]['register']
registerRequestDict = {'Attributes':{'TargetSE': fileDict['TargetSE'], 'Operation':'registerFile'},
'Files':[{'LFN': fileDict['LFN'],
'PFN':fileDict['PFN'],
'Size':fileDict['Size'],
'Addler':fileDict['Addler'],
'GUID':fileDict['GUID']}]}
gLogger.info( "TransferAgent.execute: Setting registration request for failed file." )
oRequest.addSubRequest( registerRequestDict, 'register' )
modified = True
else:
gMonitor.addMark( "Put successful", 1 )
gMonitor.addMark( "File registration successful", 1 )
self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' )
self.DataLog.addFileRecord( lfn, 'Register', diracSE, '', 'TransferAgent' )
gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['put'] ) )
gLogger.info( "TransferAgent.execute: Successfully registered %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['register'] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
errStr = "TransferAgent.execute: Failed to put and register file."
gLogger.error( errStr, "%s %s %s" % ( lfn, diracSE, res['Value']['Failed'][lfn] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Complete file failure' )
subRequestError = "Failed to put and register file"
else:
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
errStr = "TransferAgent.execute: Completely failed to put and register file."
gLogger.error( errStr, res['Message'] )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'RM call failure' )
subRequestError = operation + " RM call file"
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is a put operation
elif operation == 'put':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
diracSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark( "Put", 1 )
lfn = subRequestFile['LFN']
file = subRequestFile['PFN']
res = self.ReplicaManager.put( lfn, file, diracSE )
if res['OK']:
if res['Value']['Successful'].has_key( lfn ):
gMonitor.addMark( "Put successful", 1 )
self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' )
gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
errStr = "TransferAgent.execute: Failed to put file."
gLogger.error( errStr, "%s %s %s" % ( lfn, diracSE, res['Value']['Failed'][lfn] ) )
subRequestError = "Put operation failed for %s to %s" % ( lfn, diracSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' )
else:
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
errStr = "TransferAgent.execute: Completely failed to put file."
gLogger.error( errStr, res['Message'] )
subRequestError = "Put RM call failed for %s to %s" % ( lfn, diracSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put RM call failed' )
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is a replicate and register operation
elif operation == 'replicateAndRegister' or operation == 'replicateAndRegisterAndRemove':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
targetSE = subRequestAttributes['TargetSE']
sourceSE = subRequestAttributes['SourceSE']
if sourceSE == "None":
sourceSE = ''
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark( "Replicate and register", 1 )
lfn = subRequestFile['LFN']
res = self.ReplicaManager.replicateAndRegister( lfn, targetSE, sourceSE = sourceSE )
if res['OK']:
if res['Value']['Successful'].has_key( lfn ):
if not res['Value']['Successful'][lfn].has_key( 'replicate' ):
gLogger.info( "TransferAgent.execute: Failed to replicate %s to %s." % ( lfn, targetSE ) )
gMonitor.addMark( "Replication failed", 1 )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,"Error", "Replication failed" )
subRequestError = "Replication failed for %s to %s" % ( lfn, targetSE )
elif not res['Value']['Successful'][lfn].has_key( 'register' ):
gMonitor.addMark( "Replication successful", 1 )
gMonitor.addMark( "Replica registration failed", 1 )
gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, targetSE, res['Value']['Successful'][lfn]['replicate'] ) )
gLogger.info( "TransferAgent.execute: Failed to register %s to %s." % ( lfn, targetSE ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'Registration failed' )
subRequestError = "Registration failed for %s to %s" % ( lfn, targetSE )
fileDict = res['Value']['Failed'][lfn]['register']
registerRequestDict = {'Attributes':{'TargetSE': fileDict['TargetSE'], 'Operation':'registerReplica'}, 'Files':[{'LFN': fileDict['LFN'], 'PFN':fileDict['PFN']}]}
gLogger.info( "TransferAgent.execute: Setting registration request for failed replica." )
oRequest.addSubRequest( registerRequestDict, 'register' )
modified = True
else:
gMonitor.addMark( "Replication successful", 1 )
gMonitor.addMark( "Replica registration successful", 1 )
gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, targetSE, res['Value']['Successful'][lfn]['replicate'] ) )
gLogger.info( "TransferAgent.execute: Successfully registered %s to %s in %s seconds." % ( lfn, targetSE, res['Value']['Successful'][lfn]['register'] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gMonitor.addMark( "Replication failed", 1 )
errStr = "TransferAgent.execute: Failed to replicate and register file."
gLogger.error( errStr, "%s %s %s" % ( lfn, targetSE, res['Value']['Failed'][lfn] ) )
else:
gMonitor.addMark( "Replication failed", 1 )
errStr = "TransferAgent.execute: Completely failed to replicate and register file."
gLogger.error( errStr, res['Message'] )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'RM call failure' )
subRequestError = operation + " RM call failed"
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is a replicate operation
elif operation == 'replicate':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
targetSE = subRequestAttributes['TargetSE']
sourceSE = subRequestAttributes['SourceSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark( "Replicate", 1 )
lfn = subRequestFile['LFN']
res = self.ReplicaManager.replicate( lfn, targetSE, sourceSE = sourceSE )
if res['OK']:
if res['Value']['Successful'].has_key( lfn ):
gMonitor.addMark( "Replication successful", 1 )
gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gMonitor.addMark( "Replication failed", 1 )
errStr = "TransferAgent.execute: Failed to replicate file."
gLogger.error( errStr, "%s %s %s" % ( lfn, targetSE, res['Value']['Failed'][lfn] ) )
subRequestError = "Replicate operation failed for %s to %s" % ( lfn, targetSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' )
else:
gMonitor.addMark( "Replication failed", 1 )
errStr = "TransferAgent.execute: Completely failed to replicate file."
gLogger.error( errStr, res['Message'] )
subRequestError = "Replicate RM call failed for %s to %s" % ( lfn, targetSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Replicate RM call failed' )
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is a get operation
elif operation == 'get':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
sourceSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = str( subRequestFile['LFN'] )
pfn = str( subRequestFile['PFN'] )
got = False
if sourceSE and pfn:
res = self.ReplicaManager.getStorageFile( pfn, sourceSE )
if res['Value']['Successful'].has_key( pfn ):
got = True
else:
res = self.ReplicaManager.getFile( lfn )
if res['Value']['Successful'].has_key( lfn ):
got = False
if got:
gLogger.info( "TransferAgent.execute: Successfully got %s." % lfn )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
errStr = "TransferAgent.execute: Failed to get file."
gLogger.error( errStr, lfn )
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is none of the above types
else:
gLogger.error( "TransferAgent.execute: Operation not supported.", operation )
if subRequestError:
oRequest.setSubRequestAttributeValue( ind, 'transfer', 'Error', subRequestError )
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty( ind, 'transfer' )['Value']:
oRequest.setSubRequestStatus( ind, 'transfer', 'Done' )
gMonitor.addMark( "Done", 1 )
################################################
# If the sub-request is already in terminal state
else:
gLogger.info( "TransferAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )
if modified and jobID:
result = self.finalizeRequest( requestName, jobID, sourceServer )
return S_OK()
def executeRequest(self):
################################################
# Get a request from request DB
gMonitor.addMark("Iteration", 1)
res = self.RequestDBClient.getRequest('transfer')
if not res['OK']:
gLogger.info(
"TransferAgent.execute: Failed to get request from database.")
return S_OK()
elif not res['Value']:
gLogger.info(
"TransferAgent.execute: No requests to be executed found.")
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
try:
jobID = int(res['Value']['JobID'])
except:
jobID = 0
gLogger.info("TransferAgent.execute: Obtained request %s" %
requestName)
result = self.RequestDBClient.getCurrentExecutionOrder(
requestName, sourceServer)
if result['OK']:
currentOrder = result['Value']
else:
return S_OK('Can not get the request execution order')
oRequest = RequestContainer(request=requestString)
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('transfer')
if not res['OK']:
errStr = "TransferAgent.execute: Failed to obtain number of transfer subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info("TransferAgent.execute: Found %s sub requests." %
res['Value'])
################################################
# For all the sub-requests in the request
modified = False
for ind in range(res['Value']):
gMonitor.addMark("Execute", 1)
gLogger.info("TransferAgent.execute: Processing sub-request %s." %
ind)
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'transfer')['Value']
if subRequestAttributes['ExecutionOrder']:
subExecutionOrder = int(subRequestAttributes['ExecutionOrder'])
else:
subExecutionOrder = 0
subStatus = subRequestAttributes['Status']
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
subRequestFiles = oRequest.getSubRequestFiles(
ind, 'transfer')['Value']
operation = subRequestAttributes['Operation']
subRequestError = ''
################################################
# If the sub-request is a put and register operation
if operation == 'putAndRegister' or operation == 'putAndRegisterAndRemove':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSE = str(subRequestAttributes['TargetSE'])
catalog = ''
if subRequestAttributes.has_key('Catalogue'):
catalog = subRequestAttributes['Catalogue']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark("Put and register", 1)
lfn = str(subRequestFile['LFN'])
file = subRequestFile['PFN']
guid = subRequestFile['GUID']
addler = subRequestFile['Addler']
res = self.ReplicaManager.putAndRegister(
lfn,
file,
diracSE,
guid=guid,
checksum=addler,
catalog=catalog)
if res['OK']:
if res['Value']['Successful'].has_key(lfn):
if not res['Value']['Successful'][
lfn].has_key('put'):
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
gLogger.info(
"TransferAgent.execute: Failed to put %s to %s."
% (lfn, diracSE))
subRequestError = "Put operation failed for %s to %s" % (
lfn, diracSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Put failed')
elif not res['Value']['Successful'][
lfn].has_key('register'):
gMonitor.addMark("Put successful", 1)
gMonitor.addMark(
"File registration failed", 1)
self.DataLog.addFileRecord(
lfn, 'Put', diracSE, '',
'TransferAgent')
self.DataLog.addFileRecord(
lfn, 'RegisterFail', diracSE, '',
'TransferAgent')
gLogger.info(
"TransferAgent.execute: Successfully put %s to %s in %s seconds."
% (lfn, diracSE, res['Value']
['Successful'][lfn]['put']))
gLogger.info(
"TransferAgent.execute: Failed to register %s to %s."
% (lfn, diracSE))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status',
'Done')
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Registration failed')
subRequestError = "Registration failed for %s to %s" % (
lfn, diracSE)
fileDict = res['Value']['Failed'][lfn][
'register']
registerRequestDict = {
'Attributes': {
'TargetSE':
fileDict['TargetSE'],
'Operation': 'registerFile'
},
'Files': [{
'LFN':
fileDict['LFN'],
'PFN':
fileDict['PFN'],
'Size':
fileDict['Size'],
'Addler':
fileDict['Addler'],
'GUID':
fileDict['GUID']
}]
}
gLogger.info(
"TransferAgent.execute: Setting registration request for failed file."
)
oRequest.addSubRequest(
registerRequestDict, 'register')
modified = True
else:
gMonitor.addMark("Put successful", 1)
gMonitor.addMark(
"File registration successful", 1)
self.DataLog.addFileRecord(
lfn, 'Put', diracSE, '',
'TransferAgent')
self.DataLog.addFileRecord(
lfn, 'Register', diracSE, '',
'TransferAgent')
gLogger.info(
"TransferAgent.execute: Successfully put %s to %s in %s seconds."
% (lfn, diracSE, res['Value']
['Successful'][lfn]['put']))
gLogger.info(
"TransferAgent.execute: Successfully registered %s to %s in %s seconds."
%
(lfn, diracSE, res['Value']
['Successful'][lfn]['register']))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status',
'Done')
modified = True
else:
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
errStr = "TransferAgent.execute: Failed to put and register file."
gLogger.error(
errStr, "%s %s %s" %
(lfn, diracSE,
res['Value']['Failed'][lfn]))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Complete file failure')
subRequestError = "Failed to put and register file"
else:
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
errStr = "TransferAgent.execute: Completely failed to put and register file."
gLogger.error(errStr, res['Message'])
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'RM call failure')
subRequestError = operation + " RM call file"
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is a put operation
elif operation == 'put':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark("Put", 1)
lfn = subRequestFile['LFN']
file = subRequestFile['PFN']
res = self.ReplicaManager.put(lfn, file, diracSE)
if res['OK']:
if res['Value']['Successful'].has_key(lfn):
gMonitor.addMark("Put successful", 1)
self.DataLog.addFileRecord(
lfn, 'Put', diracSE, '',
'TransferAgent')
gLogger.info(
"TransferAgent.execute: Successfully put %s to %s in %s seconds."
% (lfn, diracSE,
res['Value']['Successful'][lfn]))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Done')
modified = True
else:
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
errStr = "TransferAgent.execute: Failed to put file."
gLogger.error(
errStr, "%s %s %s" %
(lfn, diracSE,
res['Value']['Failed'][lfn]))
subRequestError = "Put operation failed for %s to %s" % (
lfn, diracSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Put failed')
else:
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
errStr = "TransferAgent.execute: Completely failed to put file."
gLogger.error(errStr, res['Message'])
subRequestError = "Put RM call failed for %s to %s" % (
lfn, diracSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Put RM call failed')
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is a replicate and register operation
elif operation == 'replicateAndRegister' or operation == 'replicateAndRegisterAndRemove':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
targetSE = subRequestAttributes['TargetSE']
sourceSE = subRequestAttributes['SourceSE']
if sourceSE == "None":
sourceSE = ''
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark("Replicate and register", 1)
lfn = subRequestFile['LFN']
res = self.ReplicaManager.replicateAndRegister(
lfn, targetSE, sourceSE=sourceSE)
if res['OK']:
if res['Value']['Successful'].has_key(lfn):
if not res['Value']['Successful'][
lfn].has_key('replicate'):
gLogger.info(
"TransferAgent.execute: Failed to replicate %s to %s."
% (lfn, targetSE))
gMonitor.addMark(
"Replication failed", 1)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, "Error",
"Replication failed")
subRequestError = "Replication failed for %s to %s" % (
lfn, targetSE)
elif not res['Value']['Successful'][
lfn].has_key('register'):
gMonitor.addMark(
"Replication successful", 1)
gMonitor.addMark(
"Replica registration failed", 1)
gLogger.info(
"TransferAgent.execute: Successfully replicated %s to %s in %s seconds."
%
(lfn, targetSE, res['Value']
['Successful'][lfn]['replicate']))
gLogger.info(
"TransferAgent.execute: Failed to register %s to %s."
% (lfn, targetSE))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status',
'Done')
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Registration failed')
subRequestError = "Registration failed for %s to %s" % (
lfn, targetSE)
fileDict = res['Value']['Failed'][lfn][
'register']
registerRequestDict = {
'Attributes': {
'TargetSE':
fileDict['TargetSE'],
'Operation': 'registerReplica'
},
'Files': [{
'LFN': fileDict['LFN'],
'PFN': fileDict['PFN']
}]
}
gLogger.info(
"TransferAgent.execute: Setting registration request for failed replica."
)
oRequest.addSubRequest(
registerRequestDict, 'register')
modified = True
else:
gMonitor.addMark(
"Replication successful", 1)
gMonitor.addMark(
"Replica registration successful",
1)
gLogger.info(
"TransferAgent.execute: Successfully replicated %s to %s in %s seconds."
%
(lfn, targetSE, res['Value']
['Successful'][lfn]['replicate']))
gLogger.info(
"TransferAgent.execute: Successfully registered %s to %s in %s seconds."
%
(lfn, targetSE, res['Value']
['Successful'][lfn]['register']))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status',
'Done')
modified = True
else:
gMonitor.addMark("Replication failed", 1)
errStr = "TransferAgent.execute: Failed to replicate and register file."
gLogger.error(
errStr, "%s %s %s" %
(lfn, targetSE,
res['Value']['Failed'][lfn]))
else:
gMonitor.addMark("Replication failed", 1)
errStr = "TransferAgent.execute: Completely failed to replicate and register file."
gLogger.error(errStr, res['Message'])
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'RM call failure')
subRequestError = operation + " RM call failed"
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is a replicate operation
elif operation == 'replicate':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
targetSE = subRequestAttributes['TargetSE']
sourceSE = subRequestAttributes['SourceSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark("Replicate", 1)
lfn = subRequestFile['LFN']
res = self.ReplicaManager.replicate(
lfn, targetSE, sourceSE=sourceSE)
if res['OK']:
if res['Value']['Successful'].has_key(lfn):
gMonitor.addMark("Replication successful",
1)
gLogger.info(
"TransferAgent.execute: Successfully replicated %s to %s in %s seconds."
% (lfn, diracSE,
res['Value']['Successful'][lfn]))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Done')
modified = True
else:
gMonitor.addMark("Replication failed", 1)
errStr = "TransferAgent.execute: Failed to replicate file."
gLogger.error(
errStr, "%s %s %s" %
(lfn, targetSE,
res['Value']['Failed'][lfn]))
subRequestError = "Replicate operation failed for %s to %s" % (
lfn, targetSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Put failed')
else:
gMonitor.addMark("Replication failed", 1)
errStr = "TransferAgent.execute: Completely failed to replicate file."
gLogger.error(errStr, res['Message'])
subRequestError = "Replicate RM call failed for %s to %s" % (
lfn, targetSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Replicate RM call failed')
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is a get operation
elif operation == 'get':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
sourceSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = str(subRequestFile['LFN'])
pfn = str(subRequestFile['PFN'])
got = False
if sourceSE and pfn:
res = self.ReplicaManager.getStorageFile(
pfn, sourceSE)
if res['Value']['Successful'].has_key(pfn):
got = True
else:
res = self.ReplicaManager.getFile(lfn)
if res['Value']['Successful'].has_key(lfn):
got = False
if got:
gLogger.info(
"TransferAgent.execute: Successfully got %s."
% lfn)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Done')
modified = True
else:
errStr = "TransferAgent.execute: Failed to get file."
gLogger.error(errStr, lfn)
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is none of the above types
else:
gLogger.error(
"TransferAgent.execute: Operation not supported.",
operation)
if subRequestError:
oRequest.setSubRequestAttributeValue(
ind, 'transfer', 'Error', subRequestError)
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty(ind, 'transfer')['Value']:
oRequest.setSubRequestStatus(ind, 'transfer', 'Done')
gMonitor.addMark("Done", 1)
################################################
# If the sub-request is already in terminal state
else:
gLogger.info(
"TransferAgent.execute: Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest(requestName, requestString,
sourceServer)
if modified and jobID:
result = self.finalizeRequest(requestName, jobID, sourceServer)
return S_OK()
