def generateRequest(self):
""" Generate failover requests for the operations in the internal cache
"""
request = RequestContainer()
result = self.sendStoredStatusInfo()
if not result['OK']:
if result.has_key('rpcStub'):
request.addSubRequest(DISETSubRequest(result['rpcStub']).getDictionary(),'diset')
else:
return S_ERROR('Could not create job state sub-request')
result = self.sendStoredJobParameters()
if not result['OK']:
if result.has_key('rpcStub'):
request.addSubRequest(DISETSubRequest(result['rpcStub']).getDictionary(),'diset')
else:
return S_ERROR('Could not create job parameters sub-request')
if request.isEmpty()['Value']:
request = None
return S_OK(request)
def generateRequest(self):
""" Generate failover requests for the operations in the internal cache
"""
request = RequestContainer()
result = self.sendStoredStatusInfo()
if not result['OK']:
if result.has_key('rpcStub'):
request.addSubRequest(
DISETSubRequest(result['rpcStub']).getDictionary(),
'diset')
else:
return S_ERROR('Could not create job state sub-request')
result = self.sendStoredJobParameters()
if not result['OK']:
if result.has_key('rpcStub'):
request.addSubRequest(
DISETSubRequest(result['rpcStub']).getDictionary(),
'diset')
else:
return S_ERROR('Could not create job parameters sub-request')
if request.isEmpty()['Value']:
request = None
return S_OK(request)
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
request = RequestContainer()
request.setRequestAttributes( { 'OwnerDN':ownerDN, 'OwnerGroup':ownerGroup } )
requestName = os.path.basename( lfn ).strip()+'_removal_request.xml'
request.setRequestName( requestName )
request.setSourceComponent( 'JobCleaningAgent' )
removalDict = {'Attributes':{ 'Operation':'removeFile',
'TargetSE':'',
'ExecutionOrder':0
}
}
result = request.addSubRequest( removalDict, 'removal' )
if not result['OK']:
return result
index = result['Value']
fileDict = { 'LFN':lfn, 'PFN':'', 'Status':'Waiting' }
request.setSubRequestFiles( index, 'removal', [fileDict] )
client = RequestClient()
result = request.toXML()
if not result['OK']:
return result
xmlRequest = result['Value']
result = client.setRequest( requestName, xmlRequest )
return result
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup):
""" Set removal request with the given credentials
"""
request = RequestContainer()
request.setRequestAttributes({
'OwnerDN': ownerDN,
'OwnerGroup': ownerGroup
})
requestName = os.path.basename(lfn).strip() + '_removal_request.xml'
request.setRequestName(requestName)
request.setSourceComponent('JobCleaningAgent')
removalDict = {
'Attributes': {
'Operation': 'removeFile',
'TargetSE': '',
'ExecutionOrder': 0
}
}
result = request.addSubRequest(removalDict, 'removal')
if not result['OK']:
return result
index = result['Value']
fileDict = {'LFN': lfn, 'PFN': '', 'Status': 'Waiting'}
request.setSubRequestFiles(index, 'removal', [fileDict])
client = RequestClient()
result = request.toXML()
if not result['OK']:
return result
xmlRequest = result['Value']
result = client.setRequest(requestName, xmlRequest)
return result
def test__getLastOrder(self):
# no files
req = RequestContainer()
res = req._getLastOrder()
self.assertEqual( res, 0 )
self.assertEqual( req.subRequests, {} )
req.addSubRequest( {'Attributes':{'Operation':'replicateAndRegister',
'TargetSE':'SE', 'ExecutionOrder': 0}},
'transfer' )
res = req._getLastOrder()
self.assertEqual( res, 0 )
req.addSubRequest( {'Attributes':{'Operation':'replicateAndRegister',
'TargetSE':'SE', 'ExecutionOrder': 1}},
'transfer' )
res = req._getLastOrder()
self.assertEqual( res, 1 )
del( req )
# with files
req = RequestContainer()
res = req._getLastOrder( 'foo' )
self.assertEqual( res, 0 )
req.addSubRequest( {'Attributes':{'Operation':'replicateAndRegister',
'TargetSE':'SE', 'ExecutionOrder': 1}},
'transfer' )
res = req._getLastOrder( 'foo' )
self.assertEqual( res, 0 )
req.setSubRequestFiles( 0, 'transfer', [{'LFN':'foo', 'Status':'Waiting'}] )
res = req._getLastOrder( 'foo' )
self.assertEqual( res, 1 )
req.addSubRequest( {'Attributes':{'Operation':'replicateAndRegister',
'TargetSE':'SE', 'ExecutionOrder': 2}},
'removal' )
res = req._getLastOrder( 'foo' )
self.assertEqual( res, 1 )
req.setSubRequestFiles( 0, 'removal', [{'LFN':'foo', 'Status':'Waiting'}] )
res = req._getLastOrder( 'foo' )
self.assertEqual( res, 2 )
def test__getLastOrder(self):
# no files
req = RequestContainer()
res = req._getLastOrder()
self.assertEqual(res, 0)
self.assertEqual(req.subRequests, {})
req.addSubRequest(
{"Attributes": {"Operation": "replicateAndRegister", "TargetSE": "SE", "ExecutionOrder": 0}}, "transfer"
)
res = req._getLastOrder()
self.assertEqual(res, 0)
req.addSubRequest(
{"Attributes": {"Operation": "replicateAndRegister", "TargetSE": "SE", "ExecutionOrder": 1}}, "transfer"
)
res = req._getLastOrder()
self.assertEqual(res, 1)
del (req)
# with files
req = RequestContainer()
res = req._getLastOrder("foo")
self.assertEqual(res, 0)
req.addSubRequest(
{"Attributes": {"Operation": "replicateAndRegister", "TargetSE": "SE", "ExecutionOrder": 1}}, "transfer"
)
res = req._getLastOrder("foo")
self.assertEqual(res, 0)
req.setSubRequestFiles(0, "transfer", [{"LFN": "foo", "Status": "Waiting"}])
res = req._getLastOrder("foo")
self.assertEqual(res, 1)
req.addSubRequest(
{"Attributes": {"Operation": "replicateAndRegister", "TargetSE": "SE", "ExecutionOrder": 2}}, "removal"
)
res = req._getLastOrder("foo")
self.assertEqual(res, 1)
req.setSubRequestFiles(0, "removal", [{"LFN": "foo", "Status": "Waiting"}])
res = req._getLastOrder("foo")
self.assertEqual(res, 2)
def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ):
if self.getCSOption( "DelayedExternalDeletion", True ):
gLogger.info( "Setting deletion request" )
try:
request = RequestContainer()
result = request.addSubRequest( { 'Attributes' : { 'Operation' : 'removePhysicalFile',
'TargetSE' : SEName,
'ExecutionOrder' : 1
} },
'removal' )
index = result['Value']
fileDict = { 'PFN' : SEPFN, 'Status' : 'Waiting' }
request.setSubRequestFiles( index, 'removal', [ fileDict ] )
return RequestClient().setRequest( "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() ),
request.toXML()[ 'Value' ] )
except Exception, e:
gLogger.exception( "Exception while setting deletion request" )
return S_ERROR( "Cannot set deletion request: %s" % str( e ) )
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
request = RequestContainer()
result = request.addSubRequest(
{
'Attributes': {
'Operation': 'removePhysicalFile',
'TargetSE': SEName,
'ExecutionOrder': 1
}
}, 'removal')
index = result['Value']
fileDict = {'PFN': SEPFN, 'Status': 'Waiting'}
request.setSubRequestFiles(index, 'removal', [fileDict])
return RequestClient().setRequest(
"RemoteSBDeletion:%s|%s:%s" % (SEName, SEPFN, time.time()),
request.toXML()['Value'])
except Exception, e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR("Cannot set deletion request: %s" % str(e))
else:
lfnpath = "%s%s" % (path, os.path.basename(appTar))
res = rm.putAndRegister(lfnpath, appTar, ops.getValue('Software/BaseStorageElement',"CERN-SRM"))
if not res['OK']:
return res
request = RequestContainer()
request.setCreationTime()
requestClient = RequestClient()
request.setRequestName('copy_%s' % os.path.basename(appTar).replace(".tgz","").replace(".tar.gz",""))
request.setSourceComponent('ReplicateILCSoft')
copies_at = ops.getValue('Software/CopiesAt',[])
index_copy = 0
for copies in copies_at:
res = request.addSubRequest({'Attributes':{'Operation' : 'replicateAndRegister',
'TargetSE' : copies,
'ExecutionOrder' : index_copy},
'Files':[{'LFN':lfnpath}]},
'transfer')
#res = rm.replicateAndRegister("%s%s"%(path,appTar),"IN2P3-SRM")
if not res['OK']:
return res
index_copy += 1
requestxml = request.toXML()['Value']
if copies_at:
res = requestClient.setRequest(request.getRequestName()['Value'], requestxml)
if not res['OK']:
gLogger.error('Could not set replication request %s' % res['Message'])
return S_OK('Application uploaded')
return S_OK()
class FailoverTransfer:
#############################################################################
def __init__(self, requestObject=False):
""" Constructor function, can specify request object to instantiate
FailoverTransfer or a new request object is created.
"""
self.log = gLogger.getSubLogger("FailoverTransfer")
self.rm = ReplicaManager()
self.request = requestObject
if not self.request:
self.request = RequestContainer()
self.request.setRequestName('default_request.xml')
self.request.setSourceComponent('FailoverTransfer')
#############################################################################
def transferAndRegisterFile(self,
fileName,
localPath,
lfn,
destinationSEList,
fileGUID=None,
fileCatalog=None):
"""Performs the transfer and register operation with failover.
"""
errorList = []
for se in destinationSEList:
self.log.info(
'Attempting rm.putAndRegister("%s","%s","%s",guid="%s",catalog="%s")'
% (lfn, localPath, se, fileGUID, fileCatalog))
result = self.rm.putAndRegister(lfn,
localPath,
se,
guid=fileGUID,
catalog=fileCatalog)
self.log.verbose(result)
if not result['OK']:
self.log.error('rm.putAndRegister failed with message',
result['Message'])
errorList.append(result['Message'])
continue
if not result['Value']['Failed']:
self.log.info(
'rm.putAndRegister successfully uploaded %s to %s' %
(fileName, se))
return S_OK({'uploadedSE': se, 'lfn': lfn})
#Now we know something went wrong
errorDict = result['Value']['Failed'][lfn]
if not errorDict.has_key('register'):
self.log.error('rm.putAndRegister failed with unknown error',
str(errorDict))
errorList.append(
'Unknown error while attempting upload to %s' % se)
continue
fileDict = errorDict['register']
#Therefore the registration failed but the upload was successful
if not fileCatalog:
fileCatalog = ''
result = self.__setRegistrationRequest(fileDict['LFN'], se,
fileCatalog, fileDict)
if not result['OK']:
self.log.error(
'Failed to set registration request for: SE %s and metadata: \n%s'
% (se, fileDict))
errorList.append(
'Failed to set registration request for: SE %s and metadata: \n%s'
% (se, fileDict))
continue
else:
self.log.info(
'Successfully set registration request for: SE %s and metadata: \n%s'
% (se, fileDict))
metadata = {}
metadata['filedict'] = fileDict
metadata['uploadedSE'] = se
metadata['lfn'] = lfn
metadata['registration'] = 'request'
return S_OK(metadata)
self.log.error(
'Encountered %s errors during attempts to upload output data' %
len(errorList))
return S_ERROR('Failed to upload output data file')
#############################################################################
def transferAndRegisterFileFailover(self,
fileName,
localPath,
lfn,
targetSE,
failoverSEList,
fileGUID=None,
fileCatalog=None):
"""Performs the transfer and register operation to failover storage and sets the
necessary replication and removal requests to recover.
"""
failover = self.transferAndRegisterFile(fileName, localPath, lfn,
failoverSEList, fileGUID,
fileCatalog)
if not failover['OK']:
self.log.error('Could not upload file to failover SEs',
failover['Message'])
return failover
#set removal requests and replication requests
result = self.__setFileReplicationRequest(lfn, targetSE)
if not result['OK']:
self.log.error('Could not set file replication request',
result['Message'])
return result
lfn = failover['Value']['lfn']
failoverSE = failover['Value']['uploadedSE']
self.log.info(
'Attempting to set replica removal request for LFN %s at failover SE %s'
% (lfn, failoverSE))
result = self.__setReplicaRemovalRequest(lfn, failoverSE)
if not result['OK']:
self.log.error('Could not set removal request', result['Message'])
return result
return S_OK('%s uploaded to a failover SE' % fileName)
#############################################################################
def getRequestObject(self):
"""Returns the potentially modified request object in order to propagate changes.
"""
return S_OK(self.request)
#############################################################################
def __setFileReplicationRequest(self, lfn, se):
""" Sets a registration request.
"""
self.log.info('Setting replication request for %s to %s' % (lfn, se))
result = self.request.addSubRequest(
{
'Attributes': {
'Operation': 'replicateAndRegister',
'TargetSE': se,
'ExecutionOrder': 0
}
}, 'transfer')
if not result['OK']:
return result
index = result['Value']
fileDict = {'LFN': lfn, 'Status': 'Waiting'}
self.request.setSubRequestFiles(index, 'transfer', [fileDict])
return S_OK()
#############################################################################
def __setRegistrationRequest(self, lfn, se, catalog, fileDict):
""" Sets a registration request.
"""
self.log.info('Setting registration request for %s at %s.' % (lfn, se))
result = self.request.addSubRequest(
{
'Attributes': {
'Operation': 'registerFile',
'ExecutionOrder': 0,
'TargetSE': se,
'Catalogue': catalog
}
}, 'register')
if not result['OK']:
return result
index = result['Value']
if not fileDict.has_key('Status'):
fileDict['Status'] = 'Waiting'
self.request.setSubRequestFiles(index, 'register', [fileDict])
return S_OK()
#############################################################################
def __setReplicaRemovalRequest(self, lfn, se):
""" Sets a removal request for a replica.
"""
result = self.request.addSubRequest(
{
'Attributes': {
'Operation': 'replicaRemoval',
'TargetSE': se,
'ExecutionOrder': 1
}
}, 'removal')
index = result['Value']
fileDict = {'LFN': lfn, 'Status': 'Waiting'}
self.request.setSubRequestFiles(index, 'removal', [fileDict])
return S_OK()
#############################################################################
def __setFileRemovalRequest(self, lfn, se='', pfn=''):
""" Sets a removal request for a file including all replicas.
"""
result = self.request.addSubRequest(
{
'Attributes': {
'Operation': 'removeFile',
'TargetSE': se,
'ExecutionOrder': 1
}
}, 'removal')
index = result['Value']
fileDict = {'LFN': lfn, 'PFN': pfn, 'Status': 'Waiting'}
self.request.setSubRequestFiles(index, 'removal', [fileDict])
return S_OK()
class FailoverTransfer:
#############################################################################
def __init__(self,requestObject=False):
""" Constructor function, can specify request object to instantiate
FailoverTransfer or a new request object is created.
"""
self.log = gLogger.getSubLogger( "FailoverTransfer" )
self.rm = ReplicaManager()
self.request = requestObject
if not self.request:
self.request = RequestContainer()
self.request.setRequestName('default_request.xml')
self.request.setSourceComponent('FailoverTransfer')
#############################################################################
def transferAndRegisterFile(self,fileName,localPath,lfn,destinationSEList,fileGUID=None,fileCatalog=None):
"""Performs the transfer and register operation with failover.
"""
errorList = []
for se in destinationSEList:
self.log.info('Attempting rm.putAndRegister("%s","%s","%s",guid="%s",catalog="%s")' %(lfn,localPath,se,fileGUID,fileCatalog))
result = self.rm.putAndRegister(lfn,localPath,se,guid=fileGUID,catalog=fileCatalog)
self.log.verbose(result)
if not result['OK']:
self.log.error('rm.putAndRegister failed with message',result['Message'])
errorList.append(result['Message'])
continue
if not result['Value']['Failed']:
self.log.info('rm.putAndRegister successfully uploaded %s to %s' %(fileName,se))
return S_OK({'uploadedSE':se,'lfn':lfn})
#Now we know something went wrong
errorDict = result['Value']['Failed'][lfn]
if not errorDict.has_key('register'):
self.log.error('rm.putAndRegister failed with unknown error',str(errorDict))
errorList.append('Unknown error while attempting upload to %s' %se)
continue
fileDict = errorDict['register']
#Therefore the registration failed but the upload was successful
if not fileCatalog:
fileCatalog=''
result = self.__setRegistrationRequest(fileDict['LFN'],se,fileCatalog,fileDict)
if not result['OK']:
self.log.error('Failed to set registration request for: SE %s and metadata: \n%s' %(se,fileDict))
errorList.append('Failed to set registration request for: SE %s and metadata: \n%s' %(se,fileDict))
continue
else:
self.log.info('Successfully set registration request for: SE %s and metadata: \n%s' %(se,fileDict))
metadata = {}
metadata['filedict']=fileDict
metadata['uploadedSE']=se
metadata['lfn']=lfn
metadata['registration']='request'
return S_OK(metadata)
self.log.error('Encountered %s errors during attempts to upload output data' %len(errorList))
return S_ERROR('Failed to upload output data file')
#############################################################################
def transferAndRegisterFileFailover(self,fileName,localPath,lfn,targetSE,failoverSEList,fileGUID=None,fileCatalog=None):
"""Performs the transfer and register operation to failover storage and sets the
necessary replication and removal requests to recover.
"""
failover = self.transferAndRegisterFile(fileName,localPath,lfn,failoverSEList,fileGUID,fileCatalog)
if not failover['OK']:
self.log.error('Could not upload file to failover SEs',failover['Message'])
return failover
#set removal requests and replication requests
result = self.__setFileReplicationRequest(lfn,targetSE)
if not result['OK']:
self.log.error('Could not set file replication request',result['Message'])
return result
lfn = failover['Value']['lfn']
failoverSE = failover['Value']['uploadedSE']
self.log.info('Attempting to set replica removal request for LFN %s at failover SE %s' %(lfn,failoverSE))
result = self.__setReplicaRemovalRequest(lfn,failoverSE)
if not result['OK']:
self.log.error('Could not set removal request',result['Message'])
return result
return S_OK('%s uploaded to a failover SE' %fileName)
#############################################################################
def getRequestObject(self):
"""Returns the potentially modified request object in order to propagate changes.
"""
return S_OK(self.request)
#############################################################################
def __setFileReplicationRequest(self,lfn,se):
""" Sets a registration request.
"""
self.log.info('Setting replication request for %s to %s' % (lfn,se))
result = self.request.addSubRequest({'Attributes':{'Operation':'replicateAndRegister',
'TargetSE':se,'ExecutionOrder':0}},
'transfer')
if not result['OK']:
return result
index = result['Value']
fileDict = {'LFN':lfn,'Status':'Waiting'}
self.request.setSubRequestFiles(index,'transfer',[fileDict])
return S_OK()
#############################################################################
def __setRegistrationRequest(self,lfn,se,catalog,fileDict):
""" Sets a registration request.
"""
self.log.info('Setting registration request for %s at %s.' % (lfn,se))
result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0,
'TargetSE':se,'Catalogue':catalog}},'register')
if not result['OK']:
return result
index = result['Value']
if not fileDict.has_key('Status'):
fileDict['Status']='Waiting'
self.request.setSubRequestFiles(index,'register',[fileDict])
return S_OK()
#############################################################################
def __setReplicaRemovalRequest(self,lfn,se):
""" Sets a removal request for a replica.
"""
result = self.request.addSubRequest({'Attributes':{'Operation':'replicaRemoval',
'TargetSE':se,'ExecutionOrder':1}},
'removal')
index = result['Value']
fileDict = {'LFN':lfn,'Status':'Waiting'}
self.request.setSubRequestFiles(index,'removal',[fileDict])
return S_OK()
#############################################################################
def __setFileRemovalRequest(self,lfn,se='',pfn=''):
""" Sets a removal request for a file including all replicas.
"""
result = self.request.addSubRequest({'Attributes':{'Operation':'removeFile',
'TargetSE':se,'ExecutionOrder':1}},
'removal')
index = result['Value']
fileDict = {'LFN':lfn,'PFN':pfn,'Status':'Waiting'}
self.request.setSubRequestFiles(index,'removal',[fileDict])
return S_OK()
def test_addSubRequest(self):
rc_o = RequestContainer()
op1_Index = rc_o.addSubRequest(
{"Attributes": {"SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Operation": "op1"}},
"someType",
)
op1_Index = op1_Index["Value"]
subRequestExpected = {
"someType": [
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 0,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "op1",
},
"Datasets": [],
}
]
}
self.assertEqual(rc_o.subRequests, subRequestExpected)
op2_index = rc_o.addSubRequest(
{"Attributes": {"SubRequestID": "x", "CreationTime": "2012-06-06 14:53:43.763743", "Operation": "op2"}},
"someType",
)
op2_index = op2_index["Value"]
subRequestExpected = {
"someType": [
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 0,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "op1",
},
"Datasets": [],
},
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 0,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "op2",
},
"Datasets": [],
},
]
}
self.assertEqual(rc_o.subRequests, subRequestExpected)
rc_o.addSubRequest(
{
"Attributes": {
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"ExecutionOrder": "last",
}
},
"someType",
)
subRequestExpected = {
"someType": [
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 0,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "op1",
},
"Datasets": [],
},
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 0,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "op2",
},
"Datasets": [],
},
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 1,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "",
},
"Datasets": [],
},
]
}
self.assertEqual(rc_o.subRequests, subRequestExpected)
rc_o.addSubRequest(
{
"Attributes": {
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"ExecutionOrder": "last",
}
},
"someOtherType",
)
subRequestExpected = {
"someType": [
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 0,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "op1",
},
"Datasets": [],
},
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 0,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "op2",
},
"Datasets": [],
},
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 1,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "",
},
"Datasets": [],
},
],
"someOtherType": [
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 2,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "",
},
"Datasets": [],
}
],
}
self.assertEqual(rc_o.subRequests, subRequestExpected)
fileDict = {"LFN": "foo", "Status": "Waiting"}
rc_o.setSubRequestFiles(op1_Index, "someType", [fileDict])
subRequestExpected = {
"someType": [
{
"Files": [{"LFN": "foo", "Status": "Waiting"}],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 0,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "op1",
},
"Datasets": [],
},
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 0,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "op2",
},
"Datasets": [],
},
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 1,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "",
},
"Datasets": [],
},
],
"someOtherType": [
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 2,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "",
},
"Datasets": [],
}
],
}
self.assertEqual(rc_o.subRequests, subRequestExpected)
fileLastOp = rc_o._getLastOrder("foo")
rc_o.addSubRequest(
{
"Attributes": {
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"ExecutionOrder": fileLastOp + 1,
}
},
"someOtherType",
)
subRequestExpected = {
"someType": [
{
"Files": [{"LFN": "foo", "Status": "Waiting"}],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 0,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "op1",
},
"Datasets": [],
},
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 0,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "op2",
},
"Datasets": [],
},
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 1,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "",
},
"Datasets": [],
},
],
"someOtherType": [
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 2,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "",
},
"Datasets": [],
},
{
"Files": [],
"Attributes": {
"Status": "Waiting",
"LastUpdate": "",
"TargetSE": "",
"ExecutionOrder": 1,
"SubRequestID": "x",
"CreationTime": "2012-06-06 14:53:43.763743",
"Catalogue": "",
"Error": "",
"Operation": "",
},
"Datasets": [],
},
],
}
self.assertEqual(rc_o.subRequests, subRequestExpected)
class UploadOutputData(ModuleBase):
""" As name suggest: upload output data. For Production only: See L{UserJobFinalization} for User job upload.
"""
#############################################################################
def __init__(self):
"""Module initialization.
"""
super(UploadOutputData, self).__init__()
self.version = __RCSID__
self.log = gLogger.getSubLogger( "UploadOutputData" )
self.commandTimeOut = 10*60
self.enable = True
self.failoverTest = False #flag to put file to failover SE by default
self.failoverSEs = gConfig.getValue('/Resources/StorageElementGroups/Tier1-Failover', [])
self.ops = Operations()
#List all parameters here
self.outputDataFileMask = ''
self.outputMode = 'Any' #or 'Local' for reco case
self.outputList = []
self.request = None
self.PRODUCTION_ID = ""
self.prodOutputLFNs = []
self.experiment = "CLIC"
#############################################################################
def applicationSpecificInputs(self):
""" By convention the module parameters are resolved here.
"""
self.log.verbose("Workflow commons:")
self.log.verbose(self.workflow_commons)
self.log.verbose("Step commons:")
self.log.verbose(self.step_commons)
if self.step_commons.has_key('Enable'):
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
self.enable = False
if self.step_commons.has_key('TestFailover'):
self.enable = self.step_commons['TestFailover']
if not type(self.failoverTest) == type(True):
self.log.warn('Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest)
self.failoverTest = False
if self.workflow_commons.has_key("PRODUCTION_ID"):
self.PRODUCTION_ID = self.workflow_commons["PRODUCTION_ID"]
if os.environ.has_key('JOBID'):
self.log.verbose('Found WMS JobID = %s' % self.jobID)
else:
self.log.info('No WMS JobID found, disabling module via control flag')
self.enable = False
if self.workflow_commons.has_key('Request'):
self.request = self.workflow_commons['Request']
else:
self.request = RequestContainer()
self.request.setRequestName('job_%s_request.xml' % self.jobID)
self.request.setJobID(self.jobID)
self.request.setSourceComponent("Job_%s" % self.jobID)
##This is the thing that is used to establish the list of outpufiles to treat:
## Make sure that all that is in the : "listoutput" and also in the ProductionData
## is treated properly. Needed as whatever is in listoutput does not contain any reference to the
## prodID and task ID. Also if for some reason a step failed, then the corresponding data will not be there
if self.workflow_commons.has_key('outputList'):
self.outputList = self.workflow_commons['outputList']
if self.workflow_commons.has_key('ProductionOutputData'):
proddata = self.workflow_commons['ProductionOutputData'].split(";")
self.log.verbose("prod data : %s" % proddata )
olist = {}
for obj in self.outputList:
fname_in_outputlist = obj['outputFile'].lower()
extension = ''
if fname_in_outputlist.count("_sim") or fname_in_outputlist.count("_rec") or fname_in_outputlist.count("_dst"):
extension = ".slcio"
elif fname_in_outputlist.count("_gen"):
extension = ".stdhep"
fname_in_outputlist = fname_in_outputlist.replace(extension,"")
for prodfile in proddata:
prodfile = os.path.basename(prodfile)
extension = ''
if prodfile.count("_sim") or prodfile.count("_rec") or prodfile.count("_dst"):
extension = ".slcio"
elif prodfile.count("_gen"):
extension = ".stdhep"
prodfile = prodfile.replace(extension,"")
if olist.has_key(prodfile):
## This has already been treated, no need to come back to it.
continue
appdict = {}
if (fname_in_outputlist.count("_gen")):# and prodfile.lower().count("_gen_")) :
genf = obj['outputFile'].split("_gen")[0]
genf += "_gen"
if (prodfile.count(genf)):
appdict.update(obj)
appdict['outputFile'] = prodfile+extension
olist[prodfile] = appdict
if (fname_in_outputlist.count("_sim")):
simf = obj['outputFile'].split("_sim")[0]
simf += "_sim"
if (prodfile.count(simf)):
appdict.update(obj)
appdict['outputFile'] = prodfile+extension
olist[prodfile] = appdict
self.log.verbose('olist %s'%olist)
if (fname_in_outputlist.count("_rec")):
recf = obj['outputFile'].split("_rec")[0]
recf += "_rec"
if (prodfile.count(recf)):
appdict.update(obj)
appdict['outputFile'] = prodfile+extension
olist[prodfile] = appdict
break
if (fname_in_outputlist.count("_dst") and prodfile.lower().count("_dst_")):
dstf = obj['outputFile'].split("_dst")[0]
dstf += "_dst"
if (prodfile.count(dstf)):
appdict.update(obj)
appdict['outputFile'] = prodfile+extension
olist[prodfile] = appdict
break
self.outputList = olist.values()
else:
olist = []
for obj in self.outputList:
appdict = obj
appdict['outputFile'] = getProdFilename(obj['outputFile'],
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
olist.append(appdict)
self.outputList = olist
self.log.verbose("OutputList : %s" % self.outputList)
if self.workflow_commons.has_key('outputMode'):
self.outputMode = self.workflow_commons['outputMode']
if self.workflow_commons.has_key('outputDataFileMask'):
self.outputDataFileMask = self.workflow_commons['outputDataFileMask']
if not type(self.outputDataFileMask) == type([]):
self.outputDataFileMask = [i.lower().strip() for i in self.outputDataFileMask.split(';')]
#result = constructProductionLFNs(self.workflow_commons)
#if not result['OK']:
# self.log.error('Could not create production LFNs',result['Message'])
# return result
#self.prodOutputLFNs=result['Value']['ProductionOutputData']
if self.workflow_commons.has_key('ProductionOutputData'):
self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
else:
self.prodOutputLFNs = []
return S_OK('Parameters resolved')
#############################################################################
def execute(self):
""" Main execution function.
"""
self.log.info('Initializing %s' % self.version)
result = self.resolveInputVariables()
if not result['OK']:
self.log.error(result['Message'])
return result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('No output data upload attempted')
##determine the experiment
example_file = self.prodOutputLFNs[0]
if "/ilc/prod/clic" in example_file:
self.experiment = "CLIC"
elif "/ilc/prod/ilc/sid" in example_file:
self.experiment = 'ILC_SID'
elif "/ilc/prod/ilc/mc-dbd" in example_file:
self.experiment = 'ILC_ILD'
else:
self.log.warn("Failed to determine experiment, reverting to default")
#Determine the final list of possible output files for the
#workflow and all the parameters needed to upload them.
result = self.getCandidateFiles(self.outputList, self.prodOutputLFNs, self.outputDataFileMask)
if not result['OK']:
self.setApplicationStatus(result['Message'])
return result
fileDict = result['Value']
result = self.getFileMetadata(fileDict)
if not result['OK']:
self.setApplicationStatus(result['Message'])
return result
if not result['Value']:
self.log.info('No output data files were determined to be uploaded for this workflow')
return S_OK()
fileMetadata = result['Value']
#Get final, resolved SE list for files
final = {}
for fileName, metadata in fileMetadata.items():
result = getDestinationSEList(metadata['workflowSE'], DIRAC.siteName(), self.outputMode)
if not result['OK']:
self.log.error('Could not resolve output data SE', result['Message'])
self.setApplicationStatus('Failed To Resolve OutputSE')
return result
resolvedSE = result['Value']
final[fileName] = metadata
final[fileName]['resolvedSE'] = resolvedSE
self.log.info('The following files will be uploaded: %s' % (string.join(final.keys(), ', ')))
for fileName, metadata in final.items():
self.log.info('--------%s--------' % fileName)
for n, v in metadata.items():
self.log.info('%s = %s' % (n, v))
#At this point can exit and see exactly what the module would have uploaded
if not self.enable:
self.log.info('Module is disabled by control flag, would have attempted to upload the \
following files %s' % string.join(final.keys(), ', '))
return S_OK('Module is disabled by control flag')
#Disable the watchdog check in case the file uploading takes a long time
self.log.info('Creating DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK in order to disable the Watchdog prior to upload')
fopen = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK','w')
fopen.write('%s' % time.asctime())
fopen.close()
#Instantiate the failover transfer client with the global request object
failoverTransfer = FailoverTransfer(self.request)
catalogs = ['FileCatalog', 'LcgFileCatalog']
#One by one upload the files with failover if necessary
failover = {}
if not self.failoverTest:
for fileName, metadata in final.items():
self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName,
string.join(metadata['resolvedSE'],
', ')))
result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'],
metadata['lfn'], metadata['resolvedSE'],
fileGUID = metadata['guid'], fileCatalog = catalogs)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
failover[fileName] = metadata
else:
lfn = metadata['lfn']
else:
failover = final
self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs)
cleanUp = False
for fileName, metadata in failover.items():
self.log.info('Setting default catalog for failover transfer to FileCatalog')
random.shuffle(self.failoverSEs)
targetSE = metadata['resolvedSE'][0]
metadata['resolvedSE'] = self.failoverSEs
result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'],
metadata['lfn'], targetSE, metadata['resolvedSE'],
fileGUID = metadata['guid'], fileCatalog = catalogs)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
cleanUp = True
break #no point continuing if one completely fails
os.remove("DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK") #cleanup the mess
#Now after all operations, retrieve potentially modified request object
result = failoverTransfer.getRequestObject()
if not result['OK']:
self.log.error(result)
return S_ERROR('Could not retrieve modified request')
self.request = result['Value']
#If some or all of the files failed to be saved to failover
if cleanUp:
lfns = []
for fileName, metadata in final.items():
lfns.append(metadata['lfn'])
result = self.__cleanUp(lfns)
self.workflow_commons['Request'] = self.request
return S_ERROR('Failed to upload output data')
# #Can now register the successfully uploaded files in the BK
# if not performBKRegistration:
# self.log.info('There are no files to perform the BK registration for, all could be saved to failover')
# else:
# rm = ReplicaManager()
# result = rm.addCatalogFile(performBKRegistration,catalogs=['BookkeepingDB'])
# self.log.verbose(result)
# if not result['OK']:
# self.log.error(result)
# return S_ERROR('Could Not Perform BK Registration')
# if result['Value']['Failed']:
# for lfn,error in result['Value']['Failed'].items():
# self.log.info('BK registration for %s failed with message: "%s" setting failover request' %(lfn,error))
# result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0, 'Catalogue':'BookkeepingDB'}},'register')
# if not result['OK']:
# self.log.error('Could not set registerFile request:\n%s' %result)
# return S_ERROR('Could Not Set BK Registration Request')
# fileDict = {'LFN':lfn,'Status':'Waiting'}
# index = result['Value']
# self.request.setSubRequestFiles(index,'register',[fileDict])
self.workflow_commons['Request'] = self.request
return S_OK('Output data uploaded')
#############################################################################
def __cleanUp(self, lfnList):
""" Clean up uploaded data for the LFNs in the list
"""
# Clean up the current request
for req_type in ['transfer', 'register']:
for lfn in lfnList:
result = self.request.getNumSubRequests(req_type)
if result['OK']:
nreq = result['Value']
if nreq:
# Go through subrequests in reverse order in order not to spoil the numbering
ind_range = [0]
if nreq > 1:
ind_range = range(nreq-1, -1, -1)
for i in ind_range:
result = self.request.getSubRequestFiles(i, req_type)
if result['OK']:
fileList = result['Value']
if fileList[0]['LFN'] == lfn:
result = self.request.removeSubRequest(i, req_type)
# Set removal requests just in case
for lfn in lfnList:
result = self.request.addSubRequest({'Attributes': {'Operation' : 'removeFile', 'TargetSE' : '',
'ExecutionOrder' : 1}}, 'removal')
index = result['Value']
fileDict = {'LFN':lfn, 'PFN':'', 'Status':'Waiting'}
self.request.setSubRequestFiles(index, 'removal', [fileDict])
return S_OK()
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#
def test__getLastOrder(self):
# no files
req = RequestContainer()
res = req._getLastOrder()
self.assertEqual(res, 0)
self.assertEqual(req.subRequests, {})
req.addSubRequest(
{
'Attributes': {
'Operation': 'replicateAndRegister',
'TargetSE': 'SE',
'ExecutionOrder': 0
}
}, 'transfer')
res = req._getLastOrder()
self.assertEqual(res, 0)
req.addSubRequest(
{
'Attributes': {
'Operation': 'replicateAndRegister',
'TargetSE': 'SE',
'ExecutionOrder': 1
}
}, 'transfer')
res = req._getLastOrder()
self.assertEqual(res, 1)
del (req)
# with files
req = RequestContainer()
res = req._getLastOrder('foo')
self.assertEqual(res, 0)
req.addSubRequest(
{
'Attributes': {
'Operation': 'replicateAndRegister',
'TargetSE': 'SE',
'ExecutionOrder': 1
}
}, 'transfer')
res = req._getLastOrder('foo')
self.assertEqual(res, 0)
req.setSubRequestFiles(0, 'transfer', [{
'LFN': 'foo',
'Status': 'Waiting'
}])
res = req._getLastOrder('foo')
self.assertEqual(res, 1)
req.addSubRequest(
{
'Attributes': {
'Operation': 'replicateAndRegister',
'TargetSE': 'SE',
'ExecutionOrder': 2
}
}, 'removal')
res = req._getLastOrder('foo')
self.assertEqual(res, 1)
req.setSubRequestFiles(0, 'removal', [{
'LFN': 'foo',
'Status': 'Waiting'
}])
res = req._getLastOrder('foo')
self.assertEqual(res, 2)
def test_addSubRequest( self ):
rc_o = RequestContainer()
op1_Index = rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Operation': 'op1'}},
'someType' )
op1_Index = op1_Index['Value']
subRequestExpected = {'someType': [{'Files': [], 'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op1'},
'Datasets': []}]}
self.assertEqual( rc_o.subRequests, subRequestExpected )
op2_index = rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'Operation': 'op2'}},
'someType' )
op2_index = op2_index['Value']
subRequestExpected = {
'someType':
[
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op1'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op2'},
'Datasets': []
}
]
}
self.assertEqual( rc_o.subRequests, subRequestExpected )
rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'ExecutionOrder': 'last'}},
'someType' )
subRequestExpected = {
'someType':
[
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op1'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op2'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
}
]
}
self.assertEqual( rc_o.subRequests, subRequestExpected )
rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'ExecutionOrder': 'last'}},
'someOtherType' )
subRequestExpected = {
'someType':
[
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op1'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op2'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
}
],
'someOtherType':
[
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 2,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
},
]
}
self.assertEqual( rc_o.subRequests, subRequestExpected )
fileDict = {'LFN':'foo', 'Status':'Waiting'}
rc_o.setSubRequestFiles( op1_Index, 'someType', [fileDict] )
subRequestExpected = {
'someType':
[
{
'Files': [{'LFN':'foo', 'Status':'Waiting'}],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op1'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op2'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
}
],
'someOtherType':
[
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 2,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
},
]
}
self.assertEqual( rc_o.subRequests, subRequestExpected )
fileLastOp = rc_o._getLastOrder( 'foo' )
rc_o.addSubRequest( {'Attributes': {'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743', 'ExecutionOrder': fileLastOp + 1}},
'someOtherType' )
subRequestExpected = {
'someType':
[
{
'Files': [{'LFN':'foo', 'Status':'Waiting'}],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op1'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 0,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': 'op2'},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
}
],
'someOtherType':
[
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 2,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
},
{
'Files': [],
'Attributes': {'Status': 'Waiting', 'LastUpdate': '', 'TargetSE': '', 'ExecutionOrder': 1,
'SubRequestID': 'x', 'CreationTime': '2012-06-06 14:53:43.763743',
'Catalogue': '', 'Error': '', 'Operation': ''},
'Datasets': []
},
]
}
self.assertEqual( rc_o.subRequests, subRequestExpected )
def executeRequest(self):
################################################
# Get a request from request DB
gMonitor.addMark("Iteration", 1)
res = self.RequestDBClient.getRequest('transfer')
if not res['OK']:
gLogger.info(
"TransferAgent.execute: Failed to get request from database.")
return S_OK()
elif not res['Value']:
gLogger.info(
"TransferAgent.execute: No requests to be executed found.")
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
try:
jobID = int(res['Value']['JobID'])
except:
jobID = 0
gLogger.info("TransferAgent.execute: Obtained request %s" %
requestName)
result = self.RequestDBClient.getCurrentExecutionOrder(
requestName, sourceServer)
if result['OK']:
currentOrder = result['Value']
else:
return S_OK('Can not get the request execution order')
oRequest = RequestContainer(request=requestString)
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests('transfer')
if not res['OK']:
errStr = "TransferAgent.execute: Failed to obtain number of transfer subrequests."
gLogger.error(errStr, res['Message'])
return S_OK()
gLogger.info("TransferAgent.execute: Found %s sub requests." %
res['Value'])
################################################
# For all the sub-requests in the request
modified = False
for ind in range(res['Value']):
gMonitor.addMark("Execute", 1)
gLogger.info("TransferAgent.execute: Processing sub-request %s." %
ind)
subRequestAttributes = oRequest.getSubRequestAttributes(
ind, 'transfer')['Value']
if subRequestAttributes['ExecutionOrder']:
subExecutionOrder = int(subRequestAttributes['ExecutionOrder'])
else:
subExecutionOrder = 0
subStatus = subRequestAttributes['Status']
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
subRequestFiles = oRequest.getSubRequestFiles(
ind, 'transfer')['Value']
operation = subRequestAttributes['Operation']
subRequestError = ''
################################################
# If the sub-request is a put and register operation
if operation == 'putAndRegister' or operation == 'putAndRegisterAndRemove':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSE = str(subRequestAttributes['TargetSE'])
catalog = ''
if subRequestAttributes.has_key('Catalogue'):
catalog = subRequestAttributes['Catalogue']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark("Put and register", 1)
lfn = str(subRequestFile['LFN'])
file = subRequestFile['PFN']
guid = subRequestFile['GUID']
addler = subRequestFile['Addler']
res = self.ReplicaManager.putAndRegister(
lfn,
file,
diracSE,
guid=guid,
checksum=addler,
catalog=catalog)
if res['OK']:
if res['Value']['Successful'].has_key(lfn):
if not res['Value']['Successful'][
lfn].has_key('put'):
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
gLogger.info(
"TransferAgent.execute: Failed to put %s to %s."
% (lfn, diracSE))
subRequestError = "Put operation failed for %s to %s" % (
lfn, diracSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Put failed')
elif not res['Value']['Successful'][
lfn].has_key('register'):
gMonitor.addMark("Put successful", 1)
gMonitor.addMark(
"File registration failed", 1)
self.DataLog.addFileRecord(
lfn, 'Put', diracSE, '',
'TransferAgent')
self.DataLog.addFileRecord(
lfn, 'RegisterFail', diracSE, '',
'TransferAgent')
gLogger.info(
"TransferAgent.execute: Successfully put %s to %s in %s seconds."
% (lfn, diracSE, res['Value']
['Successful'][lfn]['put']))
gLogger.info(
"TransferAgent.execute: Failed to register %s to %s."
% (lfn, diracSE))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status',
'Done')
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Registration failed')
subRequestError = "Registration failed for %s to %s" % (
lfn, diracSE)
fileDict = res['Value']['Failed'][lfn][
'register']
registerRequestDict = {
'Attributes': {
'TargetSE':
fileDict['TargetSE'],
'Operation': 'registerFile'
},
'Files': [{
'LFN':
fileDict['LFN'],
'PFN':
fileDict['PFN'],
'Size':
fileDict['Size'],
'Addler':
fileDict['Addler'],
'GUID':
fileDict['GUID']
}]
}
gLogger.info(
"TransferAgent.execute: Setting registration request for failed file."
)
oRequest.addSubRequest(
registerRequestDict, 'register')
modified = True
else:
gMonitor.addMark("Put successful", 1)
gMonitor.addMark(
"File registration successful", 1)
self.DataLog.addFileRecord(
lfn, 'Put', diracSE, '',
'TransferAgent')
self.DataLog.addFileRecord(
lfn, 'Register', diracSE, '',
'TransferAgent')
gLogger.info(
"TransferAgent.execute: Successfully put %s to %s in %s seconds."
% (lfn, diracSE, res['Value']
['Successful'][lfn]['put']))
gLogger.info(
"TransferAgent.execute: Successfully registered %s to %s in %s seconds."
%
(lfn, diracSE, res['Value']
['Successful'][lfn]['register']))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status',
'Done')
modified = True
else:
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
errStr = "TransferAgent.execute: Failed to put and register file."
gLogger.error(
errStr, "%s %s %s" %
(lfn, diracSE,
res['Value']['Failed'][lfn]))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Complete file failure')
subRequestError = "Failed to put and register file"
else:
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
errStr = "TransferAgent.execute: Completely failed to put and register file."
gLogger.error(errStr, res['Message'])
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'RM call failure')
subRequestError = operation + " RM call file"
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is a put operation
elif operation == 'put':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
diracSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark("Put", 1)
lfn = subRequestFile['LFN']
file = subRequestFile['PFN']
res = self.ReplicaManager.put(lfn, file, diracSE)
if res['OK']:
if res['Value']['Successful'].has_key(lfn):
gMonitor.addMark("Put successful", 1)
self.DataLog.addFileRecord(
lfn, 'Put', diracSE, '',
'TransferAgent')
gLogger.info(
"TransferAgent.execute: Successfully put %s to %s in %s seconds."
% (lfn, diracSE,
res['Value']['Successful'][lfn]))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Done')
modified = True
else:
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
errStr = "TransferAgent.execute: Failed to put file."
gLogger.error(
errStr, "%s %s %s" %
(lfn, diracSE,
res['Value']['Failed'][lfn]))
subRequestError = "Put operation failed for %s to %s" % (
lfn, diracSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Put failed')
else:
gMonitor.addMark("Put failed", 1)
self.DataLog.addFileRecord(
lfn, 'PutFail', diracSE, '',
'TransferAgent')
errStr = "TransferAgent.execute: Completely failed to put file."
gLogger.error(errStr, res['Message'])
subRequestError = "Put RM call failed for %s to %s" % (
lfn, diracSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Put RM call failed')
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is a replicate and register operation
elif operation == 'replicateAndRegister' or operation == 'replicateAndRegisterAndRemove':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
targetSE = subRequestAttributes['TargetSE']
sourceSE = subRequestAttributes['SourceSE']
if sourceSE == "None":
sourceSE = ''
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark("Replicate and register", 1)
lfn = subRequestFile['LFN']
res = self.ReplicaManager.replicateAndRegister(
lfn, targetSE, sourceSE=sourceSE)
if res['OK']:
if res['Value']['Successful'].has_key(lfn):
if not res['Value']['Successful'][
lfn].has_key('replicate'):
gLogger.info(
"TransferAgent.execute: Failed to replicate %s to %s."
% (lfn, targetSE))
gMonitor.addMark(
"Replication failed", 1)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, "Error",
"Replication failed")
subRequestError = "Replication failed for %s to %s" % (
lfn, targetSE)
elif not res['Value']['Successful'][
lfn].has_key('register'):
gMonitor.addMark(
"Replication successful", 1)
gMonitor.addMark(
"Replica registration failed", 1)
gLogger.info(
"TransferAgent.execute: Successfully replicated %s to %s in %s seconds."
%
(lfn, targetSE, res['Value']
['Successful'][lfn]['replicate']))
gLogger.info(
"TransferAgent.execute: Failed to register %s to %s."
% (lfn, targetSE))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status',
'Done')
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Registration failed')
subRequestError = "Registration failed for %s to %s" % (
lfn, targetSE)
fileDict = res['Value']['Failed'][lfn][
'register']
registerRequestDict = {
'Attributes': {
'TargetSE':
fileDict['TargetSE'],
'Operation': 'registerReplica'
},
'Files': [{
'LFN': fileDict['LFN'],
'PFN': fileDict['PFN']
}]
}
gLogger.info(
"TransferAgent.execute: Setting registration request for failed replica."
)
oRequest.addSubRequest(
registerRequestDict, 'register')
modified = True
else:
gMonitor.addMark(
"Replication successful", 1)
gMonitor.addMark(
"Replica registration successful",
1)
gLogger.info(
"TransferAgent.execute: Successfully replicated %s to %s in %s seconds."
%
(lfn, targetSE, res['Value']
['Successful'][lfn]['replicate']))
gLogger.info(
"TransferAgent.execute: Successfully registered %s to %s in %s seconds."
%
(lfn, targetSE, res['Value']
['Successful'][lfn]['register']))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status',
'Done')
modified = True
else:
gMonitor.addMark("Replication failed", 1)
errStr = "TransferAgent.execute: Failed to replicate and register file."
gLogger.error(
errStr, "%s %s %s" %
(lfn, targetSE,
res['Value']['Failed'][lfn]))
else:
gMonitor.addMark("Replication failed", 1)
errStr = "TransferAgent.execute: Completely failed to replicate and register file."
gLogger.error(errStr, res['Message'])
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'RM call failure')
subRequestError = operation + " RM call failed"
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is a replicate operation
elif operation == 'replicate':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
targetSE = subRequestAttributes['TargetSE']
sourceSE = subRequestAttributes['SourceSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark("Replicate", 1)
lfn = subRequestFile['LFN']
res = self.ReplicaManager.replicate(
lfn, targetSE, sourceSE=sourceSE)
if res['OK']:
if res['Value']['Successful'].has_key(lfn):
gMonitor.addMark("Replication successful",
1)
gLogger.info(
"TransferAgent.execute: Successfully replicated %s to %s in %s seconds."
% (lfn, diracSE,
res['Value']['Successful'][lfn]))
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Done')
modified = True
else:
gMonitor.addMark("Replication failed", 1)
errStr = "TransferAgent.execute: Failed to replicate file."
gLogger.error(
errStr, "%s %s %s" %
(lfn, targetSE,
res['Value']['Failed'][lfn]))
subRequestError = "Replicate operation failed for %s to %s" % (
lfn, targetSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Put failed')
else:
gMonitor.addMark("Replication failed", 1)
errStr = "TransferAgent.execute: Completely failed to replicate file."
gLogger.error(errStr, res['Message'])
subRequestError = "Replicate RM call failed for %s to %s" % (
lfn, targetSE)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Error',
'Replicate RM call failed')
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is a get operation
elif operation == 'get':
gLogger.info(
"TransferAgent.execute: Attempting to execute %s sub-request."
% operation)
sourceSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = str(subRequestFile['LFN'])
pfn = str(subRequestFile['PFN'])
got = False
if sourceSE and pfn:
res = self.ReplicaManager.getStorageFile(
pfn, sourceSE)
if res['Value']['Successful'].has_key(pfn):
got = True
else:
res = self.ReplicaManager.getFile(lfn)
if res['Value']['Successful'].has_key(lfn):
got = False
if got:
gLogger.info(
"TransferAgent.execute: Successfully got %s."
% lfn)
oRequest.setSubRequestFileAttributeValue(
ind, 'transfer', lfn, 'Status', 'Done')
modified = True
else:
errStr = "TransferAgent.execute: Failed to get file."
gLogger.error(errStr, lfn)
else:
gLogger.info(
"TransferAgent.execute: File already completed."
)
################################################
# If the sub-request is none of the above types
else:
gLogger.error(
"TransferAgent.execute: Operation not supported.",
operation)
if subRequestError:
oRequest.setSubRequestAttributeValue(
ind, 'transfer', 'Error', subRequestError)
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty(ind, 'transfer')['Value']:
oRequest.setSubRequestStatus(ind, 'transfer', 'Done')
gMonitor.addMark("Done", 1)
################################################
# If the sub-request is already in terminal state
else:
gLogger.info(
"TransferAgent.execute: Sub-request %s is status '%s' and not to be executed."
% (ind, subRequestAttributes['Status']))
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest(requestName, requestString,
sourceServer)
if modified and jobID:
result = self.finalizeRequest(requestName, jobID, sourceServer)
return S_OK()
def executeRequest( self ):
################################################
# Get a request from request DB
gMonitor.addMark( "Iteration", 1 )
res = self.RequestDBClient.getRequest( 'transfer' )
if not res['OK']:
gLogger.info( "TransferAgent.execute: Failed to get request from database." )
return S_OK()
elif not res['Value']:
gLogger.info( "TransferAgent.execute: No requests to be executed found." )
return S_OK()
requestString = res['Value']['RequestString']
requestName = res['Value']['RequestName']
sourceServer = res['Value']['Server']
try:
jobID = int( res['Value']['JobID'] )
except:
jobID = 0
gLogger.info( "TransferAgent.execute: Obtained request %s" % requestName )
result = self.RequestDBClient.getCurrentExecutionOrder( requestName, sourceServer )
if result['OK']:
currentOrder = result['Value']
else:
return S_OK( 'Can not get the request execution order' )
oRequest = RequestContainer( request = requestString )
################################################
# Find the number of sub-requests from the request
res = oRequest.getNumSubRequests( 'transfer' )
if not res['OK']:
errStr = "TransferAgent.execute: Failed to obtain number of transfer subrequests."
gLogger.error( errStr, res['Message'] )
return S_OK()
gLogger.info( "TransferAgent.execute: Found %s sub requests." % res['Value'] )
################################################
# For all the sub-requests in the request
modified = False
for ind in range( res['Value'] ):
gMonitor.addMark( "Execute", 1 )
gLogger.info( "TransferAgent.execute: Processing sub-request %s." % ind )
subRequestAttributes = oRequest.getSubRequestAttributes( ind, 'transfer' )['Value']
if subRequestAttributes['ExecutionOrder']:
subExecutionOrder = int( subRequestAttributes['ExecutionOrder'] )
else:
subExecutionOrder = 0
subStatus = subRequestAttributes['Status']
if subStatus == 'Waiting' and subExecutionOrder <= currentOrder:
subRequestFiles = oRequest.getSubRequestFiles( ind, 'transfer' )['Value']
operation = subRequestAttributes['Operation']
subRequestError = ''
################################################
# If the sub-request is a put and register operation
if operation == 'putAndRegister' or operation == 'putAndRegisterAndRemove':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
diracSE = str( subRequestAttributes['TargetSE'] )
catalog = ''
if subRequestAttributes.has_key( 'Catalogue' ):
catalog = subRequestAttributes['Catalogue']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark( "Put and register", 1 )
lfn = str( subRequestFile['LFN'] )
file = subRequestFile['PFN']
guid = subRequestFile['GUID']
addler = subRequestFile['Addler']
res = self.ReplicaManager.putAndRegister( lfn, file, diracSE, guid = guid, checksum = addler, catalog = catalog )
if res['OK']:
if res['Value']['Successful'].has_key( lfn ):
if not res['Value']['Successful'][lfn].has_key( 'put' ):
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
gLogger.info( "TransferAgent.execute: Failed to put %s to %s." % ( lfn, diracSE ) )
subRequestError = "Put operation failed for %s to %s" % ( lfn, diracSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' )
elif not res['Value']['Successful'][lfn].has_key( 'register' ):
gMonitor.addMark( "Put successful", 1 )
gMonitor.addMark( "File registration failed", 1 )
self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' )
self.DataLog.addFileRecord( lfn, 'RegisterFail', diracSE, '', 'TransferAgent' )
gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['put'] ) )
gLogger.info( "TransferAgent.execute: Failed to register %s to %s." % ( lfn, diracSE ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Registration failed' )
subRequestError = "Registration failed for %s to %s" % ( lfn, diracSE )
fileDict = res['Value']['Failed'][lfn]['register']
registerRequestDict = {'Attributes':{'TargetSE': fileDict['TargetSE'], 'Operation':'registerFile'},
'Files':[{'LFN': fileDict['LFN'],
'PFN':fileDict['PFN'],
'Size':fileDict['Size'],
'Addler':fileDict['Addler'],
'GUID':fileDict['GUID']}]}
gLogger.info( "TransferAgent.execute: Setting registration request for failed file." )
oRequest.addSubRequest( registerRequestDict, 'register' )
modified = True
else:
gMonitor.addMark( "Put successful", 1 )
gMonitor.addMark( "File registration successful", 1 )
self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' )
self.DataLog.addFileRecord( lfn, 'Register', diracSE, '', 'TransferAgent' )
gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['put'] ) )
gLogger.info( "TransferAgent.execute: Successfully registered %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn]['register'] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
errStr = "TransferAgent.execute: Failed to put and register file."
gLogger.error( errStr, "%s %s %s" % ( lfn, diracSE, res['Value']['Failed'][lfn] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Complete file failure' )
subRequestError = "Failed to put and register file"
else:
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
errStr = "TransferAgent.execute: Completely failed to put and register file."
gLogger.error( errStr, res['Message'] )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'RM call failure' )
subRequestError = operation + " RM call file"
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is a put operation
elif operation == 'put':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
diracSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark( "Put", 1 )
lfn = subRequestFile['LFN']
file = subRequestFile['PFN']
res = self.ReplicaManager.put( lfn, file, diracSE )
if res['OK']:
if res['Value']['Successful'].has_key( lfn ):
gMonitor.addMark( "Put successful", 1 )
self.DataLog.addFileRecord( lfn, 'Put', diracSE, '', 'TransferAgent' )
gLogger.info( "TransferAgent.execute: Successfully put %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
errStr = "TransferAgent.execute: Failed to put file."
gLogger.error( errStr, "%s %s %s" % ( lfn, diracSE, res['Value']['Failed'][lfn] ) )
subRequestError = "Put operation failed for %s to %s" % ( lfn, diracSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' )
else:
gMonitor.addMark( "Put failed", 1 )
self.DataLog.addFileRecord( lfn, 'PutFail', diracSE, '', 'TransferAgent' )
errStr = "TransferAgent.execute: Completely failed to put file."
gLogger.error( errStr, res['Message'] )
subRequestError = "Put RM call failed for %s to %s" % ( lfn, diracSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put RM call failed' )
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is a replicate and register operation
elif operation == 'replicateAndRegister' or operation == 'replicateAndRegisterAndRemove':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
targetSE = subRequestAttributes['TargetSE']
sourceSE = subRequestAttributes['SourceSE']
if sourceSE == "None":
sourceSE = ''
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark( "Replicate and register", 1 )
lfn = subRequestFile['LFN']
res = self.ReplicaManager.replicateAndRegister( lfn, targetSE, sourceSE = sourceSE )
if res['OK']:
if res['Value']['Successful'].has_key( lfn ):
if not res['Value']['Successful'][lfn].has_key( 'replicate' ):
gLogger.info( "TransferAgent.execute: Failed to replicate %s to %s." % ( lfn, targetSE ) )
gMonitor.addMark( "Replication failed", 1 )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,"Error", "Replication failed" )
subRequestError = "Replication failed for %s to %s" % ( lfn, targetSE )
elif not res['Value']['Successful'][lfn].has_key( 'register' ):
gMonitor.addMark( "Replication successful", 1 )
gMonitor.addMark( "Replica registration failed", 1 )
gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, targetSE, res['Value']['Successful'][lfn]['replicate'] ) )
gLogger.info( "TransferAgent.execute: Failed to register %s to %s." % ( lfn, targetSE ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Error', 'Registration failed' )
subRequestError = "Registration failed for %s to %s" % ( lfn, targetSE )
fileDict = res['Value']['Failed'][lfn]['register']
registerRequestDict = {'Attributes':{'TargetSE': fileDict['TargetSE'], 'Operation':'registerReplica'}, 'Files':[{'LFN': fileDict['LFN'], 'PFN':fileDict['PFN']}]}
gLogger.info( "TransferAgent.execute: Setting registration request for failed replica." )
oRequest.addSubRequest( registerRequestDict, 'register' )
modified = True
else:
gMonitor.addMark( "Replication successful", 1 )
gMonitor.addMark( "Replica registration successful", 1 )
gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, targetSE, res['Value']['Successful'][lfn]['replicate'] ) )
gLogger.info( "TransferAgent.execute: Successfully registered %s to %s in %s seconds." % ( lfn, targetSE, res['Value']['Successful'][lfn]['register'] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gMonitor.addMark( "Replication failed", 1 )
errStr = "TransferAgent.execute: Failed to replicate and register file."
gLogger.error( errStr, "%s %s %s" % ( lfn, targetSE, res['Value']['Failed'][lfn] ) )
else:
gMonitor.addMark( "Replication failed", 1 )
errStr = "TransferAgent.execute: Completely failed to replicate and register file."
gLogger.error( errStr, res['Message'] )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'RM call failure' )
subRequestError = operation + " RM call failed"
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is a replicate operation
elif operation == 'replicate':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
targetSE = subRequestAttributes['TargetSE']
sourceSE = subRequestAttributes['SourceSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
gMonitor.addMark( "Replicate", 1 )
lfn = subRequestFile['LFN']
res = self.ReplicaManager.replicate( lfn, targetSE, sourceSE = sourceSE )
if res['OK']:
if res['Value']['Successful'].has_key( lfn ):
gMonitor.addMark( "Replication successful", 1 )
gLogger.info( "TransferAgent.execute: Successfully replicated %s to %s in %s seconds." % ( lfn, diracSE, res['Value']['Successful'][lfn] ) )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
gMonitor.addMark( "Replication failed", 1 )
errStr = "TransferAgent.execute: Failed to replicate file."
gLogger.error( errStr, "%s %s %s" % ( lfn, targetSE, res['Value']['Failed'][lfn] ) )
subRequestError = "Replicate operation failed for %s to %s" % ( lfn, targetSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Put failed' )
else:
gMonitor.addMark( "Replication failed", 1 )
errStr = "TransferAgent.execute: Completely failed to replicate file."
gLogger.error( errStr, res['Message'] )
subRequestError = "Replicate RM call failed for %s to %s" % ( lfn, targetSE )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn,'Error', 'Replicate RM call failed' )
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is a get operation
elif operation == 'get':
gLogger.info( "TransferAgent.execute: Attempting to execute %s sub-request." % operation )
sourceSE = subRequestAttributes['TargetSE']
for subRequestFile in subRequestFiles:
if subRequestFile['Status'] == 'Waiting':
lfn = str( subRequestFile['LFN'] )
pfn = str( subRequestFile['PFN'] )
got = False
if sourceSE and pfn:
res = self.ReplicaManager.getStorageFile( pfn, sourceSE )
if res['Value']['Successful'].has_key( pfn ):
got = True
else:
res = self.ReplicaManager.getFile( lfn )
if res['Value']['Successful'].has_key( lfn ):
got = False
if got:
gLogger.info( "TransferAgent.execute: Successfully got %s." % lfn )
oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' )
modified = True
else:
errStr = "TransferAgent.execute: Failed to get file."
gLogger.error( errStr, lfn )
else:
gLogger.info( "TransferAgent.execute: File already completed." )
################################################
# If the sub-request is none of the above types
else:
gLogger.error( "TransferAgent.execute: Operation not supported.", operation )
if subRequestError:
oRequest.setSubRequestAttributeValue( ind, 'transfer', 'Error', subRequestError )
################################################
# Determine whether there are any active files
if oRequest.isSubRequestEmpty( ind, 'transfer' )['Value']:
oRequest.setSubRequestStatus( ind, 'transfer', 'Done' )
gMonitor.addMark( "Done", 1 )
################################################
# If the sub-request is already in terminal state
else:
gLogger.info( "TransferAgent.execute: Sub-request %s is status '%s' and not to be executed." % ( ind, subRequestAttributes['Status'] ) )
################################################
# Generate the new request string after operation
requestString = oRequest.toXML()['Value']
res = self.RequestDBClient.updateRequest( requestName, requestString, sourceServer )
if modified and jobID:
result = self.finalizeRequest( requestName, jobID, sourceServer )
return S_OK()
