def __setRemovalRequest(self, lfn, ownerDN, ownerGroup):
""" Set removal request with the given credentials
"""
request = RequestContainer()
request.setRequestAttributes({
'OwnerDN': ownerDN,
'OwnerGroup': ownerGroup
})
requestName = os.path.basename(lfn).strip() + '_removal_request.xml'
request.setRequestName(requestName)
request.setSourceComponent('JobCleaningAgent')
removalDict = {
'Attributes': {
'Operation': 'removeFile',
'TargetSE': '',
'ExecutionOrder': 0
}
}
result = request.addSubRequest(removalDict, 'removal')
if not result['OK']:
return result
index = result['Value']
fileDict = {'LFN': lfn, 'PFN': '', 'Status': 'Waiting'}
request.setSubRequestFiles(index, 'removal', [fileDict])
client = RequestClient()
result = request.toXML()
if not result['OK']:
return result
xmlRequest = result['Value']
result = client.setRequest(requestName, xmlRequest)
return result
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
request = RequestContainer()
request.setRequestAttributes( { 'OwnerDN':ownerDN, 'OwnerGroup':ownerGroup } )
requestName = os.path.basename( lfn ).strip()+'_removal_request.xml'
request.setRequestName( requestName )
request.setSourceComponent( 'JobCleaningAgent' )
removalDict = {'Attributes':{ 'Operation':'removeFile',
'TargetSE':'',
'ExecutionOrder':0
}
}
result = request.addSubRequest( removalDict, 'removal' )
if not result['OK']:
return result
index = result['Value']
fileDict = { 'LFN':lfn, 'PFN':'', 'Status':'Waiting' }
request.setSubRequestFiles( index, 'removal', [fileDict] )
client = RequestClient()
result = request.toXML()
if not result['OK']:
return result
xmlRequest = result['Value']
result = client.setRequest( requestName, xmlRequest )
return result
def getRequest():
"""
helper fcn to build requestContainer
"""
requestContainer = RequestContainer(init=False)
## get request
requestContainer.setRequestName("00009423_00000118")
requestContainer.setJobID(0)
requestContainer.setOwnerDN("")
requestContainer.setOwnerGroup("")
requestContainer.setDIRACSetup("")
requestContainer.setSourceComponent(None)
requestContainer.setCreationTime("0000-00-00 00:00:00")
requestContainer.setLastUpdate("2011-02-19 04:57:02")
requestContainer.setStatus("Waiting")
## get subrequest
requestContainer.initiateSubRequest("transfer")
subRequestDict = {
"Status": "Waiting",
"SubRequestID": 2259916,
"Operation": "replicateAndRegister",
"Arguments": None,
"ExecutionOrder": 0,
"SourceSE": None,
"TargetSE": "GRIDKA_MC-DST,GRIDKA_MC_M-DST",
"Catalogue": None,
"CreationTime": "2011-02-19 04:57:02",
"SubmissionTime": "2011-02-19 04:57:02",
"LastUpdate": "2011-08-18 20:14:22"
}
requestContainer.setSubRequestAttributes(0, "transfer", subRequestDict)
## get subrequest files
files = [{
"FileID": 1610538,
"LFN":
"/lhcb/MC/MC10/ALLSTREAMS.DST/00009422/0000/00009422_00000171_1.allstreams.dst",
"Size": None,
"PFN": None,
"GUID": None,
"Md5": None,
"Addler": None,
"Attempt": 1,
"Status": "Scheduled"
}]
requestContainer.setSubRequestFiles(0, "transfer", files)
return {
"OK": True,
"Value": {
"RequestName": "00009423_00000118",
"RequestString": requestContainer.toXML()["Value"],
"JobID": 0,
"RequestContainer": requestContainer
}
}
def getRequest():
"""
helper fcn to build requestContainer
"""
requestContainer = RequestContainer( init = False )
## get request
requestContainer.setRequestName( "00009423_00000118" )
requestContainer.setJobID( 0 )
requestContainer.setOwnerDN( "" )
requestContainer.setOwnerGroup( "" )
requestContainer.setDIRACSetup( "" )
requestContainer.setSourceComponent( None )
requestContainer.setCreationTime( "0000-00-00 00:00:00" )
requestContainer.setLastUpdate( "2011-02-19 04:57:02" )
requestContainer.setStatus( "Waiting" )
## get subrequest
requestContainer.initiateSubRequest( "transfer" )
subRequestDict = { "Status" : "Waiting",
"SubRequestID" : 2259916,
"Operation" : "replicateAndRegister",
"Arguments" : None,
"ExecutionOrder" : 0,
"SourceSE" : None,
"TargetSE" : "GRIDKA_MC-DST,GRIDKA_MC_M-DST",
"Catalogue" : None,
"CreationTime" : "2011-02-19 04:57:02",
"SubmissionTime" : "2011-02-19 04:57:02",
"LastUpdate" : "2011-08-18 20:14:22" }
requestContainer.setSubRequestAttributes( 0, "transfer", subRequestDict )
## get subrequest files
files = [ { "FileID" : 1610538,
"LFN" : "/lhcb/MC/MC10/ALLSTREAMS.DST/00009422/0000/00009422_00000171_1.allstreams.dst",
"Size" : None,
"PFN" : None,
"GUID" : None,
"Md5" : None,
"Addler" : None,
"Attempt" : 1,
"Status" : "Scheduled" } ]
requestContainer.setSubRequestFiles( 0, "transfer", files )
return { "OK" : True,
"Value" : { "RequestName" : "00009423_00000118",
"RequestString" : requestContainer.toXML()["Value"],
"JobID" : 0,
"RequestContainer" : requestContainer } }
shutil.copy(appTar,"%s%s" % (final_path, os.path.basename(appTar)))
except Exception, x:
gLogger.error("Could not copy because %s" % x)
return S_ERROR("Could not copy because %s" % x)
elif path.find("http://") > -1:
gLogger.error("path %s was not foreseen, location not known, upload to location yourself, and publish in CS manually" % path)
return S_ERROR()
else:
lfnpath = "%s%s" % (path, os.path.basename(appTar))
res = rm.putAndRegister(lfnpath, appTar, ops.getValue('Software/BaseStorageElement',"CERN-SRM"))
if not res['OK']:
return res
request = RequestContainer()
request.setCreationTime()
requestClient = RequestClient()
request.setRequestName('copy_%s' % os.path.basename(appTar).replace(".tgz","").replace(".tar.gz",""))
request.setSourceComponent('ReplicateILCSoft')
copies_at = ops.getValue('Software/CopiesAt',[])
index_copy = 0
for copies in copies_at:
res = request.addSubRequest({'Attributes':{'Operation' : 'replicateAndRegister',
'TargetSE' : copies,
'ExecutionOrder' : index_copy},
'Files':[{'LFN':lfnpath}]},
'transfer')
#res = rm.replicateAndRegister("%s%s"%(path,appTar),"IN2P3-SRM")
if not res['OK']:
return res
index_copy += 1
requestxml = request.toXML()['Value']
if copies_at:
class FailoverTransfer:
#############################################################################
def __init__(self, requestObject=False):
""" Constructor function, can specify request object to instantiate
FailoverTransfer or a new request object is created.
"""
self.log = gLogger.getSubLogger("FailoverTransfer")
self.rm = ReplicaManager()
self.request = requestObject
if not self.request:
self.request = RequestContainer()
self.request.setRequestName('default_request.xml')
self.request.setSourceComponent('FailoverTransfer')
#############################################################################
def transferAndRegisterFile(self,
fileName,
localPath,
lfn,
destinationSEList,
fileGUID=None,
fileCatalog=None):
"""Performs the transfer and register operation with failover.
"""
errorList = []
for se in destinationSEList:
self.log.info(
'Attempting rm.putAndRegister("%s","%s","%s",guid="%s",catalog="%s")'
% (lfn, localPath, se, fileGUID, fileCatalog))
result = self.rm.putAndRegister(lfn,
localPath,
se,
guid=fileGUID,
catalog=fileCatalog)
self.log.verbose(result)
if not result['OK']:
self.log.error('rm.putAndRegister failed with message',
result['Message'])
errorList.append(result['Message'])
continue
if not result['Value']['Failed']:
self.log.info(
'rm.putAndRegister successfully uploaded %s to %s' %
(fileName, se))
return S_OK({'uploadedSE': se, 'lfn': lfn})
#Now we know something went wrong
errorDict = result['Value']['Failed'][lfn]
if not errorDict.has_key('register'):
self.log.error('rm.putAndRegister failed with unknown error',
str(errorDict))
errorList.append(
'Unknown error while attempting upload to %s' % se)
continue
fileDict = errorDict['register']
#Therefore the registration failed but the upload was successful
if not fileCatalog:
fileCatalog = ''
result = self.__setRegistrationRequest(fileDict['LFN'], se,
fileCatalog, fileDict)
if not result['OK']:
self.log.error(
'Failed to set registration request for: SE %s and metadata: \n%s'
% (se, fileDict))
errorList.append(
'Failed to set registration request for: SE %s and metadata: \n%s'
% (se, fileDict))
continue
else:
self.log.info(
'Successfully set registration request for: SE %s and metadata: \n%s'
% (se, fileDict))
metadata = {}
metadata['filedict'] = fileDict
metadata['uploadedSE'] = se
metadata['lfn'] = lfn
metadata['registration'] = 'request'
return S_OK(metadata)
self.log.error(
'Encountered %s errors during attempts to upload output data' %
len(errorList))
return S_ERROR('Failed to upload output data file')
#############################################################################
def transferAndRegisterFileFailover(self,
fileName,
localPath,
lfn,
targetSE,
failoverSEList,
fileGUID=None,
fileCatalog=None):
"""Performs the transfer and register operation to failover storage and sets the
necessary replication and removal requests to recover.
"""
failover = self.transferAndRegisterFile(fileName, localPath, lfn,
failoverSEList, fileGUID,
fileCatalog)
if not failover['OK']:
self.log.error('Could not upload file to failover SEs',
failover['Message'])
return failover
#set removal requests and replication requests
result = self.__setFileReplicationRequest(lfn, targetSE)
if not result['OK']:
self.log.error('Could not set file replication request',
result['Message'])
return result
lfn = failover['Value']['lfn']
failoverSE = failover['Value']['uploadedSE']
self.log.info(
'Attempting to set replica removal request for LFN %s at failover SE %s'
% (lfn, failoverSE))
result = self.__setReplicaRemovalRequest(lfn, failoverSE)
if not result['OK']:
self.log.error('Could not set removal request', result['Message'])
return result
return S_OK('%s uploaded to a failover SE' % fileName)
#############################################################################
def getRequestObject(self):
"""Returns the potentially modified request object in order to propagate changes.
"""
return S_OK(self.request)
#############################################################################
def __setFileReplicationRequest(self, lfn, se):
""" Sets a registration request.
"""
self.log.info('Setting replication request for %s to %s' % (lfn, se))
result = self.request.addSubRequest(
{
'Attributes': {
'Operation': 'replicateAndRegister',
'TargetSE': se,
'ExecutionOrder': 0
}
}, 'transfer')
if not result['OK']:
return result
index = result['Value']
fileDict = {'LFN': lfn, 'Status': 'Waiting'}
self.request.setSubRequestFiles(index, 'transfer', [fileDict])
return S_OK()
#############################################################################
def __setRegistrationRequest(self, lfn, se, catalog, fileDict):
""" Sets a registration request.
"""
self.log.info('Setting registration request for %s at %s.' % (lfn, se))
result = self.request.addSubRequest(
{
'Attributes': {
'Operation': 'registerFile',
'ExecutionOrder': 0,
'TargetSE': se,
'Catalogue': catalog
}
}, 'register')
if not result['OK']:
return result
index = result['Value']
if not fileDict.has_key('Status'):
fileDict['Status'] = 'Waiting'
self.request.setSubRequestFiles(index, 'register', [fileDict])
return S_OK()
#############################################################################
def __setReplicaRemovalRequest(self, lfn, se):
""" Sets a removal request for a replica.
"""
result = self.request.addSubRequest(
{
'Attributes': {
'Operation': 'replicaRemoval',
'TargetSE': se,
'ExecutionOrder': 1
}
}, 'removal')
index = result['Value']
fileDict = {'LFN': lfn, 'Status': 'Waiting'}
self.request.setSubRequestFiles(index, 'removal', [fileDict])
return S_OK()
#############################################################################
def __setFileRemovalRequest(self, lfn, se='', pfn=''):
""" Sets a removal request for a file including all replicas.
"""
result = self.request.addSubRequest(
{
'Attributes': {
'Operation': 'removeFile',
'TargetSE': se,
'ExecutionOrder': 1
}
}, 'removal')
index = result['Value']
fileDict = {'LFN': lfn, 'PFN': pfn, 'Status': 'Waiting'}
self.request.setSubRequestFiles(index, 'removal', [fileDict])
return S_OK()
class FailoverTransfer:
#############################################################################
def __init__(self,requestObject=False):
""" Constructor function, can specify request object to instantiate
FailoverTransfer or a new request object is created.
"""
self.log = gLogger.getSubLogger( "FailoverTransfer" )
self.rm = ReplicaManager()
self.request = requestObject
if not self.request:
self.request = RequestContainer()
self.request.setRequestName('default_request.xml')
self.request.setSourceComponent('FailoverTransfer')
#############################################################################
def transferAndRegisterFile(self,fileName,localPath,lfn,destinationSEList,fileGUID=None,fileCatalog=None):
"""Performs the transfer and register operation with failover.
"""
errorList = []
for se in destinationSEList:
self.log.info('Attempting rm.putAndRegister("%s","%s","%s",guid="%s",catalog="%s")' %(lfn,localPath,se,fileGUID,fileCatalog))
result = self.rm.putAndRegister(lfn,localPath,se,guid=fileGUID,catalog=fileCatalog)
self.log.verbose(result)
if not result['OK']:
self.log.error('rm.putAndRegister failed with message',result['Message'])
errorList.append(result['Message'])
continue
if not result['Value']['Failed']:
self.log.info('rm.putAndRegister successfully uploaded %s to %s' %(fileName,se))
return S_OK({'uploadedSE':se,'lfn':lfn})
#Now we know something went wrong
errorDict = result['Value']['Failed'][lfn]
if not errorDict.has_key('register'):
self.log.error('rm.putAndRegister failed with unknown error',str(errorDict))
errorList.append('Unknown error while attempting upload to %s' %se)
continue
fileDict = errorDict['register']
#Therefore the registration failed but the upload was successful
if not fileCatalog:
fileCatalog=''
result = self.__setRegistrationRequest(fileDict['LFN'],se,fileCatalog,fileDict)
if not result['OK']:
self.log.error('Failed to set registration request for: SE %s and metadata: \n%s' %(se,fileDict))
errorList.append('Failed to set registration request for: SE %s and metadata: \n%s' %(se,fileDict))
continue
else:
self.log.info('Successfully set registration request for: SE %s and metadata: \n%s' %(se,fileDict))
metadata = {}
metadata['filedict']=fileDict
metadata['uploadedSE']=se
metadata['lfn']=lfn
metadata['registration']='request'
return S_OK(metadata)
self.log.error('Encountered %s errors during attempts to upload output data' %len(errorList))
return S_ERROR('Failed to upload output data file')
#############################################################################
def transferAndRegisterFileFailover(self,fileName,localPath,lfn,targetSE,failoverSEList,fileGUID=None,fileCatalog=None):
"""Performs the transfer and register operation to failover storage and sets the
necessary replication and removal requests to recover.
"""
failover = self.transferAndRegisterFile(fileName,localPath,lfn,failoverSEList,fileGUID,fileCatalog)
if not failover['OK']:
self.log.error('Could not upload file to failover SEs',failover['Message'])
return failover
#set removal requests and replication requests
result = self.__setFileReplicationRequest(lfn,targetSE)
if not result['OK']:
self.log.error('Could not set file replication request',result['Message'])
return result
lfn = failover['Value']['lfn']
failoverSE = failover['Value']['uploadedSE']
self.log.info('Attempting to set replica removal request for LFN %s at failover SE %s' %(lfn,failoverSE))
result = self.__setReplicaRemovalRequest(lfn,failoverSE)
if not result['OK']:
self.log.error('Could not set removal request',result['Message'])
return result
return S_OK('%s uploaded to a failover SE' %fileName)
#############################################################################
def getRequestObject(self):
"""Returns the potentially modified request object in order to propagate changes.
"""
return S_OK(self.request)
#############################################################################
def __setFileReplicationRequest(self,lfn,se):
""" Sets a registration request.
"""
self.log.info('Setting replication request for %s to %s' % (lfn,se))
result = self.request.addSubRequest({'Attributes':{'Operation':'replicateAndRegister',
'TargetSE':se,'ExecutionOrder':0}},
'transfer')
if not result['OK']:
return result
index = result['Value']
fileDict = {'LFN':lfn,'Status':'Waiting'}
self.request.setSubRequestFiles(index,'transfer',[fileDict])
return S_OK()
#############################################################################
def __setRegistrationRequest(self,lfn,se,catalog,fileDict):
""" Sets a registration request.
"""
self.log.info('Setting registration request for %s at %s.' % (lfn,se))
result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0,
'TargetSE':se,'Catalogue':catalog}},'register')
if not result['OK']:
return result
index = result['Value']
if not fileDict.has_key('Status'):
fileDict['Status']='Waiting'
self.request.setSubRequestFiles(index,'register',[fileDict])
return S_OK()
#############################################################################
def __setReplicaRemovalRequest(self,lfn,se):
""" Sets a removal request for a replica.
"""
result = self.request.addSubRequest({'Attributes':{'Operation':'replicaRemoval',
'TargetSE':se,'ExecutionOrder':1}},
'removal')
index = result['Value']
fileDict = {'LFN':lfn,'Status':'Waiting'}
self.request.setSubRequestFiles(index,'removal',[fileDict])
return S_OK()
#############################################################################
def __setFileRemovalRequest(self,lfn,se='',pfn=''):
""" Sets a removal request for a file including all replicas.
"""
result = self.request.addSubRequest({'Attributes':{'Operation':'removeFile',
'TargetSE':se,'ExecutionOrder':1}},
'removal')
index = result['Value']
fileDict = {'LFN':lfn,'PFN':pfn,'Status':'Waiting'}
self.request.setSubRequestFiles(index,'removal',[fileDict])
return S_OK()
def getRequest(self, requestType):
""" Get a request of a given type eligible for execution
"""
# RG: What if requestType is not given?
# the first query will return nothing.
# KC: maybe returning S_ERROR would be enough?
# alternatively we should check if requestType is known (in 'transfer', 'removal', 'register' and 'diset')
if not requestType or type(requestType) not in types.StringTypes:
return S_ERROR("Request type not given.")
myRequestType = self._escapeString(requestType)
if not myRequestType:
return myRequestType
myRequestType = myRequestType['Value']
start = time.time()
dmRequest = RequestContainer(init=False)
requestID = 0
subIDList = []
fields = [
'RequestID', 'SubRequestID', 'Operation', 'Arguments',
'ExecutionOrder', 'SourceSE', 'TargetSE', 'Catalogue',
'CreationTime', 'SubmissionTime', 'LastUpdate'
]
# get the pending SubRequest sorted by ExecutionOrder and LastUpdate
req = "SELECT RequestID, ExecutionOrder, Status, RequestType, LastUpdate from SubRequests WHERE Status IN ( 'Waiting', 'Assigned' ) ORDER BY ExecutionOrder, LastUpdate"
# now get sorted list of RequestID (according to the above)
req = "SELECT * from ( %s ) as T1 GROUP BY RequestID" % req
# and get the 100 oldest ones of Type requestType
req = "SELECT RequestID, ExecutionOrder FROM ( %s ) as T2 WHERE RequestType = %s ORDER BY LastUpdate limit 100" % (
req, myRequestType)
# and now get all waiting SubRequest for the selected RequestID and ExecutionOrder
req = "SELECT A.%s FROM SubRequests AS A, ( %s ) AS B WHERE " % (
', A.'.join(fields), req)
req = "%s A.RequestID = B.RequestID AND A.ExecutionOrder = B.ExecutionOrder AND A.Status = 'Waiting' AND A.RequestType = %s;" % (
req, myRequestType)
result = self._query(req)
if not result['OK']:
err = 'RequestDB._getRequest: Failed to retrieve Requests'
return S_ERROR('%s\n%s' % (err, result['Message']))
if not result['Value']:
return S_OK()
# We get up to 10 Request candidates, to add some randomness
reqDict = {}
for row in result['Value']:
reqDict.setdefault(row[0], [])
reqDict[row[0]].append(row[1:])
reqIDList = reqDict.keys()
random.shuffle(reqIDList)
for reqID in reqIDList:
sidList = [x[0] for x in reqDict[reqID]]
for subID in sidList:
req = "UPDATE SubRequests SET Status='Assigned' WHERE RequestID=%s AND SubRequestID=%s;" % (
reqID, subID)
resAssigned = self._update(req)
if not resAssigned['OK']:
if subIDList:
self.__releaseSubRequests(reqID, subIDList)
return S_ERROR('Failed to assign subrequests: %s' %
resAssigned['Message'])
if resAssigned['Value'] == 0:
# Somebody has assigned this request
gLogger.warn(
'Already assigned subrequest %d of request %d' %
(subID, reqID))
else:
subIDList.append(subID)
if subIDList:
# We managed to get some requests, can continue now
requestID = reqID
break
# Haven't succeeded to get any request
if not requestID:
return S_OK()
dmRequest.setRequestID(requestID)
fields = [
'FileID', 'LFN', 'Size', 'PFN', 'GUID', 'Md5', 'Addler', 'Attempt',
'Status'
]
for subRequestID, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in reqDict[
requestID]:
if not subRequestID in subIDList: continue
res = dmRequest.initiateSubRequest(requestType)
ind = res['Value']
subRequestDict = {
'Status': 'Waiting',
'SubRequestID': subRequestID,
'Operation': operation,
'Arguments': arguments,
'ExecutionOrder': int(executionOrder),
'SourceSE': sourceSE,
'TargetSE': targetSE,
'Catalogue': catalogue,
'CreationTime': creationTime,
'SubmissionTime': submissionTime,
'LastUpdate': lastUpdate
}
res = dmRequest.setSubRequestAttributes(ind, requestType,
subRequestDict)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set subRequest attributes for RequestID %s' % requestID
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
req = "SELECT %s FROM Files WHERE SubRequestID = %s ORDER BY FileID;" % (
', '.join(fields), subRequestID)
res = self._query(req)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get File attributes for RequestID %s.%s' % (
requestID, subRequestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res[
'Value']:
fileDict = {
'FileID': fileID,
'LFN': lfn,
'Size': size,
'PFN': pfn,
'GUID': guid,
'Md5': md5,
'Addler': addler,
'Attempt': attempt,
'Status': status
}
files.append(fileDict)
res = dmRequest.setSubRequestFiles(ind, requestType, files)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set files into Request for RequestID %s.%s' % (
requestID, subRequestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
req = "SELECT Dataset,Status FROM Datasets WHERE SubRequestID = %s;" % subRequestID
res = self._query(req)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get Datasets for RequestID %s.%s' % (
requestID, subRequestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
datasets = []
for dataset, status in res['Value']:
datasets.append(dataset)
res = dmRequest.setSubRequestDatasets(ind, requestType, datasets)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set datasets into Request for RequestID %s.%s' % (
requestID, subRequestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
fields = [
'RequestName', 'JobID', 'OwnerDN', 'OwnerGroup', 'DIRACSetup',
'SourceComponent', 'CreationTime', 'SubmissionTime', 'LastUpdate'
]
req = "SELECT %s from Requests WHERE RequestID = %s;" % (
', '.join(fields), requestID)
res = self._query(req)
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve max RequestID'
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate = res[
'Value'][0]
dmRequest.setRequestName(requestName)
dmRequest.setJobID(jobID)
dmRequest.setOwnerDN(ownerDN)
dmRequest.setOwnerGroup(ownerGroup)
dmRequest.setDIRACSetup(diracSetup)
dmRequest.setSourceComponent(sourceComponent)
dmRequest.setCreationTime(str(creationTime))
dmRequest.setLastUpdate(str(lastUpdate))
res = dmRequest.toXML()
if not res['OK']:
err = 'RequestDB._getRequest: Failed to create XML for RequestID %s' % (
requestID)
self.__releaseSubRequests(requestID, subIDList)
return S_ERROR('%s\n%s' % (err, res['Message']))
requestString = res['Value']
#still have to manage the status of the dataset properly
resultDict = {}
resultDict['RequestName'] = requestName
resultDict['RequestString'] = requestString
resultDict['JobID'] = jobID
return S_OK(resultDict)
def getRequestForSubRequest(self, itself, subRequestID):
"""
Select Request given SubRequestID.
:param self: plugin reference
:param itself: patient reference for injection
:param int subRequestID: SubRequests.SubRequestID
:warn: function has to be injected to RequestDBMySQL instance
"""
## get RequestID
requestID = "SELECT RequestID FROM SubRequests WHERE SubRequestID = %s;" % str(
subRequestID)
requestID = self._query(requestID)
if not requestID["OK"]:
return requestID
requestID = requestID["Value"][0]
## create RequestContainer
requestContainer = RequestContainer(init=False)
requestContainer.setRequestID(requestID)
## put some basic infos in
requestInfo = "SELECT RequestName, JobID, OwnerDN, OwnerGroup, DIRACSetup, SourceComponent, CreationTime, SubmissionTime, LastUpdate, Status "
requestInfo += "FROM Requests WHERE RequestID = %d;" % requestID
requestInfo = self._query(requestInfo)
if not requestInfo["OK"]:
return requestInfo
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate, status = requestInfo[
'Value'][0]
requestContainer.setRequestName(requestName)
requestContainer.setJobID(jobID)
requestContainer.setOwnerDN(ownerDN)
requestContainer.setOwnerGroup(ownerGroup)
requestContainer.setDIRACSetup(diracSetup)
requestContainer.setSourceComponent(sourceComponent)
requestContainer.setCreationTime(str(creationTime))
requestContainer.setLastUpdate(str(lastUpdate))
requestContainer.setStatus(status)
## get sub-requests
subRequests = "SELECT SubRequestID, Status, RequestType, Operation, Arguments, ExecutionOrder, SourceSE, "
subRequests += "TargetSE, Catalogue, CreationTime, SubmissionTime, LastUpdate FROM SubRequests WHERE RequestID=%s;" % requestID
subRequests = self._query(subRequests)
if not subRequests["OK"]:
return subRequests
## loop over sub requests
for subRequestID, status, requestType, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in subRequests[
"Value"]:
res = requestContainer.initiateSubRequest(requestType)
ind = res["Value"]
subRequestDict = {
"Status": status,
"SubRequestID": subRequestID,
"Operation": operation,
"Arguments": arguments,
"ExecutionOrder": int(executionOrder),
"SourceSE": sourceSE,
"TargetSE": targetSE,
"Catalogue": catalogue,
"CreationTime": creationTime,
"SubmissionTime": submissionTime,
"LastUpdate": lastUpdate
}
res = requestContainer.setSubRequestAttributes(
ind, requestType, subRequestDict)
if not res["OK"]:
return res
## get files for this subrequest
req = "SELECT FileID, LFN, Size, PFN, GUID, Md5, Addler, Attempt, Status FROM Files WHERE SubRequestID = %s ORDER BY FileID;" % str(
subRequestID)
res = self._query(req)
if not res["OK"]:
return res
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res[
"Value"]:
fileDict = {
"FileID": fileID,
"LFN": lfn,
"Size": size,
"PFN": pfn,
"GUID": guid,
"Md5": md5,
"Addler": addler,
"Attempt": attempt,
"Status": status
}
files.append(fileDict)
res = requestContainer.setSubRequestFiles(ind, requestType, files)
if not res["OK"]:
return res
## dump request to XML
res = requestContainer.toXML()
if not res["OK"]:
return res
requestString = res["Value"]
## return dictonary with all info in at least
return S_OK({
"RequestName": requestName,
"RequestString": requestString,
"JobID": jobID,
"RequestContainer": requestContainer
})
class FailoverRequest(ModuleBase):
""" Handle the failover requests issued by previous steps. Used in production.
"""
#############################################################################
def __init__(self):
"""Module initialization.
"""
super(FailoverRequest, self).__init__()
self.version = __RCSID__
self.log = gLogger.getSubLogger( "FailoverRequest" )
#Internal parameters
self.enable = True
self.jobID = ''
self.productionID = None
self.prodJobID = None
#Workflow parameters
self.jobReport = None
self.fileReport = None
self.request = None
#############################################################################
def applicationSpecificInputs(self):
""" By convention the module input parameters are resolved here.
"""
self.log.debug(self.workflow_commons)
self.log.debug(self.step_commons)
if os.environ.has_key('JOBID'):
self.jobID = os.environ['JOBID']
self.log.verbose('Found WMS JobID = %s' %self.jobID)
else:
self.log.info('No WMS JobID found, disabling module via control flag')
self.enable = False
if self.step_commons.has_key('Enable'):
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
self.enable = False
#Earlier modules will have populated the report objects
if self.workflow_commons.has_key('JobReport'):
self.jobReport = self.workflow_commons['JobReport']
if self.workflow_commons.has_key('FileReport'):
self.fileReport = self.workflow_commons['FileReport']
if self.InputData:
if type(self.InputData) != type([]):
self.InputData = self.InputData.split(';')
self.InputData = [x.replace('LFN:','') for x in self.InputData]
if self.workflow_commons.has_key('Request'):
self.request = self.workflow_commons['Request']
if not self.request:
self.request = RequestContainer()
self.request.setRequestName('job_%s_request.xml' % self.jobID)
self.request.setJobID(self.jobID)
self.request.setSourceComponent("Job_%s" % self.jobID)
if self.workflow_commons.has_key('PRODUCTION_ID'):
self.productionID = self.workflow_commons['PRODUCTION_ID']
if self.workflow_commons.has_key('JOB_ID'):
self.prodJobID = self.workflow_commons['JOB_ID']
return S_OK('Parameters resolved')
#############################################################################
def execute(self):
""" Main execution function.
"""
self.log.info('Initializing %s' % self.version)
result = self.resolveInputVariables()
if not result['OK']:
self.log.error(result['Message'])
return result
if not self.fileReport:
self.fileReport = FileReport('Transformation/TransformationManager')
if self.InputData:
inputFiles = self.fileReport.getFiles()
for lfn in self.InputData:
if not lfn in inputFiles:
self.log.verbose('No status populated for input data %s, setting to "Unused"' % lfn)
result = self.fileReport.setFileStatus(int(self.productionID), lfn, 'Unused')
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.info('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
inputFiles = self.fileReport.getFiles()
for lfn in inputFiles:
if inputFiles[lfn] != 'ApplicationCrash':
self.log.info('Forcing status to "Unused" due to workflow failure for: %s' % (lfn))
self.fileReport.setFileStatus(int(self.productionID), lfn, 'Unused')
else:
inputFiles = self.fileReport.getFiles()
if inputFiles:
self.log.info('Workflow status OK, setting input file status to Processed')
for lfn in inputFiles:
self.log.info('Setting status to "Processed" for: %s' % (lfn))
self.fileReport.setFileStatus(int(self.productionID), lfn, 'Processed')
result = self.fileReport.commit()
if not result['OK']:
self.log.error('Failed to report file status to ProductionDB, request will be generated', result['Message'])
else:
self.log.info('Status of files have been properly updated in the ProcessingDB')
# Must ensure that the local job report instance is used to report the final status
# in case of failure and a subsequent failover operation
if self.workflowStatus['OK'] and self.stepStatus['OK']:
if not self.jobReport:
self.jobReport = JobReport(int(self.jobID))
jobStatus = self.jobReport.setApplicationStatus('Job Finished Successfully')
if not jobStatus['OK']:
self.log.warn(jobStatus['Message'])
# Retrieve the accumulated reporting request
reportRequest = None
if self.jobReport:
result = self.jobReport.generateRequest()
if not result['OK']:
self.log.warn('Could not generate request for job report with result:\n%s' % (result))
else:
reportRequest = result['Value']
if reportRequest:
self.log.info('Populating request with job report information')
self.request.update(reportRequest)
fileReportRequest = None
if self.fileReport:
result = self.fileReport.generateRequest()
if not result['OK']:
self.log.warn('Could not generate request for file report with result:\n%s' % (result))
else:
fileReportRequest = result['Value']
if fileReportRequest:
self.log.info('Populating request with file report information')
result = self.request.update(fileReportRequest)
accountingReport = None
if self.workflow_commons.has_key('AccountingReport'):
accountingReport = self.workflow_commons['AccountingReport']
if accountingReport:
result = accountingReport.commit()
if not result['OK']:
self.log.info('Populating request with accounting report information')
self.request.setDISETRequest(result['rpcStub'])
if self.request.isEmpty()['Value']:
self.log.info('Request is empty, nothing to do.')
return self.finalize()
request_string = self.request.toXML()['Value']
self.log.debug(request_string)
# Write out the request string
fname = '%s_%s_request.xml' % (self.productionID, self.prodJobID)
xmlfile = open(fname, 'w')
xmlfile.write(request_string)
xmlfile.close()
self.log.info('Creating failover request for deferred operations for job %s:' % self.jobID)
result = self.request.getDigest()
if result['OK']:
digest = result['Value']
self.log.info(digest)
if not self.enable:
self.log.info('Module is disabled by control flag')
return S_OK('Module is disabled by control flag')
return self.finalize()
#############################################################################
def finalize(self):
""" Finalize and report correct status for the workflow based on the workflow
or step status.
"""
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.warn('Workflow status is not ok, will not overwrite status')
self.log.info('Workflow failed, end of FailoverRequest module execution.')
return S_ERROR('Workflow failed, FailoverRequest module completed')
self.log.info('Workflow successful, end of FailoverRequest module execution.')
return S_OK('FailoverRequest module completed')
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#
class HandleProdOutputData(ModuleBase):
def __init__(self):
super(HandleProdOutputData, self).__init__()
self.result = S_ERROR()
self.fc = FileCatalogClient()
self.rm = ReplicaManager()
self.destination = ''
self.basepath = '/ilc/prod'
self.dataattributes = ''
self.attributesdict = {}
self.generatorfile = ''
self.mokkafile = ''
self.marlinfiles = ''
self.slicfile = ''
self.lcsimfiles = ''
self.request = None
self.failoverTest=False
self.log = gLogger.getSubLogger( "HandleOutputData" )
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
@return: S_OK()
"""
if self.step_commons.has_key('DataAttributes'):
self.dataattributes = self.step_commons['DataAttributes']
else:
return S_ERROR('No data attributes found, cannot proceed with registration in Catalog, ABORT!')
for attribute in self.dataattributes.split(";"):
if self.step_commons.has_key(attribute):
self.attributesdict[attribute] = self.step_commons[attribute]
if self.step_commons.has_key("destination"):
self.destination = self.step_commons['destination']
if self.step_commons.has_key('GENFile'):
self.generatorfile = self.step_commons['GENFile']
if self.step_commons.has_key('MokkaFile'):
self.mokkafile = self.step_commons['MokkaFile']
if self.step_commons.has_key('MarlinFiles'):
self.marlinfiles = self.step_commons['MarlinFiles'].split(';')
if self.step_commons.has_key('SLICFile'):
self.slicfile = self.step_commons['SLICFile']
if self.step_commons.has_key('LCSIMFiles'):
self.lcsimfiles = self.step_commons['LCSIMFiles'].split(';')
if self.workflow_commons.has_key('Request'):
self.request = self.workflow_commons['Request']
else:
self.request = RequestContainer()
self.request.setRequestName('job_%s_request.xml' % self.jobID)
self.request.setJobID(self.jobID)
self.request.setSourceComponent("Job_%s" % self.jobID)
return S_OK('Parameters resolved')
def execute(self):
#Have to work out if the module is part of the last step i.e.
#user jobs can have any number of steps and we only want
#to run the finalization once.
currentStep = int(self.step_commons['STEP_NUMBER'])
totalSteps = int(self.workflow_commons['TotalSteps'])
if currentStep==totalSteps:
self.lastStep=True
else:
self.log.verbose('Current step = %s, total steps of workflow = %s, HandleProdOutputData will enable itself only at the last workflow step.' %(currentStep,totalSteps))
if not self.lastStep:
return S_OK()
self.result =self.resolveInputVariables()
if not self.result['OK']:
self.log.error(self.result['Message'])
return self.result
###Instantiate object that will ensure that the files are registered properly
failoverTransfer = FailoverTransfer(self.request)
datatohandle = {}
if self.generatorfile:
if not os.path.exists(self.generatorfile):
return S_ERROR("File %s does not exist, something went wrong before !"%(self.generatorfile))
self.attributesdict['DataType'] = 'gen'
lfnpath = string.join([self.basepath,self.attributesdict['Machine'],self.attributesdict['Energy'],
self.attributesdict['DataType'],self.attributesdict['EvtType'],self.attributesdict['ProdID'],
self.generatorfile],"/")
datatohandle[self.generatorfile]={'lfn':lfnpath,'type':'gen','workflowSE':self.destination}
if self.mokkafile or self.slicfile:
recofile = ''
if self.mokkafile and not os.path.exists(self.mokkafile):
return S_ERROR("File %s does not exist, something went wrong before !"%(self.mokkafile))
else:
recofile = self.mokkafile
if self.slicfile and not os.path.exists(self.slicfile):
return S_ERROR("File %s does not exist, something went wrong before !"%(self.slicfile))
else:
recofile = self.slicfile
self.attributesdict['DataType'] = 'SIM'
lfnpath = string.join([self.basepath,self.attributesdict['Machine'],self.attributesdict['Energy'],
self.attributesdict['DetectorModel'],self.attributesdict['DataType'],self.attributesdict['EvtType'],
self.attributesdict['ProdID'],recofile],"/")
datatohandle[recofile]={'lfn':lfnpath,'type':'gen','workflowSE':self.destination}
##Below, look in file name if it contain REC or DST, to determine the data type.
if self.marlinfiles:
for file in self.marlinfiles:
if file.find("REC")>-1:
self.attributesdict['DataType'] = 'REC'
if file.find("DST")>-1:
self.attributesdict['DataType'] = 'DST'
lfnpath = string.join([self.basepath,self.attributesdict['Machine'],self.attributesdict['Energy'],
self.attributesdict['DetectorModel'],self.attributesdict['DataType'],self.attributesdict['EvtType'],
self.attributesdict['ProdID'],file],"/")
datatohandle[file]={'lfn':lfnpath,'type':'gen','workflowSE':self.destination}
if self.lcsimfiles:
for file in self.lcsimfiles:
if file.find("DST")>-1:
self.attributesdict['DataType'] = 'DST'
lfnpath = string.join([self.basepath,self.attributesdict['Machine'],self.attributesdict['Energy'],
self.attributesdict['DetectorModel'],self.attributesdict['DataType'],self.attributesdict['EvtType'],
self.attributesdict['ProdID'],file],"/")
datatohandle[file]={'lfn':lfnpath,'type':'gen','workflowSE':self.destination}
result = self.getFileMetadata(datatohandle)
if not result['OK']:
self.setApplicationStatus(result['Message'])
return S_OK()
fileMetadata = result['Value']
final = {}
for fileName,metadata in fileMetadata.items():
final[fileName]=metadata
final[fileName]['resolvedSE']=self.destination
#One by one upload the files with failover if necessary
replication = {}
failover = {}
uploaded = []
if not self.failoverTest:
for fileName,metadata in final.items():
self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName, string.join(metadata['resolvedSE'],', ')))
result = failoverTransfer.transferAndRegisterFile(fileName,metadata['localpath'],metadata['lfn'],metadata['resolvedSE'],fileGUID=metadata['guid'],fileCatalog=self.userFileCatalog)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' %(fileName,metadata))
failover[fileName]=metadata
else:
#Only attempt replication after successful upload
lfn = metadata['lfn']
uploaded.append(lfn)
seList = metadata['resolvedSE']
replicateSE = ''
if result['Value'].has_key('uploadedSE'):
uploadedSE = result['Value']['uploadedSE']
for se in seList:
if not se == uploadedSE:
replicateSE = se
break
if replicateSE and lfn:
self.log.info('Will attempt to replicate %s to %s' %(lfn,replicateSE))
replication[lfn]=replicateSE
else:
failover = final
cleanUp = False
for fileName,metadata in failover.items():
random.shuffle(self.failoverSEs)
targetSE = metadata['resolvedSE'][0]
metadata['resolvedSE']=self.failoverSEs
result = failoverTransfer.transferAndRegisterFileFailover(fileName,metadata['localpath'],metadata['lfn'],targetSE,metadata['resolvedSE'],fileGUID=metadata['guid'],fileCatalog=self.userFileCatalog)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' %(fileName,metadata))
cleanUp = True
continue #for users can continue even if one completely fails
else:
lfn = metadata['lfn']
uploaded.append(lfn)
#For files correctly uploaded must report LFNs to job parameters
if uploaded:
report = string.join( uploaded, ', ' )
self.jobReport.setJobParameter( 'UploadedOutputData', report )
#Now after all operations, retrieve potentially modified request object
result = failoverTransfer.getRequestObject()
if not result['OK']:
self.log.error(result)
return S_ERROR('Could Not Retrieve Modified Request')
self.request = result['Value']
#If some or all of the files failed to be saved to failover
if cleanUp:
self.workflow_commons['Request']=self.request
#Leave any uploaded files just in case it is useful for the user
#do not try to replicate any files.
return S_ERROR('Failed To Upload Output Data')
return S_OK()
class UserJobFinalization(ModuleBase):
""" User Job finalization: takes care of uploading the output data to the specified storage elements
If it does not work, it will upload to a Failover SE, then register the request to replicate and remove.
"""
#############################################################################
def __init__(self):
"""Module initialization.
"""
super(UserJobFinalization, self).__init__()
self.version = __RCSID__
self.log = gLogger.getSubLogger( "UserJobFinalization" )
self.enable = True
self.failoverTest = False #flag to put file to failover SE by default
self.defaultOutputSE = gConfig.getValue( '/Resources/StorageElementGroups/Tier1-USER', [])
self.failoverSEs = gConfig.getValue('/Resources/StorageElementGroups/Tier1-Failover', [])
#List all parameters here
self.userFileCatalog = self.ops.getValue('/UserJobs/Catalogs', ['FileCatalog'] )
self.request = None
self.lastStep = False
#Always allow any files specified by users
self.outputDataFileMask = ''
self.userOutputData = ''
self.userOutputSE = ''
self.userOutputPath = ''
self.jobReport = None
#############################################################################
def applicationSpecificInputs(self):
""" By convention the module parameters are resolved here.
"""
self.log.verbose(self.workflow_commons)
self.log.verbose(self.step_commons)
#Earlier modules may have populated the report objects
if self.workflow_commons.has_key('JobReport'):
self.jobReport = self.workflow_commons['JobReport']
if self.step_commons.has_key('Enable'):
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
self.log.warn('Enable flag set to non-boolean value %s, setting to False' %self.enable)
self.enable = False
if self.step_commons.has_key('TestFailover'):
self.enable = self.step_commons['TestFailover']
if not type(self.failoverTest) == type(True):
self.log.warn('Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest)
self.failoverTest = False
if os.environ.has_key('JOBID'):
self.jobID = os.environ['JOBID']
self.log.verbose('Found WMS JobID = %s' % self.jobID)
else:
self.log.info('No WMS JobID found, disabling module via control flag')
self.enable = False
if self.workflow_commons.has_key('Request'):
self.request = self.workflow_commons['Request']
else:
self.request = RequestContainer()
self.request.setRequestName('job_%s_request.xml' % self.jobID)
self.request.setJobID(self.jobID)
self.request.setSourceComponent("Job_%s" % self.jobID)
#Use LHCb utility for local running via dirac-jobexec
if self.workflow_commons.has_key('UserOutputData'):
self.userOutputData = self.workflow_commons['UserOutputData']
if not type(self.userOutputData) == type([]):
self.userOutputData = [i.strip() for i in self.userOutputData.split(';')]
if self.workflow_commons.has_key('UserOutputSE'):
specifiedSE = self.workflow_commons['UserOutputSE']
if not type(specifiedSE) == type([]):
self.userOutputSE = [i.strip() for i in specifiedSE.split(';')]
else:
self.log.verbose('No UserOutputSE specified, using default value: %s' % (string.join(self.defaultOutputSE,
', ')))
self.userOutputSE = self.defaultOutputSE
if self.workflow_commons.has_key('UserOutputPath'):
self.userOutputPath = self.workflow_commons['UserOutputPath']
return S_OK('Parameters resolved')
#############################################################################
def execute(self):
""" Main execution function.
"""
#Have to work out if the module is part of the last step i.e.
#user jobs can have any number of steps and we only want
#to run the finalization once.
currentStep = int(self.step_commons['STEP_NUMBER'])
totalSteps = int(self.workflow_commons['TotalSteps'])
if currentStep == totalSteps:
self.lastStep = True
else:
self.log.verbose('Current step = %s, total steps of workflow = %s, UserJobFinalization will enable itself only \
at the last workflow step.' % (currentStep, totalSteps))
if not self.lastStep:
return S_OK()
result = self.resolveInputVariables()
if not result['OK']:
self.log.error(result['Message'])
return result
self.log.info('Initializing %s' % self.version)
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'],
self.stepStatus['OK']))
return S_OK('No output data upload attempted')
if not self.userOutputData:
self.log.info('No user output data is specified for this job, nothing to do')
return S_OK('No output data to upload')
#Determine the final list of possible output files for the
#workflow and all the parameters needed to upload them.
outputList = []
for i in self.userOutputData:
outputList.append({'outputPath' : string.upper(string.split(i, '.')[-1]),
'outputDataSE' : self.userOutputSE,
'outputFile' : os.path.basename(i)})
userOutputLFNs = []
if self.userOutputData:
self.log.info('Constructing user output LFN(s) for %s' % (string.join(self.userOutputData, ', ')))
if not self.jobID:
self.jobID = 12345
owner = ''
if self.workflow_commons.has_key('Owner'):
owner = self.workflow_commons['Owner']
else:
res = self.getCurrentOwner()
if not res['OK']:
return S_ERROR('Could not obtain owner from proxy')
owner = res['Value']
vo = ''
if self.workflow_commons.has_key('VO'):
vo = self.workflow_commons['VO']
else:
res = self.getCurrentVO()
if not res['OK']:
return S_ERROR('Could not obtain VO from proxy')
vo = res['Value']
result = constructUserLFNs(int(self.jobID), vo, owner, self.userOutputData, self.userOutputPath)
if not result['OK']:
self.log.error('Could not create user LFNs', result['Message'])
return result
userOutputLFNs = result['Value']
self.log.verbose('Calling getCandidateFiles( %s, %s, %s)' % (outputList, userOutputLFNs, self.outputDataFileMask))
result = self.getCandidateFiles(outputList, userOutputLFNs, self.outputDataFileMask)
if not result['OK']:
if not self.ignoreapperrors:
self.setApplicationStatus(result['Message'])
return S_OK()
fileDict = result['Value']
result = self.getFileMetadata(fileDict)
if not result['OK']:
if not self.ignoreapperrors:
self.setApplicationStatus(result['Message'])
return S_OK()
if not result['Value']:
if not self.ignoreapperrors:
self.log.info('No output data files were determined to be uploaded for this workflow')
self.setApplicationStatus('No Output Data Files To Upload')
return S_OK()
fileMetadata = result['Value']
#First get the local (or assigned) SE to try first for upload and others in random fashion
result = getDestinationSEList('Tier1-USER', DIRAC.siteName(), outputmode='local')
if not result['OK']:
self.log.error('Could not resolve output data SE', result['Message'])
self.setApplicationStatus('Failed To Resolve OutputSE')
return result
localSE = result['Value']
self.log.verbose('Site Local SE for user outputs is: %s' % (localSE))
orderedSEs = self.defaultOutputSE
for se in localSE:
if se in orderedSEs:
orderedSEs.remove(se)
for se in self.userOutputSE:
if se in orderedSEs:
orderedSEs.remove(se)
orderedSEs = localSE + List.randomize(orderedSEs)
if self.userOutputSE:
prependSEs = []
for userSE in self.userOutputSE:
if not userSE in orderedSEs:
prependSEs.append(userSE)
orderedSEs = prependSEs + orderedSEs
self.log.info('Ordered list of output SEs is: %s' % (string.join(orderedSEs, ', ')))
final = {}
for fileName, metadata in fileMetadata.items():
final[fileName] = metadata
final[fileName]['resolvedSE'] = orderedSEs
#At this point can exit and see exactly what the module will upload
if not self.enable:
self.log.info('Module is disabled by control flag, would have attempted \
to upload the following files %s' % string.join(final.keys(), ', '))
for fileName, metadata in final.items():
self.log.info('--------%s--------' % fileName)
for n, v in metadata.items():
self.log.info('%s = %s' %(n, v))
return S_OK('Module is disabled by control flag')
#Instantiate the failover transfer client with the global request object
failoverTransfer = FailoverTransfer(self.request)
#One by one upload the files with failover if necessary
replication = {}
failover = {}
uploaded = []
if not self.failoverTest:
for fileName, metadata in final.items():
self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName,
string.join(metadata['resolvedSE'],
', ')))
result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'], metadata['lfn'],
metadata['resolvedSE'], fileGUID = metadata['guid'],
fileCatalog = self.userFileCatalog)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
failover[fileName] = metadata
else:
#Only attempt replication after successful upload
lfn = metadata['lfn']
uploaded.append(lfn)
seList = metadata['resolvedSE']
replicateSE = ''
if result['Value'].has_key('uploadedSE'):
uploadedSE = result['Value']['uploadedSE']
for se in seList:
if not se == uploadedSE:
replicateSE = se
break
if replicateSE and lfn:
self.log.info('Will attempt to replicate %s to %s' % (lfn, replicateSE))
replication[lfn] = replicateSE
else:
failover = final
cleanUp = False
for fileName, metadata in failover.items():
random.shuffle(self.failoverSEs)
targetSE = metadata['resolvedSE'][0]
metadata['resolvedSE'] = self.failoverSEs
result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'], metadata['lfn'],
targetSE, metadata['resolvedSE'],
fileGUID = metadata['guid'],
fileCatalog = self.userFileCatalog)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
cleanUp = True
continue #for users can continue even if one completely fails
else:
lfn = metadata['lfn']
uploaded.append(lfn)
#For files correctly uploaded must report LFNs to job parameters
if uploaded:
report = string.join( uploaded, ', ' )
self.jobReport.setJobParameter( 'UploadedOutputData', report )
#Now after all operations, retrieve potentially modified request object
result = failoverTransfer.getRequestObject()
if not result['OK']:
self.log.error(result)
return S_ERROR('Could Not Retrieve Modified Request')
self.request = result['Value']
#If some or all of the files failed to be saved to failover
if cleanUp:
self.workflow_commons['Request'] = self.request
#Leave any uploaded files just in case it is useful for the user
#do not try to replicate any files.
return S_ERROR('Failed To Upload Output Data')
#If there is now at least one replica for uploaded files can trigger replication
rm = ReplicaManager()
self.log.info('Sleeping for 10 seconds before attempting replication of recently uploaded files')
time.sleep(10)
for lfn, repSE in replication.items():
result = rm.replicateAndRegister(lfn, repSE, catalog = self.userFileCatalog)
if not result['OK']:
self.log.info('Replication failed with below error but file already exists in Grid storage with \
at least one replica:\n%s' % (result))
self.workflow_commons['Request'] = self.request
#Now must ensure if any pending requests are generated that these are propagated to the job wrapper
reportRequest = None
if self.jobReport:
result = self.jobReport.generateRequest()
if not result['OK']:
self.log.warn('Could not generate request for job report with result:\n%s' % (result))
else:
reportRequest = result['Value']
if reportRequest:
self.log.info('Populating request with job report information')
self.request.update(reportRequest)
if not self.request.isEmpty()['Value']:
request_string = self.request.toXML()['Value']
# Write out the request string
fname = 'user_job_%s_request.xml' % (self.jobID)
xmlfile = open(fname, 'w')
xmlfile.write(request_string)
xmlfile.close()
self.log.info('Creating failover request for deferred operations for job %s:' % self.jobID)
result = self.request.getDigest()
if result['OK']:
digest = result['Value']
self.log.info(digest)
self.setApplicationStatus('Job Finished Successfully')
return S_OK('Output data uploaded')
#############################################################################
def getCurrentOwner(self):
"""Simple function to return current DIRAC username.
"""
result = getProxyInfo()
if not result['OK']:
return S_ERROR('Could not obtain proxy information')
if not result['Value'].has_key('username'):
return S_ERROR('Could not get username from proxy')
username = result['Value']['username']
return S_OK(username)
#############################################################################
def getCurrentVO(self):
"""Simple function to return current DIRAC username.
"""
result = getProxyInfo()
if not result['OK']:
return S_ERROR('Could not obtain proxy information')
if not result['Value'].has_key('group'):
return S_ERROR('Could not get group from proxy')
group = result['Value']['group']
vo = group.split("_")[0]
return S_OK(vo)
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#
class UploadLogFile(ModuleBase):
""" Handle log file uploads in the production jobs
"""
#############################################################################
def __init__(self):
"""Module initialization.
"""
super(UploadLogFile, self).__init__()
self.version = __RCSID__
self.log = gLogger.getSubLogger("UploadLogFile")
self.PRODUCTION_ID = None
self.JOB_ID = None
self.workflow_commons = None
self.request = None
self.logFilePath = ""
self.logLFNPath = ""
self.logdir = ""
self.logSE = self.ops.getValue("/LogStorage/LogSE", "LogSE")
self.root = gConfig.getValue("/LocalSite/Root", os.getcwd())
self.logSizeLimit = self.ops.getValue("/LogFiles/SizeLimit", 20 * 1024 * 1024)
self.logExtensions = []
self.failoverSEs = gConfig.getValue("/Resources/StorageElementGroups/Tier1-Failover", [])
self.diracLogo = self.ops.getValue(
"/SAM/LogoURL", "https://lhcbweb.pic.es/DIRAC/images/logos/DIRAC-logo-transp.png"
)
self.rm = ReplicaManager()
self.experiment = "CLIC"
self.enable = True
self.failoverTest = False # flag to put log files to failover by default
self.jobID = ""
######################################################################
def applicationSpecificInputs(self):
if self.step_commons.has_key("Enable"):
self.enable = self.step_commons["Enable"]
if not type(self.enable) == type(True):
self.log.warn("Enable flag set to non-boolean value %s, setting to False" % self.enable)
self.enable = False
if self.step_commons.has_key("TestFailover"):
self.enable = self.step_commons["TestFailover"]
if not type(self.failoverTest) == type(True):
self.log.warn("Test failover flag set to non-boolean value %s, setting to False" % self.failoverTest)
self.failoverTest = False
if os.environ.has_key("JOBID"):
self.jobID = os.environ["JOBID"]
self.log.verbose("Found WMS JobID = %s" % self.jobID)
else:
self.log.info("No WMS JobID found, disabling module via control flag")
self.enable = False
if self.workflow_commons.has_key("LogFilePath") and self.workflow_commons.has_key("LogTargetPath"):
self.logFilePath = self.workflow_commons["LogFilePath"]
self.logLFNPath = self.workflow_commons["LogTargetPath"]
else:
self.log.info("LogFilePath parameter not found, creating on the fly")
result = getLogPath(self.workflow_commons)
if not result["OK"]:
self.log.error("Could not create LogFilePath", result["Message"])
return result
self.logFilePath = result["Value"]["LogFilePath"][0]
self.logLFNPath = result["Value"]["LogTargetPath"][0]
if not type(self.logFilePath) == type(" "):
self.logFilePath = self.logFilePath[0]
if not type(self.logLFNPath) == type(" "):
self.logLFNPath = self.logLFNPath[0]
example_file = self.logFilePath
if "/ilc/prod/clic" in example_file:
self.experiment = "CLIC"
elif "/ilc/prod/ilc/sid" in example_file:
self.experiment = "ILC_SID"
elif "/ilc/prod/ilc/mc-dbd" in example_file:
self.experiment = "ILC_ILD"
else:
self.log.warn("Failed to determine experiment, reverting to default: %s" % self.experiment)
if self.workflow_commons.has_key("Request"):
self.request = self.workflow_commons["Request"]
else:
self.request = RequestContainer()
self.request.setRequestName("job_%s_request.xml" % self.jobID)
self.request.setJobID(self.jobID)
self.request.setSourceComponent("Job_%s" % self.jobID)
return S_OK("Parameters resolved")
######################################################################
def execute(self):
""" Main execution method
"""
self.log.info("Initializing %s" % self.version)
# Add global reporting tool
self.resolveInputVariables()
res = shellCall(0, "ls -al")
if res["OK"] and res["Value"][0] == 0:
self.log.info("The contents of the working directory...")
self.log.info(str(res["Value"][1]))
else:
self.log.error("Failed to list the log directory", str(res["Value"][2]))
self.log.info("Job root is found to be %s" % (self.root))
self.log.info("PRODUCTION_ID = %s, JOB_ID = %s " % (self.PRODUCTION_ID, self.JOB_ID))
self.logdir = os.path.realpath("./job/log/%s/%s" % (self.PRODUCTION_ID, self.JOB_ID))
self.log.info("Selected log files will be temporarily stored in %s" % self.logdir)
res = self.finalize()
self.workflow_commons["Request"] = self.request
return res
#############################################################################
def finalize(self):
""" finalize method performs final operations after all the job
steps were executed. Only production jobs are treated.
"""
self.log.verbose("Starting UploadLogFile finalize")
##########################################
# First determine the files which should be saved
self.log.info("Determining the files to be saved in the logs.")
res = self.determineRelevantFiles()
if not res["OK"]:
self.log.error("Completely failed to select relevant log files.", res["Message"])
return S_OK() # because if the logs are lost, it's not the end of the world.
selectedFiles = res["Value"]
self.log.info(
"The following %s files were selected to be saved:\n%s"
% (len(selectedFiles), string.join(selectedFiles, "\n"))
)
#########################################
# Create a temporary directory containing these files
self.log.info("Populating a temporary directory for selected files.")
res = self.populateLogDirectory(selectedFiles)
if not res["OK"]:
self.log.error("Completely failed to populate temporary log file directory.", res["Message"])
self.setApplicationStatus("Failed To Populate Log Dir")
return S_OK() # because if the logs are lost, it's not the end of the world.
self.log.info("%s populated with log files." % self.logdir)
#########################################
# Create a tailored index page
# self.log.info('Creating an index page for the logs')
# result = self.__createLogIndex(selectedFiles)
# if not result['OK']:
# self.log.error('Failed to create index page for logs', res['Message'])
if not self.enable:
self.log.info("Module is disabled by control flag")
return S_OK("Module is disabled by control flag")
#########################################
# Make sure all the files in the log directory have the correct permissions
result = self.__setLogFilePermissions(self.logdir)
if not result["OK"]:
self.log.error("Could not set permissions of log files to 0755 with message:\n%s" % (result["Message"]))
#########################################
# Attempt to uplaod logs to the LogSE
self.log.info("Transferring log files to the %s" % self.logSE)
res = S_ERROR()
if not self.failoverTest:
self.log.info("PutDirectory %s %s %s" % (self.logFilePath, os.path.realpath(self.logdir), self.logSE))
res = self.rm.putStorageDirectory(
{self.logFilePath: os.path.realpath(self.logdir)}, self.logSE, singleDirectory=True
)
self.log.verbose(res)
if res["OK"]:
self.log.info("Successfully upload log directory to %s" % self.logSE)
# TODO: The logURL should be constructed using the LogSE and StorageElement()
# storageElement = StorageElement(self.logSE)
# pfn = storageElement.getPfnForLfn(self.logFilePath)['Value']
# logURL = getPfnForProtocol(res['Value'],'http')['Value']
logURL = "%s" % self.logFilePath
self.setJobParameter("Log LFN", logURL)
self.log.info("Logs for this job may be retrieved with dirac-ilc-get-prod-log -F %s" % logURL)
return S_OK()
#########################################
# Recover the logs to a failover storage element
self.log.error(
"Completely failed to upload log files to %s, will attempt upload to failover SE" % self.logSE,
res["Message"],
)
tarFileDir = os.path.dirname(self.logdir)
self.logLFNPath = "%s.gz" % self.logLFNPath
tarFileName = os.path.basename(self.logLFNPath)
start = os.getcwd()
os.chdir(self.logdir)
logTarFiles = os.listdir(self.logdir)
# comm = 'tar czvf %s %s' % (tarFileName,string.join(logTarFiles,' '))
tfile = tarfile.open(tarFileName, "w:gz")
for item in logTarFiles:
tfile.add(item)
tfile.close()
# res = shellCall(0,comm)
if not os.path.exists(tarFileName):
res = S_ERROR("File was not created")
os.chdir(start)
if not res["OK"]:
self.log.error("Failed to create tar file from directory", "%s %s" % (self.logdir, res["Message"]))
self.setApplicationStatus("Failed To Create Log Tar Dir")
return S_OK() # because if the logs are lost, it's not the end of the world.
# if res['Value'][0]: #i.e. non-zero status
# self.log.error('Failed to create tar file from directory','%s %s' % (self.logdir,res['Value']))
# self.setApplicationStatus('Failed To Create Log Tar Dir')
# return S_OK()#because if the logs are lost, it's not the end of the world.
############################################################
# Instantiate the failover transfer client with the global request object
failoverTransfer = FailoverTransfer(self.request)
##determine the experiment
self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs)
random.shuffle(self.failoverSEs)
self.log.info(
"Attempting to store file %s to the following SE(s):\n%s"
% (tarFileName, string.join(self.failoverSEs, ", "))
)
result = failoverTransfer.transferAndRegisterFile(
tarFileName,
"%s/%s" % (tarFileDir, tarFileName),
self.logLFNPath,
self.failoverSEs,
fileGUID=None,
fileCatalog=["FileCatalog", "LcgFileCatalog"],
)
if not result["OK"]:
self.log.error("Failed to upload logs to all destinations")
self.setApplicationStatus("Failed To Upload Logs")
return S_OK() # because if the logs are lost, it's not the end of the world.
# Now after all operations, retrieve potentially modified request object
result = failoverTransfer.getRequestObject()
if not result["OK"]:
self.log.error(result)
return S_ERROR("Could not retrieve modified request")
self.request = result["Value"]
res = self.createLogUploadRequest(self.logSE, self.logLFNPath)
if not res["OK"]:
self.log.error("Failed to create failover request", res["Message"])
self.setApplicationStatus("Failed To Upload Logs To Failover")
else:
self.log.info("Successfully created failover request")
self.workflow_commons["Request"] = self.request
return S_OK()
#############################################################################
def determineRelevantFiles(self):
""" The files which are below a configurable size will be stored in the logs.
This will typically pick up everything in the working directory minus the output data files.
"""
logFileExtensions = ["*.txt", "*.log", "*.out", "*.output", "*.xml", "*.sh", "*.info", "*.err", "*.root"]
self.logExtensions = self.ops.getValue("/LogFiles/%s/Extensions" % self.experiment, [])
if self.logExtensions:
self.log.info("Using list of log extensions from CS:\n%s" % (", ".join(self.logExtensions)))
logFileExtensions = self.logExtensions
else:
self.log.info("Using default list of log extensions:\n%s" % (", ".join(logFileExtensions)))
candidateFiles = []
for ext in logFileExtensions:
self.log.debug("Looking at log file wildcard: %s" % ext)
globList = glob.glob(ext)
for check in globList:
if os.path.isfile(check):
self.log.debug("Found locally existing log file: %s" % check)
candidateFiles.append(check)
selectedFiles = []
try:
for candidate in candidateFiles:
fileSize = os.stat(candidate)[6]
if fileSize < self.logSizeLimit:
selectedFiles.append(candidate)
else:
self.log.error(
"Log file found to be greater than maximum of %s bytes" % self.logSizeLimit, candidate
)
return S_OK(selectedFiles)
except Exception, x:
self.log.exception("Exception while determining files to save.", "", str(x))
return S_ERROR("Could not determine log files")
class UploadOutputData(ModuleBase):
""" As name suggest: upload output data. For Production only: See L{UserJobFinalization} for User job upload.
"""
#############################################################################
def __init__(self):
"""Module initialization.
"""
super(UploadOutputData, self).__init__()
self.version = __RCSID__
self.log = gLogger.getSubLogger( "UploadOutputData" )
self.commandTimeOut = 10*60
self.enable = True
self.failoverTest = False #flag to put file to failover SE by default
self.failoverSEs = gConfig.getValue('/Resources/StorageElementGroups/Tier1-Failover', [])
self.ops = Operations()
#List all parameters here
self.outputDataFileMask = ''
self.outputMode = 'Any' #or 'Local' for reco case
self.outputList = []
self.request = None
self.PRODUCTION_ID = ""
self.prodOutputLFNs = []
self.experiment = "CLIC"
#############################################################################
def applicationSpecificInputs(self):
""" By convention the module parameters are resolved here.
"""
self.log.verbose("Workflow commons:")
self.log.verbose(self.workflow_commons)
self.log.verbose("Step commons:")
self.log.verbose(self.step_commons)
if self.step_commons.has_key('Enable'):
self.enable = self.step_commons['Enable']
if not type(self.enable) == type(True):
self.log.warn('Enable flag set to non-boolean value %s, setting to False' % self.enable)
self.enable = False
if self.step_commons.has_key('TestFailover'):
self.enable = self.step_commons['TestFailover']
if not type(self.failoverTest) == type(True):
self.log.warn('Test failover flag set to non-boolean value %s, setting to False' % self.failoverTest)
self.failoverTest = False
if self.workflow_commons.has_key("PRODUCTION_ID"):
self.PRODUCTION_ID = self.workflow_commons["PRODUCTION_ID"]
if os.environ.has_key('JOBID'):
self.log.verbose('Found WMS JobID = %s' % self.jobID)
else:
self.log.info('No WMS JobID found, disabling module via control flag')
self.enable = False
if self.workflow_commons.has_key('Request'):
self.request = self.workflow_commons['Request']
else:
self.request = RequestContainer()
self.request.setRequestName('job_%s_request.xml' % self.jobID)
self.request.setJobID(self.jobID)
self.request.setSourceComponent("Job_%s" % self.jobID)
##This is the thing that is used to establish the list of outpufiles to treat:
## Make sure that all that is in the : "listoutput" and also in the ProductionData
## is treated properly. Needed as whatever is in listoutput does not contain any reference to the
## prodID and task ID. Also if for some reason a step failed, then the corresponding data will not be there
if self.workflow_commons.has_key('outputList'):
self.outputList = self.workflow_commons['outputList']
if self.workflow_commons.has_key('ProductionOutputData'):
proddata = self.workflow_commons['ProductionOutputData'].split(";")
self.log.verbose("prod data : %s" % proddata )
olist = {}
for obj in self.outputList:
fname_in_outputlist = obj['outputFile'].lower()
extension = ''
if fname_in_outputlist.count("_sim") or fname_in_outputlist.count("_rec") or fname_in_outputlist.count("_dst"):
extension = ".slcio"
elif fname_in_outputlist.count("_gen"):
extension = ".stdhep"
fname_in_outputlist = fname_in_outputlist.replace(extension,"")
for prodfile in proddata:
prodfile = os.path.basename(prodfile)
extension = ''
if prodfile.count("_sim") or prodfile.count("_rec") or prodfile.count("_dst"):
extension = ".slcio"
elif prodfile.count("_gen"):
extension = ".stdhep"
prodfile = prodfile.replace(extension,"")
if olist.has_key(prodfile):
## This has already been treated, no need to come back to it.
continue
appdict = {}
if (fname_in_outputlist.count("_gen")):# and prodfile.lower().count("_gen_")) :
genf = obj['outputFile'].split("_gen")[0]
genf += "_gen"
if (prodfile.count(genf)):
appdict.update(obj)
appdict['outputFile'] = prodfile+extension
olist[prodfile] = appdict
if (fname_in_outputlist.count("_sim")):
simf = obj['outputFile'].split("_sim")[0]
simf += "_sim"
if (prodfile.count(simf)):
appdict.update(obj)
appdict['outputFile'] = prodfile+extension
olist[prodfile] = appdict
self.log.verbose('olist %s'%olist)
if (fname_in_outputlist.count("_rec")):
recf = obj['outputFile'].split("_rec")[0]
recf += "_rec"
if (prodfile.count(recf)):
appdict.update(obj)
appdict['outputFile'] = prodfile+extension
olist[prodfile] = appdict
break
if (fname_in_outputlist.count("_dst") and prodfile.lower().count("_dst_")):
dstf = obj['outputFile'].split("_dst")[0]
dstf += "_dst"
if (prodfile.count(dstf)):
appdict.update(obj)
appdict['outputFile'] = prodfile+extension
olist[prodfile] = appdict
break
self.outputList = olist.values()
else:
olist = []
for obj in self.outputList:
appdict = obj
appdict['outputFile'] = getProdFilename(obj['outputFile'],
int(self.workflow_commons["PRODUCTION_ID"]),
int(self.workflow_commons["JOB_ID"]))
olist.append(appdict)
self.outputList = olist
self.log.verbose("OutputList : %s" % self.outputList)
if self.workflow_commons.has_key('outputMode'):
self.outputMode = self.workflow_commons['outputMode']
if self.workflow_commons.has_key('outputDataFileMask'):
self.outputDataFileMask = self.workflow_commons['outputDataFileMask']
if not type(self.outputDataFileMask) == type([]):
self.outputDataFileMask = [i.lower().strip() for i in self.outputDataFileMask.split(';')]
#result = constructProductionLFNs(self.workflow_commons)
#if not result['OK']:
# self.log.error('Could not create production LFNs',result['Message'])
# return result
#self.prodOutputLFNs=result['Value']['ProductionOutputData']
if self.workflow_commons.has_key('ProductionOutputData'):
self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";")
else:
self.prodOutputLFNs = []
return S_OK('Parameters resolved')
#############################################################################
def execute(self):
""" Main execution function.
"""
self.log.info('Initializing %s' % self.version)
result = self.resolveInputVariables()
if not result['OK']:
self.log.error(result['Message'])
return result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('No output data upload attempted')
##determine the experiment
example_file = self.prodOutputLFNs[0]
if "/ilc/prod/clic" in example_file:
self.experiment = "CLIC"
elif "/ilc/prod/ilc/sid" in example_file:
self.experiment = 'ILC_SID'
elif "/ilc/prod/ilc/mc-dbd" in example_file:
self.experiment = 'ILC_ILD'
else:
self.log.warn("Failed to determine experiment, reverting to default")
#Determine the final list of possible output files for the
#workflow and all the parameters needed to upload them.
result = self.getCandidateFiles(self.outputList, self.prodOutputLFNs, self.outputDataFileMask)
if not result['OK']:
self.setApplicationStatus(result['Message'])
return result
fileDict = result['Value']
result = self.getFileMetadata(fileDict)
if not result['OK']:
self.setApplicationStatus(result['Message'])
return result
if not result['Value']:
self.log.info('No output data files were determined to be uploaded for this workflow')
return S_OK()
fileMetadata = result['Value']
#Get final, resolved SE list for files
final = {}
for fileName, metadata in fileMetadata.items():
result = getDestinationSEList(metadata['workflowSE'], DIRAC.siteName(), self.outputMode)
if not result['OK']:
self.log.error('Could not resolve output data SE', result['Message'])
self.setApplicationStatus('Failed To Resolve OutputSE')
return result
resolvedSE = result['Value']
final[fileName] = metadata
final[fileName]['resolvedSE'] = resolvedSE
self.log.info('The following files will be uploaded: %s' % (string.join(final.keys(), ', ')))
for fileName, metadata in final.items():
self.log.info('--------%s--------' % fileName)
for n, v in metadata.items():
self.log.info('%s = %s' % (n, v))
#At this point can exit and see exactly what the module would have uploaded
if not self.enable:
self.log.info('Module is disabled by control flag, would have attempted to upload the \
following files %s' % string.join(final.keys(), ', '))
return S_OK('Module is disabled by control flag')
#Disable the watchdog check in case the file uploading takes a long time
self.log.info('Creating DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK in order to disable the Watchdog prior to upload')
fopen = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK','w')
fopen.write('%s' % time.asctime())
fopen.close()
#Instantiate the failover transfer client with the global request object
failoverTransfer = FailoverTransfer(self.request)
catalogs = ['FileCatalog', 'LcgFileCatalog']
#One by one upload the files with failover if necessary
failover = {}
if not self.failoverTest:
for fileName, metadata in final.items():
self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName,
string.join(metadata['resolvedSE'],
', ')))
result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'],
metadata['lfn'], metadata['resolvedSE'],
fileGUID = metadata['guid'], fileCatalog = catalogs)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
failover[fileName] = metadata
else:
lfn = metadata['lfn']
else:
failover = final
self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs)
cleanUp = False
for fileName, metadata in failover.items():
self.log.info('Setting default catalog for failover transfer to FileCatalog')
random.shuffle(self.failoverSEs)
targetSE = metadata['resolvedSE'][0]
metadata['resolvedSE'] = self.failoverSEs
result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'],
metadata['lfn'], targetSE, metadata['resolvedSE'],
fileGUID = metadata['guid'], fileCatalog = catalogs)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
cleanUp = True
break #no point continuing if one completely fails
os.remove("DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK") #cleanup the mess
#Now after all operations, retrieve potentially modified request object
result = failoverTransfer.getRequestObject()
if not result['OK']:
self.log.error(result)
return S_ERROR('Could not retrieve modified request')
self.request = result['Value']
#If some or all of the files failed to be saved to failover
if cleanUp:
lfns = []
for fileName, metadata in final.items():
lfns.append(metadata['lfn'])
result = self.__cleanUp(lfns)
self.workflow_commons['Request'] = self.request
return S_ERROR('Failed to upload output data')
# #Can now register the successfully uploaded files in the BK
# if not performBKRegistration:
# self.log.info('There are no files to perform the BK registration for, all could be saved to failover')
# else:
# rm = ReplicaManager()
# result = rm.addCatalogFile(performBKRegistration,catalogs=['BookkeepingDB'])
# self.log.verbose(result)
# if not result['OK']:
# self.log.error(result)
# return S_ERROR('Could Not Perform BK Registration')
# if result['Value']['Failed']:
# for lfn,error in result['Value']['Failed'].items():
# self.log.info('BK registration for %s failed with message: "%s" setting failover request' %(lfn,error))
# result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0, 'Catalogue':'BookkeepingDB'}},'register')
# if not result['OK']:
# self.log.error('Could not set registerFile request:\n%s' %result)
# return S_ERROR('Could Not Set BK Registration Request')
# fileDict = {'LFN':lfn,'Status':'Waiting'}
# index = result['Value']
# self.request.setSubRequestFiles(index,'register',[fileDict])
self.workflow_commons['Request'] = self.request
return S_OK('Output data uploaded')
#############################################################################
def __cleanUp(self, lfnList):
""" Clean up uploaded data for the LFNs in the list
"""
# Clean up the current request
for req_type in ['transfer', 'register']:
for lfn in lfnList:
result = self.request.getNumSubRequests(req_type)
if result['OK']:
nreq = result['Value']
if nreq:
# Go through subrequests in reverse order in order not to spoil the numbering
ind_range = [0]
if nreq > 1:
ind_range = range(nreq-1, -1, -1)
for i in ind_range:
result = self.request.getSubRequestFiles(i, req_type)
if result['OK']:
fileList = result['Value']
if fileList[0]['LFN'] == lfn:
result = self.request.removeSubRequest(i, req_type)
# Set removal requests just in case
for lfn in lfnList:
result = self.request.addSubRequest({'Attributes': {'Operation' : 'removeFile', 'TargetSE' : '',
'ExecutionOrder' : 1}}, 'removal')
index = result['Value']
fileDict = {'LFN':lfn, 'PFN':'', 'Status':'Waiting'}
self.request.setSubRequestFiles(index, 'removal', [fileDict])
return S_OK()
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#
def getRequest( self, requestType = '' ):
""" Get a request of a given type
"""
# RG: What if requestType is not given?
# the first query will return nothing.
start = time.time()
dmRequest = RequestContainer( init = False )
requestID = 0
req = "SELECT RequestID,SubRequestID FROM SubRequests WHERE Status = 'Waiting' AND RequestType = '%s' ORDER BY LastUpdate ASC LIMIT 50;" % requestType
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve max RequestID'
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
if not res['Value']:
return S_OK()
reqIDList = [ x[0] for x in res['Value'] ]
random.shuffle( reqIDList )
count = 0
for reqID in reqIDList:
count += 1
if requestType:
req = "SELECT SubRequestID,Operation,Arguments,ExecutionOrder,SourceSE,TargetSE,Catalogue,CreationTime,SubmissionTime,LastUpdate \
from SubRequests WHERE RequestID=%s AND RequestType='%s' AND Status='%s'" % ( reqID, requestType, 'Waiting' )
else:
# RG: What if requestType is not given?
# we should never get there, and it misses the "AND Status='Waiting'"
req = "SELECT SubRequestID,Operation,Arguments,ExecutionOrder,SourceSE,TargetSE,Catalogue,CreationTime,SubmissionTime,LastUpdate \
from SubRequests WHERE RequestID=%s" % reqID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve SubRequests for RequestID %s' % reqID
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
subIDList = []
for tuple in res['Value']:
subID = tuple[0]
# RG: We should set the condition "AND Status='Waiting'"
# if the subrequest has got assigned it will failed
req = "UPDATE SubRequests SET Status='Assigned' WHERE RequestID=%s AND SubRequestID=%s;" % ( reqID, subID )
resAssigned = self._update( req )
if not resAssigned['OK']:
if subIDList:
self.__releaseSubRequests( reqID, subIDList )
return S_ERROR( 'Failed to assign subrequests: %s' % resAssigned['Message'] )
if resAssigned['Value'] == 0:
# Somebody has assigned this request
gLogger.warn( 'Already assigned subrequest %d of request %d' % ( subID, reqID ) )
else:
subIDList.append( subID )
# RG: We need to check that all subRequest with smaller ExecutionOrder are "Done"
if subIDList:
# We managed to get some requests, can continue now
requestID = reqID
break
# Haven't succeeded to get any request
if not requestID:
return S_OK()
dmRequest.setRequestID( requestID )
# RG: We have this list in subIDList, can different queries get part of the subrequets of the same type?
subRequestIDs = []
for subRequestID, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in res['Value']:
if not subRequestID in subIDList: continue
subRequestIDs.append( subRequestID )
# RG: res['Value'] is the range of the loop and it gets redefined here !!!!!!
res = dmRequest.initiateSubRequest( requestType )
ind = res['Value']
subRequestDict = {
'Status' : 'Waiting',
'SubRequestID' : subRequestID,
'Operation' : operation,
'Arguments' : arguments,
'ExecutionOrder': int( executionOrder ),
'SourceSE' : sourceSE,
'TargetSE' : targetSE,
'Catalogue' : catalogue,
'CreationTime' : creationTime,
'SubmissionTime': submissionTime,
'LastUpdate' : lastUpdate
}
res = dmRequest.setSubRequestAttributes( ind, requestType, subRequestDict )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set subRequest attributes for RequestID %s' % requestID
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT FileID,LFN,Size,PFN,GUID,Md5,Addler,Attempt,Status \
from Files WHERE SubRequestID = %s ORDER BY FileID;" % subRequestID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get File attributes for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res['Value']:
fileDict = {'FileID':fileID, 'LFN':lfn, 'Size':size, 'PFN':pfn, 'GUID':guid, 'Md5':md5, 'Addler':addler, 'Attempt':attempt, 'Status':status}
files.append( fileDict )
res = dmRequest.setSubRequestFiles( ind, requestType, files )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set files into Request for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT Dataset,Status FROM Datasets WHERE SubRequestID = %s;" % subRequestID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get Datasets for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
datasets = []
for dataset, status in res['Value']:
datasets.append( dataset )
res = dmRequest.setSubRequestDatasets( ind, requestType, datasets )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set datasets into Request for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT RequestName,JobID,OwnerDN,OwnerGroup,DIRACSetup,SourceComponent,CreationTime,SubmissionTime,LastUpdate from Requests WHERE RequestID = %s;" % requestID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve max RequestID'
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate = res['Value'][0]
dmRequest.setRequestName( requestName )
dmRequest.setJobID( jobID )
dmRequest.setOwnerDN( ownerDN )
dmRequest.setOwnerGroup( ownerGroup )
dmRequest.setDIRACSetup( diracSetup )
dmRequest.setSourceComponent( sourceComponent )
dmRequest.setCreationTime( str( creationTime ) )
dmRequest.setLastUpdate( str( lastUpdate ) )
res = dmRequest.toXML()
if not res['OK']:
err = 'RequestDB._getRequest: Failed to create XML for RequestID %s' % ( requestID )
self.__releaseSubRequests( requestID, subRequestIDs )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
requestString = res['Value']
#still have to manage the status of the dataset properly
resultDict = {}
resultDict['RequestName'] = requestName
resultDict['RequestString'] = requestString
resultDict['JobID'] = jobID
return S_OK( resultDict )
def getRequest( self, requestType ):
""" Get a request of a given type eligible for execution
"""
# RG: What if requestType is not given?
# the first query will return nothing.
# KC: maybe returning S_ERROR would be enough?
# alternatively we should check if requestType is known (in 'transfer', 'removal', 'register' and 'diset')
if not requestType or type( requestType ) not in types.StringTypes:
return S_ERROR( "Request type not given." )
myRequestType = self._escapeString( requestType )
if not myRequestType:
return myRequestType
myRequestType = myRequestType['Value']
start = time.time()
dmRequest = RequestContainer( init = False )
requestID = 0
subIDList = []
fields = ['RequestID', 'SubRequestID', 'Operation', 'Arguments',
'ExecutionOrder', 'SourceSE', 'TargetSE', 'Catalogue',
'CreationTime', 'SubmissionTime', 'LastUpdate', 'Status', 'RequestType']
# get the pending SubRequest sorted by ExecutionOrder and LastUpdate
req = "SELECT `RequestID`,`ExecutionOrder`,`Status`,`RequestType`,`LastUpdate` FROM `SubRequests` "\
"WHERE `Status` IN ( 'Waiting', 'Assigned' ) ORDER BY `ExecutionOrder`,`LastUpdate`"
# now get sorted list of RequestID (according to the above)
req = "SELECT * FROM ( %s ) as T1 GROUP BY `RequestID`" % req
# and get the 100 oldest ones of Type requestType
req = "SELECT `RequestID`,`ExecutionOrder` FROM ( %s ) as T2 WHERE `RequestType`=%s "\
"ORDER BY `LastUpdate` LIMIT 100" % ( req, myRequestType )
# and now get all waiting SubRequest for the selected RequestID and ExecutionOrder
req = "SELECT A.%s FROM SubRequests AS A, ( %s ) AS B WHERE " % ( ', A.'.join( fields ), req )
req = "%s A.RequestID=B.RequestID AND A.ExecutionOrder=B.ExecutionOrder" % ( req )
result = self._query( req )
if not result['OK']:
err = 'RequestDB._getRequest: Failed to retrieve Requests'
return S_ERROR( '%s\n%s' % ( err, result['Message'] ) )
if not result['Value']:
return S_OK()
# We get up to 10 Request candidates, to add some randomness
reqDict = {}
for row in result['Value']:
if ('"%s"' % row[-1]) != myRequestType:
continue
if row[-2] != 'Waiting':
continue
reqDict.setdefault( row[0], [] )
reqDict[row[0]].append( row[1:-2] )
reqIDList = reqDict.keys()
random.shuffle( reqIDList )
for reqID in reqIDList:
sidList = [ x[0] for x in reqDict[reqID] ]
for subID in sidList:
req = "UPDATE SubRequests SET Status='Assigned' WHERE RequestID=%s AND SubRequestID=%s;" % ( reqID, subID )
resAssigned = self._update( req )
if not resAssigned['OK']:
if subIDList:
self.__releaseSubRequests( reqID, subIDList )
return S_ERROR( 'Failed to assign subrequests: %s' % resAssigned['Message'] )
if resAssigned['Value'] == 0:
# Somebody has assigned this request
gLogger.warn( 'Already assigned subrequest %d of request %d' % ( subID, reqID ) )
else:
subIDList.append( subID )
if subIDList:
# We managed to get some requests, can continue now
requestID = reqID
break
# Haven't succeeded to get any request
if not requestID:
return S_OK()
dmRequest.setRequestID( requestID )
fields = ['FileID', 'LFN', 'Size', 'PFN', 'GUID', 'Md5', 'Addler', 'Attempt', 'Status' ]
for subRequestID, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, \
creationTime, submissionTime, lastUpdate in reqDict[requestID]:
if not subRequestID in subIDList: continue
res = dmRequest.initiateSubRequest( requestType )
ind = res['Value']
subRequestDict = {
'Status' : 'Waiting',
'SubRequestID' : subRequestID,
'Operation' : operation,
'Arguments' : arguments,
'ExecutionOrder': int( executionOrder ),
'SourceSE' : sourceSE,
'TargetSE' : targetSE,
'Catalogue' : catalogue,
'CreationTime' : creationTime,
'SubmissionTime': submissionTime,
'LastUpdate' : lastUpdate
}
res = dmRequest.setSubRequestAttributes( ind, requestType, subRequestDict )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set subRequest attributes for RequestID %s' % requestID
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT %s FROM `Files` WHERE `SubRequestID`=%s ORDER BY `FileID`;" % ( ', '.join( fields ),
subRequestID )
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get File attributes for RequestID %s.%s' % ( requestID,
subRequestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res['Value']:
fileDict = {'FileID':fileID, 'LFN':lfn, 'Size':size, 'PFN':pfn, 'GUID':guid,
'Md5':md5, 'Addler':addler, 'Attempt':attempt, 'Status':status}
files.append( fileDict )
res = dmRequest.setSubRequestFiles( ind, requestType, files )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set files into Request for RequestID %s.%s' % ( requestID,
subRequestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
req = "SELECT Dataset,Status FROM Datasets WHERE SubRequestID = %s;" % subRequestID
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to get Datasets for RequestID %s.%s' % ( requestID, subRequestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
datasets = []
for dataset, status in res['Value']:
datasets.append( dataset )
res = dmRequest.setSubRequestDatasets( ind, requestType, datasets )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to set datasets into Request for RequestID %s.%s' % ( requestID,
subRequestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
fields = ['RequestName', 'JobID', 'OwnerDN', 'OwnerGroup',
'DIRACSetup', 'SourceComponent', 'CreationTime',
'SubmissionTime', 'LastUpdate']
req = "SELECT %s FROM `Requests` WHERE `RequestID`=%s;" % ( ', '.join( fields ), requestID )
res = self._query( req )
if not res['OK']:
err = 'RequestDB._getRequest: Failed to retrieve max RequestID'
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, \
creationTime, submissionTime, lastUpdate = res['Value'][0]
dmRequest.setRequestName( requestName )
dmRequest.setJobID( jobID )
dmRequest.setOwnerDN( ownerDN )
dmRequest.setOwnerGroup( ownerGroup )
dmRequest.setDIRACSetup( diracSetup )
dmRequest.setSourceComponent( sourceComponent )
dmRequest.setCreationTime( str( creationTime ) )
dmRequest.setLastUpdate( str( lastUpdate ) )
res = dmRequest.toXML()
if not res['OK']:
err = 'RequestDB._getRequest: Failed to create XML for RequestID %s' % ( requestID )
self.__releaseSubRequests( requestID, subIDList )
return S_ERROR( '%s\n%s' % ( err, res['Message'] ) )
requestString = res['Value']
#still have to manage the status of the dataset properly
resultDict = {}
resultDict['RequestName'] = requestName
resultDict['RequestString'] = requestString
resultDict['JobID'] = jobID
return S_OK( resultDict )
def getRequestForSubRequest(self, itself, subRequestID ):
"""
Select Request given SubRequestID.
:param self: plugin reference
:param itself: patient reference for injection
:param int subRequestID: SubRequests.SubRequestID
:warn: function has to be injected to RequestDBMySQL instance
"""
## get RequestID
requestID = "SELECT RequestID FROM SubRequests WHERE SubRequestID = %s;" % str(subRequestID)
requestID = self._query( requestID )
if not requestID["OK"]:
return requestID
requestID = requestID["Value"][0]
## create RequestContainer
requestContainer = RequestContainer( init = False )
requestContainer.setRequestID( requestID )
## put some basic infos in
requestInfo = "SELECT RequestName, JobID, OwnerDN, OwnerGroup, DIRACSetup, SourceComponent, CreationTime, SubmissionTime, LastUpdate, Status "
requestInfo += "FROM Requests WHERE RequestID = %d;" % requestID
requestInfo = self._query( requestInfo )
if not requestInfo["OK"]:
return requestInfo
requestName, jobID, ownerDN, ownerGroup, diracSetup, sourceComponent, creationTime, submissionTime, lastUpdate, status = requestInfo['Value'][0]
requestContainer.setRequestName( requestName )
requestContainer.setJobID( jobID )
requestContainer.setOwnerDN( ownerDN )
requestContainer.setOwnerGroup( ownerGroup )
requestContainer.setDIRACSetup( diracSetup )
requestContainer.setSourceComponent( sourceComponent )
requestContainer.setCreationTime( str( creationTime ) )
requestContainer.setLastUpdate( str( lastUpdate ) )
requestContainer.setStatus( status )
## get sub-requests
subRequests = "SELECT SubRequestID, Status, RequestType, Operation, Arguments, ExecutionOrder, SourceSE, "
subRequests += "TargetSE, Catalogue, CreationTime, SubmissionTime, LastUpdate FROM SubRequests WHERE RequestID=%s;" % requestID
subRequests = self._query( subRequests )
if not subRequests["OK"]:
return subRequests
## loop over sub requests
for subRequestID, status, requestType, operation, arguments, executionOrder, sourceSE, targetSE, catalogue, creationTime, submissionTime, lastUpdate in subRequests["Value"]:
res = requestContainer.initiateSubRequest( requestType )
ind = res["Value"]
subRequestDict = { "Status" : status, "SubRequestID" : subRequestID, "Operation" : operation, "Arguments" : arguments,
"ExecutionOrder" : int( executionOrder ), "SourceSE" : sourceSE, "TargetSE" : targetSE,
"Catalogue" : catalogue, "CreationTime" : creationTime, "SubmissionTime" : submissionTime,
"LastUpdate" : lastUpdate }
res = requestContainer.setSubRequestAttributes( ind, requestType, subRequestDict )
if not res["OK"]:
return res
## get files for this subrequest
req = "SELECT FileID, LFN, Size, PFN, GUID, Md5, Addler, Attempt, Status FROM Files WHERE SubRequestID = %s ORDER BY FileID;" % str(subRequestID)
res = self._query( req )
if not res["OK"]:
return res
files = []
for fileID, lfn, size, pfn, guid, md5, addler, attempt, status in res["Value"]:
fileDict = { "FileID" : fileID, "LFN" : lfn, "Size" : size,
"PFN" : pfn, "GUID" : guid, "Md5" : md5,
"Addler" : addler, "Attempt" : attempt,
"Status" : status }
files.append( fileDict )
res = requestContainer.setSubRequestFiles( ind, requestType, files )
if not res["OK"]:
return res
## dump request to XML
res = requestContainer.toXML()
if not res["OK"]:
return res
requestString = res["Value"]
## return dictonary with all info in at least
return S_OK( {
"RequestName" : requestName,
"RequestString" : requestString,
"JobID" : jobID,
"RequestContainer" : requestContainer
} )
