def test_getdestinationse_no_assoc_alias(self):
options_dict = {
'/Resources/StorageElements/myOutputSE': S_ERROR('bla')
}
option_dict = {
'/Resources/Countries/country/AssignedTo':
S_OK('MyTestCountry.Assigned'),
'/Resources/Countries/MyTestCountry.Assigned/AssociatedSEs':
S_OK('something')
}
values_dict = {
'/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE':
[],
'/Resources/StorageElementGroups/myOutputSE':
['some_elements', 'ignore_me'],
'/Resources/Countries/MyTestCountry.Assigned/AssociatedSEs/myOutputSE':
[],
'/Resources/Countries/country/AssociatedSEs/myOutputSE':
''
}
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getOption.side_effect = lambda path: option_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \
patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList('myOutputSE',
'pre.myTestSite.country', 'Local')
assertDiracFailsWith(result, 'failed to resolve se', self)
def test_getdestinationse_alias_found( self ):
options_dict = { '/Resources/StorageElements/myOutputSE' : S_ERROR( 'bla' ) }
values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE' :
'myTest_Alias_value' }
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
with patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList( 'myOutputSE', 'pre.myTestSite.country', 'AnyOutputmodeTest' )
assertDiracSucceedsWith( result, 'myTest_Alias_value', self )
def test_getdestinationse( self ):
options_dict = { '/Resources/StorageElements/myOutputSE' : S_ERROR( 'bla' ) }
values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE' : [],
'/Resources/StorageElementGroups/myOutputSE' : [ 'some_grp_member', 'GroupElement', 'StorEl', 'LocalElement' ] }
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'StorEl', 'LocalElement' ]))), \
patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList( 'myOutputSE', 'pre.myTestSite.country', 'AnyOutputmodeTest' )
assertDiracSucceedsWith_equals( result, [ 'StorEl', 'LocalElement', 'some_grp_member', 'GroupElement' ], self )
def test_getdestinationse_associatedse( self ):
options_dict = { '/Resources/StorageElements/myOutputSE' : S_ERROR( 'bla' ) }
values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE' : [],
'/Resources/StorageElementGroups/myOutputSE' : [ 'some_elements', 'ignore_me' ],
'/Resources/Countries/country/AssociatedSEs/myOutputSE' : 'myassociated_se' }
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \
patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList( 'myOutputSE', 'pre.myTestSite.country', 'Local' )
assertDiracSucceedsWith_equals( result, [ 'myassociated_se' ], self )
def test_getdestinationse_nogroupse( self ):
options_dict = { '/Resources/StorageElements/myOutputSE' : S_ERROR( 'bla' ) }
values_dict = { '/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE' : [],
'/Resources/StorageElementGroups/myOutputSE' : [] }
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
#DIRAC.gConfig = conf_mock
with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \
patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList( 'myOutputSE', 'pre.myTestSite.country', 'AnyOutputmodeTest' )
assertDiracFailsWith( result, 'failed to resolve se myoutputse', self )
def test_getdestinationse_no_assoc_list( self ):
options_dict = { '/Resources/StorageElements/myOutputSE' : S_ERROR( 'bla' ) }
option_dict = { '/Resources/Countries//AssignedTo' : S_OK( '' ) }
values_dict = { '/Resources/Sites/pre/pre.myTestSite./AssociatedSEs/myOutputSE' : [],
'/Resources/StorageElementGroups/myOutputSE' : [ 'some_elements', 'ignore_me' ],
'/Resources/Countries/MyTestCountry.Assigned/AssociatedSEs/myOutputSE' :
[ 'myassociated_alias_se' ],
'/Resources/Countries//AssociatedSEs/myOutputSE' : '' }
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getOption.side_effect = lambda path: option_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \
patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList( 'myOutputSE', 'pre.myTestSite.', 'Local' )
assertDiracFailsWith( result, 'could not determine associated se list', self )
def test_getdestinationse_alias_found(self):
options_dict = {
'/Resources/StorageElements/myOutputSE': S_ERROR('bla')
}
values_dict = {
'/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE':
'myTest_Alias_value'
}
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
with patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList('myOutputSE',
'pre.myTestSite.country',
'AnyOutputmodeTest')
assertDiracSucceedsWith(result, 'myTest_Alias_value', self)
def test_getdestinationse_localgroupse(self):
options_dict = {
'/Resources/StorageElements/myOutputSE': S_ERROR('bla')
}
values_dict = {
'/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE':
[],
'/Resources/StorageElementGroups/myOutputSE':
['some_elements', 'ignore_me', 'otherStorageElem2']
}
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \
patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList('myOutputSE',
'pre.myTestSite.country', 'Local')
assertDiracSucceedsWith_equals(result, ['otherStorageElem2'], self)
def test_getdestinationse_nogroupse(self):
options_dict = {
'/Resources/StorageElements/myOutputSE': S_ERROR('bla')
}
values_dict = {
'/Resources/Sites/pre/pre.myTestSite.country/AssociatedSEs/myOutputSE':
[],
'/Resources/StorageElementGroups/myOutputSE': []
}
conf_mock = Mock()
conf_mock.getOptions.side_effect = lambda path: options_dict[path]
conf_mock.getValue.side_effect = lambda path, _: values_dict[path]
#DIRAC.gConfig = conf_mock
with patch('%s.getSEsForSite' % MODULE_NAME, new=Mock(return_value=S_OK([ 'myTestStorageElement1', 'otherStorageElem2' ]))), \
patch('%s.gConfig' % MODULE_NAME, conf_mock):
result = getDestinationSEList('myOutputSE',
'pre.myTestSite.country',
'AnyOutputmodeTest')
assertDiracFailsWith(result, 'failed to resolve se myoutputse',
self)
def execute(self):
""" Main execution function.
"""
#Have to work out if the module is part of the last step i.e.
#user jobs can have any number of steps and we only want
#to run the finalization once.
currentStep = int(self.step_commons['STEP_NUMBER'])
totalSteps = int(self.workflow_commons['TotalSteps'])
if currentStep == totalSteps:
self.lastStep = True
else:
self.log.verbose('Current step = %s, total steps of workflow = %s, UserJobFinalization will enable itself only \
at the last workflow step.' % (currentStep, totalSteps))
if not self.lastStep:
return S_OK()
result = self.resolveInputVariables()
if not result['OK']:
self.log.error(result['Message'])
return result
self.log.info('Initializing %s' % self.version)
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'],
self.stepStatus['OK']))
return S_OK('No output data upload attempted')
if not self.userOutputData:
self.log.info('No user output data is specified for this job, nothing to do')
return S_OK('No output data to upload')
#Determine the final list of possible output files for the
#workflow and all the parameters needed to upload them.
outputList = []
for i in self.userOutputData:
outputList.append({'outputPath' : string.upper(string.split(i, '.')[-1]),
'outputDataSE' : self.userOutputSE,
'outputFile' : os.path.basename(i)})
userOutputLFNs = []
if self.userOutputData:
self.log.info('Constructing user output LFN(s) for %s' % (string.join(self.userOutputData, ', ')))
if not self.jobID:
self.jobID = 12345
owner = ''
if self.workflow_commons.has_key('Owner'):
owner = self.workflow_commons['Owner']
else:
res = self.getCurrentOwner()
if not res['OK']:
return S_ERROR('Could not obtain owner from proxy')
owner = res['Value']
vo = ''
if self.workflow_commons.has_key('VO'):
vo = self.workflow_commons['VO']
else:
res = self.getCurrentVO()
if not res['OK']:
return S_ERROR('Could not obtain VO from proxy')
vo = res['Value']
result = constructUserLFNs(int(self.jobID), vo, owner, self.userOutputData, self.userOutputPath)
if not result['OK']:
self.log.error('Could not create user LFNs', result['Message'])
return result
userOutputLFNs = result['Value']
self.log.verbose('Calling getCandidateFiles( %s, %s, %s)' % (outputList, userOutputLFNs, self.outputDataFileMask))
result = self.getCandidateFiles(outputList, userOutputLFNs, self.outputDataFileMask)
if not result['OK']:
if not self.ignoreapperrors:
self.setApplicationStatus(result['Message'])
return S_OK()
fileDict = result['Value']
result = self.getFileMetadata(fileDict)
if not result['OK']:
if not self.ignoreapperrors:
self.setApplicationStatus(result['Message'])
return S_OK()
if not result['Value']:
if not self.ignoreapperrors:
self.log.info('No output data files were determined to be uploaded for this workflow')
self.setApplicationStatus('No Output Data Files To Upload')
return S_OK()
fileMetadata = result['Value']
#First get the local (or assigned) SE to try first for upload and others in random fashion
result = getDestinationSEList('Tier1-USER', DIRAC.siteName(), outputmode='local')
if not result['OK']:
self.log.error('Could not resolve output data SE', result['Message'])
self.setApplicationStatus('Failed To Resolve OutputSE')
return result
localSE = result['Value']
self.log.verbose('Site Local SE for user outputs is: %s' % (localSE))
orderedSEs = self.defaultOutputSE
for se in localSE:
if se in orderedSEs:
orderedSEs.remove(se)
for se in self.userOutputSE:
if se in orderedSEs:
orderedSEs.remove(se)
orderedSEs = localSE + List.randomize(orderedSEs)
if self.userOutputSE:
prependSEs = []
for userSE in self.userOutputSE:
if not userSE in orderedSEs:
prependSEs.append(userSE)
orderedSEs = prependSEs + orderedSEs
self.log.info('Ordered list of output SEs is: %s' % (string.join(orderedSEs, ', ')))
final = {}
for fileName, metadata in fileMetadata.items():
final[fileName] = metadata
final[fileName]['resolvedSE'] = orderedSEs
#At this point can exit and see exactly what the module will upload
if not self.enable:
self.log.info('Module is disabled by control flag, would have attempted \
to upload the following files %s' % string.join(final.keys(), ', '))
for fileName, metadata in final.items():
self.log.info('--------%s--------' % fileName)
for n, v in metadata.items():
self.log.info('%s = %s' %(n, v))
return S_OK('Module is disabled by control flag')
#Instantiate the failover transfer client with the global request object
failoverTransfer = FailoverTransfer(self.request)
#One by one upload the files with failover if necessary
replication = {}
failover = {}
uploaded = []
if not self.failoverTest:
for fileName, metadata in final.items():
self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName,
string.join(metadata['resolvedSE'],
', ')))
result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'], metadata['lfn'],
metadata['resolvedSE'], fileGUID = metadata['guid'],
fileCatalog = self.userFileCatalog)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
failover[fileName] = metadata
else:
#Only attempt replication after successful upload
lfn = metadata['lfn']
uploaded.append(lfn)
seList = metadata['resolvedSE']
replicateSE = ''
if result['Value'].has_key('uploadedSE'):
uploadedSE = result['Value']['uploadedSE']
for se in seList:
if not se == uploadedSE:
replicateSE = se
break
if replicateSE and lfn:
self.log.info('Will attempt to replicate %s to %s' % (lfn, replicateSE))
replication[lfn] = replicateSE
else:
failover = final
cleanUp = False
for fileName, metadata in failover.items():
random.shuffle(self.failoverSEs)
targetSE = metadata['resolvedSE'][0]
metadata['resolvedSE'] = self.failoverSEs
result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'], metadata['lfn'],
targetSE, metadata['resolvedSE'],
fileGUID = metadata['guid'],
fileCatalog = self.userFileCatalog)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
cleanUp = True
continue #for users can continue even if one completely fails
else:
lfn = metadata['lfn']
uploaded.append(lfn)
#For files correctly uploaded must report LFNs to job parameters
if uploaded:
report = string.join( uploaded, ', ' )
self.jobReport.setJobParameter( 'UploadedOutputData', report )
#Now after all operations, retrieve potentially modified request object
result = failoverTransfer.getRequestObject()
if not result['OK']:
self.log.error(result)
return S_ERROR('Could Not Retrieve Modified Request')
self.request = result['Value']
#If some or all of the files failed to be saved to failover
if cleanUp:
self.workflow_commons['Request'] = self.request
#Leave any uploaded files just in case it is useful for the user
#do not try to replicate any files.
return S_ERROR('Failed To Upload Output Data')
#If there is now at least one replica for uploaded files can trigger replication
rm = ReplicaManager()
self.log.info('Sleeping for 10 seconds before attempting replication of recently uploaded files')
time.sleep(10)
for lfn, repSE in replication.items():
result = rm.replicateAndRegister(lfn, repSE, catalog = self.userFileCatalog)
if not result['OK']:
self.log.info('Replication failed with below error but file already exists in Grid storage with \
at least one replica:\n%s' % (result))
self.workflow_commons['Request'] = self.request
#Now must ensure if any pending requests are generated that these are propagated to the job wrapper
reportRequest = None
if self.jobReport:
result = self.jobReport.generateRequest()
if not result['OK']:
self.log.warn('Could not generate request for job report with result:\n%s' % (result))
else:
reportRequest = result['Value']
if reportRequest:
self.log.info('Populating request with job report information')
self.request.update(reportRequest)
if not self.request.isEmpty()['Value']:
request_string = self.request.toXML()['Value']
# Write out the request string
fname = 'user_job_%s_request.xml' % (self.jobID)
xmlfile = open(fname, 'w')
xmlfile.write(request_string)
xmlfile.close()
self.log.info('Creating failover request for deferred operations for job %s:' % self.jobID)
result = self.request.getDigest()
if result['OK']:
digest = result['Value']
self.log.info(digest)
self.setApplicationStatus('Job Finished Successfully')
return S_OK('Output data uploaded')
def execute(self):
""" Main execution function.
"""
#Have to work out if the module is part of the last step i.e.
#user jobs can have any number of steps and we only want
#to run the finalization once. Not a problem if this is not the last step so return S_OK()
resultLS = self.isLastStep()
if not resultLS['OK']:
return S_OK()
self.logWorkingDirectory()
resultIV = self.resolveInputVariables()
if not resultIV['OK']:
self.log.error("Failed to resolve input parameters:", resultIV['Message'])
return resultIV
self.log.info('Initializing %s' % self.version)
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'],
self.stepStatus['OK']))
return S_OK('No output data upload attempted')
if not self.userOutputData:
self.log.info('No user output data is specified for this job, nothing to do')
return S_OK('No output data to upload')
#Determine the final list of possible output files for the
#workflow and all the parameters needed to upload them.
outputList = self.getOutputList()
userOutputLFNs = []
if self.userOutputData:
resultOLfn = self.constructOutputLFNs()
if not resultOLfn['OK']:
self.log.error('Could not create user LFNs', resultOLfn['Message'])
return resultOLfn
userOutputLFNs = resultOLfn['Value']
self.log.verbose('Calling getCandidateFiles( %s, %s, %s)' % (outputList, userOutputLFNs, self.outputDataFileMask))
self.log.debug("IgnoreAppErrors? '%s' " % self.ignoreapperrors)
resultCF = self.getCandidateFiles(outputList, userOutputLFNs, self.outputDataFileMask)
if not resultCF['OK']:
if not self.ignoreapperrors:
self.log.error(resultCF['Message'])
self.setApplicationStatus(resultCF['Message'])
return S_OK()
fileDict = resultCF['Value']
resultFMD = self.getFileMetadata(fileDict)
if not resultFMD['OK']:
if not self.ignoreapperrors:
self.log.error(resultFMD['Message'])
self.setApplicationStatus(resultFMD['Message'])
return S_OK()
if not resultFMD['Value']:
if not self.ignoreapperrors:
self.log.info('No output data files were determined to be uploaded for this workflow')
self.setApplicationStatus('No Output Data Files To Upload')
return S_OK()
fileMetadata = resultFMD['Value']
#First get the local (or assigned) SE to try first for upload and others in random fashion
resultSEL = getDestinationSEList('Tier1-USER', DIRAC.siteName(), outputmode='local')
if not resultSEL['OK']:
self.log.error('Could not resolve output data SE', resultSEL['Message'])
self.setApplicationStatus('Failed To Resolve OutputSE')
return resultSEL
localSE = resultSEL['Value']
orderedSEs = [ se for se in self.defaultOutputSE if se not in localSE and se not in self.userOutputSE]
orderedSEs = localSE + List.randomize(orderedSEs)
if self.userOutputSE:
prependSEs = []
for userSE in self.userOutputSE:
if not userSE in orderedSEs:
prependSEs.append(userSE)
orderedSEs = prependSEs + orderedSEs
self.log.info('Ordered list of output SEs is: %s' % (', '.join(orderedSEs)))
final = {}
for fileName, metadata in fileMetadata.iteritems():
final[fileName] = metadata
final[fileName]['resolvedSE'] = orderedSEs
#At this point can exit and see exactly what the module will upload
self.printOutputInfo(final)
if not self.enable:
return S_OK('Module is disabled by control flag')
#Instantiate the failover transfer client with the global request object
failoverTransfer = FailoverTransfer(self._getRequestContainer())
#One by one upload the files with failover if necessary
filesToReplicate = {}
filesToFailover = {}
filesUploaded = []
if not self.failoverTest:
self.transferAndRegisterFiles(final, failoverTransfer, filesToFailover, filesUploaded, filesToReplicate)
else:
filesToFailover = final
##if there are files to be failovered, we do it now
resultTRFF = self.transferRegisterAndFailoverFiles(failoverTransfer, filesToFailover, filesUploaded)
cleanUp = resultTRFF['Value']['cleanUp']
#For files correctly uploaded must report LFNs to job parameters
if filesUploaded:
report = ', '.join( filesUploaded )
self.jobReport.setJobParameter( 'UploadedOutputData', report )
self.workflow_commons['Request'] = failoverTransfer.request
#If some or all of the files failed to be saved to failover
if cleanUp:
#Leave any uploaded files just in case it is useful for the user
#do not try to replicate any files.
return S_ERROR('Failed To Upload Output Data')
#If there is now at least one replica for uploaded files can trigger replication
datMan = DataManager( catalogs = self.userFileCatalog )
self.log.info('Sleeping for 10 seconds before attempting replication of recently uploaded files')
time.sleep(10)
for lfn, repSE in filesToReplicate.items():
resultRAR = datMan.replicateAndRegister(lfn, repSE)
if not resultRAR['OK']:
self.log.info('Replication failed with below error but file already exists in Grid storage with \
at least one replica:\n%s' % (resultRAR))
self.generateFailoverFile()
self.setApplicationStatus('Job Finished Successfully')
return S_OK('Output data uploaded')
def execute(self):
""" Main execution function.
"""
self.log.info('Initializing %s' % self.version)
result = self.resolveInputVariables()
if not result['OK']:
self.log.error("Failed to resolve input parameters:", result['Message'])
return result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('No output data upload attempted')
##determine the experiment
example_file = self.prodOutputLFNs[0]
if "/ilc/prod/clic" in example_file:
self.experiment = "CLIC"
elif "/ilc/prod/ilc/sid" in example_file:
self.experiment = 'ILC_SID'
elif "/ilc/prod/ilc/mc-dbd" in example_file:
self.experiment = 'ILC_ILD'
else:
self.log.warn("Failed to determine experiment, reverting to default")
#Determine the final list of possible output files for the
#workflow and all the parameters needed to upload them.
result = self.getCandidateFiles(self.outputList, self.prodOutputLFNs, self.outputDataFileMask)
if not result['OK']:
self.log.error(result['Message'])
self.setApplicationStatus(result['Message'])
return result
fileDict = result['Value']
result = self.getFileMetadata(fileDict)
if not result['OK']:
self.log.error(result['Message'])
self.setApplicationStatus(result['Message'])
return result
if not result['Value']:
self.log.info('No output data files were determined to be uploaded for this workflow')
return S_OK()
fileMetadata = result['Value']
#Get final, resolved SE list for files
final = {}
for fileName, metadata in fileMetadata.items():
result = getDestinationSEList(metadata['workflowSE'], DIRAC.siteName(), self.outputMode)
if not result['OK']:
self.log.error('Could not resolve output data SE', result['Message'])
self.setApplicationStatus('Failed To Resolve OutputSE')
return result
resolvedSE = result['Value']
final[fileName] = metadata
final[fileName]['resolvedSE'] = resolvedSE
self.log.info('The following files will be uploaded: %s' % (', '.join(final.keys() )))
for fileName, metadata in final.items():
self.log.info('--------%s--------' % fileName)
for metaName, metaValue in metadata.items():
self.log.info('%s = %s' % (metaName, metaValue))
#At this point can exit and see exactly what the module would have uploaded
if not self.enable:
self.log.info('Module is disabled by control flag, would have attempted to upload the \
following files %s' % ', '.join(final.keys()))
return S_OK('Module is disabled by control flag')
#Disable the watchdog check in case the file uploading takes a long time
self.log.info('Creating DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK in order to disable the Watchdog prior to upload')
fopen = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK','w')
fopen.write('%s' % time.asctime())
fopen.close()
#Instantiate the failover transfer client with the global request object
failoverTransfer = FailoverTransfer(self._getRequestContainer())
catalogs = self.ops.getValue('Production/%s/Catalogs' % self.experiment,
['FileCatalog', 'LcgFileCatalog'])
#One by one upload the files with failover if necessary
failover = {}
if not self.failoverTest:
for fileName, metadata in final.iteritems():
self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName,
', '.join(metadata['resolvedSE'])))
result = failoverTransfer.transferAndRegisterFile(fileName,
metadata['localpath'],
metadata['lfn'],
metadata['resolvedSE'],
fileMetaDict = metadata['filedict'],
fileCatalog = catalogs)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata['filedict']))
failover[fileName] = metadata
else:
#lfn = metadata['lfn']
pass
else:
failover = final
self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs)
cleanUp = False
for fileName, metadata in failover.iteritems():
self.log.info('Setting default catalog for failover transfer to FileCatalog')
failovers = self.failoverSEs
targetSE = metadata['resolvedSE'][0]
try:#remove duplicate site, otherwise it will do nasty things where processing the request
failovers.remove(targetSE)
except ValueError:
pass
random.shuffle(failovers)
metadata['resolvedSE'] = failovers
result = failoverTransfer.transferAndRegisterFileFailover(fileName,
metadata['localpath'],
metadata['lfn'],
targetSE,
metadata['resolvedSE'],
fileMetaDict = metadata['filedict'],
fileCatalog = catalogs)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata['filedict']))
cleanUp = True
break #no point continuing if one completely fails
os.remove("DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK") #cleanup the mess
self.workflow_commons['Request'] = failoverTransfer.request
#If some or all of the files failed to be saved to failover
if cleanUp:
lfns = []
for fileName, metadata in final.items():
lfns.append(metadata['lfn'])
result = self._cleanUp(lfns)
return S_ERROR('Failed to upload output data')
return S_OK('Output data uploaded')
def execute(self):
""" Main execution function.
"""
self.log.info('Initializing %s' % self.version)
result = self.resolveInputVariables()
if not result['OK']:
self.log.error(result['Message'])
return result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('No output data upload attempted')
##determine the experiment
example_file = self.prodOutputLFNs[0]
if "/ilc/prod/clic" in example_file:
self.experiment = "CLIC"
elif "/ilc/prod/ilc/sid" in example_file:
self.experiment = 'ILC_SID'
elif "/ilc/prod/ilc/mc-dbd" in example_file:
self.experiment = 'ILC_ILD'
else:
self.log.warn("Failed to determine experiment, reverting to default")
#Determine the final list of possible output files for the
#workflow and all the parameters needed to upload them.
result = self.getCandidateFiles(self.outputList, self.prodOutputLFNs, self.outputDataFileMask)
if not result['OK']:
self.setApplicationStatus(result['Message'])
return result
fileDict = result['Value']
result = self.getFileMetadata(fileDict)
if not result['OK']:
self.setApplicationStatus(result['Message'])
return result
if not result['Value']:
self.log.info('No output data files were determined to be uploaded for this workflow')
return S_OK()
fileMetadata = result['Value']
#Get final, resolved SE list for files
final = {}
for fileName, metadata in fileMetadata.items():
result = getDestinationSEList(metadata['workflowSE'], DIRAC.siteName(), self.outputMode)
if not result['OK']:
self.log.error('Could not resolve output data SE', result['Message'])
self.setApplicationStatus('Failed To Resolve OutputSE')
return result
resolvedSE = result['Value']
final[fileName] = metadata
final[fileName]['resolvedSE'] = resolvedSE
self.log.info('The following files will be uploaded: %s' % (string.join(final.keys(), ', ')))
for fileName, metadata in final.items():
self.log.info('--------%s--------' % fileName)
for n, v in metadata.items():
self.log.info('%s = %s' % (n, v))
#At this point can exit and see exactly what the module would have uploaded
if not self.enable:
self.log.info('Module is disabled by control flag, would have attempted to upload the \
following files %s' % string.join(final.keys(), ', '))
return S_OK('Module is disabled by control flag')
#Disable the watchdog check in case the file uploading takes a long time
self.log.info('Creating DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK in order to disable the Watchdog prior to upload')
fopen = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK','w')
fopen.write('%s' % time.asctime())
fopen.close()
#Instantiate the failover transfer client with the global request object
failoverTransfer = FailoverTransfer(self.request)
catalogs = ['FileCatalog', 'LcgFileCatalog']
#One by one upload the files with failover if necessary
failover = {}
if not self.failoverTest:
for fileName, metadata in final.items():
self.log.info("Attempting to store file %s to the following SE(s):\n%s" % (fileName,
string.join(metadata['resolvedSE'],
', ')))
result = failoverTransfer.transferAndRegisterFile(fileName, metadata['localpath'],
metadata['lfn'], metadata['resolvedSE'],
fileGUID = metadata['guid'], fileCatalog = catalogs)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
failover[fileName] = metadata
else:
lfn = metadata['lfn']
else:
failover = final
self.failoverSEs = self.ops.getValue("Production/%s/FailOverSE" % self.experiment, self.failoverSEs)
cleanUp = False
for fileName, metadata in failover.items():
self.log.info('Setting default catalog for failover transfer to FileCatalog')
random.shuffle(self.failoverSEs)
targetSE = metadata['resolvedSE'][0]
metadata['resolvedSE'] = self.failoverSEs
result = failoverTransfer.transferAndRegisterFileFailover(fileName, metadata['localpath'],
metadata['lfn'], targetSE, metadata['resolvedSE'],
fileGUID = metadata['guid'], fileCatalog = catalogs)
if not result['OK']:
self.log.error('Could not transfer and register %s with metadata:\n %s' % (fileName, metadata))
cleanUp = True
break #no point continuing if one completely fails
os.remove("DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK") #cleanup the mess
#Now after all operations, retrieve potentially modified request object
result = failoverTransfer.getRequestObject()
if not result['OK']:
self.log.error(result)
return S_ERROR('Could not retrieve modified request')
self.request = result['Value']
#If some or all of the files failed to be saved to failover
if cleanUp:
lfns = []
for fileName, metadata in final.items():
lfns.append(metadata['lfn'])
result = self.__cleanUp(lfns)
self.workflow_commons['Request'] = self.request
return S_ERROR('Failed to upload output data')
# #Can now register the successfully uploaded files in the BK
# if not performBKRegistration:
# self.log.info('There are no files to perform the BK registration for, all could be saved to failover')
# else:
# rm = ReplicaManager()
# result = rm.addCatalogFile(performBKRegistration,catalogs=['BookkeepingDB'])
# self.log.verbose(result)
# if not result['OK']:
# self.log.error(result)
# return S_ERROR('Could Not Perform BK Registration')
# if result['Value']['Failed']:
# for lfn,error in result['Value']['Failed'].items():
# self.log.info('BK registration for %s failed with message: "%s" setting failover request' %(lfn,error))
# result = self.request.addSubRequest({'Attributes':{'Operation':'registerFile','ExecutionOrder':0, 'Catalogue':'BookkeepingDB'}},'register')
# if not result['OK']:
# self.log.error('Could not set registerFile request:\n%s' %result)
# return S_ERROR('Could Not Set BK Registration Request')
# fileDict = {'LFN':lfn,'Status':'Waiting'}
# index = result['Value']
# self.request.setSubRequestFiles(index,'register',[fileDict])
self.workflow_commons['Request'] = self.request
return S_OK('Output data uploaded')
def execute(self):
""" Main execution function.
"""
#Have to work out if the module is part of the last step i.e.
#user jobs can have any number of steps and we only want
#to run the finalization once. Not a problem if this is not the last step so return S_OK()
resultLS = self.isLastStep()
if not resultLS['OK']:
return S_OK()
self.logWorkingDirectory()
resultIV = self.resolveInputVariables()
if not resultIV['OK']:
self.log.error("Failed to resolve input parameters:", resultIV['Message'])
return resultIV
self.log.info('Initializing %s' % self.version)
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'],
self.stepStatus['OK']))
return S_OK('No output data upload attempted')
if not self.userOutputData:
self.log.info('No user output data is specified for this job, nothing to do')
return S_OK('No output data to upload')
#Determine the final list of possible output files for the
#workflow and all the parameters needed to upload them.
outputList = self.getOutputList()
userOutputLFNs = []
if self.userOutputData:
resultOLfn = self.constructOutputLFNs()
if not resultOLfn['OK']:
self.log.error('Could not create user LFNs', resultOLfn['Message'])
return resultOLfn
userOutputLFNs = resultOLfn['Value']
self.log.verbose('Calling getCandidateFiles( %s, %s, %s)' % (outputList, userOutputLFNs, self.outputDataFileMask))
self.log.debug("IgnoreAppErrors? '%s' " % self.ignoreapperrors)
resultCF = self.getCandidateFiles(outputList, userOutputLFNs, self.outputDataFileMask)
if not resultCF['OK']:
if not self.ignoreapperrors:
self.log.error(resultCF['Message'])
self.setApplicationStatus(resultCF['Message'])
return S_OK()
fileDict = resultCF['Value']
resultFMD = self.getFileMetadata(fileDict)
if not resultFMD['OK']:
if not self.ignoreapperrors:
self.log.error(resultFMD['Message'])
self.setApplicationStatus(resultFMD['Message'])
return S_OK()
if not resultFMD['Value']:
if not self.ignoreapperrors:
self.log.info('No output data files were determined to be uploaded for this workflow')
self.setApplicationStatus('No Output Data Files To Upload')
return S_OK()
fileMetadata = resultFMD['Value']
#First get the local (or assigned) SE to try first for upload and others in random fashion
resultSEL = getDestinationSEList('Tier1-USER', DIRAC.siteName(), outputmode='local')
if not resultSEL['OK']:
self.log.error('Could not resolve output data SE', resultSEL['Message'])
self.setApplicationStatus('Failed To Resolve OutputSE')
return resultSEL
localSE = resultSEL['Value']
orderedSEs = [ se for se in self.defaultOutputSE if se not in localSE and se not in self.userOutputSE]
orderedSEs = localSE + List.randomize(orderedSEs)
if self.userOutputSE:
prependSEs = []
for userSE in self.userOutputSE:
if userSE not in orderedSEs:
prependSEs.append(userSE)
orderedSEs = prependSEs + orderedSEs
self.log.info('Ordered list of output SEs is: %s' % (', '.join(orderedSEs)))
final = {}
for fileName, metadata in fileMetadata.iteritems():
final[fileName] = metadata
final[fileName]['resolvedSE'] = orderedSEs
#At this point can exit and see exactly what the module will upload
self.printOutputInfo(final)
if not self.enable:
return S_OK('Module is disabled by control flag')
self.injectJobIndex( final )
#Instantiate the failover transfer client with the global request object
failoverTransfer = FailoverTransfer(self._getRequestContainer())
#One by one upload the files with failover if necessary
filesToReplicate = {}
filesToFailover = {}
filesUploaded = []
if not self.failoverTest:
self.transferAndRegisterFiles(final, failoverTransfer, filesToFailover, filesUploaded, filesToReplicate)
else:
filesToFailover = final
##if there are files to be failovered, we do it now
resultTRFF = self.transferRegisterAndFailoverFiles(failoverTransfer, filesToFailover, filesUploaded)
cleanUp = resultTRFF['Value']['cleanUp']
#For files correctly uploaded must report LFNs to job parameters
if filesUploaded:
report = ', '.join( filesUploaded )
self.jobReport.setJobParameter( 'UploadedOutputData', report )
self.workflow_commons['Request'] = failoverTransfer.request
#If some or all of the files failed to be saved to failover
if cleanUp:
#Leave any uploaded files just in case it is useful for the user
#do not try to replicate any files.
return S_ERROR('Failed To Upload Output Data')
#If there is now at least one replica for uploaded files can trigger replication
datMan = DataManager( catalogs = self.userFileCatalog )
self.log.info('Sleeping for 10 seconds before attempting replication of recently uploaded files')
time.sleep(10)
for lfn, repSE in filesToReplicate.items():
resultRAR = datMan.replicateAndRegister(lfn, repSE)
if not resultRAR['OK']:
self.log.info('Replication failed with below error but file already exists in Grid storage with \
at least one replica:\n%s' % (resultRAR))
self.generateFailoverFile()
self.setApplicationStatus('Job Finished Successfully')
return S_OK('Output data uploaded')
