class UserJob(Job):
""" User job class. To be used by users, not for production.
"""
def __init__(self, script = None):
super(UserJob, self).__init__( script )
self.type = 'User'
self.diracinstance = None
def submit(self, diracinstance = None, mode = "wms"):
""" Submit call: when your job is defined, and all applications are set, you need to call this to
add the job to DIRAC.
"""
res = self._addToWorkflow()
if not res['OK']:
return res
self.oktosubmit = True
if not diracinstance:
self.diracinstance = DiracILC()
else:
self.diracinstance = diracinstance
return self.diracinstance.submit(self, mode)
#############################################################################
def setInputData( self, lfns ):
"""Helper function.
Specify input data by Logical File Name (LFN).
Example usage:
>>> job = Job()
>>> job.setInputData(['/ilc/prod/whizard/processlist.whiz'])
@param lfns: Logical File Names
@type lfns: Single LFN string or list of LFNs
"""
if type( lfns ) == list and len( lfns ):
for i in xrange( len( lfns ) ):
lfns[i] = lfns[i].replace( 'LFN:', '' )
inputData = map( lambda x: 'LFN:' + x, lfns )
inputDataStr = string.join( inputData, ';' )
description = 'List of input data specified by LFNs'
self._addParameter( self.workflow, 'InputData', 'JDL', inputDataStr, description )
elif type( lfns ) == type( ' ' ): #single LFN
description = 'Input data specified by LFN'
self._addParameter( self.workflow, 'InputData', 'JDL', lfns, description )
else:
kwargs = {'lfns':lfns}
return self._reportError( 'Expected lfn string or list of lfns for input data', **kwargs )
return S_OK()
def setInputSandbox(self, flist):
""" Mostly inherited from DIRAC.Job
"""
if type(flist) == type(""):
flist = [flist]
if not type(flist) == type([]) :
return self._reportError("File passed must be either single file or list of files.")
self.inputsandbox.extend(flist)
return S_OK()
#############################################################################
def setOutputData(self, lfns, OutputPath = '', OutputSE = ['']):
"""Helper function, used in preference to Job.setOutputData() for ILC.
For specifying output data to be registered in Grid storage. If a list
of OutputSEs are specified the job wrapper will try each in turn until
successful.
Example usage:
>>> job = Job()
>>> job.setOutputData(['Ntuple.root'])
@param lfns: Output data file or files
@type lfns: Single string or list of strings ['','']
@param OutputSE: Optional parameter to specify the Storage
@param OutputPath: Optional parameter to specify the Path in the Storage, postpented to /ilc/user/u/username/
Element to store data or files, e.g. CERN-tape
@type OutputSE: string or list
@type OutputPath: string
"""
kwargs = {'lfns' : lfns, 'OutputSE' : OutputSE, 'OutputPath' : OutputPath}
if type(lfns) == list and len(lfns):
outputDataStr = string.join(lfns, ';')
description = 'List of output data files'
self._addParameter(self.workflow, 'UserOutputData', 'JDL', outputDataStr, description)
elif type(lfns) == type(" "):
description = 'Output data file'
self._addParameter(self.workflow, 'UserOutputData', 'JDL', lfns, description)
else:
return self._reportError('Expected file name string or list of file names for output data', **kwargs)
if OutputSE:
description = 'User specified Output SE'
if type(OutputSE) in types.StringTypes:
OutputSE = [OutputSE]
elif type(OutputSE) != types.ListType:
return self._reportError('Expected string or list for OutputSE', **kwargs)
OutputSE = ';'.join(OutputSE)
self._addParameter(self.workflow, 'UserOutputSE', 'JDL', OutputSE, description)
if OutputPath:
description = 'User specified Output Path'
if not type(OutputPath) in types.StringTypes:
return self._reportError('Expected string for OutputPath', **kwargs)
# Remove leading "/" that might cause problems with os.path.join
while OutputPath[0] == '/':
OutputPath = OutputPath[1:]
if OutputPath.count("ilc/user"):
return self._reportError('Output path contains /ilc/user/ which is not what you want', **kwargs)
self._addParameter(self.workflow, 'UserOutputPath', 'JDL', OutputPath, description)
return S_OK()
#############################################################################
def setOutputSandbox( self, files ):
"""Helper function.
Specify output sandbox files. If specified files are over 10MB, these
may be uploaded to Grid storage with a notification returned in the
output sandbox.
Example usage:
>>> job = Job()
>>> job.setOutputSandbox(['*.log','myfile.slcio'])
@param files: Output sandbox files
@type files: Single string or list of strings ['','']
"""
if type( files ) == list and len( files ):
fileList = string.join( files, ";" )
description = 'Output sandbox file list'
self._addParameter( self.workflow, 'OutputSandbox', 'JDL', fileList, description )
elif type( files ) == type( " " ):
description = 'Output sandbox file'
self._addParameter( self.workflow, 'OutputSandbox', 'JDL', files, description )
else:
kwargs = {'files' : files}
return self._reportError( 'Expected file string or list of files for output sandbox contents', **kwargs )
return S_OK()
def setILDConfig(self,Version):
""" Define the Configuration package to obtain
"""
appName = 'ILDConfig'
self._addSoftware(appName.lower(), Version)
self._addParameter( self.workflow, 'ILDConfigPackage', 'JDL', appName+Version, 'ILDConfig package' )
return S_OK()
class UserJob(Job):
""" User job class. To be used by users, not for production.
"""
def __init__(self, script=None):
super(UserJob, self).__init__(script)
self.type = 'User'
self.diracinstance = None
self.usergroup = ['ilc_user', 'calice_user']
self.proxyinfo = getProxyInfo()
########## SPLITTING STUFF: ATTRIBUTES ##########
self._data = []
self.splittingOption = None
self._switch = {}
self.numberOfJobs = None
self.totalNumberOfEvents = None
self.eventsPerJob = None
self.numberOfFilesPerJob = 1
def submit(self, diracinstance=None, mode="wms"):
""" Submit call: when your job is defined, and all applications are set, you need to call this to
add the job to DIRAC.
:param diracinstance: DiracILC instance
:type diracinstance: ~ILCDIRAC.Interfaces.API.DiracILC.DiracILC
:param str mode: "wms" (default), "agent", or "local"
.. note ::
The *local* mode means that the job will be run on the submission machine. Use this mode for testing of submission scripts
"""
if self.splittingOption:
result = self._split()
if 'OK' in result and not result['OK']:
return result
#Check the credentials. If no proxy or not user proxy, return an error
if not self.proxyinfo['OK']:
self.log.error(
"Not allowed to submit a job, you need a %s proxy." %
self.usergroup)
return self._reportError(
"Not allowed to submit a job, you need a %s proxy." %
self.usergroup, self.__class__.__name__)
if 'group' in self.proxyinfo['Value']:
group = self.proxyinfo['Value']['group']
if group not in self.usergroup:
self.log.error(
"Not allowed to submit a job, you need a %s proxy." %
self.usergroup)
return self._reportError(
"Not allowed to submit job, you need a %s proxy." %
self.usergroup, self.__class__.__name__)
else:
self.log.error(
"Could not determine group, you do not have the right proxy.")
return self._reportError(
"Could not determine group, you do not have the right proxy.")
res = self._addToWorkflow()
if not res['OK']:
return res
self.oktosubmit = True
if not diracinstance:
self.diracinstance = DiracILC()
else:
self.diracinstance = diracinstance
return self.diracinstance.submit(self, mode)
#############################################################################
def setInputData(self, lfns):
"""Specify input data by Logical File Name (LFN).
Input files specified via this function will be automatically staged if necessary.
Example usage:
>>> job = UserJob()
>>> job.setInputData(['/ilc/prod/whizard/processlist.whiz'])
:param lfns: Logical File Names
:type lfns: Single LFN string or list of LFNs
"""
if isinstance(lfns, list) and lfns:
for i, lfn in enumerate(lfns):
lfns[i] = lfn.replace('LFN:', '')
#inputData = map( lambda x: 'LFN:' + x, lfns )
inputData = lfns #because we don't need the LFN: for inputData, and it breaks the
#resolution of the metadata in the InputFilesUtilities
inputDataStr = ';'.join(inputData)
description = 'List of input data specified by LFNs'
self._addParameter(self.workflow, 'InputData', 'JDL', inputDataStr,
description)
elif isinstance(lfns, basestring): #single LFN
description = 'Input data specified by LFN'
self._addParameter(self.workflow, 'InputData', 'JDL', lfns,
description)
else:
kwargs = {'lfns': lfns}
return self._reportError(
'Expected lfn string or list of lfns for input data', **kwargs)
return S_OK()
def setInputSandbox(self, flist):
""" Add files to the input sandbox, can be on the local machine or on the grid
>>> job = UserJob()
>>> job.setInputSandbox( ['LFN:/ilc/user/u/username/libraries.tar.gz',
>>> 'mySteeringFile.xml'] )
:param flist: Files for the inputsandbox
:type flist: `python:list` or `str`
"""
if isinstance(flist, basestring):
flist = [flist]
if not isinstance(flist, list):
return self._reportError(
"File passed must be either single file or list of files.")
self.inputsandbox.extend(flist)
return S_OK()
#############################################################################
def setOutputData(self, lfns, OutputPath='', OutputSE=''):
"""For specifying output data to be registered in Grid storage. If a list
of OutputSEs are specified the job wrapper will try each in turn until
successful.
Example usage:
>>> job = UserJob()
>>> job.setOutputData(['Ntuple.root'])
:param lfns: Output data file or files
:type lfns: Single `str` or `python:list` of strings ['','']
:param str OutputPath: Optional parameter to specify the Path in the Storage, postpended to /ilc/user/u/username/
:param OutputSE: Optional parameter to specify the Storage Element to store data or files, e.g. CERN-SRM
:type OutputSE: `python:list` or `str`
"""
kwargs = {'lfns': lfns, 'OutputSE': OutputSE, 'OutputPath': OutputPath}
if isinstance(lfns, list) and lfns:
outputDataStr = ';'.join(lfns)
description = 'List of output data files'
self._addParameter(self.workflow, 'UserOutputData', 'JDL',
outputDataStr, description)
elif isinstance(lfns, basestring):
description = 'Output data file'
self._addParameter(self.workflow, 'UserOutputData', 'JDL', lfns,
description)
else:
return self._reportError(
'Expected file name string or list of file names for output data',
**kwargs)
if OutputSE:
description = 'User specified Output SE'
if isinstance(OutputSE, basestring):
OutputSE = [OutputSE]
elif not isinstance(OutputSE, list):
return self._reportError(
'Expected string or list for OutputSE', **kwargs)
OutputSE = ';'.join(OutputSE)
self._addParameter(self.workflow, 'UserOutputSE', 'JDL', OutputSE,
description)
if OutputPath:
description = 'User specified Output Path'
if not isinstance(OutputPath, basestring):
return self._reportError('Expected string for OutputPath',
**kwargs)
# Remove leading "/" that might cause problems with os.path.join
while OutputPath[0] == '/':
OutputPath = OutputPath[1:]
if OutputPath.count("ilc/user"):
return self._reportError(
'Output path contains /ilc/user/ which is not what you want',
**kwargs)
self._addParameter(self.workflow, 'UserOutputPath', 'JDL',
OutputPath, description)
return S_OK()
#############################################################################
def setOutputSandbox(self, files):
"""Specify output sandbox files. If specified files are over 10MB, these
may be uploaded to Grid storage with a notification returned in the
output sandbox.
.. Note ::
Sandbox files are removed after 2 weeks.
Example usage:
>>> job = UserJob()
>>> job.setOutputSandbox(['*.log','*.sh', 'myfile.txt'])
Use the output sandbox only for small files. Larger files should be stored
on the grid and downloaded later if necessary. See :func:`setOutputData`
:param files: Output sandbox files
:type files: Single `str` or `python:list` of strings ['','']
"""
if isinstance(files, list) and files:
fileList = ";".join(files)
description = 'Output sandbox file list'
self._addParameter(self.workflow, 'OutputSandbox', 'JDL', fileList,
description)
elif isinstance(files, basestring):
description = 'Output sandbox file'
self._addParameter(self.workflow, 'OutputSandbox', 'JDL', files,
description)
else:
kwargs = {'files': files}
return self._reportError(
'Expected file string or list of files for output sandbox contents',
**kwargs)
return S_OK()
def setILDConfig(self, version):
""" Define the Configuration package to obtain
"""
appName = 'ILDConfig'
self._addSoftware(appName.lower(), version)
self._addParameter(self.workflow, 'ILDConfigPackage', 'JDL',
appName + version, 'ILDConfig package')
return S_OK()
def setCLICConfig(self, version):
"""Define the CLIC Configuration package to obtain, copies steering files
from CLIC Configuration folder to working directory
:param str version: version string, e.g.: 'ILCSoft-2017-07-27'
"""
appName = 'ClicConfig'
self._addSoftware(appName.lower(), version)
self._addParameter(self.workflow, 'ClicConfigPackage', 'JDL',
appName + version, 'CLIC Config package')
return S_OK()
############################## SPLITTING STUFF: METHODS ################################
# Some methods have been added :
#
# * _atomicSubmission
# * _checkJobConsistency
# * setSplitEvents
# * setSplitInputData
# * setSplitDoNotAlterOutputFilename
# * _split
# * _splitByData
# * _splitByEvents
# * _toInt
#
# Given the type of splitting (Events or Data), these methods compute
# the right parameters of the method 'Job.setParameterSequence()'
##########################################################################################
def setSplitEvents(self,
eventsPerJob=None,
numberOfJobs=None,
totalNumberOfEvents=None):
"""This function sets split parameters for doing splitting over events
Example usage:
>>> job = UserJob()
>>> job.setSplitEvents( numberOfJobs=42, totalNumberOfEvents=126 )
Exactly two of the parmeters should be set
:param int eventsPerJob: The events processed by a single job
:param int numberOfJobs: The number of jobs
:param int totalNumberOfEvents: The total number of events processed by all jobs
"""
self.totalNumberOfEvents = totalNumberOfEvents
self.eventsPerJob = eventsPerJob
self.numberOfJobs = numberOfJobs
self._addParameter(self.workflow, 'NbOfEvts', 'JDL', -1,
'Number of Events')
self.splittingOption = "byEvents"
def setSplitInputData(self, lfns, numberOfFilesPerJob=1):
"""sets split parameters for doing splitting over input data
Example usage:
>>> job = UserJob()
>>> job.setSplitInputData( listOfLFNs )
:param lfns: Logical File Names
:type lfns: list of LFNs
:param int numberOfFilesPerJob: The number of input data processed by a single job
"""
self._data = lfns if isinstance(lfns, list) else [lfns]
self.numberOfFilesPerJob = numberOfFilesPerJob
self.splittingOption = "byData"
def setSplitDoNotAlterOutputFilename(self, value=True):
"""if this option is set the output data lfns will _not_ include the JobIndex
:param bool value: if *True* disable the changing of the output data
filenames. If *False* the JobIndex will be added at the end of
OutputData LFNs before the extension. Or replace '%n' with the jobIndex
in the fileName. See :func:`Core.Utilities.Splitting.addJobIndexToFilename`
"""
self._addParameter(self.workflow, 'DoNotAlterOutputData', 'JDL', value,
'Do Not Change Output Data')
def _split(self):
"""checks the consistency of the job and call the right split method.
:return: The success or the failure of the consistency checking
:rtype: DIRAC.S_OK, DIRAC.S_ERROR
"""
self.eventsPerJob = self._toInt(self.eventsPerJob)
self.numberOfJobs = self._toInt(self.numberOfJobs)
if self.numberOfJobs is False or self.eventsPerJob is False:
return self._reportError("Splitting: Invalid values for splitting")
# FIXME: move somewhere more prominent
self._switch = {
"byEvents": self._splitByEvents,
"byData": self._splitByData,
None: self._atomicSubmission,
}
self.log.info("Job splitting...")
if not self._checkJobConsistency():
errorMessage = "Job._checkJobConsistency() failed"
self.log.error(errorMessage)
return self._reportError(errorMessage)
sequence = self._switch[self.splittingOption]()
if not sequence:
errorMessage = "Job._splitBySomething() failed"
self.log.error(errorMessage)
return self._reportError(errorMessage)
sequenceType, sequenceList, addToWorkflow = sequence[0], sequence[
1], sequence[2]
if sequenceType != "Atomic":
self.setParameterSequence(sequenceType, sequenceList,
addToWorkflow)
self.setParameterSequence('JobIndexList',
range(len(sequenceList)),
addToWorkflow='JobIndex')
self._addParameter(self.workflow, 'JobIndex', 'int', 0, 'JobIndex')
self.log.info("Job splitting successful")
return S_OK()
#############################################################################
def _atomicSubmission(self):
"""called when no splitting is necessary, do not return valid parameters fot setParameterSequence().
:return: parameter name and parameter values for setParameterSequence(), addToWorkflow flag
:rtype: tuple of (str, list, bool/str)
"""
self.log.verbose(
"Job splitting: No splitting to apply, 'atomic submission' will be used"
)
return "Atomic", [], False
#############################################################################
def _checkJobConsistency(self):
"""checks if Job parameters are valid.
:return: The success or the failure of the consistency checking
:rtype: bool
:Example:
>>> self._checkJobConsistency()
"""
self.log.info("Job consistency: _checkJobConsistency()...")
if self.splittingOption not in self._switch:
splitOptions = ",".join(self._switch.keys())
errorMessage = "checkJobConsistency failed: Bad split value: possible values are %s" % splitOptions
self.log.error(errorMessage)
return False
# All applications should have the same number of events
# We can get this number from the first application for example
sameNumberOfEvents = next(iter(self.applicationlist)).numberOfEvents
if not all(app.numberOfEvents == sameNumberOfEvents
for app in self.applicationlist):
self.log.warn(
"Job: Applications should all have the same number of events")
if (self.totalNumberOfEvents == -1
or sameNumberOfEvents == -1) and not self._data:
self.log.warn(
"Job: Number of events is -1 without input data. Was that intentional?"
)
self.log.info("job._checkJobConsistency successful")
return True
#############################################################################
def _splitByData(self):
"""a job is submitted per input data.
:return: parameter name and parameter values for setParameterSequence()
:rtype: tuple of (str, list, bool/str)
"""
# reset split attribute to avoid infinite loop
self.splittingOption = None
self.log.info("Job splitting: Splitting 'byData' method...")
# Ensure that data have been specified by setInputData() method
if not self._data:
errorMessage = "Job splitting: missing input data"
self.log.error(errorMessage)
return False
if self.numberOfFilesPerJob > len(self._data):
errorMessage = "Job splitting: 'numberOfFilesPerJob' must be less/equal than the number of input data"
self.log.error(errorMessage)
return False
self._data = breakListIntoChunks(self._data, self.numberOfFilesPerJob)
self.log.info("Job splitting: submission consists of %d job(s)" %
len(self._data))
return ["InputData", self._data, 'ParametricInputData']
#############################################################################
def _splitByEvents(self):
"""a job is submitted per subset of events.
:return: parameter name and parameter values for setParameterSequence()
:rtype: tuple of (str, list, bool/str)
"""
# reset split attribute to avoid infinite loop
self.splittingOption = None
self.log.info("Job splitting: splitting 'byEvents' method...")
if self.eventsPerJob and self.numberOfJobs:
# 1st case: (numberOfJobs=3, eventsPerJob=10)
# trivial case => each job (total of 3) run applications of 10 events each
self.log.debug("Job splitting: events per job and number of jobs")
mapEventJob = [self.eventsPerJob] * self.numberOfJobs
elif self.eventsPerJob and self.totalNumberOfEvents:
# 2nd case: (split="byEvents", eventsPerJob=10, totalNumberOfEvents=10)
# Given the number of events per job and total of number of event we want,
# we can compute the unknown which is the number of jobs.
self.log.debug(
"Job splitting: Events per job and total number of events")
if self.eventsPerJob > self.totalNumberOfEvents:
self.log.error(
"Job splitting: The number of events per job has to be lower than or equal to the total number of events"
)
return False
numberOfJobsIntDiv = self.totalNumberOfEvents / self.eventsPerJob
numberOfJobsRest = self.totalNumberOfEvents % self.eventsPerJob
mapEventJob = [self.eventsPerJob] * numberOfJobsIntDiv
mapEventJob += [numberOfJobsRest] if numberOfJobsRest != 0 else []
else:
# 3rd case: (split='byEvents', njobs=10, totalNumberOfEvents=10)
# Then compute the right number of events per job
self.log.debug(
"Job splitting: The number of jobs and the total number of events"
)
if (not self.totalNumberOfEvents) or (self.totalNumberOfEvents <
self.numberOfJobs):
self.log.error(
"Job splitting: The number of events has to be greater than or equal to the number of jobs"
)
return False
eventPerJobIntDiv = self.totalNumberOfEvents / self.numberOfJobs
eventPerJobRest = self.totalNumberOfEvents % self.numberOfJobs
mapEventJob = [eventPerJobIntDiv] * self.numberOfJobs
if eventPerJobRest != 0:
for suplement in xrange(eventPerJobRest):
mapEventJob[suplement] += 1
self.log.debug("Job splitting: events over the jobs: %s" % mapEventJob)
self.log.info("Job splitting: submission consists of %d job(s)" %
len(mapEventJob))
return ['NumberOfEvents', mapEventJob, 'NbOfEvts']
#############################################################################
def _toInt(self, number):
"""casts number parameter to an integer.
It also accepts 'string integer' parameter.
:param number: the number to cast (number of events, number of jobs)
:type number: str or int
:return: The success or the failure of the casting
:rtype: bool, int or None
:Example:
>>> number = self._toInt("1000")
"""
if number is None:
return number
try:
number = int(number)
if number <= 0:
raise ValueError
except ValueError:
self.log.error(
"Job splitting: arguments must be positive integers")
return False
return number
class UserJob(Job):
""" User job class. To be used by users, not for production.
"""
def __init__(self, script = None):
super(UserJob, self).__init__( script )
self.type = 'User'
self.diracinstance = None
self.usergroup = ['ilc_user', 'calice_user']
self.proxyinfo = getProxyInfo()
def submit(self, diracinstance = None, mode = "wms"):
""" Submit call: when your job is defined, and all applications are set, you need to call this to
add the job to DIRAC.
:param diracinstance: :any:`DiracILC <ILCDIRAC.Interfaces.API.DiracILC.DiracILC>` instance
:param string mode: "wms" (default), "agent", or "local"
.. note ::
The *local* mode means that the job will be run on the submission machine. Use this mode for testing of submission scripts
"""
#Check the credentials. If no proxy or not user proxy, return an error
if not self.proxyinfo['OK']:
self.log.error("Not allowed to submit a job, you need a %s proxy." % self.usergroup)
return self._reportError("Not allowed to submit a job, you need a %s proxy." % self.usergroup,
self.__class__.__name__)
if self.proxyinfo['Value'].has_key('group'):
group = self.proxyinfo['Value']['group']
if not group in self.usergroup:
self.log.error("Not allowed to submit a job, you need a %s proxy." % self.usergroup)
return self._reportError("Not allowed to submit job, you need a %s proxy." % self.usergroup,
self.__class__.__name__)
else:
self.log.error("Could not determine group, you do not have the right proxy.")
return self._reportError("Could not determine group, you do not have the right proxy.")
res = self._addToWorkflow()
if not res['OK']:
return res
self.oktosubmit = True
if not diracinstance:
self.diracinstance = DiracILC()
else:
self.diracinstance = diracinstance
return self.diracinstance.submit(self, mode)
#############################################################################
def setInputData( self, lfns ):
"""Specify input data by Logical File Name (LFN).
Input files specified via this function will be automatically staged if necessary.
Example usage:
>>> job = UserJob()
>>> job.setInputData(['/ilc/prod/whizard/processlist.whiz'])
:param lfns: Logical File Names
:type lfns: Single LFN string or list of LFNs
"""
if type( lfns ) == list and len( lfns ):
for i in xrange( len( lfns ) ):
lfns[i] = lfns[i].replace( 'LFN:', '' )
#inputData = map( lambda x: 'LFN:' + x, lfns )
inputData = lfns #because we don't need the LFN: for inputData, and it breaks the
#resolution of the metadata in the InputFilesUtilities
inputDataStr = ';'.join( inputData )
description = 'List of input data specified by LFNs'
self._addParameter( self.workflow, 'InputData', 'JDL', inputDataStr, description )
elif type( lfns ) == type( ' ' ): #single LFN
description = 'Input data specified by LFN'
self._addParameter( self.workflow, 'InputData', 'JDL', lfns, description )
else:
kwargs = {'lfns':lfns}
return self._reportError( 'Expected lfn string or list of lfns for input data', **kwargs )
return S_OK()
def setInputSandbox(self, flist):
""" Add files to the input sandbox, can be on the local machine or on the grid
>>> job = UserJob()
>>> job.setInputSandbox( ['LFN:/ilc/user/u/username/libraries.tar.gz',
>>> 'mySteeringFile.xml'] )
:param flist: Files for the inputsandbox
:type flist: list or string
"""
if type(flist) == type(""):
flist = [flist]
if not type(flist) == type([]) :
return self._reportError("File passed must be either single file or list of files.")
self.inputsandbox.extend(flist)
return S_OK()
#############################################################################
def setOutputData(self, lfns, OutputPath = '', OutputSE = ''):
"""For specifying output data to be registered in Grid storage. If a list
of OutputSEs are specified the job wrapper will try each in turn until
successful.
Example usage:
>>> job = UserJob()
>>> job.setOutputData(['Ntuple.root'])
:param lfns: Output data file or files
:type lfns: Single string or list of strings ['','']
:param string OutputPath: Optional parameter to specify the Path in the Storage, postpended to /ilc/user/u/username/
:param OutputSE: Optional parameter to specify the Storage Element to store data or files, e.g. CERN-SRM
:type OutputSE: string or list
"""
kwargs = {'lfns' : lfns, 'OutputSE' : OutputSE, 'OutputPath' : OutputPath}
if type(lfns) == list and len(lfns):
outputDataStr = ';'.join(lfns)
description = 'List of output data files'
self._addParameter(self.workflow, 'UserOutputData', 'JDL', outputDataStr, description)
elif type(lfns) == type(" "):
description = 'Output data file'
self._addParameter(self.workflow, 'UserOutputData', 'JDL', lfns, description)
else:
return self._reportError('Expected file name string or list of file names for output data', **kwargs)
if OutputSE:
description = 'User specified Output SE'
if type(OutputSE) in types.StringTypes:
OutputSE = [OutputSE]
elif type(OutputSE) != types.ListType:
return self._reportError('Expected string or list for OutputSE', **kwargs)
OutputSE = ';'.join(OutputSE)
self._addParameter(self.workflow, 'UserOutputSE', 'JDL', OutputSE, description)
if OutputPath:
description = 'User specified Output Path'
if not type(OutputPath) in types.StringTypes:
return self._reportError('Expected string for OutputPath', **kwargs)
# Remove leading "/" that might cause problems with os.path.join
while OutputPath[0] == '/':
OutputPath = OutputPath[1:]
if OutputPath.count("ilc/user"):
return self._reportError('Output path contains /ilc/user/ which is not what you want', **kwargs)
self._addParameter(self.workflow, 'UserOutputPath', 'JDL', OutputPath, description)
return S_OK()
#############################################################################
def setOutputSandbox( self, files ):
"""Specify output sandbox files. If specified files are over 10MB, these
may be uploaded to Grid storage with a notification returned in the
output sandbox.
.. Note ::
Sandbox files are removed after 2 weeks.
Example usage:
>>> job = UserJob()
>>> job.setOutputSandbox(['*.log','*.sh', 'myfile.txt'])
Use the output sandbox only for small files. Larger files should be stored
on the grid and downloaded later if necessary. See :func:`setOutputData`
:param files: Output sandbox files
:type files: Single string or list of strings ['','']
"""
if type( files ) == list and len( files ):
fileList = ";".join( files )
description = 'Output sandbox file list'
self._addParameter( self.workflow, 'OutputSandbox', 'JDL', fileList, description )
elif type( files ) == type( " " ):
description = 'Output sandbox file'
self._addParameter( self.workflow, 'OutputSandbox', 'JDL', files, description )
else:
kwargs = {'files' : files}
return self._reportError( 'Expected file string or list of files for output sandbox contents', **kwargs )
return S_OK()
def setILDConfig(self,version):
""" Define the Configuration package to obtain
"""
appName = 'ILDConfig'
self._addSoftware(appName.lower(), version)
self._addParameter( self.workflow, 'ILDConfigPackage', 'JDL', appName+version, 'ILDConfig package' )
return S_OK()
