class RootJob(Job):
def __init__(self,
script,
parameters=None,
softwarePackage='HESS/v0.1/root',
compiled=False,
cpuTime=3600):
Job.__init__(self)
self.workflow = Workflow()
self.executable = '$DIRACROOT/scripts/cta-root-macro'
self.setName(os.path.basename(script))
self.script = os.path.basename(script)
self.setInputSandbox([script])
self.setCPUTime(cpuTime)
arguments = []
toCompile = ''
if compiled:
toCompile = '+'
if parameters:
arguments = [
repr(k).replace('"', "\\\\'").replace("'", "\\\\'")
for k in parameters
]
argumentStr = "%s%s %s" % (os.path.basename(script), toCompile,
' '.join(arguments))
self.setConfigArgs(argumentStr)
self.__addSoftwarePackage(softwarePackage)
def __addSoftwarePackage(self, package):
swPackages = 'SoftwarePackages'
description = 'List of Software Packages to be installed'
if not self.workflow.findParameter(swPackages):
self._addParameter(self.workflow, swPackages, 'JDL', package,
description)
else:
apps = self.workflow.findParameter(swPackages).getValue()
if not package in string.split(apps, ';'):
apps += ';' + package
self._addParameter(self.workflow, swPackages, 'JDL', apps,
description)
def __getProductionParameters(self,
prodXMLFile,
prodID,
groupDescription='',
bkPassInfo={},
derivedProd=0,
reqID=0):
""" This method will publish production parameters.
"""
prodWorkflow = Workflow(prodXMLFile)
parameters = {}
info = []
for parameterName in ('Priority', 'CondDBTag', 'DDDBTag', 'DQTag',
'eventType', 'processingPass',
'FractionToProcess', 'MinFilesToProcess',
'configName', 'configVersion',
'outputDataFileMask', 'JobType',
'MaxNumberOfTasks'):
try:
parameters[parameterName] = prodWorkflow.findParameter(
parameterName).getValue()
info.append(
"%s: %s" %
(parameterName,
prodWorkflow.findParameter(parameterName).getValue()))
except AttributeError:
continue
parameters['SizeGroup'] = self.jobFileGroupSize
if prodWorkflow.findParameter(
'InputData'): # now only comes from BK query
prodWorkflow.findParameter('InputData').setValue('')
gLogger.verbose(
'Resetting input data for production to null, this comes from a BK query...'
)
prodXMLFile = self.__createWorkflow(prodXMLFile)
# prodWorkflow.toXMLFile(prodXMLFile)
if self.transformationFamily:
parameters['TransformationFamily'] = self.transformationFamily
if not bkPassInfo:
bkPassInfo = prodWorkflow.findParameter(
'BKProcessingPass').getValue()
if not groupDescription:
groupDescription = prodWorkflow.findParameter(
'groupDescription').getValue()
parameters['BKCondition'] = prodWorkflow.findParameter(
'conditions').getValue()
parameters['BKProcessingPass'] = bkPassInfo
parameters['groupDescription'] = groupDescription
parameters['RequestID'] = reqID
parameters['DerivedProduction'] = derivedProd
result = self.__getOutputLFNs(prodID, '99999999', prodXMLFile)
if not result['OK']:
gLogger.error('Could not create production LFNs', result)
outputLFNs = result['Value']
parameters['OutputLFNs'] = outputLFNs
# the list of output directories is later used for consistency check and for removing output data
outputDirectories = []
del outputLFNs[
'BookkeepingLFNs'] # since ProductionOutputData uses the file mask
for i in outputLFNs.values():
for j in i:
outputDir = '%s%s' % (j.split(str(prodID))[0], prodID)
if outputDir not in outputDirectories:
outputDirectories.append(outputDir)
parameters['OutputDirectories'] = outputDirectories
# Now for the steps of the workflow
stepKeys = sorted(bkPassInfo.keys())
for step in stepKeys:
info.append('====> %s %s %s' %
(bkPassInfo[step]['ApplicationName'],
bkPassInfo[step]['ApplicationVersion'], step))
info.append('%s Option Files:' %
(bkPassInfo[step]['ApplicationName']))
if bkPassInfo[step]['OptionFiles']:
for opts in bkPassInfo[step]['OptionFiles'].split(';'):
info.append('%s' % opts)
if bkPassInfo[step]['ExtraPackages']:
info.append('ExtraPackages: %s' %
(bkPassInfo[step]['ExtraPackages']))
try:
if parameters['BkQuery']:
info.append('\nBK Input Data Query:')
for bkn, bkv in parameters['BkQuery'].iteritems():
info.append('%s= %s' % (bkn, bkv))
except KeyError:
pass
# BK output directories (very useful)
bkPaths = []
bkOutputPath = '%s/%s/%s/%s/%s/%s' % (
parameters['configName'], parameters['configVersion'],
parameters['BKCondition'], parameters.get('processingPass', ''),
parameters['groupDescription'], parameters['eventType'])
fileTypes = parameters['outputDataFileMask']
fileTypes = [a.upper() for a in fileTypes.split(';')]
# Annoying that histograms are extension root
if 'ROOT' in fileTypes:
fileTypes.remove('ROOT')
fileTypes.append('HIST')
for ft in fileTypes:
bkPaths.append('%s/%s' % (bkOutputPath, ft))
parameters['BKPaths'] = bkPaths
info.append('\nBK Browsing Paths:\n%s' % ('\n'.join(bkPaths)))
infoString = '\n'.join(info)
parameters['DetailedInfo'] = infoString
gLogger.verbose('Parameters that will be added: %s' % parameters)
return parameters
