def createSingleModuleWorkflow(module, name):
''' Creates a workflow based on a single module definition
'''
moduleType = module.getType()
moduleName = name
workflow = Workflow()
step = StepDefinition(moduleType + '_step')
step.addModule(module)
moduleInstance = step.createModuleInstance(moduleType, moduleName)
step.addParameter(moduleInstance.parameters.getInput())
workflow.addParameter(moduleInstance.parameters.getInput())
workflow.addStep(step)
stepInstance = workflow.createStepInstance(moduleType + '_step',
moduleName + '_step')
# Propagate the module input parameters to the workflow level
moduleInstance.linkParameterUp(moduleInstance.parameters.getInput())
stepInstance.linkParameterUp(moduleInstance.parameters.getInput())
workflow.setName(name)
workflow.setDescription('Single module workflow from ' + moduleType +
' type module')
workflow.setDescrShort(moduleType + ' workflow')
return workflow
def __init__(self,
script,
parameters=None,
softwarePackage='HESS/v0.1/root',
compiled=False,
cpuTime=3600):
Job.__init__(self)
self.workflow = Workflow()
self.executable = '$DIRACROOT/scripts/cta-root-macro'
self.setName(os.path.basename(script))
self.script = os.path.basename(script)
self.setInputSandbox([script])
self.setCPUTime(cpuTime)
arguments = []
toCompile = ''
if compiled:
toCompile = '+'
if parameters:
arguments = [
repr(k).replace('"', "\\\\'").replace("'", "\\\\'")
for k in parameters
]
argumentStr = "%s%s %s" % (os.path.basename(script), toCompile,
' '.join(arguments))
self.setConfigArgs(argumentStr)
self.__addSoftwarePackage(softwarePackage)
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ):
"""Instantiates the Workflow object and some default parameters.
"""
super( Job, self ).__init__()
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
#gConfig.getValue('Tier0SE-tape','SEName')
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo( disableVOMS = True )
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup( ret['Value']['group'] )
self.group = vo
self.site = 'ANY' #ANY
#self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
self.systemConfig = 'ANY'
self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE',
'MinCPUTime': 'other.NAME<=VALUE',
'Site': 'other.NAME=="VALUE"',
'Platform': 'other.NAME=="VALUE"',
#'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately
'BannedSites': 'other.Site!="VALUE"',
'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'}
##Add member to handle Parametric jobs
self.parametric = {}
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow( script )
def __init__(self, cpuTime=3600):
Job.__init__(self)
self.workflow = Workflow()
self.executable = '$DIRACROOT/scripts/cta-corsikasimteluser'
self.setCPUTime(cpuTime)
global argumentStr
def __init__(self, script=None, stdout='std.out', stderr='std.err'):
"""Instantiates the Workflow object and some default parameters.
"""
super(Job, self).__init__()
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo(disableVOMS=True)
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup(ret['Value']['group'])
self.group = vo
self.site = None
# self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
# $DIRACROOT is set by the JobWrapper at execution time
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
# #Add member to handle Parametric jobs
self.numberOfParameters = 0
self.parameterSeqs = {}
self.wfArguments = {}
self.parametricWFArguments = {}
# loading the function that will be used to determine the platform (it can be VO specific)
res = ObjectLoader().loadObject("ConfigurationSystem.Client.Helpers.Resources", 'getDIRACPlatforms')
if not res['OK']:
self.log.fatal(res['Message'])
self.getDIRACPlatforms = res['Value']
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow(script)
self._siteSet = set(getSites().get('Value', []))
def __init__(self, parameters=None, cpuTime=3600):
Job.__init__(self)
self.workflow = Workflow()
self.executable = '$DIRACROOT/scripts/cta-hap-wf-param'
self.setCPUTime(cpuTime)
argumentStr = "%s" % (' '.join(parameters))
self.setConfigArgs(argumentStr)
def __init__(self, cpuTime=3600):
Job.__init__(self)
self.workflow = Workflow()
self.executable = '$DIRACROOT/scripts/cta-read-cta'
# self.executable = '$DIRACROOT/CTADIRAC/Core/scripts/cta-read_cta.py'
self.setCPUTime(cpuTime)
global argumentStr
def __init__(self, cpuTime=3600):
Job.__init__(self)
self.workflow = Workflow()
self.executable = '$DIRACROOT/scripts/cta-evndispstandalone'
self.setCPUTime(cpuTime)
global parfileList
parfileList = []
global argumentStr
def __init__(self, script, parameters=None, cpuTime=3600):
Job.__init__(self)
self.workflow = Workflow()
self.executable = '$DIRACROOT/scripts/cta-hap-application'
self.setName(os.path.basename(script))
self.setCPUTime(cpuTime)
global argumentStr
argumentStr = "%s" % (' '.join(parameters))
self.setConfigArgs(argumentStr)
def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ):
"""Instantiates the Workflow object and some default parameters.
"""
super( Job, self ).__init__()
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
#gConfig.getValue('Tier0SE-tape','SEName')
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo( disableVOMS = True )
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup( ret['Value']['group'] )
self.group = vo
self.site = 'ANY' #ANY
#self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
##Add member to handle Parametric jobs
self.parametric = {}
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow( script )
def startElement(self, name, attrs):
# print name ,"startElement", "attr=", attrs.getLength(), attrs.getNames()
self.clearCharacters(
) # clear to remove empty or nonprintable characters
if name == "Workflow":
if self.root is None: # if root not defined by constractor
self.root = Workflow()
self.stack.append(self.root)
elif name == "StepDefinition":
obj = StepDefinition("TemporaryXMLObject_StepDefinition")
if self.root is None: # in case we are saving Step only
self.root = obj
self.stack.append(obj)
elif name == "StepInstance":
obj = StepInstance("TemporaryXMLObject_StepInstance")
self.stack.append(obj)
elif name == "ModuleDefinition":
obj = ModuleDefinition("TemporaryXMLObject_ModuleDefinition")
if self.root is None: # in case we are saving Module only
self.root = obj
self.stack.append(obj)
elif name == "ModuleInstance":
obj = ModuleInstance("TemporaryXMLObject_ModuleInstance")
self.stack.append(obj)
elif name == "Parameter":
obj = Parameter(str(attrs['name']), None, str(attrs['type']),
str(attrs['linked_module']),
str(attrs['linked_parameter']), str(attrs['in']),
str(attrs['out']), str(attrs['description']))
self.stack.append(obj)
# TEMPORARY CODE
# 2020-03-27: this is temporary code since 2007-06-26
elif name == "origin" or name == "version" or name == "name"\
or name == "type" or name == "value"\
or name == "required" or name == "descr_short"\
or name == "name" or name == "type" or name == "description" or name == "body":
pass
else:
print("UNTREATED! startElement name=", name, "attr=",
attrs.getLength(), attrs.getNames())
pass
def __getProductionParameters(self,
prodXMLFile,
prodID,
groupDescription='',
bkPassInfo={},
derivedProd=0,
reqID=0):
""" This method will publish production parameters.
"""
prodWorkflow = Workflow(prodXMLFile)
parameters = {}
info = []
for parameterName in ('Priority', 'CondDBTag', 'DDDBTag', 'DQTag',
'eventType', 'processingPass',
'FractionToProcess', 'MinFilesToProcess',
'configName', 'configVersion',
'outputDataFileMask', 'JobType',
'MaxNumberOfTasks'):
try:
parameters[parameterName] = prodWorkflow.findParameter(
parameterName).getValue()
info.append(
"%s: %s" %
(parameterName,
prodWorkflow.findParameter(parameterName).getValue()))
except AttributeError:
continue
parameters['SizeGroup'] = self.jobFileGroupSize
if prodWorkflow.findParameter(
'InputData'): # now only comes from BK query
prodWorkflow.findParameter('InputData').setValue('')
gLogger.verbose(
'Resetting input data for production to null, this comes from a BK query...'
)
prodXMLFile = self.__createWorkflow(prodXMLFile)
# prodWorkflow.toXMLFile(prodXMLFile)
if self.transformationFamily:
parameters['TransformationFamily'] = self.transformationFamily
if not bkPassInfo:
bkPassInfo = prodWorkflow.findParameter(
'BKProcessingPass').getValue()
if not groupDescription:
groupDescription = prodWorkflow.findParameter(
'groupDescription').getValue()
parameters['BKCondition'] = prodWorkflow.findParameter(
'conditions').getValue()
parameters['BKProcessingPass'] = bkPassInfo
parameters['groupDescription'] = groupDescription
parameters['RequestID'] = reqID
parameters['DerivedProduction'] = derivedProd
result = self.__getOutputLFNs(prodID, '99999999', prodXMLFile)
if not result['OK']:
gLogger.error('Could not create production LFNs', result)
outputLFNs = result['Value']
parameters['OutputLFNs'] = outputLFNs
# the list of output directories is later used for consistency check and for removing output data
outputDirectories = []
del outputLFNs[
'BookkeepingLFNs'] # since ProductionOutputData uses the file mask
for i in outputLFNs.values():
for j in i:
outputDir = '%s%s' % (j.split(str(prodID))[0], prodID)
if outputDir not in outputDirectories:
outputDirectories.append(outputDir)
parameters['OutputDirectories'] = outputDirectories
# Now for the steps of the workflow
stepKeys = sorted(bkPassInfo.keys())
for step in stepKeys:
info.append('====> %s %s %s' %
(bkPassInfo[step]['ApplicationName'],
bkPassInfo[step]['ApplicationVersion'], step))
info.append('%s Option Files:' %
(bkPassInfo[step]['ApplicationName']))
if bkPassInfo[step]['OptionFiles']:
for opts in bkPassInfo[step]['OptionFiles'].split(';'):
info.append('%s' % opts)
if bkPassInfo[step]['ExtraPackages']:
info.append('ExtraPackages: %s' %
(bkPassInfo[step]['ExtraPackages']))
try:
if parameters['BkQuery']:
info.append('\nBK Input Data Query:')
for bkn, bkv in parameters['BkQuery'].iteritems():
info.append('%s= %s' % (bkn, bkv))
except KeyError:
pass
# BK output directories (very useful)
bkPaths = []
bkOutputPath = '%s/%s/%s/%s/%s/%s' % (
parameters['configName'], parameters['configVersion'],
parameters['BKCondition'], parameters.get('processingPass', ''),
parameters['groupDescription'], parameters['eventType'])
fileTypes = parameters['outputDataFileMask']
fileTypes = [a.upper() for a in fileTypes.split(';')]
# Annoying that histograms are extension root
if 'ROOT' in fileTypes:
fileTypes.remove('ROOT')
fileTypes.append('HIST')
for ft in fileTypes:
bkPaths.append('%s/%s' % (bkOutputPath, ft))
parameters['BKPaths'] = bkPaths
info.append('\nBK Browsing Paths:\n%s' % ('\n'.join(bkPaths)))
infoString = '\n'.join(info)
parameters['DetailedInfo'] = infoString
gLogger.verbose('Parameters that will be added: %s' % parameters)
return parameters
