def __init__( self, script = None, stdout = 'std.out', stderr = 'std.err' ):
"""Instantiates the Workflow object and some default parameters.
"""
self.log = gLogger
self.section = COMPONENT_NAME
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
#gConfig.getValue('Tier0SE-tape','SEName')
self.stepCount = 0
self.owner = 'NotSpecified'
self.name = 'Name'
self.type = 'User'
self.priority = 1
vo = ''
ret = getProxyInfo( disableVOMS = True )
if ret['OK'] and 'group' in ret['Value']:
vo = getVOForGroup( ret['Value']['group'] )
self.group = vo
self.site = 'ANY' #ANY
#self.setup = 'Development'
self.origin = 'DIRAC'
self.stdout = stdout
self.stderr = stderr
self.logLevel = 'info'
self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified
self.addToInputSandbox = []
self.addToOutputSandbox = []
self.addToInputData = []
self.systemConfig = 'ANY'
self.reqParams = {'MaxCPUTime': 'other.NAME>=VALUE',
'MinCPUTime': 'other.NAME<=VALUE',
'Site': 'other.NAME=="VALUE"',
'Platform': 'other.NAME=="VALUE"',
#'BannedSites': '!Member(other.Site,BannedSites)', #doesn't work unfortunately
'BannedSites': 'other.Site!="VALUE"',
'SystemConfig': 'Member("VALUE",other.CompatiblePlatforms)'}
##Add member to handle Parametric jobs
self.parametric = {}
self.script = script
if not script:
self.workflow = Workflow()
self.__setJobDefaults()
else:
self.workflow = Workflow( script )
#Global error dictionary
self.errorDict = {}
def startElement(self, name, attrs):
#print name ,"startElement", "attr=", attrs.getLength(), attrs.getNames()
self.clearCharacters() # clear to remove empty or nonprintable characters
if name == "Workflow":
if self.root == None: #if root not defined by constractor
self.root = Workflow()
self.stack.append(self.root)
elif name == "StepDefinition":
obj = StepDefinition("TemporaryXMLObject_StepDefinition")
if self.root == None: # in case we are saving Step only
self.root = obj
self.stack.append(obj)
elif name == "StepInstance":
obj = StepInstance("TemporaryXMLObject_StepInstance")
self.stack.append(obj)
elif name == "ModuleDefinition":
obj = ModuleDefinition("TemporaryXMLObject_ModuleDefinition")
if self.root == None: # in case we are saving Module only
self.root = obj
self.stack.append(obj)
elif name == "ModuleInstance":
obj = ModuleInstance("TemporaryXMLObject_ModuleInstance")
self.stack.append(obj)
elif name == "Parameter":
obj = Parameter(str(attrs['name']), None, str(attrs['type']), str(attrs['linked_module']), str(attrs['linked_parameter']), str(attrs['in']), str(attrs['out']), str(attrs['description']))
self.stack.append(obj)
# TEMPORARY CODE
elif name=="origin" or name == "version" or name == "name" or name == "type" or name == "value" or\
name == "required" or name == "descr_short" or name == "name" or name == "type" or name == "description" or name == "body":
pass
else:
print "UNTREATED! startElement name=", name, "attr=", attrs.getLength(), attrs.getNames()
pass
mbool = False
mi11.findParameter('message').setValue(mbool, 'bool')
mint = 12672
mi12.findParameter('message').setValue(mint, 'int')
mlistdict = [{
'SORTIE_@{inparam4}': 4098,
'sape_@{inpar2}': 4139
}, {
"@{inparam4}jj@{inpar2}": 234
}]
mi12.findParameter('message').setValue(mlistdict, 'list')
sd1.findParameter('result').link('mi5', 'result')
#sd1.findParameter('message').link('self','inparam4') # taken from the level of step
w1 = Workflow(name='main')
w1.setOrigin('/home/user/blablabla')
w1.setDescription("Pretty long description\n several lines of text")
w1.setDescrShort("Oooooo short description")
w1.addStep(sd1)
w1.addParameter(
Parameter("final", "0.0", "float", "", "", False, True, "Final result"))
w1.addParameter(
Parameter("debug", "False", "bool", "", "", True, False, "Debug switch"))
w1.addParameter(
Parameter("message", "vv@{inparam4}jj@{inpar2}ge", "string", "", "", True,
False, ""))
w1.addParameter(Parameter("inparam4", "VER", "string", "", "", True, False,
""))
w1.addParameter(
def execute_module(self, script_to_source, fcc_executable, fcc_conf_file,
fcc_input_files, fcc_output_file, fccsw,
number_of_events):
#define FCC Module
fcc_module = ModuleDefinition(
'ExecutionModule'
) #during constraction class creates duplicating copies of the params
fcc_module.setDescription('FCC module')
fcc_module.setBody(
'from ILCDIRAC.Workflow.Modules.fcc_execution_module import ExecutionModule\n'
)
fcc_module.addParameter(
Parameter("script_to_source", script_to_source, "string", "", "",
True, False, "The script to source"))
fcc_module.addParameter(
Parameter(
"fcc_executable", fcc_executable, "string", "", "", True,
False,
"The executable you want to use ie. FCCSW,FCCPHYSICS..."))
fcc_module.addParameter(
Parameter("fcc_conf_file", fcc_conf_file, "string", "", "", True,
False, "The configuration file you want to use."))
fcc_module.addParameter(
Parameter(
"fcc_input_files", fcc_input_files, "string", "", "", True,
False,
"The input files the job needs ie. --ifiles file1 file2 file3 ..."
))
fcc_module.addParameter(
Parameter("fcc_output_file", fcc_output_file, "string", "", "",
True, False, "The output file you want to create."))
fcc_module.addParameter(
Parameter("fccsw", fccsw, "string", "", "", True, False,
"The use of fccsw software."))
fcc_module.addParameter(
Parameter("number_of_events", number_of_events, "string", "", "",
True, False, "Number of events you want to generate"))
fcc_step = StepDefinition('FCC_App_Step')
fcc_step.addModule(
fcc_module) # Creating instance of the module 'FCC_App_Step'
fcc_module_instance = fcc_step.createModuleInstance(
'ExecutionModule', 'fcc_module')
#fcc_step.addParameterLinked(fcc_module.parameters)
fcc_workflow = Workflow(name='FCC Workflow')
fcc_workflow.setDescription('Workflow of FCC Applications')
fcc_workflow.addStep(fcc_step)
fcc_step_instance = fcc_workflow.createStepInstance(
'FCC_App_Step', 'fcc')
#fcc_step_instance.addParameter(Parameter("fcc_executable","","string","","",True,False,"first event number"))
#fcc_step_instance.setValue("fcc_executable", fcc_executable)
print fcc_workflow.createCode()
fcc_workflow.execute()
print( 'Test application execution' )
print 'Module MODULE.getDescrShort() =', self.MODULE.getDescrShort()
print 'Module MODULE_DEFINITION_NAME =', self.MODULE_DEFINITION_NAME
print 'Module MODULE_INSTANCE_NAME =', self.MODULE_INSTANCE_NAME
print 'Module STEP.getDescrShort() =', self.STEP.getDescrShort()
print 'Module STEP_DEFINITION_NAME =', self.STEP_DEFINITION_NAME
print 'Module STEP_INSTANCE_NAME =', self.STEP_INSTANCE_NAME
return 1"""
md1 = ModuleDefinition('TestAppModule')
md1.addParameter(
Parameter("enable", "True", "bool", "", "", True, False,
"If False execution is disabled"))
md1.setBody(bodyTestApp)
md1.setDescrShort('Empty Module')
md1.setDescription('Empty Module to do some testing')
md1.setRequired('')
md1.setOrigin('')
md1.setVersion(0.1)
sd1 = StepDefinition('TestAppStep')
sd1.addModule(md1)
mi1 = sd1.createModuleInstance('TestAppModule', 'testappmod1')
wf1 = Workflow('TestAppWF')
wf1.addStep(sd1)
si1 = wf1.createStepInstance('TestAppStep', 'testappstep1')
print(wf1.createCode())
#eval(compile(wf1.createCode(),'<string>','exec'))
wf1.execute()
opt_brunel = opt_brunel + ";EventLoopMgr.OutputLevel = 3"
opt_brunel = opt_brunel + ";DstWriter.Output = \"DATAFILE=\'PFN:@{outputData}\' TYP=\'POOL_ROOTTREE\' OPT=\'RECREATE\'\""
opt_brunel = opt_brunel + ";EvtTupleSvc.Output = {\"EVTTAGS2 DATAFILE=\'PFN:@{etcf}\' TYP=\'POOL_ROOTTREE\' OPT=\'RECREATE\'\"}"
#opt_brunel = opt_brunel+";EventSelector.Input = {\"COLLECTION=\'TagCreator/1\' DATAFILE=\'@{InputData}\' TYPE=\'POOL_ROOTTREE\' SEL=\'(GlobalOr>=1)\' OPT=\'READ\'\"}"
opt_brunel = opt_brunel + ";EventPersistencySvc.CnvServices += { \"LHCb::RawDataCnvSvc\" }"
opt_brunel = opt_brunel + ";ApplicationMgr.TopAlg += {\"StoreExplorerAlg\"}"
opt_brunel = opt_brunel + ";StoreExplorerAlg.Load = 1"
opt_brunel = opt_brunel + ";StoreExplorerAlg.PrintFreq = 0.99"
opt_brunel = opt_brunel + ";IODataManager.AgeLimit = 2"
step3 = StepDefinition('Job_Finalization')
step3.addModule(module6)
moduleInstance6 = step3.createModuleInstance('JobFinalization', 'module6')
############## WORKFLOW #################################
workflow1 = Workflow(name='CCRC-joel-test')
workflow1.setDescription('Workflow of GaudiApplication')
workflow1.addStep(step1)
step1_prefix = "step1_"
stepInstance1 = workflow1.createStepInstance('Gaudi_App_Step', 'Step1')
# lets link all parameters them up with the level of workflow
stepInstance1.setLink("systemConfig", "self",
"SystemConfig") # capital letter corrected
# except "STEP_ID", "appLog"
stepInstance1.setValue("appName", "DaVinci")
stepInstance1.setValue("appType", "root")
stepInstance1.setValue("outputData",
"@{PRODUCTION_ID}_@{JOB_ID}_@{STEP_NUMBER}.@{appType}")
stepInstance1.setValue("optionsFile", "DVOfficialStrippingFile.opts")
stepInstance1.setValue("optionsLine", opt_dav)
opt_brunel = opt_brunel + ';EventPersistencySvc.CnvServices += { "LHCb::RawDataCnvSvc" }'
opt_brunel = opt_brunel + ';ApplicationMgr.TopAlg += {"StoreExplorerAlg"}'
opt_brunel = opt_brunel + ";StoreExplorerAlg.Load = 1"
opt_brunel = opt_brunel + ";StoreExplorerAlg.PrintFreq = 0.99"
opt_brunel = opt_brunel + ";IODataManager.AgeLimit = 2"
step3 = StepDefinition("Job_Finalization")
step3.addModule(module6)
moduleInstance6 = step3.createModuleInstance("JobFinalization", "module6")
step3.addParameterLinked(module6.parameters)
step3.unlink([
"poolXMLCatName", "SourceData", "DataType", "CONFIG_NAME", "CONFIG_VERSION"
])
############## WORKFLOW #################################
workflow1 = Workflow(name="CCRC-strip-test")
workflow1.setDescription("Workflow of GaudiApplication")
workflow1.addStep(step1)
step1_prefix = "step1_"
stepInstance1 = workflow1.createStepInstance("Gaudi_App_Step", "Step1")
# lets link all parameters them up with the level of workflow
stepInstance1.linkUp(stepInstance1.parameters, step1_prefix)
stepInstance1.setLink("systemConfig", "self",
"SystemConfig") # capital letter corrected
stepInstance1.unlink([
"listoutput",
"STEP_ID",
"optionsFile",
"optionsLine",
"appLog",
mbool = False
mi11.findParameter("message").setValue(mbool, "bool")
mint = 12672
mi12.findParameter("message").setValue(mint, "int")
mlistdict = [{
"SORTIE_@{inparam4}": 4098,
"sape_@{inpar2}": 4139
}, {
"@{inparam4}jj@{inpar2}": 234
}]
mi12.findParameter("message").setValue(mlistdict, "list")
sd1.findParameter("result").link("mi5", "result")
# sd1.findParameter('message').link('self','inparam4') # taken from the level of step
w1 = Workflow(name="main")
w1.setOrigin("/home/user/blablabla")
w1.setDescription("Pretty long description\n several lines of text")
w1.setDescrShort("Oooooo short description")
w1.addStep(sd1)
w1.addParameter(
Parameter("final", "0.0", "float", "", "", False, True, "Final result"))
w1.addParameter(
Parameter("debug", "False", "bool", "", "", True, False, "Debug switch"))
w1.addParameter(
Parameter("message", "vv@{inparam4}jj@{inpar2}ge", "string", "", "", True,
False, ""))
w1.addParameter(Parameter("inparam4", "VER", "string", "", "", True, False,
""))
w1.addParameter(
)
# opt_brunel = opt_brunel+
# ";EventSelector.Input = {\"COLLECTION=\'TagCreator/1\' DATAFILE=\'@{InputData}\'
# TYPE=\'POOL_ROOTTREE\' SEL=\'(GlobalOr>=1)\' OPT=\'READ\'\"}"
opt_brunel = opt_brunel + ';EventPersistencySvc.CnvServices += { "LHCb::RawDataCnvSvc" }'
opt_brunel = opt_brunel + ';ApplicationMgr.TopAlg += {"StoreExplorerAlg"}'
opt_brunel = opt_brunel + ";StoreExplorerAlg.Load = 1"
opt_brunel = opt_brunel + ";StoreExplorerAlg.PrintFreq = 0.99"
opt_brunel = opt_brunel + ";IODataManager.AgeLimit = 2"
step3 = StepDefinition("Job_Finalization")
step3.addModule(module6)
moduleInstance6 = step3.createModuleInstance("JobFinalization", "module6")
############## WORKFLOW #################################
workflow1 = Workflow(name="CCRC-joel-test")
workflow1.setDescription("Workflow of GaudiApplication")
workflow1.addStep(step1)
step1_prefix = "step1_"
stepInstance1 = workflow1.createStepInstance("Gaudi_App_Step", "Step1")
# lets link all parameters them up with the level of workflow
stepInstance1.setLink("systemConfig", "self",
"SystemConfig") # capital letter corrected
# except "STEP_ID", "appLog"
stepInstance1.setValue("appName", "DaVinci")
stepInstance1.setValue("appType", "root")
stepInstance1.setValue("outputData",
"@{PRODUCTION_ID}_@{JOB_ID}_@{STEP_NUMBER}.@{appType}")
stepInstance1.setValue("optionsFile", "DVOfficialStrippingFile.opts")
stepInstance1.setValue("optionsLine", opt_dav)
#opt_brunel = opt_brunel+";EventSelector.Input = {\"COLLECTION=\'TagCreator/1\' DATAFILE=\'@{InputData}\' TYPE=\'POOL_ROOTTREE\' SEL=\'(GlobalOr>=1)\' OPT=\'READ\'\"}"
opt_brunel = opt_brunel+";EventPersistencySvc.CnvServices += { \"LHCb::RawDataCnvSvc\" }"
opt_brunel = opt_brunel+";ApplicationMgr.TopAlg += {\"StoreExplorerAlg\"}"
opt_brunel = opt_brunel+";StoreExplorerAlg.Load = 1"
opt_brunel = opt_brunel+";StoreExplorerAlg.PrintFreq = 0.99"
opt_brunel = opt_brunel+";IODataManager.AgeLimit = 2"
step3 = StepDefinition('Job_Finalization')
step3.addModule(module6)
moduleInstance6 = step3.createModuleInstance('JobFinalization','module6')
step3.addParameterLinked(module6.parameters)
step3.unlink(["poolXMLCatName", "SourceData", "DataType", "CONFIG_NAME","CONFIG_VERSION"])
############## WORKFLOW #################################
workflow1 = Workflow(name='CCRC-strip-test')
workflow1.setDescription('Workflow of GaudiApplication')
workflow1.addStep(step1)
step1_prefix="step1_"
stepInstance1 = workflow1.createStepInstance('Gaudi_App_Step', 'Step1')
# lets link all parameters them up with the level of workflow
stepInstance1.linkUp(stepInstance1.parameters, step1_prefix)
stepInstance1.setLink("systemConfig","self", "SystemConfig") # capital letter corrected
stepInstance1.unlink(["listoutput","STEP_ID", "optionsFile", "optionsLine", "appLog","appName", "etcf", "appType", "outputData", "EVENTTYPE"])
stepInstance1.setValue("appName", "DaVinci")
stepInstance1.setValue("appType", "root")
stepInstance1.setValue("outputData","@{PRODUCTION_ID}_@{JOB_ID}_@{STEP_NUMBER}.@{appType}")
stepInstance1.setValue("optionsFile", "DVOfficialStrippingFile.opts")
stepInstance1.setValue("optionsLine",opt_dav)
stepInstance1.linkUp("CONFIG_NAME")
print( 'Test application execution' )
print 'Module MODULE.getDescrShort() =', self.MODULE.getDescrShort()
print 'Module MODULE_DEFINITION_NAME =', self.MODULE_DEFINITION_NAME
print 'Module MODULE_INSTANCE_NAME =', self.MODULE_INSTANCE_NAME
print 'Module STEP.getDescrShort() =', self.STEP.getDescrShort()
print 'Module STEP_DEFINITION_NAME =', self.STEP_DEFINITION_NAME
print 'Module STEP_INSTANCE_NAME =', self.STEP_INSTANCE_NAME
return 1"""
md1 = ModuleDefinition("TestAppModule")
md1.addParameter(
Parameter("enable", "True", "bool", "", "", True, False,
"If False execution is disabled"))
md1.setBody(bodyTestApp)
md1.setDescrShort("Empty Module")
md1.setDescription("Empty Module to do some testing")
md1.setRequired("")
md1.setOrigin("")
md1.setVersion(0.1)
sd1 = StepDefinition("TestAppStep")
sd1.addModule(md1)
mi1 = sd1.createModuleInstance("TestAppModule", "testappmod1")
wf1 = Workflow("TestAppWF")
wf1.addStep(sd1)
si1 = wf1.createStepInstance("TestAppStep", "testappstep1")
print(wf1.createCode())
# eval(compile(wf1.createCode(),'<string>','exec'))
wf1.execute()
