def addStageOutNode(cmsRunNode, nodeName, *nodes):
"""
_addStageOutNode_
Given a cmsRun Node add a StageOut node to it with the name provided
"""
if not nodes:
nodes = [cmsRunNode.name]
stageOut = cmsRunNode.newNode(nodeName)
stageOut.type = "StageOut"
stageOut.application["Project"] = ""
stageOut.application["Version"] = ""
stageOut.application["Architecture"] = ""
stageOut.application["Executable"] = "RuntimeStageOut.py" # binary name
config = IMProvNode("StageOutConfiguration")
for node in nodes:
config.addNode(IMProvNode("StageOutFor", None, NodeName=str(node)))
config.addNode(IMProvNode("NumberOfRetries", None, Value=3))
config.addNode(IMProvNode("RetryPauseTime", None, Value=600))
stageOut.configuration = config.makeDOMElement().toprettyxml()
return
def addStageOutNode(cmsRunNode, nodeName, *nodes):
"""
_addStageOutNode_
Given a cmsRun Node add a StageOut node to it with the name provided
"""
if not nodes:
nodes = [cmsRunNode.name]
stageOut = cmsRunNode.newNode(nodeName)
stageOut.type = "StageOut"
stageOut.application["Project"] = ""
stageOut.application["Version"] = ""
stageOut.application["Architecture"] = ""
stageOut.application["Executable"] = "RuntimeStageOut.py" # binary name
config = IMProvNode("StageOutConfiguration")
for node in nodes:
config.addNode(IMProvNode("StageOutFor", None, NodeName = str(node)))
config.addNode(IMProvNode("NumberOfRetries", None, Value = 3))
config.addNode(IMProvNode("RetryPauseTime", None, Value = 600))
stageOut.configuration = config.makeDOMElement().toprettyxml()
return
def createLogCollectorJobSpec(workflowSpec, originalWf, site, lfnBase,
stageOutParams, *lfns):
"""
createLogCollectorJobSpec
Create a LogArchive JobSpec definition, using the LogArchive
workflow template, site name and the list of LFNs to be
removed
"""
jobSpec = workflowSpec.createJobSpec()
jobName = "%s-%s" % (workflowSpec.workflowName(), makeUUID())
jobSpec.setJobName(jobName)
jobSpec.setJobType("LogCollect")
jobSpec.addWhitelistSite(site)
confNode = IMProvNode("LogCollectorConfig")
# add site and workflow to collect
confNode.addNode(IMProvNode("wf", originalWf))
confNode.addNode(IMProvNode("se", site))
confNode.addNode(IMProvNode("lfnBase", lfnBase))
# add logs to collect
logNode = IMProvNode("LogsToCollect")
for lfn in lfns:
logNode.addNode(IMProvNode("lfn", lfn))
confNode.addNode(logNode)
# stageout
if stageOutParams:
stageOutNode = IMProvNode("Override")
# WorkflowTools.addStageOutOverride(confNode, stageOutParams['command'],
# stageOutParams['option'],
# stageOutParams['se-name'],
# stageOutParams['lfnPrefix'])
stageOutNode.addNode(IMProvNode("command", stageOutParams['command']))
stageOutNode.addNode(IMProvNode("option", stageOutParams['option']))
stageOutNode.addNode(IMProvNode("se-name", stageOutParams['se-name']))
stageOutNode.addNode(
IMProvNode("lfn-prefix", stageOutParams['lfnPrefix']))
confNode.addNode(stageOutNode)
#jobSpec.payload.configuration = logNode.makeDOMElement().toprettyxml()
jobSpec.payload.configuration = confNode.makeDOMElement().toprettyxml()
return jobSpec
def persistRuns(filename, *runTables):
"""
_persistRuns_
Util to save a list of RunTable instances to a file
"""
topNode = IMProvNode("Runs")
[ topNode.addNode(x.save()) for x in runTables ]
handle = open(filename, 'w')
handle.write(topNode.makeDOMElement().toprettyxml())
handle.close()
return
def createLogCollectorJobSpec(workflowSpec, originalWf, site, lfnBase, stageOutParams, *lfns):
"""
createLogCollectorJobSpec
Create a LogArchive JobSpec definition, using the LogArchive
workflow template, site name and the list of LFNs to be
removed
"""
jobSpec = workflowSpec.createJobSpec()
jobName = "%s-%s" % (workflowSpec.workflowName(), makeUUID())
jobSpec.setJobName(jobName)
jobSpec.setJobType("LogCollect")
jobSpec.addWhitelistSite(site)
confNode = IMProvNode("LogCollectorConfig")
# add site and workflow to collect
confNode.addNode(IMProvNode("wf", originalWf))
confNode.addNode(IMProvNode("se", site))
confNode.addNode(IMProvNode("lfnBase", lfnBase))
# add logs to collect
logNode = IMProvNode("LogsToCollect")
for lfn in lfns:
logNode.addNode(IMProvNode("lfn", lfn))
confNode.addNode(logNode)
# stageout
if stageOutParams:
stageOutNode = IMProvNode("Override")
# WorkflowTools.addStageOutOverride(confNode, stageOutParams['command'],
# stageOutParams['option'],
# stageOutParams['se-name'],
# stageOutParams['lfnPrefix'])
stageOutNode.addNode(IMProvNode("command", stageOutParams["command"]))
stageOutNode.addNode(IMProvNode("option", stageOutParams["option"]))
stageOutNode.addNode(IMProvNode("se-name", stageOutParams["se-name"]))
stageOutNode.addNode(IMProvNode("lfn-prefix", stageOutParams["lfnPrefix"]))
confNode.addNode(stageOutNode)
# jobSpec.payload.configuration = logNode.makeDOMElement().toprettyxml()
jobSpec.payload.configuration = confNode.makeDOMElement().toprettyxml()
return jobSpec
def addStageOutOverride(stageOutNode, command, option, seName, lfnPrefix):
"""
_addStageOutOverride_
Given the stageout node provided, add an Override to its configuration
attribute
"""
if len(stageOutNode.configuration.strip()) == 0:
config = IMProvNode("StageOutConfiguration")
else:
config = loadIMProvString(stageOutNode.configuration)
override = IMProvNode("Override")
override.addNode(IMProvNode("command", command))
override.addNode(IMProvNode("option", option))
override.addNode(IMProvNode("se-name", seName))
override.addNode(IMProvNode("lfn-prefix", lfnPrefix))
config.addNode(override)
stageOutNode.configuration = config.makeDOMElement().toprettyxml()
return
def addStageOutOverride(stageOutNode, command, option, seName, lfnPrefix):
"""
_addStageOutOverride_
Given the stageout node provided, add an Override to its configuration
attribute
"""
if len(stageOutNode.configuration.strip()) == 0:
config = IMProvNode("StageOutConfiguration")
else:
config = loadIMProvString(stageOutNode.configuration)
override = IMProvNode("Override")
override.addNode(IMProvNode("command", command))
override.addNode(IMProvNode("option" , option))
override.addNode(IMProvNode("se-name" , seName))
override.addNode(IMProvNode("lfn-prefix", lfnPrefix))
config.addNode(override)
stageOutNode.configuration = config.makeDOMElement().toprettyxml()
return
