def save(self):
"""save to improv node"""
result = IMProvNode("Run", None, ID = str(self.run))
for lumi in self:
result.addNode(IMProvNode("LumiSection", None, ID = str(lumi)))
return result
def save(self, filename):
"""
_save_
Write this Configuration out as an XML file to the
file provided by the argument
Args --
- *filename* : Path where file will be created. Will overwrite any
existing file
"""
# //
# // Serialise the Task Tree
#//
if self._TaskTree == None:
msg = "Error: Task Tree is empty\n"
msg += "You must provide a set of ShREEKTasks to execute!\n"
raise ShREEKException(msg, ClassInstance = self)
taskNode = IMProvNode("ShREEKTaskTree")
self.addNode(taskNode)
taskNode.addNode(self._TaskTree.makeIMProv())
# //
# // Save Config to file as XML Document
#//
doc = IMProvDoc("ShREEKConfiguration")
doc.addNode(self)
handle = open(filename, "w")
handle.write(doc.makeDOMDocument().toprettyxml())
handle.close()
return
def save(self):
"""
_save_
Save the tfc by converting it into
an XML IMProv Object
"""
result = IMProvNode("storage-mapping")
for maping in self.lfnToPfn:
node = IMProvNode("lfn-to-pfn", None)
node.attrs.update(maping)
del node.attrs['path-match-regexp']
if not maping.get('chain', None):
del node.attrs['chain']
result.addNode(node)
for maping in self.pfnToLfn:
node = IMProvNode("pfn-to-lfn", None)
node.attrs.update(maping)
del node.attrs['path-match-regexp']
if not maping.get('chain', None):
del node.attrs['chain']
result.addNode(node)
return result
def save(self):
"""save to improv node"""
result = IMProvNode("Run", None, ID=str(self.run))
for lumi in self:
result.addNode(IMProvNode("LumiSection", None, ID=str(lumi)))
return result
def __init__(self):
IMProvNode.__init__(self, self.__class__.__name__)
self.addNode(IMProvNode("ShREEKPlugins"))
self.addNode(IMProvNode("ShREEKMonitors"))
self.addNode(IMProvNode("ShREEKUpdators"))
self.addPluginModule("ShREEK.ShREEK_common")
self.addPluginModule("ShREEK.ControlPoints.ActionImpl")
self.addPluginModule("ShREEK.ControlPoints.CondImpl")
self._TaskTree = None
def makeIMProv(self):
"""
_makeIMProv_
Make IMProv XML Node structure from this conditional
"""
result = IMProvNode(self.name, str(self.content).strip(), **self.attrs)
for child in self._Children:
result.addNode(child.makeIMProv())
return result
def save(self):
"""
_save_
Serialise this to XML compatible with PhEDEx injection
"""
result = IMProvNode("block")
result.attrs['name'] = self.fileblockName
result.attrs['is-open'] = self.isOpen
for lfn, checksums, size in self:
# checksums is a comma separated list of key:value pair
checksum = ",".join(["%s:%s" % (x, y) for x, y \
in checksums.items() \
if y not in (None, '')])
for lfn, checksums, size in self:
# checksums is a comma separated list of key:value pair
formattedChecksums = ",".join(["%s:%s" % (x.lower(), y) for x, y \
in checksums.items() \
if y not in (None, '')])
file = IMProvNode("file")
file.attrs['name'] = lfn
file.attrs['checksum'] = formattedChecksums
file.attrs['bytes'] = size
result.addNode(file)
return result
def save(self):
"""
_save_
return IMProv structure of self
"""
result = IMProvNode("BulkSpecs")
for key, value in self.items():
result.addNode(IMProvNode("BulkSpec", value, ID=key))
return result
def save(self):
"""
_save_
Serialise self into IMProvNode structure
"""
result = IMProvNode(self.__class__.__name__)
for value in self.values():
result.addNode(value.save())
return result
def save(self):
"""
_save_
return IMProv structure of self
"""
result = IMProvNode("BulkSpecs")
for key, value in self.items():
result.addNode(IMProvNode("BulkSpec", value, ID = key))
return result
def save(self):
"""
_save_
Convert this instance into an IMProv structure
"""
result = IMProvNode("RunTable")
result.attrs['RunNumber'] = self['RunNumber']
for dataset in self.primaryDatasets.values():
result.addNode(dataset.save())
return result
def save(self):
"""self to improv"""
result = IMProvNode("Test", self["CfgUrl"],
Events = str(self['Events']),
Name = str(self['Name']),
)
if self['FactionSelected'] != None:
result.attrs['FractionSelected'] = str(self['FractionSelected'])
if self['InputDataset'] != None:
result.attrs['InputDataset'] = self['InputDataset']
return result
def save(self):
"""
_save_
convert this object to an improv Node
"""
result = IMProvNode("PrimaryDataset")
result.attrs['PrimaryName'] = self['PrimaryName']
result.attrs['RepackAlgorithm'] = self['RepackAlgorithm']
for tpath in self.triggerPaths.values():
result.addNode(tpath.save())
return result
def persistRuns(filename, *runTables):
"""
_persistRuns_
Util to save a list of RunTable instances to a file
"""
topNode = IMProvNode("Runs")
[ topNode.addNode(x.save()) for x in runTables ]
handle = open(filename, 'w')
handle.write(topNode.makeDOMElement().toprettyxml())
handle.close()
return
def dictToNode(dictRef):
results = []
for key, value in dictRef.items():
if type(value) == type({}):
keyNode = IMProvNode(str(key), None, Type = "Path")
for item in dictToNode(value):
keyNode.addNode(item)
results.append(keyNode)
if type(value) == type([]):
for item in value:
dataNode = IMProvNode(str(key), str(item), Type = "Data")
results.append(dataNode)
return results
def save(self):
"""
_save_
Save the tfc by converting it into
an XML IMProv Object
"""
result = IMProvNode("storage-mapping")
for maping in self.lfnToPfn:
node = IMProvNode("lfn-to-pfn", None)
node.attrs.update(maping)
del node.attrs['path-match-regexp']
if not maping.get('chain', None):
del node.attrs['chain']
result.addNode(node)
for maping in self.pfnToLfn:
node = IMProvNode("pfn-to-lfn", None)
node.attrs.update(maping)
del node.attrs['path-match-regexp']
if not maping.get('chain', None):
del node.attrs['chain']
result.addNode(node)
return result
def addStageOutNode(cmsRunNode, nodeName, *nodes):
"""
_addStageOutNode_
Given a cmsRun Node add a StageOut node to it with the name provided
"""
if not nodes:
nodes = [cmsRunNode.name]
stageOut = cmsRunNode.newNode(nodeName)
stageOut.type = "StageOut"
stageOut.application["Project"] = ""
stageOut.application["Version"] = ""
stageOut.application["Architecture"] = ""
stageOut.application["Executable"] = "RuntimeStageOut.py" # binary name
config = IMProvNode("StageOutConfiguration")
for node in nodes:
config.addNode(IMProvNode("StageOutFor", None, NodeName = str(node)))
config.addNode(IMProvNode("NumberOfRetries", None, Value = 3))
config.addNode(IMProvNode("RetryPauseTime", None, Value = 600))
stageOut.configuration = config.makeDOMElement().toprettyxml()
return
def save(self):
"""
_save_
Return an improvNode structure containing details
of this object so it can be saved to a file
"""
improvNode = IMProvNode(self.__class__.__name__)
for key, val in self.items():
if val == None:
continue
node = IMProvNode("Entry", str(val), Name=key)
improvNode.addNode(node)
return improvNode
def save(self):
"""
_save_
Generate an IMProvNode object to save this object to XML
"""
result = IMProvNode("ProdAgentConfiguration")
for item in self.components:
result.addNode(IMProvNode("Component", None,
Name = item))
for item in self.values():
result.addNode(item.save())
return result
def save(self):
"""
_save_
Serialise to IMProvNode
"""
result = IMProvNode("AnalysisFile")
for key, val in self.items():
if key == "FileName":
result.addNode(IMProvNode("FileName", val))
else:
result.addNode(IMProvNode(key, None, Value = str(val)))
return result
def save(self):
"""
_save_
Create IMProvNode repr of self
"""
result = IMProvNode("TriggerPath", None, Name = self['Name'])
for key, val in self.items():
if key == "Name":
continue
if val == None:
continue
result.addNode(IMProvNode(key, None, Value = str(val)))
return result
def save(self):
"""
_save_
Return an improvNode structure containing details
of this object so it can be saved to a file
"""
improvNode = IMProvNode(self.__class__.__name__)
for key, val in self.items():
if val == None:
continue
node = IMProvNode("Entry", str(val), Name = key)
improvNode.addNode(node)
return improvNode
def save(self):
"""
_save_
convert to improv node
"""
node = IMProvNode("RelValTest")
for key, val in self.items():
if key in ["Name", "JobSpecs", "BadTest"]:
continue
if val != None:
node.addNode(IMProvNode(key, None, Value = str(val)))
return node
def save(self):
"""
_save_
Generate an IMProvNode containing the configuration for this
Component
"""
result = IMProvNode("ConfigBlock", None,
Name = self.name)
if self.comment != None:
result.addNode(IMProvNode("Comment", self.comment))
for key, val in self.items():
result.addNode(IMProvNode("Parameter", None,
Name = str(key), Value = str(val))
)
return result
def save(self):
"""
this object -> improvNode
"""
result = IMProvDoc("PileupDataset")
node = IMProvNode("Dataset", None,
Name = self.dataset,
FilesPerJob = str(self.filesPerJob))
if self.overlap:
node.attrs['Overlap'] = "True"
else:
node.attrs['Overlap'] = "False"
result.addNode(node)
for lfn in self:
node.addNode(IMProvNode("LFN", lfn))
return result
def startElement(self, name, attrs):
"""
_startElement_
Override SAX startElement handler
"""
if self._ParentDoc == None:
self._ParentDoc = IMProvDoc(str(name))
self._NodeStack.append(self._ParentDoc)
return
plainAttrs = {}
self._CharCache = ""
for key, value in attrs.items():
plainAttrs[str(key)] = str(value)
newnode = IMProvNode(str(name))
for key, value in attrs.items():
newnode.attrs[key] = value
self._NodeStack[-1].addNode(newnode)
self._NodeStack.append(newnode)
return
def save(self):
"""
_save_
Convert this object into an IMProvNode structure
"""
result = IMProvDoc("RequestSpec")
details = IMProvNode("RequestDetails")
for key, val in self.requestDetails.items():
details.addNode(IMProvNode(key, str(val)))
policies = IMProvNode("Policies")
for key, val in self.policies.items():
policies.addNode(IMProvNode(key, str(val)))
result.addNode(details)
result.addNode(policies)
result.addNode(self.workflow.makeIMProv())
return result
def startElement(self, name, attrs):
"""
_startElement_
Override SAX startElement handler
"""
plainAttrs = {}
for key, value in attrs.items():
plainAttrs[str(key)] = str(value)
if self._ParentDoc == None:
self._ParentDoc = IMProvDoc(str(name))
self._ParentDoc.attrs.update(plainAttrs)
self._NodeStack.append(self._ParentDoc)
return
self._CharCache = []
newnode = IMProvNode(str(name))
for key, value in attrs.items():
newnode.attrs[key] = value
self._NodeStack[-1].addNode(newnode)
self._NodeStack.append(newnode)
return
def createLogger(self, values):
"""
_createLogger_
create the xml file with first entry
"""
dictions = {}
if type(values) == StringType:
eve = self.loadEventPkl(values)
dictions = eve.fields
else:
dictions = values
keytag = {"tags" : str(dictions.keys())}
result = IMProvNode( self.rootname )
key4report = IMProvNode( self.keystag, None, **keytag )
report = IMProvNode( self.entrname, None, **dictions)
result.addNode(key4report)
result.addNode(report)
outfile = file( self.filepath, 'w').write(str(result))
def save(self):
"""
_save_
Serialise to IMProvNode
"""
result = IMProvNode("AnalysisFile")
for key, val in self.items():
if key == "FileName":
result.addNode(IMProvNode("FileName", val))
else:
result.addNode(IMProvNode(key, None, Value=str(val)))
return result
def save(self):
"""
_save_
serialise object into PhEDEx injection XML format
"""
result = IMProvNode("data")
# hard coded as version 2. might need to change
result.attrs['version'] = '2'
dbs = IMProvNode("dbs")
dbs.attrs['name'] = self.dbs
dbs.attrs['dls'] = 'dbs'
result.addNode(dbs)
for dataset in self.datasetPaths.values():
dbs.addNode(dataset.save())
return result
def startElement(self, name, attrs):
"""
_startElement_
Override SAX startElement handler
"""
if name == "RunResDB":
self._Active = True
if not self._Active:
return
if self._ParentDoc == None:
self._ParentDoc = IMProvNode(str(name))
self._NodeStack.append(self._ParentDoc)
return
self._CharCache = ""
newnode = IMProvNode(str(name))
for key, value in attrs.items():
newnode.attrs[str(key)] = str(value)
self._NodeStack[-1].addNode(newnode)
self._NodeStack.append(newnode)
return
def save(self):
"""
_save_
Generate an IMProvNode containing the configuration for this
Component
"""
result = IMProvNode("ConfigBlock", None,
Name = self.name)
if self.comment != None:
result.addNode(IMProvNode("Comment", self.comment))
for key, val in self.items():
result.addNode(IMProvNode("Parameter", None,
Name = str(key), Value = str(val))
)
return result
def save(self):
"""
_save_
Generate an IMProvNode object to save this object to XML
"""
result = IMProvNode("ProdCommonConfiguration")
for item in self.components:
result.addNode(IMProvNode("Component", None,
Name = item))
for item in self.values():
result.addNode(item.save())
return result
def save(self):
"""
_save_
serialise object into PhEDEx injection XML format
"""
dataset = IMProvNode("dataset")
dataset.attrs['name'] = self.datasetName
dataset.attrs['is-open'] = self.datasetIsOpen
dataset.attrs['is-transient'] = self.datasetIsTransient
for block in self.fileblocks.values():
dataset.addNode(block.save())
return dataset
def save(self):
"""
_save_
serialise object into PhEDEx injection XML format
"""
dataset = IMProvNode("dataset")
dataset.attrs['name'] = self.datasetName
dataset.attrs['is-open'] = self.datasetIsOpen
dataset.attrs['is-transient'] = self.datasetIsTransient
for block in self.fileblocks.values():
dataset.addNode(block.save())
return dataset
def save(self):
"""
_save_
Serialise this to XML compatible with PhEDEx injection
"""
result = IMProvNode("block")
result.attrs['name'] = self.fileblockName
result.attrs['is-open'] = self.isOpen
for lfn, checksums, size in self:
# checksums is a comma separated list of key:value pair
checksum = ",".join(["%s:%s" % (x, y) for x, y \
in checksums.items() \
if y not in (None, '')])
result.addNode(
IMProvNode("file", None,
lfn = lfn,
checksum = checksum,
size = size)
)
return result
def save(self):
"""
_save_
Convert this object into an IMProvNode structure
"""
result = IMProvDoc("RequestSpec")
details = IMProvNode("RequestDetails")
for key, val in self.requestDetails.items():
details.addNode(IMProvNode(key, str(val)))
policies = IMProvNode("Policies")
for key, val in self.policies.items():
policies.addNode(IMProvNode(key, str(val)))
result.addNode(details)
result.addNode(policies)
result.addNode(self.workflow.makeIMProv())
return result
def addStageOutNode(cmsRunNode, nodeName, *nodes):
"""
_addStageOutNode_
Given a cmsRun Node add a StageOut node to it with the name provided
"""
if not nodes:
nodes = [cmsRunNode.name]
stageOut = cmsRunNode.newNode(nodeName)
stageOut.type = "StageOut"
stageOut.application["Project"] = ""
stageOut.application["Version"] = ""
stageOut.application["Architecture"] = ""
stageOut.application["Executable"] = "RuntimeStageOut.py" # binary name
config = IMProvNode("StageOutConfiguration")
for node in nodes:
config.addNode(IMProvNode("StageOutFor", None, NodeName=str(node)))
config.addNode(IMProvNode("NumberOfRetries", None, Value=3))
config.addNode(IMProvNode("RetryPauseTime", None, Value=600))
stageOut.configuration = config.makeDOMElement().toprettyxml()
return
def save(self):
"""
_save_
Return an improvNode structure containing details
of this object so it can be saved to a file
"""
if self.isInput == True:
improvNode = IMProvNode("InputFile")
if self.isInput == False:
improvNode = IMProvNode("File")
# //
# // General keys
#//
for key, val in self.items():
if val == None:
continue
node = IMProvNode(str(key), str(val))
improvNode.addNode(node)
# //
# // Checksums
#//
for key, val in self.checksums.items():
improvNode.addNode(IMProvNode("Checksum", val, Algorithm = key) )
# //
# // State
#//
improvNode.addNode(IMProvNode("State", None, Value = self.state))
# //
# // Inputs
#//
if not self.isInput:
inputs = IMProvNode("Inputs")
improvNode.addNode(inputs)
for inputFile in self.inputFiles:
inpNode = IMProvNode("Input")
for key, value in inputFile.items():
inpNode.addNode(IMProvNode(key, value))
inputs.addNode(inpNode)
# //
# // Runs
#//
runs = IMProvNode("Runs")
improvNode.addNode(runs)
for run in self.runs.values():
runs.addNode(run.save())
# //
# // Dataset info
#//
if not self.isInput:
datasets = IMProvNode("Datasets")
improvNode.addNode(datasets)
for datasetEntry in self.dataset:
datasets.addNode(datasetEntry.save())
# //
# // Branches
#//
branches = IMProvNode("Branches")
improvNode.addNode(branches)
for branch in self.branches:
branches.addNode(IMProvNode("Branch", branch))
return improvNode
def makeIMProv(self):
"""
_makeIMProv_
Generate IMProvNode based persistency object
"""
baseNode = PayloadNode.makeIMProv(self)
baseNode.attrs['JobName'] = self.jobName
baseNode.attrs['JobType'] = self.jobType
specNode = IMProvNode("JobSpecification")
paramsNode = IMProvNode("Parameters")
for item in self.parameters:
paramsNode.addNode(IMProvNode(item[0], str(item[1])))
outNode = IMProvNode("OutputModules")
for outMod in self.outputModules:
outNode.addNode(IMProvNode("OutputModule", str(outMod)))
specNode.addNode(paramsNode)
specNode.addNode(IMProvNode("InputSource", str(self.inputModule)))
specNode.addNode(outNode)
baseNode.addNode(specNode)
return baseNode
def addStageOutOverride(stageOutNode, command, option, seName, lfnPrefix):
"""
_addStageOutOverride_
Given the stageout node provided, add an Override to its configuration
attribute
"""
if len(stageOutNode.configuration.strip()) == 0:
config = IMProvNode("StageOutConfiguration")
else:
config = loadIMProvString(stageOutNode.configuration)
override = IMProvNode("Override")
override.addNode(IMProvNode("command", command))
override.addNode(IMProvNode("option", option))
override.addNode(IMProvNode("se-name", seName))
override.addNode(IMProvNode("lfn-prefix", lfnPrefix))
config.addNode(override)
stageOutNode.configuration = config.makeDOMElement().toprettyxml()
return
def save(self):
"""
_save_
Save the Framework Job Report by converting it into
an XML IMProv Object
"""
result = IMProvNode("FrameworkJobReport")
if self.name != None:
result.attrs['Name'] = self.name
if self.status != None:
result.attrs['Status'] = str(self.status)
if self.jobSpecId != None:
result.attrs['JobSpecID'] = self.jobSpecId
if self.workflowSpecId != None:
result.attrs['WorkflowSpecID'] = self.workflowSpecId
if self.jobType != None:
result.attrs['JobType'] = self.jobType
if self.dashboardId != None:
result.attrs['DashboardId'] = self.dashboardId
# //
# // Save ExitCode
#//
result.addNode(
IMProvNode("ExitCode",
None,
Value = str(self.exitCode)
)
)
# //
# // Save Site details
#//
for key, value in self.siteDetails.items():
siteDetail = IMProvNode("SiteDetail", None,
Parameter = key,
Value = str(value))
result.addNode(siteDetail)
# //
# // Save PSetHash
#//
if self.psetHash != None:
result.addNode(IMProvNode("PSetHash", str(self.psetHash)))
# //
# // Save Files
#//
for fileInfo in self.files:
result.addNode(fileInfo.save())
# //
# // Save Input Files
#//
for infileInfo in self.inputFiles:
result.addNode(infileInfo.save())
# //
# // Save Analysis Files
#//
for aFileInfo in self.analysisFiles:
result.addNode(aFileInfo.save())
# //
# // Save Skipped Events
#//
for skipped in self.skippedEvents:
result.addNode(IMProvNode("SkippedEvent", None,
Run = skipped['Run'],
Event = skipped['Event']))
# //
# // Save Skipped Files
#//
for skipped in self.skippedFiles:
result.addNode(IMProvNode("SkippedFile", None,
Pfn = skipped['Pfn'],
Lfn = skipped['Lfn']))
# //
# // Save Removed Files
#//
for remLfn, remSE in self.removedFiles.items():
result.addNode(IMProvNode("RemovedFile", remLfn, SEName=remSE))
# //
# // Save Unremoved Files
#//
for unremLfn, unremSE in self.unremovedFiles.items():
result.addNode(IMProvNode("UnremovedFile", unremLfn, SEName=unremSE))
# //
# // Save Log Files
#//
for logLfn, logSE in self.logFiles.items():
result.addNode(IMProvNode("LogFile", logLfn, SEName=logSE))
# //
# // Save Errors
#//
for error in self.errors:
result.addNode(
IMProvNode("FrameworkError", error['Description'],
ExitStatus = error['ExitStatus'],
Type = error['Type'])
)
# //
# // Save Timing Info
#//
timing = IMProvNode("TimingService")
result.addNode(timing)
for key, value in self.timing.items():
timing.addNode(IMProvNode(key, None, Value=str(value) ))
# //
# // Save ReadBranches Info
#//
readBranches = IMProvNode("ReadBranches")
result.addNode(readBranches)
for key, value in self.readBranches.items():
readBranches.addNode(IMProvNode("Branch", None, Name= key, ReadCount=str(value) ))
# //
# // Save Storage Statistics
#//
if self.storageStatistics != None:
result.addNode(
IMProvNode("StorageStatistics", self.storageStatistics))
genInfo = IMProvNode("GeneratorInfo")
result.addNode(genInfo)
for key, val in self.generatorInfo.items():
genInfo.addNode(IMProvNode("Data", None, Name = key,
Value = str(val)))
# //
# // Save Performance Report
#//
result.addNode(self.performance.save())
return result
def createLogCollectorJobSpec(workflowSpec, originalWf, site, lfnBase,
stageOutParams, *lfns):
"""
createLogCollectorJobSpec
Create a LogArchive JobSpec definition, using the LogArchive
workflow template, site name and the list of LFNs to be
removed
"""
jobSpec = workflowSpec.createJobSpec()
jobName = "%s-%s" % (workflowSpec.workflowName(), makeUUID())
jobSpec.setJobName(jobName)
jobSpec.setJobType("LogCollect")
jobSpec.addWhitelistSite(site)
confNode = IMProvNode("LogCollectorConfig")
# add site and workflow to collect
confNode.addNode(IMProvNode("wf", originalWf))
confNode.addNode(IMProvNode("se", site))
confNode.addNode(IMProvNode("lfnBase", lfnBase))
# add logs to collect
logNode = IMProvNode("LogsToCollect")
for lfn in lfns:
logNode.addNode(IMProvNode("lfn", lfn))
confNode.addNode(logNode)
# stageout
if stageOutParams:
stageOutNode = IMProvNode("Override")
# WorkflowTools.addStageOutOverride(confNode, stageOutParams['command'],
# stageOutParams['option'],
# stageOutParams['se-name'],
# stageOutParams['lfnPrefix'])
stageOutNode.addNode(IMProvNode("command", stageOutParams['command']))
stageOutNode.addNode(IMProvNode("option", stageOutParams['option']))
stageOutNode.addNode(IMProvNode("se-name", stageOutParams['se-name']))
stageOutNode.addNode(
IMProvNode("lfn-prefix", stageOutParams['lfnPrefix']))
confNode.addNode(stageOutNode)
#jobSpec.payload.configuration = logNode.makeDOMElement().toprettyxml()
jobSpec.payload.configuration = confNode.makeDOMElement().toprettyxml()
return jobSpec
def __init__(self, baseNodeName="IMProvDoc"):
IMProvNode.__init__(self, baseNodeName)
def save(self):
"""
_save_
Save as XML via IMProvNode
returns IMProvNode
"""
result = IMProvNode("PerformanceReport")
cpu = IMProvNode("CPU")
result.addNode(cpu)
for cpuName, cpuInfo in self.cpus.items():
cpuNode = IMProvNode("CPUCore", None, Core=cpuName)
[
cpuNode.addNode(IMProvNode("Property", y, Name=x))
for x, y in cpuInfo.items()
]
cpu.addNode(cpuNode)
mem = IMProvNode("Memory")
result.addNode(mem)
[
mem.addNode(IMProvNode("Property", y, Name=x))
for x, y in self.memory.items()
]
for metName, metSumm in self.summaries.items():
summaryNode = IMProvNode("PerformanceSummary",
None,
Metric=metName)
[
summaryNode.addNode(IMProvNode("Metric", None, Name=x,
Value=y))
for x, y in metSumm.items()
]
result.addNode(summaryNode)
for modName in self.modules.keys():
for metricName, metrics in self.modules[modName].items():
modNode = IMProvNode("PerformanceModule",
None,
Metric=metricName,
Module=modName)
[[
modNode.addNode(IMProvNode("Metric", None, Name=x,
Value=y)) for x, y in m.items()
] for m in metrics]
result.addNode(modNode)
return result
def makeIMProv(self):
"""
_makeIMProv_
Serialise the WorkflowSpec instance into an XML IMProv structure
"""
node = IMProvNode("WorkflowSpec")
for key, val in self.parameters.items():
paramNode = IMProvNode("Parameter", str(val), Name=str(key))
node.addNode(paramNode)
libs = IMProvNode("PythonLibraries")
for lib in self.pythonLibs:
libs.addNode(IMProvNode("PythonLibrary", None, Name=str(lib)))
node.addNode(libs)
payload = IMProvNode("Payload")
payload.addNode(self.payload.makeIMProv())
node.addNode(payload)
return node
def makeIMProv(self):
"""
_makeIMProv_
Serialise the WorkflowSpec instance into an XML IMProv structure
"""
node = IMProvNode("JobSpec")
for key, val in self.parameters.items():
paramNode = IMProvNode("Parameter", str(val), Name = str(key))
node.addNode(paramNode)
whitelist = IMProvNode("SiteWhitelist")
blacklist = IMProvNode("SiteBlacklist")
node.addNode(whitelist)
node.addNode(blacklist)
for site in self.siteWhitelist:
whitelist.addNode(
IMProvNode("Site", None, Name = site)
)
for site in self.siteBlacklist:
blacklist.addNode(
IMProvNode("Site", None, Name = site)
)
if len(self.associatedFiles.keys()) > 0:
assocFiles = IMProvNode("AssociatedFiles")
for key, val in self.associatedFiles.items():
assocList = IMProvNode("AssocFileList", None, Name = key)
assocFiles.addNode(assocList)
for fileEntry in val:
fileNode = IMProvNode("AssocFile", fileEntry['LFN'])
for fileAttr, fileVal in fileEntry.items():
if fileAttr == "LFN":
continue
fileNode.attrs[fileAttr] = str(fileVal)
assocList.addNode(fileNode)
node.addNode(assocFiles)
if self.isBulkSpec():
node.addNode(self.bulkSpecs.save())
payload = IMProvNode("Payload")
payload.addNode(self.payload.makeIMProv())
node.addNode(payload)
return node
def save(self):
"""
_save_
Serialise this to XML compatible with PhEDEx injection
"""
result = IMProvNode("block")
result.attrs['name'] = self.fileblockName
result.attrs['is-open'] = self.isOpen
for lfn, checksums, size in self:
# checksums is a comma separated list of key:value pair
checksum = ",".join(["%s:%s" % (x, y) for x, y \
in checksums.items() \
if y not in (None, '')])
for lfn, checksums, size in self:
# checksums is a comma separated list of key:value pair
formattedChecksums = ",".join(["%s:%s" % (x.lower(), y) for x, y \
in checksums.items() \
if y not in (None, '')])
file = IMProvNode("file")
file.attrs['name'] = lfn
file.attrs['checksum'] = formattedChecksums
file.attrs['bytes'] = size
result.addNode(file)
return result
def save(self):
"""self to improv"""
result = IMProvNode("Validate", None, Version = self.release)
for item in self:
result.addNode(item.save())
return result
def save(self):
"""
_save_
serialise object into PhEDEx injection XML format
"""
result = IMProvNode("data")
# hard coded as version 2. might need to change
result.attrs['version'] = '2'
dbs = IMProvNode("dbs")
dbs.attrs['name'] = self.dbs
dbs.attrs['dls'] = 'dbs'
result.addNode(dbs)
for dataset in self.datasetPaths.values():
dbs.addNode(dataset.save())
return result
def save(self):
result = IMProvNode("InputLink")
for key, value in self.items():
result.addNode(IMProvNode(key, None, Value=value))
return result
def makeIMProv(self):
"""
_makeIMProv_
Serialise self and children into an XML DOM friendly node structure
"""
node = IMProvNode(self.__class__.__name__,
None,
Name=str(self.name),
Type=str(self.type),
Workflow=str(self.workflow))
appNode = IMProvNode("Application")
for key, val in self.application.items():
appNode.addNode(IMProvNode(key, None, Value=val))
appConNode = IMProvNode("ApplicationControl")
for key, val in self.applicationControls.items():
if val == None:
continue
appConNode.addNode(IMProvNode(key, None, Value=val))
inputNode = IMProvNode("InputDatasets")
for inpDS in self._InputDatasets:
inputNode.addNode(inpDS.save())
outputNode = IMProvNode("OutputDatasets")
for outDS in self._OutputDatasets:
outputNode.addNode(outDS.save())
pileupNode = IMProvNode("PileupDatasets")
for puDS in self._PileupDatasets:
pileupNode.addNode(puDS.save())
inpLinksNode = IMProvNode("InputLinks")
for iLink in self._InputLinks:
inpLinksNode.addNode(iLink.save())
scriptsNode = IMProvNode("ScriptControls")
for key, scriptList in self.scriptControls.items():
scriptListNode = IMProvNode("ScriptList", None, Name=key)
[
scriptListNode.addNode(IMProvNode("Script", None, Value=x))
for x in scriptList
]
scriptsNode.addNode(scriptListNode)
if self.cfgInterface == None:
configNode = IMProvNode("Configuration",
base64.encodestring(self.configuration),
Encoding="base64")
else:
configNode = self.cfgInterface.save()
node.addNode(appNode)
node.addNode(appConNode)
node.addNode(scriptsNode)
node.addNode(inputNode)
node.addNode(outputNode)
node.addNode(pileupNode)
node.addNode(inpLinksNode)
node.addNode(configNode)
if self.userSandbox != None:
sandboxNode = IMProvNode("UserSandbox", self.userSandbox)
node.addNode(sandboxNode)
for child in self.children:
node.addNode(child.makeIMProv())
return node
# a controlling terminal.
pid = os.fork() # Fork a second child.
except OSError, e:
raise Exception, "%s [%d]" % (e.strerror, e.errno)
if (pid == 0): # The second child.
# Since the current working directory may be a mounted filesystem, we
# avoid the issue of not being able to unmount the filesystem at
# shutdown time by changing it to the root directory.
os.chdir(workdir)
# We probably don't want the file mode creation mask inherited from
# the parent, so we give the child complete control over permissions.
os.umask(UMASK)
doc = IMProvDoc("Daemon")
doc.addNode(IMProvNode("ProcessID", None, Value=os.getpid()))
doc.addNode(IMProvNode("ParentProcessID", None,
Value=os.getppid()))
doc.addNode(IMProvNode("ProcessGroupID", None, Value=os.getpgrp()))
doc.addNode(IMProvNode("UserID", None, Value=os.getuid()))
doc.addNode(IMProvNode("EffectiveUserID", None,
Value=os.geteuid()))
doc.addNode(IMProvNode("GroupID", None, Value=os.getgid()))
doc.addNode(
IMProvNode("EffectiveGroupID", None, Value=os.getegid()))
open("Daemon.xml", "w").write(doc.makeDOMDocument().toprettyxml())
print "Started Daemon: Process %s" % os.getpid()
else:
# exit() or _exit()? See below.
def save(self):
"""
_save_
Return an improvNode structure containing details
of this object so it can be saved to a file
"""
if self.isInput == True:
improvNode = IMProvNode("InputFile")
if self.isInput == False:
improvNode = IMProvNode("File")
# //
# // General keys
#//
for key, val in self.items():
if val == None:
continue
node = IMProvNode(str(key), str(val))
improvNode.addNode(node)
# //
# // Checksums
#//
for key, val in self.checksums.items():
improvNode.addNode(IMProvNode("Checksum", val, Algorithm=key))
# //
# // State
#//
improvNode.addNode(IMProvNode("State", None, Value=self.state))
# //
# // Inputs
#//
if not self.isInput:
inputs = IMProvNode("Inputs")
improvNode.addNode(inputs)
for inputFile in self.inputFiles:
inpNode = IMProvNode("Input")
for key, value in inputFile.items():
inpNode.addNode(IMProvNode(key, value))
inputs.addNode(inpNode)
# //
# // Runs
#//
runs = IMProvNode("Runs")
improvNode.addNode(runs)
for run in self.runs.values():
runs.addNode(run.save())
# //
# // Dataset info
#//
if not self.isInput:
datasets = IMProvNode("Datasets")
improvNode.addNode(datasets)
for datasetEntry in self.dataset:
datasets.addNode(datasetEntry.save())
# //
# // Branches
#//
branches = IMProvNode("Branches")
improvNode.addNode(branches)
for branch in self.branches:
branches.addNode(IMProvNode("Branch", branch))
return improvNode
