def generateTFC(self):
"""
_generateTFC_
Method to generate on the fly TFC. Each LFN would be mapped to local PFN
"""
workingDir = os.path.join(os.getcwd(),'prestage')
if not os.path.exists(workingDir):
os.mkdirs(workingDir)
tfcFile = IMProvDoc ('storage-mapping')
for item in self.localFiles.keys():
if item.startswith('/'):
temp = item.split('/',1)[1]
temp = os.path.split(temp)
params = {'protocol':'local-stage-in','path-match':'/%s/(%s)' % (temp[0],temp[1]),'result':'file:%s/$1' % workingDir}
node = IMProvNode('lfn-to-pfn',None,**params)
tfcFile.addNode(node)
handle = open('%s/prestageTFC.xml' % workingDir, 'w')
handle.write (tfcFile.makeDOMDocument().toprettyxml() )
handle.close()
return
def write(self, filename):
"""
_write_
Write out an XML file containing all of the RunResDB nodes
currently in memory, removing any URL references embedded in them
to avoid loading duplicates when the file produced is read
"""
masterNode = IMProvDoc("RunResOutput")
nodelist = []
for key in self.keys():
if self[key] == None:
continue
nodelist.extend(self[key])
for node in nodelist:
if "RunResDBURL" in node.keys():
del node["RunResDBURL"]
for i in range(0, len(node.children)):
item = node.children[i]
if item.name == "RunResDBURL":
node.children.pop(i)
masterNode.addNode(node)
handle = open(filename, 'w')
handle.write(masterNode.makeDOMDocument().toprettyxml())
handle.close()
return
def save(self, filename):
"""
_save_
Write this Configuration out as an XML file to the
file provided by the argument
Args --
- *filename* : Path where file will be created. Will overwrite any
existing file
"""
# //
# // Serialise the Task Tree
#//
if self._TaskTree == None:
msg = "Error: Task Tree is empty\n"
msg += "You must provide a set of ShREEKTasks to execute!\n"
raise ShREEKException(msg, ClassInstance = self)
taskNode = IMProvNode("ShREEKTaskTree")
self.addNode(taskNode)
taskNode.addNode(self._TaskTree.makeIMProv())
# //
# // Save Config to file as XML Document
#//
doc = IMProvDoc("ShREEKConfiguration")
doc.addNode(self)
handle = open(filename, "w")
handle.write(doc.makeDOMDocument().toprettyxml())
handle.close()
return
def saveToFile(self, filename):
"""
_saveToFile_
Save this instance to the file provided
"""
doc = IMProvDoc("ProdCommonConfig")
doc.addNode(self.save())
handle = open(filename, 'w')
handle.write(doc.makeDOMDocument().toprettyxml())
handle.close()
return
def writeToFile(self, filename):
"""
_writeToFile_
Write data in this instance to file specified as XML
"""
doc = IMProvDoc("ProdAgentPluginConfig")
doc.addNode(self.save())
handle = open(filename, 'w')
handle.write(doc.makeDOMDocument().toprettyxml())
handle.close()
return
def saveToFile(self, filename):
"""
_saveToFile_
Save this instance to the file provided
"""
doc = IMProvDoc("ProdAgentConfig")
doc.addNode(self.save())
handle = open(filename, 'w')
handle.write(doc.makeDOMDocument().toprettyxml())
handle.close()
return
def writeSpecFile(filename, *specInstances):
"""
_writeSpecFile_
Util to write multiple specs to a single file
"""
doc = IMProvDoc("RequestSpecs")
for spec in specInstances:
doc.addNode(spec.save())
handle = open(filename, 'w')
handle.write(doc.makeDOMDocument().toprettyxml())
handle.close()
return
def writeSpecFile(filename, *tests):
"""
_writeSpecFile_
Gather a list of tests into a RelVal Spec file suitable for
injection into a PA
"""
doc = IMProvDoc("RelValSpec")
for test in tests:
doc.addNode(test.save())
handle = open(filename, 'w')
handle.write(doc.makeDOMDocument().toprettyxml())
handle.close()
return
def writeSpec(filename, *relValInstances):
"""
_writeSpec_
Write out the RelValidation instances provided to the filename
given. Will overwrite file if present
"""
masterDoc = IMProvDoc("ReleaseValidation")
for item in relValInstances:
masterDoc.addNode(item.save())
handle = open(filename, 'w')
handle.write(masterDoc.makeDOMDocument().toprettyxml())
handle.close()
return
def writeSpecFile(filename, *specInstances):
"""
_writeSpecFile_
Util to write multiple specs to a single file
"""
doc = IMProvDoc("RequestSpecs")
for spec in specInstances:
doc.addNode(spec.save())
handle = open(filename, 'w')
handle.write(doc.makeDOMDocument().toprettyxml())
handle.close()
return
def merge_reports(self):
reports = []
tdir = os.path.dirname(self.__jobdir)
for path in self.__rpaths:
f = gzip.open(os.path.join(os.path.dirname(tdir), path, 'report.xml.gz'))
report = readJobReport(f)
reports.extend(report)
f.close()
output = IMProvDoc("JobReports")
for item in reports:
output.addNode(item.save())
outfile = gzip.open(os.path.join(self.__jobdir, 'report.xml.gz'), 'wb')
outfile.write(output.makeDOMDocument().toprettyxml())
outfile.close()
def updateReport(reportFile, newReportInstance):
"""
_updateReport_
Given a file containing several reports: reportFile,
find the report in there whose name matches the newReportInstance's
name and replace that report with the new Report instance.
Returns a boolean: True if report name was matched and updated,
False if the report was not found and updated. (False may indicate that
the new report file needs to be merged with the main report file)
"""
if not os.path.exists(reportFile):
existingReports = []
else:
existingReports = readJobReport(reportFile)
updatedReport = False
output = IMProvDoc("JobReports")
for report in existingReports:
if report.name == newReportInstance.name:
output.addNode(newReportInstance.save())
updatedReport = True
else:
output.addNode(report.save())
handle = open(reportFile, 'w')
handle.write(output.makeDOMDocument().toprettyxml())
handle.close()
return updatedReport
def writeMainDB(self, targetFile):
"""
_writeMainDB_
Write the main RunResDB file in the location provided
so that all the components can be welded into a single RunResDB
structure
"""
db = IMProvDoc("RunResDB")
for url in self.runresFiles:
entry = IMProvNode("RunResDBURL", None, URL = url)
db.addNode(entry)
handle = open(targetFile, 'w')
handle.write(db.makeDOMDocument().toprettyxml())
handle.close()
return
def save(self):
"""
this object -> improvNode
"""
result = IMProvDoc("PileupDataset")
node = IMProvNode("Dataset", None,
Name = self.dataset,
FilesPerJob = str(self.filesPerJob))
if self.overlap:
node.attrs['Overlap'] = "True"
else:
node.attrs['Overlap'] = "False"
result.addNode(node)
for lfn in self:
node.addNode(IMProvNode("LFN", lfn))
return result
def save(self, directory):
"""
_save_
Save details of this object to the dir provided using
the basename of the workflow file
"""
doc = IMProvDoc("DatasetIterator")
node = IMProvNode(self.workflowSpec.workflowName())
doc.addNode(node)
node.addNode(IMProvNode("Run", None, Value = str(self.count)))
node.addNode(IMProvNode("SplitType", None, Value = str(self.splitType)))
node.addNode(IMProvNode("SplitSize", None, Value = str(self.splitSize)))
pu = IMProvNode("Pileup")
node.addNode(pu)
for key, value in self.pileupDatasets.items():
puNode = value.save()
puNode.attrs['PayloadNode'] = key
pu.addNode(puNode)
specs = IMProvNode("JobSpecs")
node.addNode(specs)
for key, val in self.ownedJobSpecs.items():
specs.addNode(IMProvNode("JobSpec", val, ID = key))
fname = os.path.join(
directory,
"%s-Persist.xml" % self.workflowSpec.workflowName()
)
handle = open(fname, 'w')
handle.write(doc.makeDOMDocument().toprettyxml())
handle.close()
return
def updateReport(reportFile, newReportInstance):
"""
_updateReport_
Given a file containing several reports: reportFile,
find the report in there whose name matches the newReportInstance's
name and replace that report with the new Report instance.
Returns a boolean: True if report name was matched and updated,
False if the report was not found and updated. (False may indicate that
the new report file needs to be merged with the main report file)
"""
if not os.path.exists(reportFile):
existingReports = []
else:
existingReports = readJobReport(reportFile)
updatedReport = False
output = IMProvDoc("JobReports")
for report in existingReports:
if report.name == newReportInstance.name:
output.addNode(newReportInstance.save())
updatedReport = True
else:
output.addNode(report.save())
handle = open(reportFile, 'w')
handle.write(output.makeDOMDocument().toprettyxml())
handle.close()
return updatedReport
def save(self, directory):
"""
_save_
Persist this objects state into an XML file and save
it in the directory provided
"""
doc = IMProvDoc("RequestIterator")
node = IMProvNode(self.workflowSpec.workflowName())
doc.addNode(node)
node.addNode(IMProvNode("Run", None, Value = str(self.count)))
node.addNode(
IMProvNode("EventsPerJob", None, Value = str(self.eventsPerJob))
)
node.addNode(IMProvNode("SitePref", None, Value = str(self.sitePref)))
pu = IMProvNode("Pileup")
node.addNode(pu)
for key, value in self.pileupDatasets.items():
puNode = value.save()
puNode.attrs['PayloadNode'] = key
pu.addNode(puNode)
specs = IMProvNode("JobSpecs")
node.addNode(specs)
for key, val in self.ownedJobSpecs.items():
specs.addNode(IMProvNode("JobSpec", val, ID = key))
fname = os.path.join(
directory,
"%s-Persist.xml" % self.workflowSpec.workflowName()
)
handle = open(fname, 'w')
handle.write(doc.makeDOMDocument().toprettyxml())
handle.close()
return
def mergeReports(reportFile1, reportFile2):
"""
_mergeReports_
Load job reports from both files, and combine them into a
single file.
The output will be written to the first file provided.
(IE JobReports from reportFile2 will be added to reportFile1)
If reportFile1 does not exist, a new report will be created, containing
the contents of reportFile2.
If reportFile2 does not exist, then a RuntimeError is thrown.
"""
if not os.path.exists(reportFile1):
reports1 = []
else:
reports1 = readJobReport(reportFile1)
if not os.path.exists(reportFile2):
msg = "Report file to be merged does not exist:\n"
msg += reportFile2
raise RuntimeError, msg
reports2 = readJobReport(reportFile2)
reports1.extend(reports2)
output = IMProvDoc("JobReports")
for item in reports1:
output.addNode(item.save())
handle = open(reportFile1, 'w')
handle.write(output.makeDOMDocument().toprettyxml())
handle.close()
return
def mergeReports(reportFile1, reportFile2):
"""
_mergeReports_
Load job reports from both files, and combine them into a
single file.
The output will be written to the first file provided.
(IE JobReports from reportFile2 will be added to reportFile1)
If reportFile1 does not exist, a new report will be created, containing
the contents of reportFile2.
If reportFile2 does not exist, then a RuntimeError is thrown.
"""
if not os.path.exists(reportFile1):
reports1 = []
else:
reports1 = readJobReport(reportFile1)
if not os.path.exists(reportFile2):
msg = "Report file to be merged does not exist:\n"
msg += reportFile2
raise RuntimeError, msg
reports2 = readJobReport(reportFile2)
reports1.extend(reports2)
output = IMProvDoc("JobReports")
for item in reports1:
output.addNode(item.save())
handle = open(reportFile1, 'w')
handle.write(output.makeDOMDocument().toprettyxml())
handle.close()
return
def save(self):
"""
_save_
Convert this object into an IMProvNode structure
"""
result = IMProvDoc("RequestSpec")
details = IMProvNode("RequestDetails")
for key, val in self.requestDetails.items():
details.addNode(IMProvNode(key, str(val)))
policies = IMProvNode("Policies")
for key, val in self.policies.items():
policies.addNode(IMProvNode(key, str(val)))
result.addNode(details)
result.addNode(policies)
result.addNode(self.workflow.makeIMProv())
return result
def startElement(self, name, attrs):
"""
_startElement_
Override SAX startElement handler
"""
plainAttrs = {}
for key, value in attrs.items():
plainAttrs[str(key)] = str(value)
if self._ParentDoc == None:
self._ParentDoc = IMProvDoc(str(name))
self._ParentDoc.attrs.update(plainAttrs)
self._NodeStack.append(self._ParentDoc)
return
self._CharCache = []
newnode = IMProvNode(str(name))
for key, value in attrs.items():
newnode.attrs[key] = value
self._NodeStack[-1].addNode(newnode)
self._NodeStack.append(newnode)
return
def save(self):
"""
_save_
Convert this object into an IMProvNode structure
"""
result = IMProvDoc("RequestSpec")
details = IMProvNode("RequestDetails")
for key, val in self.requestDetails.items():
details.addNode(IMProvNode(key, str(val)))
policies = IMProvNode("Policies")
for key, val in self.policies.items():
policies.addNode(IMProvNode(key, str(val)))
result.addNode(details)
result.addNode(policies)
result.addNode(self.workflow.makeIMProv())
return result
# longer a session leader, preventing the daemon from ever acquiring
# a controlling terminal.
pid = os.fork() # Fork a second child.
except OSError, e:
raise Exception, "%s [%d]" % (e.strerror, e.errno)
if (pid == 0): # The second child.
# Since the current working directory may be a mounted filesystem, we
# avoid the issue of not being able to unmount the filesystem at
# shutdown time by changing it to the root directory.
os.chdir(workdir)
# We probably don't want the file mode creation mask inherited from
# the parent, so we give the child complete control over permissions.
os.umask(UMASK)
doc = IMProvDoc("Daemon")
doc.addNode(IMProvNode("ProcessID", None, Value=os.getpid()))
doc.addNode(IMProvNode("ParentProcessID", None,
Value=os.getppid()))
doc.addNode(IMProvNode("ProcessGroupID", None, Value=os.getpgrp()))
doc.addNode(IMProvNode("UserID", None, Value=os.getuid()))
doc.addNode(IMProvNode("EffectiveUserID", None,
Value=os.geteuid()))
doc.addNode(IMProvNode("GroupID", None, Value=os.getgid()))
doc.addNode(
IMProvNode("EffectiveGroupID", None, Value=os.getegid()))
open("Daemon.xml", "w").write(doc.makeDOMDocument().toprettyxml())
print "Started Daemon: Process %s" % os.getpid()
else:
def combineReports(reportFile, reportNames, newReportInstance):
"""
Combine reports, take some fields from report with given name
in reportFile and then overwrite with newReportInstance
Note: newReportInstance is modified, and should be written back as the
task fjr - else subsequent tasks will take the wrong one!!!
"""
if not os.path.exists(reportFile):
existingReports = []
else:
existingReports = readJobReport(reportFile)
if not isinstance(reportNames, list):
reportNames = [reportNames]
reportFound = False
output = IMProvDoc("JobReports")
#wipe old values ready for new ones
newReportInstance.inputFiles = []
newReportInstance.generatorInfo = {} #how to handle multiple?
for report in existingReports:
if report.name in reportNames:
reportFound = True
# copy some values across from old report
newReportInstance.inputFiles.extend(report.inputFiles)
newReportInstance.skippedEvents.extend(report.skippedEvents)
newReportInstance.skippedFiles.extend(report.skippedFiles)
# loop over output files and change provenance to 1st node's
for outfile in newReportInstance.files:
oldinputfiles = outfile.inputFiles
outfile.inputFiles = [
] #clear ready for correct provenance info
for infile in oldinputfiles:
# find the ancestor input files in previous report
for ancestor in report.files:
if ancestor['LFN'] == infile['LFN']:
outfile.inputFiles.extend(ancestor.inputFiles)
print "Updated InputFiles %s for %s" % (
ancestor.inputFiles, outfile['LFN'])
# No LFN, use PFN (Needed for parent forwarding)
elif not infile['LFN'] and \
ancestor['PFN'] == infile['PFN']:
outfile.inputFiles.extend(ancestor.inputFiles)
print "Updated InputFiles %s for %s" % (
ancestor.inputFiles, outfile['LFN'])
if report.timing.has_key('AppStartTime') and \
report.timing['AppStartTime'] < newReportInstance.timing.get('AppStartTime', time.time()):
newReportInstance.timing['AppStartTime'] = report.timing[
'AppStartTime']
continue
# // if here either this report is not one of the inputs
# // or the report contained a staged out file
#// - in either case it must be saved
output.addNode(report.save())
if not reportFound:
raise RuntimeError, "Reports not combined: %s not found in %s" % \
(str(reportNames), reportFile)
output.addNode(newReportInstance.save())
handle = open(reportFile, 'w')
handle.write(output.makeDOMDocument().toprettyxml())
handle.close()
return newReportInstance
# longer a session leader, preventing the daemon from ever acquiring
# a controlling terminal.
pid = os.fork() # Fork a second child.
except OSError, e:
raise Exception, "%s [%d]" % (e.strerror, e.errno)
if (pid == 0): # The second child.
# Since the current working directory may be a mounted filesystem, we
# avoid the issue of not being able to unmount the filesystem at
# shutdown time by changing it to the root directory.
os.chdir(workdir)
# We probably don't want the file mode creation mask inherited from
# the parent, so we give the child complete control over permissions.
os.umask(UMASK)
doc = IMProvDoc("Daemon")
doc.addNode(IMProvNode("ProcessID", None, Value = os.getpid()))
doc.addNode(IMProvNode("ParentProcessID", None, Value = os.getppid()))
doc.addNode(IMProvNode("ProcessGroupID", None, Value = os.getpgrp()))
doc.addNode(IMProvNode("UserID", None, Value = os.getuid()))
doc.addNode(IMProvNode("EffectiveUserID", None, Value = os.geteuid()))
doc.addNode(IMProvNode("GroupID", None, Value = os.getgid()))
doc.addNode(IMProvNode("EffectiveGroupID", None,
Value = os.getegid()))
open("Daemon.xml", "w").write(doc.makeDOMDocument().toprettyxml())
print "Started Daemon: Process %s" % os.getpid()
else:
# exit() or _exit()? See below.
os._exit(0) # Exit parent (the first child) of the second child.
def combineReports(reportFile, reportNames, newReportInstance):
"""
Combine reports, take some fields from report with given name
in reportFile and then overwrite with newReportInstance
Note: newReportInstance is modified, and should be written back as the
task fjr - else subsequent tasks will take the wrong one!!!
"""
if not os.path.exists(reportFile):
existingReports = []
else:
existingReports = readJobReport(reportFile)
if not isinstance(reportNames, list):
reportNames = [reportNames]
reportFound = False
output = IMProvDoc("JobReports")
#wipe old values ready for new ones
newReportInstance.inputFiles = []
newReportInstance.generatorInfo = {} #how to handle multiple?
for report in existingReports:
if report.name in reportNames:
reportFound = True
# copy some values across from old report
newReportInstance.inputFiles.extend(report.inputFiles)
newReportInstance.skippedEvents.extend(report.skippedEvents)
newReportInstance.skippedFiles.extend(report.skippedFiles)
# loop over output files and change provenance to 1st node's
for outfile in newReportInstance.files:
oldinputfiles = outfile.inputFiles
outfile.inputFiles = [] #clear ready for correct provenance info
for infile in oldinputfiles:
# find the ancestor input files in previous report
for ancestor in report.files:
if ancestor['LFN'] == infile['LFN']:
outfile.inputFiles.extend(ancestor.inputFiles)
print "Updated InputFiles %s for %s" % (
ancestor.inputFiles, outfile['LFN'])
# No LFN, use PFN (Needed for parent forwarding)
elif not infile['LFN'] and \
ancestor['PFN'] == infile['PFN']:
outfile.inputFiles.extend(ancestor.inputFiles)
print "Updated InputFiles %s for %s" % (
ancestor.inputFiles, outfile['LFN'])
if report.timing.has_key('AppStartTime') and \
report.timing['AppStartTime'] < newReportInstance.timing.get('AppStartTime', time.time()):
newReportInstance.timing['AppStartTime'] = report.timing['AppStartTime']
continue
# // if here either this report is not one of the inputs
# // or the report contained a staged out file
#// - in either case it must be saved
output.addNode(report.save())
if not reportFound:
raise RuntimeError, "Reports not combined: %s not found in %s" % \
(str(reportNames), reportFile)
output.addNode(newReportInstance.save())
handle = open(reportFile, 'w')
handle.write(output.makeDOMDocument().toprettyxml())
handle.close()
return newReportInstance