def readSpecFile(filename):
"""
_readSpecFile_
Tool for extracting multiple RequestSpecs from a file
Returns a list of spec instances found
"""
result = []
# ignore failed parsed requests
ignore = 0
node = loadIMProvFile(filename)
node = loadIMProvFile(filename)
specQ = IMProvQuery("RequestSpec")
specNodes = specQ(node)
for snode in specNodes:
newSpec = RequestSpec()
try:
newSpec.load(snode)
result.append(newSpec)
except Exception, ex:
logging.debug(
'ERROR loading a requestspec, will ignore this requestspec :' +
str(ex))
ignore += 1
def readSpecFile(filename):
"""
_readSpecFile_
Tool for extracting multiple RequestSpecs from a file
Returns a list of spec instances found
"""
result = []
# ignore failed parsed requests
ignore = 0
node = loadIMProvFile(filename)
node = loadIMProvFile(filename)
specQ = IMProvQuery("RequestSpec")
specNodes = specQ(node)
for snode in specNodes:
newSpec = RequestSpec()
try:
newSpec.load(snode)
result.append(newSpec)
except Exception,ex:
logging.debug('ERROR loading a requestspec, will ignore this requestspec :'+str(ex))
ignore+=1
def readJobReport(filename):
"""
_readJobReport_
Load an XML FwkJobReport Document into a FwkJobReport instance.
return the FwkJobReport instances.
Instantiate a new ContentHandler and run it over the file producing
a list of FwkJobReport instances
"""
#handler = FwkJobRepHandler()
#parser = make_parser()
#parser.setContentHandler(handler)
#try:
# parser.parse(filename)
#except SAXParseException, ex:
# msg = "Error parsing JobReport File: %s\n" % filename
# msg += str(ex)
# print msg
# return []
#print handler.results[0]
#return handler.results
try:
improvDoc = loadIMProvFile(filename)
except Exception, ex:
return []
def loadPayload(self, payloadFile):
"""
_loadPayload_
Load the XML input file
"""
if not os.path.exists(payloadFile):
raise RuntimeError, "Payload not found: %s" % payload
improv = loadIMProvFile(payloadFile)
workflowQ = IMProvQuery("SoupKitchen/WorkflowSpec[text()]")
workflowFile = workflowQ(improv)[-1]
logging.info("SoupKitchen: Loading Workflow: %s\n" % workflowFile)
self.workflow = self.loadWorkflow(workflowFile)
siteQ = IMProvQuery("SoupKitchen/Site[text()]")
self.siteName = str(siteQ(improv)[-1])
dbsUrlQ = IMProvQuery("SoupKitchen/DBSURL[text()]")
self.dbsUrl = str(dbsUrlQ(improv)[-1])
datasetsQ = IMProvQuery("SoupKitchen/Dataset[text()]")
datasets = datasetsQ(improv)
self.datasets = [ str(x) for x in datasets ]
msg = "Datasets to be scrambled:\n"
for d in self.datasets:
msg += " %s\n" % d
logging.info(msg)
return
def loadRelValSpec(self):
"""
_loadRelValSpec_
Load the RelVal Spec file.
Populate the list of tests
"""
improv = loadIMProvFile(self.relvalSpecFile)
testQ = IMProvQuery("/RelValSpec/RelValTest")
testNodes = testQ(improv)
for test in testNodes:
for site in self.sites:
newTest = RelValTest()
newTest.load(test)
newTest['Site'] = site
self.tests.append(newTest)
logging.info("Loaded %s tests from file:\n %s\n" % (
len(self.tests),
self.relvalSpecFile))
return
def loadFromFile(self, filename):
"""
_loadFromFile_
Load the configuration saved in the file provided
into this Config Object
"""
if not os.path.exists(filename):
msg = "Error: Attempted to load non-existent file:\n"
msg += "\t%s\n" % filename
msg += "Argument must be a valid path\n"
raise ShREEKException(msg, ClassInstance = self,
MissingFile = filename)
try:
result = loadIMProvFile(filename)
except StandardError, ex:
msg = "Error Loading Configuration File:\n"
msg += "\t%s\n" % filename
msg += "This file must be a valid XML Document\n"
msg += "Containing a saved ShREEKConfig Object\n"
msg += str(ex)
raise ShREEKException(msg, ClassInstance = self,
Filename = filename,
ExceptionInstance = ex)
def readJobReport(filename):
"""
_readJobReport_
Load an XML FwkJobReport Document into a FwkJobReport instance.
return the FwkJobReport instances.
Instantiate a new ContentHandler and run it over the file producing
a list of FwkJobReport instances
"""
#handler = FwkJobRepHandler()
#parser = make_parser()
#parser.setContentHandler(handler)
#try:
# parser.parse(filename)
#except SAXParseException, ex:
# msg = "Error parsing JobReport File: %s\n" % filename
# msg += str(ex)
# print msg
# return []
#print handler.results[0]
#return handler.results
try:
improvDoc = loadIMProvFile(filename)
except Exception, ex:
return []
def showStatus(self, filepath):
from IMProv.IMProvLoader import loadIMProvFile
from IMProv.IMProvQuery import IMProvQuery
rootag = "TaskTracking"
headtag = "TaskReport"
eventag = "Job"
gotit = loadIMProvFile(filepath)
param = IMProvQuery(eventag)
nodiparam = param(gotit)
html = "<html><body><h2>Staus of task : " + str(self.taskname) + "</h2>\n "
html += '<table cellspacing="10">\n'
st = ["Job", "Status", "Action", "Destination", "Job_exit_code", "Exe_exit_code", "Ended"]
html += "<tr>"
for s in st:
html += '<th align="left">%s</b></th>\n' % s
html += "</tr>"
for item in nodiparam:
status = item.attrs.get("status")
action = item.attrs.get("action")
site = item.attrs.get("site")
ended = item.attrs.get("ended")
jec = item.attrs.get("job_exit")
eec = item.attrs.get("exe_exit")
jobid = item.attrs.get("id")
if ended == "Y":
ended = "Yes"
elif ended == "N":
ended = "No"
## temporary workaround
if action == "Terminated":
status = "Done"
if action == "SubRequested":
status = "Submitting"
html += "<tr>"
html += "<td align='left'>%s</td>" % (jobid)
html += "<td align='left'>%s</td>" % (status)
html += "<td align='left'>%s</td>" % (action)
html += "<td align='left'>%s</td>" % (site)
if jec != "None":
html += "<td align='left'>%s</td>" % (jec)
else:
html += "<td align='left'> </td>"
if eec != "None":
html += "<td align='left'>%s</td>" % (eec)
else:
html += "<td align='left'> </td>"
html += "<td align='left'>%s</td>" % (ended)
html += "</tr>"
html += "</table>"
html += "</body>"
return html
def queryStatusXML(filename):
try:
report = loadIMProvFile(filename)
except StandardError, ex:
msg = "Error reading CRAB Status Report: %s\n" % filename
msg += str(ex)
raise RuntimeError, msg
def queryStatusXML(filename):
try:
report = loadIMProvFile(filename)
except StandardError, ex:
msg = "Error reading CRAB Status Report: %s\n" % filename
msg += str(ex)
raise RuntimeError, msg
def read(self, filename):
"""
_read_
Given the filename, read the contents and populate self
"""
node = loadIMProvFile(filename)
self.load(node)
return
def read(self, filename):
"""
_read_
Given the filename, read the contents and populate self
"""
node = loadIMProvFile(filename)
self.load(node)
return
def read(self, filename):
"""
_read_
load file -> this object
"""
node = loadIMProvFile(filename)
self.load(node)
return
def load(self, filename):
"""
_load_
Load a saved WorkflowSpec from a File
"""
node = loadIMProvFile(filename)
self.loadFromNode(node)
return
def load(self, filename):
"""
_load_
Load a saved WorkflowSpec from a File
"""
node = loadIMProvFile(filename)
self.loadFromNode(node)
return
def load(self, filename):
"""
_load_
Load a saved JobSpec object and install its information
into this instance
"""
node = loadIMProvFile(filename)
self.loadFromNode(node)
return
def load(self, filename):
"""
_load_
Load a saved JobSpec object and install its information
into this instance
"""
node = loadIMProvFile(filename)
self.loadFromNode(node)
return
def loadFromFile(self, filename):
"""
_loadFromFile_
Read the file provided to extract the configuration
"""
try:
improv = loadIMProvFile(filename)
except StandardError, ex:
msg = "Cannot read file: %s\n" % filename
msg += "Failed to load ProdCommonConfiguration\n"
raise RuntimeError, msg
def read(self):
"""
_read_
Load data from SiteLocal Config file and populate this object
"""
try:
node = loadIMProvFile(self.siteConfigFile)
except StandardError, ex:
msg = "Unable to read SiteConfigFile: %s\n" % self.siteConfigFile
msg += str(ex)
raise SiteConfigError, msg
def read(self):
"""
_read_
Load data from SiteLocal Config file and populate this object
"""
try:
node = loadIMProvFile(self.siteConfigFile)
except StandardError, ex:
msg = "Unable to read SiteConfigFile: %s\n" % self.siteConfigFile
msg += str(ex)
raise SiteConfigError, msg
def loadFromFile(self, filename):
"""
_loadFromFile_
Read data from File and populate self
"""
try:
improv = loadIMProvFile(filename)
except StandardError, ex:
msg = "Cannot read file: %s\n" % filename
msg += "Failed to load ProdAgent PluginConfiguration\n"
raise RuntimeError, msg
def queryStatusXML(filename):
try:
report = loadIMProvFile(filename)
except Exception as ex:
msg = "Error reading CRAB Status Report: %s\n" % filename
msg += str(ex)
raise RuntimeError(msg)
query = IMProvQuery("Task/TaskJobs/Job/RunningJob")
Jobs = query(report)
return Jobs
def loadFromFile(self, filename):
"""
_loadFromFile_
Read the file provided to extract the configuration
"""
try:
improv = loadIMProvFile(filename)
except StandardError, ex:
msg = "Cannot read file: %s\n" % filename
msg += "Failed to load ProdAgentConfiguration\n"
raise RuntimeError, msg
def load(self, directory):
"""
_load_
For this instance, search for a params file in the dir provided
using the workflow name in this instance, and if present, load its
settings
"""
fname = os.path.join(
directory,
"%s-Persist.xml" % self.workflowSpec.workflowName()
)
node = loadIMProvFile(fname)
qbase = "/DatasetIterator/%s" % self.workflowSpec.workflowName()
runQ = IMProvQuery("%s/Run[attribute(\"Value\")]" % qbase)
splitTQ = IMProvQuery("%s/SplitType[attribute(\"Value\")]" % qbase)
splitSQ = IMProvQuery("%s/SplitSize[attribute(\"Value\")]" % qbase)
runVal = int(runQ(node)[-1])
splitT = str(splitTQ(node)[-1])
splitS = int(splitSQ(node)[-1])
self.count = runVal
self.splitType = splitT
self.splitSize = splitS
puQ = IMProvQuery("%s/Pileup/*" % qbase)
puNodes = puQ(node)
for puNode in puNodes:
payloadNode = str(puNode.attrs.get("PayloadNode"))
puDataset = PileupDataset("dummy", 1)
puDataset.load(puNode)
self.pileupDatasets[payloadNode] = puDataset
specQ = IMProvQuery("%s/JobSpecs/*" % qbase)
specNodes = specQ(node)
for specNode in specNodes:
specId = str(specNode.attrs['ID'])
specFile = str(specNode.chardata).strip()
self.ownedJobSpecs[specId] = specFile
return
def loadRunsFile(filename):
"""
_loadRunsFile_
Load a file containing a set of run tables as XML
Returns a list of RunTable instances
"""
improvNode = loadIMProvFile(filename)
runTableQ = IMProvQuery("RunTable")
runTables = runTableQ(improvNode)
result = []
for runTable in runTables:
newTable = RunTable()
newTable.load(runTable)
result.append(newTable)
return result
def readTFC(filename):
"""
_readTFC_
Read the file provided and return a TrivialFileCatalog
instance containing the details found in it
"""
if not os.path.exists(filename):
msg = "TrivialFileCatalog not found: %s" % filename
raise RuntimeError, msg
try:
node = loadIMProvFile(filename)
except StandardError, ex:
msg = "Error reading TrivialFileCatalog: %s\n" % filename
msg += str(ex)
raise RuntimeError, msg
def load(self, xmlFile):
"""
_load_
Load the XML values into this instance
"""
content = loadIMProvFile(xmlFile)
for node in content.children:
name = str(node.name)
value = node.attrs.get("Value", None)
if value == None:
continue
try:
intVal = int(value)
except ValueError:
continue
self[name] = intVal
return
def load(self, xmlFile):
"""
_load_
Load the XML values into this instance
"""
content = loadIMProvFile(xmlFile)
for node in content.children:
name = str(node.name)
value = node.attrs.get("Value", None)
if value == None:
continue
try:
intVal = int(value)
except ValueError:
continue
self[name] = intVal
return
def showLogging(self, filepath):
from IMProv.IMProvLoader import loadIMProvFile
from IMProv.IMProvQuery import IMProvQuery
rootag = "InternalLogInfo"
keystag = "Keys"
eventag = "Event"
gotit = loadIMProvFile(filepath)
k4v = IMProvQuery(keystag)
nodiparam = k4v(gotit)
tags = []
for item in nodiparam:
tags = eval(item.attrs.get("tags"))
param = IMProvQuery(eventag)
nodiparam = param(gotit)
html = "<html><body><h2>Internal server logging for task: " + str(self.taskname) + "</h2>\n "
html += '<table cellspacing="10">\n'
textcolor = ""
for item in nodiparam:
if item.attrs.has_key("ev"):
html += '<tr><td colspan="2">Event: <b>' + item.attrs.get("ev") + "</b></td></tr>\n"
else:
html += '<tr><td colspan="2">Event: <b>Unknown</b></td></tr>\n'
for key in tags:
value = item.attrs.get(key)
if key is not "ev" and value is not None and len(str(value)) > 0:
if key in ["exc", "error"]:
textcolor = "<font color='FF0000'>%s<font>" % (value)
else:
textcolor = value
html += '<tr> <td align="right">%s: </td><td>%s</td></tr>\n' % (key, textcolor)
html += "</table>"
html += "</body>"
return html
def load(self, directory):
"""
_load_
Load this instance given the workflow and directory containing
the persistency file
"""
fname = os.path.join(
directory,
"%s-Persist.xml" % self.workflowSpec.workflowName()
)
try:
node = loadIMProvFile(fname)
except Exception, ex:
msg = "ERROR: Corrupted Persistency File:\n"
msg += " => %s\n" % fname
msg += "Cannot be read:\n => %s\n" % str(ex)
logging.error(msg)
return
def loadRunResDB(self):
"""
_loadRunResDB_
If the RunResDB file exists, load it
"""
if not os.path.exists(self.runresdb):
return
improvNode = loadIMProvFile(self.runresdb)
self._RunResDB = RunResComponent()
self._RunResDB.children = improvNode.children
self.runresLoaded = True
dbDict = self._RunResDB.toDictionary()
self.taskAttrs['Name'] = dbDict.keys()[0]
tName = self.taskAttrs['Name']
self.taskAttrs['WorkflowSpecID'] = \
dbDict[tName]['WorkflowSpecID'][0]
self.taskAttrs['JobSpecID'] = \
dbDict[tName]['JobSpecID'][0]
self.taskAttrs['JobType'] = \
dbDict[tName]['JobType'][0]
if not dbDict[tName].has_key('SizeBasedMerge'):
return
doSizeMerge = dbDict[tName]['SizeBasedMerge'].get("DoSizeMerge", [])
if len(doSizeMerge) > 0:
if str(doSizeMerge[0]).lower() == "true":
self.taskAttrs["DoSizeMerge"] = True
mergeSize = dbDict[tName]['SizeBasedMerge'].get("MinMergeFileSize", [])
if len(mergeSize) > 0:
size = int(mergeSize[0])
self.taskAttrs["MinMergeFileSize"] = size
return
def load(self, url):
"""
_load_
Read a tfc into this instance
"""
self.lfnToPfn = []
self.pfnToLfn = []
self.preferredProtocol = tfcProtocol(url)
filename = tfcFilename(url)
if not os.path.exists(filename):
msg = "TrivialFileCatalog not found: %s" % filename
raise RuntimeError, msg
try:
node = loadIMProvFile(filename)
except StandardError, ex:
msg = "Error reading TrivialFileCatalog: %s\n" % filename
msg += str(ex)
raise RuntimeError, msg
def load(self, url):
"""
_load_
Read a tfc into this instance
"""
self.lfnToPfn = []
self.pfnToLfn = []
self.preferredProtocol = tfcProtocol(url)
filename = tfcFilename(url)
if not os.path.exists(filename):
msg = "TrivialFileCatalog not found: %s" % filename
raise RuntimeError, msg
try:
node = loadIMProvFile(filename)
except StandardError, ex:
msg = "Error reading TrivialFileCatalog: %s\n" % filename
msg += str(ex)
raise RuntimeError, msg
def loadRunResDB(self):
"""
_loadRunResDB_
If the RunResDB file exists, load it
"""
if not os.path.exists(self.runresdb):
return
improvNode = loadIMProvFile(self.runresdb)
self._RunResDB = RunResComponent()
self._RunResDB.children = improvNode.children
self.runresLoaded = True
dbDict = self._RunResDB.toDictionary()
self.taskAttrs['Name'] = dbDict.keys()[0]
tName = self.taskAttrs['Name']
self.taskAttrs['WorkflowSpecID'] = \
dbDict[tName]['WorkflowSpecID'][0]
self.taskAttrs['JobSpecID'] = \
dbDict[tName]['JobSpecID'][0]
self.taskAttrs['JobType'] = \
dbDict[tName]['JobType'][0]
if not dbDict[tName].has_key('SizeBasedMerge'):
return
doSizeMerge = dbDict[tName]['SizeBasedMerge'].get("DoSizeMerge", [])
if len(doSizeMerge) > 0:
if str(doSizeMerge[0]).lower() == "true":
self.taskAttrs["DoSizeMerge"] = True
mergeSize = dbDict[tName]['SizeBasedMerge'].get("MinMergeFileSize", [])
if len(mergeSize) > 0:
size = int(mergeSize[0])
self.taskAttrs["MinMergeFileSize"] = size
return