def load(self, improvNode):
"""
_load_
Extract information for this object from the improv instance provided
"""
wfQuery = IMProvQuery("/RequestSpec/WorkflowSpec")
wfnode = wfQuery(improvNode)[0]
wfspec = WorkflowSpec()
wfspec.loadFromNode(wfnode)
self.workflow = wfspec
policyQuery = IMProvQuery("/RequestSpec/Policies/*")
detailQuery = IMProvQuery("/RequestSpec/RequestDetails/*")
preferredPAQuery = IMProvQuery("/RequestSpec/PreferredPA")
policies = policyQuery(improvNode)
details = detailQuery(improvNode)
preferredPAs = preferredPAQuery(improvNode)
for policy in policies:
self.policies[str(policy.name)] = str(policy.chardata)
for detail in improvNode.attrs.keys():
self.requestDetails[detail] = str(improvNode.attrs[detail])
for preferredPA in preferredPAs:
self.preferredPAs[str(preferredPA.attrs['id'])] = \
str(preferredPA.attrs['priority'])
return
def load(self, node):
"""load into self from improv node"""
runQuery = IMProvQuery("/Run[attribute(\"ID\")]")
runInfo = runQuery(node)
if len(runInfo) == 0:
return
self.run = int(runInfo[-1])
lumiQuery = IMProvQuery("/Run/LumiSection[attribute(\"ID\")]")
lumiInfo = lumiQuery(node)
self.extend([int(x) for x in lumiInfo])
return
def legacyLumiInfo(self, improvNode, queryBase):
"""
_legacyLumiInfo_
handle legacy lumi section information
"""
# //
# // Lumi Sections
#//
lumiQ = IMProvQuery("/%s/LumiSections/LumiSection" % queryBase)
for lumiSect in lumiQ(improvNode):
newLumi = {}
[
newLumi.__setitem__(x.name, x.attrs['Value'])
for x in lumiSect.children
]
run = newLumi.get("RunNumber", None)
lumi = newLumi.get("LumiSectionNumber", None)
if run == None or lumi == None:
continue
run = int(run)
lumi = int(lumi)
if not self.runs.has_key(run):
newRun = RunInfo()
newRun.run = run
self.runs[run] = newRun
runInfo = self.runs.get(run)
runInfo.append(lumi)
return
def readSpecFile(filename):
"""
_readSpecFile_
Tool for extracting multiple RequestSpecs from a file
Returns a list of spec instances found
"""
result = []
# ignore failed parsed requests
ignore = 0
node = loadIMProvFile(filename)
node = loadIMProvFile(filename)
specQ = IMProvQuery("RequestSpec")
specNodes = specQ(node)
for snode in specNodes:
newSpec = RequestSpec()
try:
newSpec.load(snode)
result.append(newSpec)
except Exception, ex:
logging.debug(
'ERROR loading a requestspec, will ignore this requestspec :' +
str(ex))
ignore += 1
def load(self, improvNode):
"""improv to self"""
self.release = str(improvNode.attrs['Version'])
query = IMProvQuery("Validate/Test")
nodes = query(improvNode)
for node in nodes:
newTest = RelValTest()
newTest.load(node)
self.append(newTest)
return
def load(self, improvNode):
"""
_load_
Populate self based on content of improvNode instance
"""
componentQ = IMProvQuery(
"ProdCommonConfiguration/Component[attribute(\"Name\")]"
)
configQ = IMProvQuery(
"ProdCommonConfiguration/ConfigBlock"
)
components = componentQ(improvNode)
configs = configQ(improvNode)
for comp in components:
self.components.append(comp)
for config in configs:
cfgBlock = ConfigBlock("temp")
cfgBlock.load(config)
self[cfgBlock.name] = cfgBlock
return
def queryStatusXML(filename):
try:
report = loadIMProvFile(filename)
except Exception as ex:
msg = "Error reading CRAB Status Report: %s\n" % filename
msg += str(ex)
raise RuntimeError(msg)
query = IMProvQuery("Task/TaskJobs/Job/RunningJob")
Jobs = query(report)
return Jobs
def load(self, improvNode):
"""
_load_
Extract Performance reports from the improvNode provided
Uses a Relative path query
"""
cpuQ = IMProvQuery("PerformanceReport/CPU/CPUCore")
memQ = IMProvQuery("PerformanceReport/Memory/Property")
summaryQ = IMProvQuery("PerformanceReport/PerformanceSummary")
moduleQ = IMProvQuery("PerformanceReport/PerformanceModule")
for cpu in cpuQ(improvNode):
core = cpu.attrs.get("Core", None)
if core == None:
continue
core = str(core)
speed = None
desc = None
for child in cpu.children:
if child.attrs.get('Name', None) == "Description":
desc = str(child.chardata)
if child.attrs.get('Name', None) == 'Speed':
speed = str(child.chardata)
self.addCPU(core, speed, desc)
for mem in memQ(improvNode):
name = mem.attrs.get("Name", None)
if name == None: continue
self.memory[str(name)] = str(mem.chardata)
for summNode in summaryQ(improvNode):
metricName = summNode.attrs.get("Metric", None)
if metricName == None: continue
metrics = {}
for metric in IMProvQuery("Metric")(summNode):
metName = metric.attrs.get("Name", None)
metVal = metric.attrs.get("Value", None)
if metName == None: continue
metrics[str(metName)] = str(metVal)
self.addSummary(str(metricName), **metrics)
for modNode in moduleQ(improvNode):
metricName = modNode.attrs.get("Metric", None)
modName = modNode.attrs.get("Module", None)
if metricName == None: continue
if modName == None: continue
metrics = {}
for metric in IMProvQuery("Metric")(modNode):
metName = metric.attrs.get("Name", None)
metVal = metric.attrs.get("Value", None)
if metName == None: continue
metrics[str(metName)] = str(metVal)
self.addModule(str(metricName), modName, **metrics)
return
def load(self, improvNode):
dataQ = IMProvQuery("/InputLink/*")
dataNodes = dataQ(improvNode)
for node in dataNodes:
key = str(node.name)
val = str(node.attrs['Value'])
self[key] = val
if self.get("AppearStandalone", "false").lower() in ("true", "yes"):
self["AppearStandalone"] = True
else:
self["AppearStandalone"] = False
return
def load(self, improvNode):
"""
_load_
Populate self based on content of improvNode instance
"""
nameQ = IMProvQuery("/ConfigBlock[attribute(\"Name\")]")
paramQ = IMProvQuery("/ConfigBlock/Parameter")
commentQ = IMProvQuery("/ConfigBlock/Comment[text()]")
self.name = str(nameQ(improvNode)[0])
comments = commentQ(improvNode)
if len(comments) > 0:
self.comment = str(comments[0])
else:
self.comment = None
params = paramQ(improvNode)
for paramNode in params:
paramName = str(paramNode.attrs['Name'])
paramVal = str(paramNode.attrs['Value'])
self[paramName] = paramVal
return
def loadFromNode(self, improvNode):
"""
_loadFromNode_
Populate this object based on content of improvNode provided
"""
paramQ = IMProvQuery("/WorkflowSpec/Parameter")
payloadQ = IMProvQuery("/WorkflowSpec/Payload/PayloadNode")
libsQ = IMProvQuery(
"/WorkflowSpec/PythonLibraries/PythonLibrary[attribute(\"Name\")]")
# //
# // Extract Params
#//
paramNodes = paramQ(improvNode)
for item in paramNodes:
paramName = item.attrs.get("Name", None)
if paramName == None:
continue
paramValue = str(item.chardata)
self.parameters[str(paramName)] = paramValue
# //
# // Extract python lib list
#//
libNames = libsQ(improvNode)
for item in libNames:
self.addPythonLibrary(str(item))
# //
# // Extract Payload Nodes
#//
payload = payloadQ(improvNode)[0]
self.payload = PayloadNode()
self.payload.populate(payload)
return
def load(self, improvNode):
"""
_load_
Load from IMProvNode structure & populate self
"""
nodesQ = IMProvQuery("/BulkSpecs/BulkSpec")
nodes = nodesQ(improvNode)
for node in nodes:
id = node.attrs.get("ID", None)
if id == None:
continue
self[str(id)] = str(node.chardata)
return
def populate(self, improvNode):
"""
_populate_
Extract details of this node from improvNode and
instantiate and populate any children found
"""
self.unpackPayloadNodeData(improvNode)
# //
# // Recursively handle children
#//
childQ = IMProvQuery("/PayloadNode/PayloadNode")
childNodes = childQ(improvNode)
for item in childNodes:
newChild = PayloadNode()
self.addNode(newChild)
newChild.populate(item)
return
def load(self, improvNode):
"""
_load_
Populate this instance with data extracted from the improvNode
provided. The Argument should be an improvNode created with the
same structure as the result of the save method of this class
"""
if improvNode.name != self.__class__.__name__:
# //
# // Not the right node type
#//
return
entryQ = IMProvQuery("/%s/Entry" % self.__class__.__name__)
entries = entryQ(improvNode)
for entry in entries:
key = str(entry.attrs["Name"])
value = str(entry.chardata)
self[key] = value
return
def readRelValSpec(url, topNode = "ReleaseValidation"):
"""
_readRelValSpec_
Open the given URL and read the contents, parse it and return a list of
RelValidation objects
"""
handler = IMProvHandler()
parser = make_parser()
parser.setContentHandler(handler)
parser.parse(url)
mainNode = handler._ParentDoc
result = []
query = IMProvQuery("/%s/Validate" % topNode)
nodes = query(mainNode)
for node in nodes:
relval = RelValidation("")
relval.load(node)
result.append(relval)
return result
def loadFromNode(self, improvNode):
paramQ = IMProvQuery("/JobSpec/Parameter")
payloadQ = IMProvQuery("/JobSpec/Payload/JobSpecNode")
whitelistQ = IMProvQuery("/JobSpec/SiteWhitelist/Site")
blacklistQ = IMProvQuery("/JobSpec/SiteBlacklist/Site")
assocFileQ = IMProvQuery("/JobSpec/AssociatedFiles/AssocFileList")
bulkSpecQ = IMProvQuery("/JobSpec/BulkSpecs")
# //
# // Extract Params
#//
paramNodes = paramQ(improvNode)
for item in paramNodes:
paramName = item.attrs.get("Name", None)
if paramName == None:
continue
paramValue = str(item.chardata)
self.parameters[str(paramName)] = paramValue
# //
# // Extract site lists
#//
whiteNodes = whitelistQ(improvNode)
for wnode in whiteNodes:
value = wnode.attrs.get("Name", None)
if value != None:
self.siteWhitelist.append(str(value))
blackNodes = blacklistQ(improvNode)
for bnode in blackNodes:
value = bnode.attrs.get("Name", None)
if value != None:
self.siteBlacklist.append(str(value))
# //
# // Extract Associated Files
#//
assocFiles = assocFileQ(improvNode)
if len(assocFiles) > 0:
for assocFileList in assocFiles:
assocListName = str(assocFileList.attrs['Name'])
assocList = []
for aFile in assocFileList.children:
fileEntry = {
"LFN" : str(aFile.chardata),
}
for attrName, attrVal in aFile.attrs.items():
fileEntry[str(attrName)] = str(attrVal)
assocList.append(fileEntry)
self.addAssociatedFiles(assocListName, *assocList)
bulkNodes = bulkSpecQ(improvNode)
if len(bulkNodes) > 0:
self.bulkSpecs.load(bulkNodes[-1])
# //
# // Extract Payload Nodes
#//
payload = payloadQ(improvNode)[0]
self.payload = JobSpecNode()
self.payload.populate(payload)
return
def populate(self, improvNode):
"""
_populate_
Override PayloadNode.populate to load a saved JobSpecNode
"""
# //
# // Unpack base class data
#//
self.unpackPayloadNodeData(improvNode)
jobName = improvNode.attrs.get("JobName", None)
if jobName != None:
self.jobName = str(jobName)
jobType = improvNode.attrs.get("JobType", None)
if jobType != None:
self.jobType = str(jobType)
# //
# // Unpack JobSpecNode additional Data
#// Parameters
paramQ = IMProvQuery("/JobSpecNode/JobSpecification/Parameters/*")
params = paramQ(improvNode)
for paramNode in params:
key = str(paramNode.name)
value = str(paramNode.chardata)
self.addParameter(key, value)
# //
# // InputModule
#//
inpQ = IMProvQuery("/JobSpecNode/JobSpecification/InputModule")
inputModNode = inpQ(improvNode)
if len(inputModNode) > 0:
inputModNode = inputModNode[0]
self.inputModule = str(inputModNode.chardata)
# //
# // OutputModules
#//
outQ = IMProvQuery(
"/JobSpecNode/JobSpecification/OutputModules/OutputModule")
outMods = outQ(improvNode)
for outMod in outMods:
self.outputModules.append(str(outMod.chardata))
# //
# // Recursively handle children
#//
childQ = IMProvQuery("/JobSpecNode/JobSpecNode")
childNodes = childQ(improvNode)
for item in childNodes:
newChild = JobSpecNode()
self.addNode(newChild)
newChild.populate(item)
return
'Cancelled by user': '******',
'Unknown': 'U',
'Done(failed)': 'D'
}
def queryStatusXML(filename):
try:
report = loadIMProvFile(filename)
except StandardError, ex:
msg = "Error reading CRAB Status Report: %s\n" % filename
msg += str(ex)
raise RuntimeError, msg
query = IMProvQuery("Task/TaskJobs/Job/RunningJob")
Jobs = query(report)
return Jobs
def printCrabStatusFromReport(filename):
Jobs = queryStatusXML(filename)
print "Crab Id: StatusScheduler | Status | ProcessStatus | State | GridId |"
for j in Jobs:
crabId = int(j.attrs.get("jobId", None))
statusScheduler = str(j.attrs.get("statusScheduler", None))
status = str(j.attrs.get("status", None))
processStatus = str(j.attrs.get("processStatus", None))
state = str(j.attrs.get("state", None))
gridId = str(j.attrs.get("schedulerId", None))
instance containing the details found in it
"""
if not os.path.exists(filename):
msg = "TrivialFileCatalog not found: %s" % filename
raise RuntimeError, msg
try:
node = loadIMProvFile(filename)
except StandardError, ex:
msg = "Error reading TrivialFileCatalog: %s\n" % filename
msg += str(ex)
raise RuntimeError, msg
query = IMProvQuery("storage-mapping/lfn-to-pfn")
mappings = query(node)
tfcInstance = TrivialFileCatalog()
for mapping in mappings:
protocol = mapping.attrs.get("protocol", None)
match = mapping.attrs.get("path-match", None)
result = mapping.attrs.get("result", None)
chain = mapping.attrs.get("chain", None)
if True in (protocol, match, mapping == None):
continue
tfcInstance.addMapping(str(protocol), str(match), str(result), chain)
return tfcInstance
def unpackPayloadNodeData(self, improvNode):
"""
_unpackPayloadNodeData_
Unpack PayloadNode data from improv Node provided and
add information to self
"""
self.name = str(improvNode.attrs["Name"])
self.type = str(improvNode.attrs["Type"])
workflowName = improvNode.attrs.get('Workflow', None)
if workflowName != None:
self.workflow = str(workflowName)
# //
# // Unpack data for this instance
#// App details
appDataQ = IMProvQuery("/%s/Application" % self.__class__.__name__)
appData = appDataQ(improvNode)[0]
for appField in appData.children:
field = str(appField.name)
value = str(appField.attrs['Value'])
self.application[field] = value
# //
# // App Control details
#//
appConDataQ = IMProvQuery("/%s/ApplicationControl/*" %
self.__class__.__name__)
appConData = appConDataQ(improvNode)
for appConField in appConData:
field = str(appConField.name)
value = str(appConField.attrs['Value'])
self.applicationControls[field] = value
# //
# // Script Controls
#//
scriptConQ = IMProvQuery("/%s/ScriptControls/ScriptList" %
self.__class__.__name__)
scriptLists = scriptConQ(improvNode)
for scriptList in scriptLists:
listName = scriptList.attrs.get("Name", None)
if listName == None: continue
listName = str(listName)
for script in scriptList.children:
scriptName = script.attrs.get("Value", None)
if scriptName == None: continue
self.scriptControls[listName].append(str(scriptName))
# //
# // Dataset details
#// Input Datasets
inputDSQ = IMProvQuery("/%s/InputDatasets/DatasetInfo" %
self.__class__.__name__)
inputDS = inputDSQ(improvNode)
# print improvNode
for item in inputDS:
newDS = DatasetInfo()
newDS.load(item)
self._InputDatasets.append(newDS)
# //
# // Output Datasets
#//
outputDSQ = IMProvQuery("/%s/OutputDatasets/DatasetInfo" %
self.__class__.__name__)
outputDS = outputDSQ(improvNode)
for item in outputDS:
newDS = DatasetInfo()
newDS.load(item)
self._OutputDatasets.append(newDS)
# //
# // Pileup Datasets
#//
pileupDSQ = IMProvQuery("/%s/PileupDatasets/DatasetInfo" %
self.__class__.__name__)
pileupDS = pileupDSQ(improvNode)
for item in pileupDS:
newDS = DatasetInfo()
newDS.load(item)
self._PileupDatasets.append(newDS)
# //
# // Input Links
#//
inpLinkQ = IMProvQuery("/%s/InputLinks/InputLink" %
self.__class__.__name__)
inpLinks = inpLinkQ(improvNode)
for ilink in inpLinks:
newLink = InputLink()
newLink.load(ilink)
self._InputLinks.append(newLink)
# //
# // Configuration
#//
configQ = IMProvQuery("/%s/Configuration" % self.__class__.__name__)
configNodes = configQ(improvNode)
if len(configNodes) > 0:
configNode = configNodes[0]
self.configuration = base64.decodestring(str(configNode.chardata))
cfgIntQ = IMProvQuery("/%s/CMSSWConfig" % self.__class__.__name__)
cfgNodes = cfgIntQ(improvNode)
if len(cfgNodes) > 0:
cfgNode = cfgNodes[0]
self.cfgInterface = CMSSWConfig()
self.cfgInterface.load(cfgNode)
# //
# // User sandbox
#//
sandboxQ = IMProvQuery("/%s/UserSandbox" % self.__class__.__name__)
sandboxNodes = sandboxQ(improvNode)
if len(sandboxNodes) > 0:
sandboxNode = sandboxNodes[-1]
self.userSandbox = str(sandboxNode.chardata)
return
class TrivialFileCatalog:
"""
_FwkJobReport_
Framework Job Report container and interface object
"""
def __init__(self, url=None):
self.preferredProtocol = None # attribute for preferred protocol
self.lfnToPfn = []
self.pfnToLfn = []
if url:
self.load(url)
def addLfnToPfnRule(self, protocol, pathMatch, result, chain=None):
"""
Add a LFN-To-PFN mapping rule that will match for the given
protocol and destination
"""
self.lfnToPfn.append({
"protocol": protocol,
"path-match": pathMatch,
"path-match-regexp": re.compile(pathMatch),
"chain": chain,
"result": result
})
return
def matchLFN(self, protocol, lfn):
"""
_matchLFN_
Return the result for the LFN provided if the LFN
matches the path-match for that protocol
Return None if no match
"""
if not protocol:
protocol = self.preferredProtocol
for mapping in self.lfnToPfn:
if mapping['protocol'] != protocol:
continue
if mapping['path-match-regexp'].match(lfn):
if mapping['chain'] != None:
lfn = self.matchLFN(mapping['chain'], lfn)
try:
splitLFN = mapping['path-match-regexp'].split(lfn, 1)[1]
except IndexError:
continue
result = mapping['result'].replace("$1", splitLFN)
return result
return None
def addPfnToLfnRule(self, protocol, pathMatch, result, chain=None):
"""
Add a PFN-To-LFN mapping rule that will match for the given
protocol and destination
"""
self.pfnToLfn.append({
"protocol": protocol,
"path-match": pathMatch,
"path-match-regexp": re.compile(pathMatch),
"chain": chain,
"result": result
})
return
def matchPFN(self, protocol, pfn):
"""
_matchPFN_
Return the result for the PFN provided if the LFN
matches the path-match for that protocol
Return None if no match
"""
if not protocol:
protocol = self.preferredProtocol
for mapping in self.pfnToLfn:
if mapping['protocol'] != protocol:
continue
if mapping['path-match-regexp'].match(pfn):
if mapping['chain'] != None:
pfn = self.matchPFN(mapping['chain'], pfn)
try:
splitPFN = mapping['path-match-regexp'].split(pfn, 1)[1]
except IndexError:
continue
result = mapping['result'].replace("$1", splitPFN)
return result
return None
def save(self):
"""
_save_
Save the tfc by converting it into
an XML IMProv Object
"""
result = IMProvNode("storage-mapping")
for maping in self.lfnToPfn:
node = IMProvNode("lfn-to-pfn", None)
node.attrs.update(maping)
del node.attrs['path-match-regexp']
if not maping.get('chain', None):
del node.attrs['chain']
result.addNode(node)
for maping in self.pfnToLfn:
node = IMProvNode("pfn-to-lfn", None)
node.attrs.update(maping)
del node.attrs['path-match-regexp']
if not maping.get('chain', None):
del node.attrs['chain']
result.addNode(node)
return result
def write(self, filename):
"""
_write_
Write the tfc to an XML file
"""
handle = open(filename, 'w')
handle.write(self.save().makeDOMElement().toprettyxml())
handle.close()
return
def __str__(self):
"""string representation of instance"""
return str(self.save())
def load(self, url):
"""
_load_
Read a tfc into this instance
"""
self.lfnToPfn = []
self.pfnToLfn = []
self.preferredProtocol = tfcProtocol(url)
filename = tfcFilename(url)
if not os.path.exists(filename):
msg = "TrivialFileCatalog not found: %s" % filename
raise RuntimeError, msg
try:
node = loadIMProvFile(filename)
except StandardError, ex:
msg = "Error reading TrivialFileCatalog: %s\n" % filename
msg += str(ex)
raise RuntimeError, msg
query = IMProvQuery("storage-mapping/lfn-to-pfn")
mappings = query(node)
for mapping in mappings:
protocol = mapping.attrs.get("protocol", None)
match = mapping.attrs.get("path-match", None)
result = mapping.attrs.get("result", None)
chain = mapping.attrs.get("chain", None)
if True in (protocol, match, mapping == None):
continue
self.addLfnToPfnRule(str(protocol), str(match), \
str(result), chain)
query = IMProvQuery("storage-mapping/pfn-to-lfn")
mappings = query(node)
for mapping in mappings:
protocol = mapping.attrs.get("protocol", None)
match = mapping.attrs.get("path-match", None)
result = mapping.attrs.get("result", None)
chain = mapping.attrs.get("chain", None)
if True in (protocol, match, mapping == None):
continue
self.addPfnToLfnRule(str(protocol), str(match), \
str(result), chain)
return
def load(self, improvNode):
"""
_load_
Populate this object from the improvNode provided
"""
# //
# // Input or Output?
#//
queryBase = improvNode.name
if queryBase == "InputFile":
self.isInput = True
else:
self.isInput = False
# //
# // Parameters
#//
paramQ = IMProvQuery("/%s/*" % queryBase)
for paramNode in paramQ(improvNode):
if paramNode.name not in self.keys():
continue
self[paramNode.name] = paramNode.chardata
# //
# // State
#//
stateQ = IMProvQuery("/%s/State[attribute(\"Value\")]" % queryBase)
self.state = stateQ(improvNode)[-1]
# //
# // Checksums
#//
cksumQ = IMProvQuery("/%s/Checksum" % queryBase)
for cksum in cksumQ(improvNode):
algo = cksum.attrs.get('Algorithm', None)
if algo == None: continue
self.addChecksum(str(algo), str(cksum.chardata))
# //
# // Inputs
#//
inputFileQ = IMProvQuery("/%s/Inputs/Input" % queryBase)
for inputFile in inputFileQ(improvNode):
lfn = IMProvQuery("/Input/LFN[text()]")(inputFile)[-1]
pfn = IMProvQuery("/Input/PFN[text()]")(inputFile)[-1]
self.addInputFile(pfn, lfn)
# //
# // Datasets
#//
datasetQ = IMProvQuery("/%s/Datasets/DatasetInfo" % queryBase)
for dataset in datasetQ(improvNode):
newDataset = self.newDataset()
newDataset.load(dataset)
# //
# // Branches
#//
branchQ = IMProvQuery("/%s/Branches/Branch[text()]" % queryBase)
for branch in branchQ(improvNode):
self.branches.append(str(branch))
runQ = IMProvQuery("/%s/Runs/Run" % queryBase)
for run in runQ(improvNode):
newRun = RunInfo()
newRun.load(run)
if newRun.run == None:
continue
self.runs[newRun.run] = newRun
self["Runs"] = self.runs
self.legacyLumiInfo(improvNode, queryBase)
return
def handleStorageStats(improvNode, reportRef):
"""
_handleStorageStats_
Look for the detailed storage stats information and
collapse it to a set of performance report instances
"""
# //
# // Backwards compatibility
#//
if len(improvNode.chardata.strip()) > 0:
reportRef.storageStatistics = str(improvNode.chardata.strip())
return
# //
# // new format report
#//
base = "/StorageStatistics/storage-factory-summary/"
statsBase = "%sstorage-factory-stats" % base
statsQ = IMProvQuery("%s/storage-timing-summary/counter-value" % statsBase)
rootQ = IMProvQuery("%s/storage-root-summary/counter-value" % statsBase)
paramBase = "%sstorage-factory-params/param" % base
paramQ = IMProvQuery(paramBase)
allStats = {}
for stats in statsQ(improvNode):
subsystem = stats.attrs.get("subsystem", None)
if subsystem == None:
continue
subsystem = str(subsystem)
counterName = stats.attrs.get("counter-name", None)
if counterName == None:
continue
counterName = str(counterName)
keyName = "%s-%s" % (subsystem, counterName)
for statKey, statVal in stats.attrs.items():
statKey = str(statKey)
statVal = str(statVal)
if statKey in ("subsystem", "counter-name"):
continue
dataKey = "%s-%s" % (keyName, statKey)
allStats[dataKey] = statVal
#print allStats
allRoots = {}
for root in rootQ(improvNode):
subsystem = root.attrs.get("subsystem", None)
if subsystem == None:
continue
subsystem = str(subsystem)
counterName = root.attrs.get("counter-name", None)
if counterName == None:
continue
counterName = str(counterName)
keyName = "%s-%s" % (subsystem, counterName)
for rootKey, rootVal in root.attrs.items():
rootKey = str(rootKey)
rootVal = str(rootVal)
if rootKey in ("subsystem", "counter-name"):
continue
dataKey = "%s-%s" % (keyName, rootKey)
allRoots[dataKey] = rootVal
allParams = {}
for param in paramQ(improvNode):
paramName = param.attrs.get("name", None)
if paramName == None:
continue
paramName = str(paramName)
paramVal = param.attrs.get("value", None)
if paramVal == None:
continue
allParams[paramName] = str(paramVal)
reportRef.performance.addSummary("StorageTiming", **allStats)
reportRef.performance.addSummary("StorageRoot", **allRoots)
reportRef.performance.addSummary("StorageParams", **allParams)
return
def load(self, improvNode):
"""
_load_
Unpack improvNode into this instance
"""
self.name = improvNode.attrs.get("Name", None)
self.status = improvNode.attrs.get("Status", None)
self.jobSpecId = improvNode.attrs.get("JobSpecID", None)
self.jobType = improvNode.attrs.get("JobType", None)
self.workflowSpecId =improvNode.attrs.get("WorkflowSpecID", None)
self.dashboardId =improvNode.attrs.get("DashboardId", None)
exitQ = IMProvQuery(
"/FrameworkJobReport/ExitCode[attribute(\"Value\")]")
exitVals = exitQ(improvNode)
if len(exitVals) > 0:
self.exitCode = int(exitVals[-1])
# //
# // Site details
#//
siteQ = IMProvQuery("/FrameworkJobReport/SiteDetail")
[ self.siteDetails.__setitem__(x.attrs['Parameter'], x.attrs['Value'])
for x in siteQ(improvNode) ]
# //
# // PSetHash
#//
hashQ = IMProvQuery("/FrameworkJobReport/PSetHash[text()]")
hashVals = hashQ(improvNode)
if len(hashVals) > 0:
hashData = hashVals[-1]
self.psetHash = str(hashData)
# //
# // output files
#//
fileQ = IMProvQuery("/FrameworkJobReport/File")
fileList = fileQ(improvNode)
for fileEntry in fileList:
newFile = self.newFile()
newFile.load(fileEntry)
# //
# // input files
#//
infileQ = IMProvQuery("/FrameworkJobReport/InputFile")
for infileEntry in infileQ(improvNode):
newInFile = self.newInputFile()
newInFile.load(infileEntry)
# //
# // Skipped Events & Files
#//
skipFileQ = IMProvQuery("/FrameworkJobReport/SkippedFile")
skipEventQ = IMProvQuery("/FrameworkJobReport/SkippedEvent")
[ self.addSkippedEvent(int(skipEv.attrs['Run']),
int(skipEv.attrs['Event']))
for skipEv in skipEventQ(improvNode)]
[ self.addSkippedFile(skipF.attrs['Pfn'], skipF.attrs['Lfn'])
for skipF in skipFileQ(improvNode) ]
# //
# // removed files
#//
remFileQ = IMProvQuery("/FrameworkJobReport/RemovedFile")
[ self.addRemovedFile(str(remF.chardata), remF.attrs['SEName'])
for remF in remFileQ(improvNode) ]
# //
# // unremoved files
#//
unremFileQ = IMProvQuery("/FrameworkJobReport/UnremovedFile")
[ self.addUnremovedFile(str(remF.chardata), remF.attrs['SEName'])
for remF in unremFileQ(improvNode) ]
# //
# // analysis files
#//
afileQ = IMProvQuery("/FrameworkJobReport/AnalysisFile")
for afileEntry in afileQ(improvNode):
newAFile = self.newAnalysisFile()
newAFile.load(afileEntry)
# //
# // log files
#//
logFileQ = IMProvQuery("/FrameworkJobReport/LogFile")
[ self.addLogFile(str(logF.chardata), logF.attrs['SEName'])
for logF in logFileQ(improvNode) ]
# //
# // Timing, Storage and generator info
#//
timingQ = IMProvQuery("/FrameworkJobReport/TimingService/*")
[ self.timing.__setitem__(x.name, x.attrs['Value'])
for x in timingQ(improvNode) ]
# //
# // ReadBranches tag from cmsRun
#//
readBranchesQ = IMProvQuery("/FrameworkJobReport/ReadBranches/*")
[ self.readBranches.__setitem__(x.attrs['Name'], x.attrs['ReadCount'])
for x in readBranchesQ(improvNode) ]
storageQ = IMProvQuery("/FrameworkJobReport/StorageStatistics")
storageInfo = storageQ(improvNode)
if len(storageInfo) > 0:
#self.storageStatistics = storageInfo[-1]
StorageStats.handleStorageStats(storageInfo[-1], self)
genQ = IMProvQuery("/FrameworkJobReport/GeneratorInfo/Data")
[ self.generatorInfo.__setitem__(x.attrs['Name'], x.attrs['Value'])
for x in genQ(improvNode)]
errQ = IMProvQuery("/FrameworkJobReport/FrameworkError")
errors = errQ(improvNode)
for err in errors:
newErr = self.addError(
int(err.attrs['ExitStatus']),
err.attrs['Type'])
newErr['Description'] = err.chardata
# //
# // Performance reports
#//
self.performance.load(improvNode)
return
Instantiate a new ContentHandler and run it over the file producing
a list of FwkJobReport instances
"""
#handler = FwkJobRepHandler()
#parser = make_parser()
#parser.setContentHandler(handler)
#try:
# parser.parse(filename)
#except SAXParseException, ex:
# msg = "Error parsing JobReport File: %s\n" % filename
# msg += str(ex)
# print msg
# return []
#print handler.results[0]
#return handler.results
try:
improvDoc = loadIMProvFile(filename)
except Exception, ex:
return []
result = []
reportQ = IMProvQuery("FrameworkJobReport")
reportInstances = reportQ(improvDoc)
for reportInstance in reportInstances:
newReport = FwkJobReport()
newReport.load(reportInstance)
result.append(newReport)
return result
_read_
Load data from SiteLocal Config file and populate this object
"""
try:
node = loadIMProvFile(self.siteConfigFile)
except StandardError, ex:
msg = "Unable to read SiteConfigFile: %s\n" % self.siteConfigFile
msg += str(ex)
raise SiteConfigError, msg
# //
# // site name
#//
nameQ = IMProvQuery("/site-local-config/site")
nameNodes = nameQ(node)
if len(nameNodes) == 0:
msg = "Unable to find site name in SiteConfigFile:\n"
msg += self.siteConfigFile
raise SiteConfigError, msg
self.siteName = str(nameNodes[0].attrs.get("name"))
# //
# // event data (Read Trivial Catalog location)
#//
catalogQ = IMProvQuery("/site-local-config/site/event-data/catalog")
catNodes = catalogQ(node)
if len(catNodes) == 0:
msg = "Unable to find catalog entry for event data in:\n"