def load(self, improvNode): """ _load_ Extract information for this object from the improv instance provided """ wfQuery = IMProvQuery("/RequestSpec/WorkflowSpec") wfnode = wfQuery(improvNode)[0] wfspec = WorkflowSpec() wfspec.loadFromNode(wfnode) self.workflow = wfspec policyQuery = IMProvQuery("/RequestSpec/Policies/*") detailQuery = IMProvQuery("/RequestSpec/RequestDetails/*") preferredPAQuery = IMProvQuery("/RequestSpec/PreferredPA") policies = policyQuery(improvNode) details = detailQuery(improvNode) preferredPAs = preferredPAQuery(improvNode) for policy in policies: self.policies[str(policy.name)] = str(policy.chardata) for detail in improvNode.attrs.keys(): self.requestDetails[detail] = str(improvNode.attrs[detail]) for preferredPA in preferredPAs: self.preferredPAs[str(preferredPA.attrs['id'])] = \ str(preferredPA.attrs['priority']) return
def load(self, node): """load into self from improv node""" runQuery = IMProvQuery("/Run[attribute(\"ID\")]") runInfo = runQuery(node) if len(runInfo) == 0: return self.run = int(runInfo[-1]) lumiQuery = IMProvQuery("/Run/LumiSection[attribute(\"ID\")]") lumiInfo = lumiQuery(node) self.extend([int(x) for x in lumiInfo]) return
def legacyLumiInfo(self, improvNode, queryBase): """ _legacyLumiInfo_ handle legacy lumi section information """ # // # // Lumi Sections #// lumiQ = IMProvQuery("/%s/LumiSections/LumiSection" % queryBase) for lumiSect in lumiQ(improvNode): newLumi = {} [ newLumi.__setitem__(x.name, x.attrs['Value']) for x in lumiSect.children ] run = newLumi.get("RunNumber", None) lumi = newLumi.get("LumiSectionNumber", None) if run == None or lumi == None: continue run = int(run) lumi = int(lumi) if not self.runs.has_key(run): newRun = RunInfo() newRun.run = run self.runs[run] = newRun runInfo = self.runs.get(run) runInfo.append(lumi) return
def readSpecFile(filename): """ _readSpecFile_ Tool for extracting multiple RequestSpecs from a file Returns a list of spec instances found """ result = [] # ignore failed parsed requests ignore = 0 node = loadIMProvFile(filename) node = loadIMProvFile(filename) specQ = IMProvQuery("RequestSpec") specNodes = specQ(node) for snode in specNodes: newSpec = RequestSpec() try: newSpec.load(snode) result.append(newSpec) except Exception, ex: logging.debug( 'ERROR loading a requestspec, will ignore this requestspec :' + str(ex)) ignore += 1
def load(self, improvNode): """improv to self""" self.release = str(improvNode.attrs['Version']) query = IMProvQuery("Validate/Test") nodes = query(improvNode) for node in nodes: newTest = RelValTest() newTest.load(node) self.append(newTest) return
def load(self, improvNode): """ _load_ Populate self based on content of improvNode instance """ componentQ = IMProvQuery( "ProdCommonConfiguration/Component[attribute(\"Name\")]" ) configQ = IMProvQuery( "ProdCommonConfiguration/ConfigBlock" ) components = componentQ(improvNode) configs = configQ(improvNode) for comp in components: self.components.append(comp) for config in configs: cfgBlock = ConfigBlock("temp") cfgBlock.load(config) self[cfgBlock.name] = cfgBlock return
def queryStatusXML(filename): try: report = loadIMProvFile(filename) except Exception as ex: msg = "Error reading CRAB Status Report: %s\n" % filename msg += str(ex) raise RuntimeError(msg) query = IMProvQuery("Task/TaskJobs/Job/RunningJob") Jobs = query(report) return Jobs
def load(self, improvNode): """ _load_ Extract Performance reports from the improvNode provided Uses a Relative path query """ cpuQ = IMProvQuery("PerformanceReport/CPU/CPUCore") memQ = IMProvQuery("PerformanceReport/Memory/Property") summaryQ = IMProvQuery("PerformanceReport/PerformanceSummary") moduleQ = IMProvQuery("PerformanceReport/PerformanceModule") for cpu in cpuQ(improvNode): core = cpu.attrs.get("Core", None) if core == None: continue core = str(core) speed = None desc = None for child in cpu.children: if child.attrs.get('Name', None) == "Description": desc = str(child.chardata) if child.attrs.get('Name', None) == 'Speed': speed = str(child.chardata) self.addCPU(core, speed, desc) for mem in memQ(improvNode): name = mem.attrs.get("Name", None) if name == None: continue self.memory[str(name)] = str(mem.chardata) for summNode in summaryQ(improvNode): metricName = summNode.attrs.get("Metric", None) if metricName == None: continue metrics = {} for metric in IMProvQuery("Metric")(summNode): metName = metric.attrs.get("Name", None) metVal = metric.attrs.get("Value", None) if metName == None: continue metrics[str(metName)] = str(metVal) self.addSummary(str(metricName), **metrics) for modNode in moduleQ(improvNode): metricName = modNode.attrs.get("Metric", None) modName = modNode.attrs.get("Module", None) if metricName == None: continue if modName == None: continue metrics = {} for metric in IMProvQuery("Metric")(modNode): metName = metric.attrs.get("Name", None) metVal = metric.attrs.get("Value", None) if metName == None: continue metrics[str(metName)] = str(metVal) self.addModule(str(metricName), modName, **metrics) return
def load(self, improvNode): dataQ = IMProvQuery("/InputLink/*") dataNodes = dataQ(improvNode) for node in dataNodes: key = str(node.name) val = str(node.attrs['Value']) self[key] = val if self.get("AppearStandalone", "false").lower() in ("true", "yes"): self["AppearStandalone"] = True else: self["AppearStandalone"] = False return
def load(self, improvNode): """ _load_ Populate self based on content of improvNode instance """ nameQ = IMProvQuery("/ConfigBlock[attribute(\"Name\")]") paramQ = IMProvQuery("/ConfigBlock/Parameter") commentQ = IMProvQuery("/ConfigBlock/Comment[text()]") self.name = str(nameQ(improvNode)[0]) comments = commentQ(improvNode) if len(comments) > 0: self.comment = str(comments[0]) else: self.comment = None params = paramQ(improvNode) for paramNode in params: paramName = str(paramNode.attrs['Name']) paramVal = str(paramNode.attrs['Value']) self[paramName] = paramVal return
def loadFromNode(self, improvNode): """ _loadFromNode_ Populate this object based on content of improvNode provided """ paramQ = IMProvQuery("/WorkflowSpec/Parameter") payloadQ = IMProvQuery("/WorkflowSpec/Payload/PayloadNode") libsQ = IMProvQuery( "/WorkflowSpec/PythonLibraries/PythonLibrary[attribute(\"Name\")]") # // # // Extract Params #// paramNodes = paramQ(improvNode) for item in paramNodes: paramName = item.attrs.get("Name", None) if paramName == None: continue paramValue = str(item.chardata) self.parameters[str(paramName)] = paramValue # // # // Extract python lib list #// libNames = libsQ(improvNode) for item in libNames: self.addPythonLibrary(str(item)) # // # // Extract Payload Nodes #// payload = payloadQ(improvNode)[0] self.payload = PayloadNode() self.payload.populate(payload) return
def load(self, improvNode): """ _load_ Load from IMProvNode structure & populate self """ nodesQ = IMProvQuery("/BulkSpecs/BulkSpec") nodes = nodesQ(improvNode) for node in nodes: id = node.attrs.get("ID", None) if id == None: continue self[str(id)] = str(node.chardata) return
def populate(self, improvNode): """ _populate_ Extract details of this node from improvNode and instantiate and populate any children found """ self.unpackPayloadNodeData(improvNode) # // # // Recursively handle children #// childQ = IMProvQuery("/PayloadNode/PayloadNode") childNodes = childQ(improvNode) for item in childNodes: newChild = PayloadNode() self.addNode(newChild) newChild.populate(item) return
def load(self, improvNode): """ _load_ Populate this instance with data extracted from the improvNode provided. The Argument should be an improvNode created with the same structure as the result of the save method of this class """ if improvNode.name != self.__class__.__name__: # // # // Not the right node type #// return entryQ = IMProvQuery("/%s/Entry" % self.__class__.__name__) entries = entryQ(improvNode) for entry in entries: key = str(entry.attrs["Name"]) value = str(entry.chardata) self[key] = value return
def readRelValSpec(url, topNode = "ReleaseValidation"): """ _readRelValSpec_ Open the given URL and read the contents, parse it and return a list of RelValidation objects """ handler = IMProvHandler() parser = make_parser() parser.setContentHandler(handler) parser.parse(url) mainNode = handler._ParentDoc result = [] query = IMProvQuery("/%s/Validate" % topNode) nodes = query(mainNode) for node in nodes: relval = RelValidation("") relval.load(node) result.append(relval) return result
def loadFromNode(self, improvNode): paramQ = IMProvQuery("/JobSpec/Parameter") payloadQ = IMProvQuery("/JobSpec/Payload/JobSpecNode") whitelistQ = IMProvQuery("/JobSpec/SiteWhitelist/Site") blacklistQ = IMProvQuery("/JobSpec/SiteBlacklist/Site") assocFileQ = IMProvQuery("/JobSpec/AssociatedFiles/AssocFileList") bulkSpecQ = IMProvQuery("/JobSpec/BulkSpecs") # // # // Extract Params #// paramNodes = paramQ(improvNode) for item in paramNodes: paramName = item.attrs.get("Name", None) if paramName == None: continue paramValue = str(item.chardata) self.parameters[str(paramName)] = paramValue # // # // Extract site lists #// whiteNodes = whitelistQ(improvNode) for wnode in whiteNodes: value = wnode.attrs.get("Name", None) if value != None: self.siteWhitelist.append(str(value)) blackNodes = blacklistQ(improvNode) for bnode in blackNodes: value = bnode.attrs.get("Name", None) if value != None: self.siteBlacklist.append(str(value)) # // # // Extract Associated Files #// assocFiles = assocFileQ(improvNode) if len(assocFiles) > 0: for assocFileList in assocFiles: assocListName = str(assocFileList.attrs['Name']) assocList = [] for aFile in assocFileList.children: fileEntry = { "LFN" : str(aFile.chardata), } for attrName, attrVal in aFile.attrs.items(): fileEntry[str(attrName)] = str(attrVal) assocList.append(fileEntry) self.addAssociatedFiles(assocListName, *assocList) bulkNodes = bulkSpecQ(improvNode) if len(bulkNodes) > 0: self.bulkSpecs.load(bulkNodes[-1]) # // # // Extract Payload Nodes #// payload = payloadQ(improvNode)[0] self.payload = JobSpecNode() self.payload.populate(payload) return
def populate(self, improvNode): """ _populate_ Override PayloadNode.populate to load a saved JobSpecNode """ # // # // Unpack base class data #// self.unpackPayloadNodeData(improvNode) jobName = improvNode.attrs.get("JobName", None) if jobName != None: self.jobName = str(jobName) jobType = improvNode.attrs.get("JobType", None) if jobType != None: self.jobType = str(jobType) # // # // Unpack JobSpecNode additional Data #// Parameters paramQ = IMProvQuery("/JobSpecNode/JobSpecification/Parameters/*") params = paramQ(improvNode) for paramNode in params: key = str(paramNode.name) value = str(paramNode.chardata) self.addParameter(key, value) # // # // InputModule #// inpQ = IMProvQuery("/JobSpecNode/JobSpecification/InputModule") inputModNode = inpQ(improvNode) if len(inputModNode) > 0: inputModNode = inputModNode[0] self.inputModule = str(inputModNode.chardata) # // # // OutputModules #// outQ = IMProvQuery( "/JobSpecNode/JobSpecification/OutputModules/OutputModule") outMods = outQ(improvNode) for outMod in outMods: self.outputModules.append(str(outMod.chardata)) # // # // Recursively handle children #// childQ = IMProvQuery("/JobSpecNode/JobSpecNode") childNodes = childQ(improvNode) for item in childNodes: newChild = JobSpecNode() self.addNode(newChild) newChild.populate(item) return
'Cancelled by user': '******', 'Unknown': 'U', 'Done(failed)': 'D' } def queryStatusXML(filename): try: report = loadIMProvFile(filename) except StandardError, ex: msg = "Error reading CRAB Status Report: %s\n" % filename msg += str(ex) raise RuntimeError, msg query = IMProvQuery("Task/TaskJobs/Job/RunningJob") Jobs = query(report) return Jobs def printCrabStatusFromReport(filename): Jobs = queryStatusXML(filename) print "Crab Id: StatusScheduler | Status | ProcessStatus | State | GridId |" for j in Jobs: crabId = int(j.attrs.get("jobId", None)) statusScheduler = str(j.attrs.get("statusScheduler", None)) status = str(j.attrs.get("status", None)) processStatus = str(j.attrs.get("processStatus", None)) state = str(j.attrs.get("state", None)) gridId = str(j.attrs.get("schedulerId", None))
instance containing the details found in it """ if not os.path.exists(filename): msg = "TrivialFileCatalog not found: %s" % filename raise RuntimeError, msg try: node = loadIMProvFile(filename) except StandardError, ex: msg = "Error reading TrivialFileCatalog: %s\n" % filename msg += str(ex) raise RuntimeError, msg query = IMProvQuery("storage-mapping/lfn-to-pfn") mappings = query(node) tfcInstance = TrivialFileCatalog() for mapping in mappings: protocol = mapping.attrs.get("protocol", None) match = mapping.attrs.get("path-match", None) result = mapping.attrs.get("result", None) chain = mapping.attrs.get("chain", None) if True in (protocol, match, mapping == None): continue tfcInstance.addMapping(str(protocol), str(match), str(result), chain) return tfcInstance
def unpackPayloadNodeData(self, improvNode): """ _unpackPayloadNodeData_ Unpack PayloadNode data from improv Node provided and add information to self """ self.name = str(improvNode.attrs["Name"]) self.type = str(improvNode.attrs["Type"]) workflowName = improvNode.attrs.get('Workflow', None) if workflowName != None: self.workflow = str(workflowName) # // # // Unpack data for this instance #// App details appDataQ = IMProvQuery("/%s/Application" % self.__class__.__name__) appData = appDataQ(improvNode)[0] for appField in appData.children: field = str(appField.name) value = str(appField.attrs['Value']) self.application[field] = value # // # // App Control details #// appConDataQ = IMProvQuery("/%s/ApplicationControl/*" % self.__class__.__name__) appConData = appConDataQ(improvNode) for appConField in appConData: field = str(appConField.name) value = str(appConField.attrs['Value']) self.applicationControls[field] = value # // # // Script Controls #// scriptConQ = IMProvQuery("/%s/ScriptControls/ScriptList" % self.__class__.__name__) scriptLists = scriptConQ(improvNode) for scriptList in scriptLists: listName = scriptList.attrs.get("Name", None) if listName == None: continue listName = str(listName) for script in scriptList.children: scriptName = script.attrs.get("Value", None) if scriptName == None: continue self.scriptControls[listName].append(str(scriptName)) # // # // Dataset details #// Input Datasets inputDSQ = IMProvQuery("/%s/InputDatasets/DatasetInfo" % self.__class__.__name__) inputDS = inputDSQ(improvNode) # print improvNode for item in inputDS: newDS = DatasetInfo() newDS.load(item) self._InputDatasets.append(newDS) # // # // Output Datasets #// outputDSQ = IMProvQuery("/%s/OutputDatasets/DatasetInfo" % self.__class__.__name__) outputDS = outputDSQ(improvNode) for item in outputDS: newDS = DatasetInfo() newDS.load(item) self._OutputDatasets.append(newDS) # // # // Pileup Datasets #// pileupDSQ = IMProvQuery("/%s/PileupDatasets/DatasetInfo" % self.__class__.__name__) pileupDS = pileupDSQ(improvNode) for item in pileupDS: newDS = DatasetInfo() newDS.load(item) self._PileupDatasets.append(newDS) # // # // Input Links #// inpLinkQ = IMProvQuery("/%s/InputLinks/InputLink" % self.__class__.__name__) inpLinks = inpLinkQ(improvNode) for ilink in inpLinks: newLink = InputLink() newLink.load(ilink) self._InputLinks.append(newLink) # // # // Configuration #// configQ = IMProvQuery("/%s/Configuration" % self.__class__.__name__) configNodes = configQ(improvNode) if len(configNodes) > 0: configNode = configNodes[0] self.configuration = base64.decodestring(str(configNode.chardata)) cfgIntQ = IMProvQuery("/%s/CMSSWConfig" % self.__class__.__name__) cfgNodes = cfgIntQ(improvNode) if len(cfgNodes) > 0: cfgNode = cfgNodes[0] self.cfgInterface = CMSSWConfig() self.cfgInterface.load(cfgNode) # // # // User sandbox #// sandboxQ = IMProvQuery("/%s/UserSandbox" % self.__class__.__name__) sandboxNodes = sandboxQ(improvNode) if len(sandboxNodes) > 0: sandboxNode = sandboxNodes[-1] self.userSandbox = str(sandboxNode.chardata) return
class TrivialFileCatalog: """ _FwkJobReport_ Framework Job Report container and interface object """ def __init__(self, url=None): self.preferredProtocol = None # attribute for preferred protocol self.lfnToPfn = [] self.pfnToLfn = [] if url: self.load(url) def addLfnToPfnRule(self, protocol, pathMatch, result, chain=None): """ Add a LFN-To-PFN mapping rule that will match for the given protocol and destination """ self.lfnToPfn.append({ "protocol": protocol, "path-match": pathMatch, "path-match-regexp": re.compile(pathMatch), "chain": chain, "result": result }) return def matchLFN(self, protocol, lfn): """ _matchLFN_ Return the result for the LFN provided if the LFN matches the path-match for that protocol Return None if no match """ if not protocol: protocol = self.preferredProtocol for mapping in self.lfnToPfn: if mapping['protocol'] != protocol: continue if mapping['path-match-regexp'].match(lfn): if mapping['chain'] != None: lfn = self.matchLFN(mapping['chain'], lfn) try: splitLFN = mapping['path-match-regexp'].split(lfn, 1)[1] except IndexError: continue result = mapping['result'].replace("$1", splitLFN) return result return None def addPfnToLfnRule(self, protocol, pathMatch, result, chain=None): """ Add a PFN-To-LFN mapping rule that will match for the given protocol and destination """ self.pfnToLfn.append({ "protocol": protocol, "path-match": pathMatch, "path-match-regexp": re.compile(pathMatch), "chain": chain, "result": result }) return def matchPFN(self, protocol, pfn): """ _matchPFN_ Return the result for the PFN provided if the LFN matches the path-match for that protocol Return None if no match """ if not protocol: protocol = self.preferredProtocol for mapping in self.pfnToLfn: if mapping['protocol'] != protocol: continue if mapping['path-match-regexp'].match(pfn): if mapping['chain'] != None: pfn = self.matchPFN(mapping['chain'], pfn) try: splitPFN = mapping['path-match-regexp'].split(pfn, 1)[1] except IndexError: continue result = mapping['result'].replace("$1", splitPFN) return result return None def save(self): """ _save_ Save the tfc by converting it into an XML IMProv Object """ result = IMProvNode("storage-mapping") for maping in self.lfnToPfn: node = IMProvNode("lfn-to-pfn", None) node.attrs.update(maping) del node.attrs['path-match-regexp'] if not maping.get('chain', None): del node.attrs['chain'] result.addNode(node) for maping in self.pfnToLfn: node = IMProvNode("pfn-to-lfn", None) node.attrs.update(maping) del node.attrs['path-match-regexp'] if not maping.get('chain', None): del node.attrs['chain'] result.addNode(node) return result def write(self, filename): """ _write_ Write the tfc to an XML file """ handle = open(filename, 'w') handle.write(self.save().makeDOMElement().toprettyxml()) handle.close() return def __str__(self): """string representation of instance""" return str(self.save()) def load(self, url): """ _load_ Read a tfc into this instance """ self.lfnToPfn = [] self.pfnToLfn = [] self.preferredProtocol = tfcProtocol(url) filename = tfcFilename(url) if not os.path.exists(filename): msg = "TrivialFileCatalog not found: %s" % filename raise RuntimeError, msg try: node = loadIMProvFile(filename) except StandardError, ex: msg = "Error reading TrivialFileCatalog: %s\n" % filename msg += str(ex) raise RuntimeError, msg query = IMProvQuery("storage-mapping/lfn-to-pfn") mappings = query(node) for mapping in mappings: protocol = mapping.attrs.get("protocol", None) match = mapping.attrs.get("path-match", None) result = mapping.attrs.get("result", None) chain = mapping.attrs.get("chain", None) if True in (protocol, match, mapping == None): continue self.addLfnToPfnRule(str(protocol), str(match), \ str(result), chain) query = IMProvQuery("storage-mapping/pfn-to-lfn") mappings = query(node) for mapping in mappings: protocol = mapping.attrs.get("protocol", None) match = mapping.attrs.get("path-match", None) result = mapping.attrs.get("result", None) chain = mapping.attrs.get("chain", None) if True in (protocol, match, mapping == None): continue self.addPfnToLfnRule(str(protocol), str(match), \ str(result), chain) return
def load(self, improvNode): """ _load_ Populate this object from the improvNode provided """ # // # // Input or Output? #// queryBase = improvNode.name if queryBase == "InputFile": self.isInput = True else: self.isInput = False # // # // Parameters #// paramQ = IMProvQuery("/%s/*" % queryBase) for paramNode in paramQ(improvNode): if paramNode.name not in self.keys(): continue self[paramNode.name] = paramNode.chardata # // # // State #// stateQ = IMProvQuery("/%s/State[attribute(\"Value\")]" % queryBase) self.state = stateQ(improvNode)[-1] # // # // Checksums #// cksumQ = IMProvQuery("/%s/Checksum" % queryBase) for cksum in cksumQ(improvNode): algo = cksum.attrs.get('Algorithm', None) if algo == None: continue self.addChecksum(str(algo), str(cksum.chardata)) # // # // Inputs #// inputFileQ = IMProvQuery("/%s/Inputs/Input" % queryBase) for inputFile in inputFileQ(improvNode): lfn = IMProvQuery("/Input/LFN[text()]")(inputFile)[-1] pfn = IMProvQuery("/Input/PFN[text()]")(inputFile)[-1] self.addInputFile(pfn, lfn) # // # // Datasets #// datasetQ = IMProvQuery("/%s/Datasets/DatasetInfo" % queryBase) for dataset in datasetQ(improvNode): newDataset = self.newDataset() newDataset.load(dataset) # // # // Branches #// branchQ = IMProvQuery("/%s/Branches/Branch[text()]" % queryBase) for branch in branchQ(improvNode): self.branches.append(str(branch)) runQ = IMProvQuery("/%s/Runs/Run" % queryBase) for run in runQ(improvNode): newRun = RunInfo() newRun.load(run) if newRun.run == None: continue self.runs[newRun.run] = newRun self["Runs"] = self.runs self.legacyLumiInfo(improvNode, queryBase) return
def handleStorageStats(improvNode, reportRef): """ _handleStorageStats_ Look for the detailed storage stats information and collapse it to a set of performance report instances """ # // # // Backwards compatibility #// if len(improvNode.chardata.strip()) > 0: reportRef.storageStatistics = str(improvNode.chardata.strip()) return # // # // new format report #// base = "/StorageStatistics/storage-factory-summary/" statsBase = "%sstorage-factory-stats" % base statsQ = IMProvQuery("%s/storage-timing-summary/counter-value" % statsBase) rootQ = IMProvQuery("%s/storage-root-summary/counter-value" % statsBase) paramBase = "%sstorage-factory-params/param" % base paramQ = IMProvQuery(paramBase) allStats = {} for stats in statsQ(improvNode): subsystem = stats.attrs.get("subsystem", None) if subsystem == None: continue subsystem = str(subsystem) counterName = stats.attrs.get("counter-name", None) if counterName == None: continue counterName = str(counterName) keyName = "%s-%s" % (subsystem, counterName) for statKey, statVal in stats.attrs.items(): statKey = str(statKey) statVal = str(statVal) if statKey in ("subsystem", "counter-name"): continue dataKey = "%s-%s" % (keyName, statKey) allStats[dataKey] = statVal #print allStats allRoots = {} for root in rootQ(improvNode): subsystem = root.attrs.get("subsystem", None) if subsystem == None: continue subsystem = str(subsystem) counterName = root.attrs.get("counter-name", None) if counterName == None: continue counterName = str(counterName) keyName = "%s-%s" % (subsystem, counterName) for rootKey, rootVal in root.attrs.items(): rootKey = str(rootKey) rootVal = str(rootVal) if rootKey in ("subsystem", "counter-name"): continue dataKey = "%s-%s" % (keyName, rootKey) allRoots[dataKey] = rootVal allParams = {} for param in paramQ(improvNode): paramName = param.attrs.get("name", None) if paramName == None: continue paramName = str(paramName) paramVal = param.attrs.get("value", None) if paramVal == None: continue allParams[paramName] = str(paramVal) reportRef.performance.addSummary("StorageTiming", **allStats) reportRef.performance.addSummary("StorageRoot", **allRoots) reportRef.performance.addSummary("StorageParams", **allParams) return
def load(self, improvNode): """ _load_ Unpack improvNode into this instance """ self.name = improvNode.attrs.get("Name", None) self.status = improvNode.attrs.get("Status", None) self.jobSpecId = improvNode.attrs.get("JobSpecID", None) self.jobType = improvNode.attrs.get("JobType", None) self.workflowSpecId =improvNode.attrs.get("WorkflowSpecID", None) self.dashboardId =improvNode.attrs.get("DashboardId", None) exitQ = IMProvQuery( "/FrameworkJobReport/ExitCode[attribute(\"Value\")]") exitVals = exitQ(improvNode) if len(exitVals) > 0: self.exitCode = int(exitVals[-1]) # // # // Site details #// siteQ = IMProvQuery("/FrameworkJobReport/SiteDetail") [ self.siteDetails.__setitem__(x.attrs['Parameter'], x.attrs['Value']) for x in siteQ(improvNode) ] # // # // PSetHash #// hashQ = IMProvQuery("/FrameworkJobReport/PSetHash[text()]") hashVals = hashQ(improvNode) if len(hashVals) > 0: hashData = hashVals[-1] self.psetHash = str(hashData) # // # // output files #// fileQ = IMProvQuery("/FrameworkJobReport/File") fileList = fileQ(improvNode) for fileEntry in fileList: newFile = self.newFile() newFile.load(fileEntry) # // # // input files #// infileQ = IMProvQuery("/FrameworkJobReport/InputFile") for infileEntry in infileQ(improvNode): newInFile = self.newInputFile() newInFile.load(infileEntry) # // # // Skipped Events & Files #// skipFileQ = IMProvQuery("/FrameworkJobReport/SkippedFile") skipEventQ = IMProvQuery("/FrameworkJobReport/SkippedEvent") [ self.addSkippedEvent(int(skipEv.attrs['Run']), int(skipEv.attrs['Event'])) for skipEv in skipEventQ(improvNode)] [ self.addSkippedFile(skipF.attrs['Pfn'], skipF.attrs['Lfn']) for skipF in skipFileQ(improvNode) ] # // # // removed files #// remFileQ = IMProvQuery("/FrameworkJobReport/RemovedFile") [ self.addRemovedFile(str(remF.chardata), remF.attrs['SEName']) for remF in remFileQ(improvNode) ] # // # // unremoved files #// unremFileQ = IMProvQuery("/FrameworkJobReport/UnremovedFile") [ self.addUnremovedFile(str(remF.chardata), remF.attrs['SEName']) for remF in unremFileQ(improvNode) ] # // # // analysis files #// afileQ = IMProvQuery("/FrameworkJobReport/AnalysisFile") for afileEntry in afileQ(improvNode): newAFile = self.newAnalysisFile() newAFile.load(afileEntry) # // # // log files #// logFileQ = IMProvQuery("/FrameworkJobReport/LogFile") [ self.addLogFile(str(logF.chardata), logF.attrs['SEName']) for logF in logFileQ(improvNode) ] # // # // Timing, Storage and generator info #// timingQ = IMProvQuery("/FrameworkJobReport/TimingService/*") [ self.timing.__setitem__(x.name, x.attrs['Value']) for x in timingQ(improvNode) ] # // # // ReadBranches tag from cmsRun #// readBranchesQ = IMProvQuery("/FrameworkJobReport/ReadBranches/*") [ self.readBranches.__setitem__(x.attrs['Name'], x.attrs['ReadCount']) for x in readBranchesQ(improvNode) ] storageQ = IMProvQuery("/FrameworkJobReport/StorageStatistics") storageInfo = storageQ(improvNode) if len(storageInfo) > 0: #self.storageStatistics = storageInfo[-1] StorageStats.handleStorageStats(storageInfo[-1], self) genQ = IMProvQuery("/FrameworkJobReport/GeneratorInfo/Data") [ self.generatorInfo.__setitem__(x.attrs['Name'], x.attrs['Value']) for x in genQ(improvNode)] errQ = IMProvQuery("/FrameworkJobReport/FrameworkError") errors = errQ(improvNode) for err in errors: newErr = self.addError( int(err.attrs['ExitStatus']), err.attrs['Type']) newErr['Description'] = err.chardata # // # // Performance reports #// self.performance.load(improvNode) return
Instantiate a new ContentHandler and run it over the file producing a list of FwkJobReport instances """ #handler = FwkJobRepHandler() #parser = make_parser() #parser.setContentHandler(handler) #try: # parser.parse(filename) #except SAXParseException, ex: # msg = "Error parsing JobReport File: %s\n" % filename # msg += str(ex) # print msg # return [] #print handler.results[0] #return handler.results try: improvDoc = loadIMProvFile(filename) except Exception, ex: return [] result = [] reportQ = IMProvQuery("FrameworkJobReport") reportInstances = reportQ(improvDoc) for reportInstance in reportInstances: newReport = FwkJobReport() newReport.load(reportInstance) result.append(newReport) return result
_read_ Load data from SiteLocal Config file and populate this object """ try: node = loadIMProvFile(self.siteConfigFile) except StandardError, ex: msg = "Unable to read SiteConfigFile: %s\n" % self.siteConfigFile msg += str(ex) raise SiteConfigError, msg # // # // site name #// nameQ = IMProvQuery("/site-local-config/site") nameNodes = nameQ(node) if len(nameNodes) == 0: msg = "Unable to find site name in SiteConfigFile:\n" msg += self.siteConfigFile raise SiteConfigError, msg self.siteName = str(nameNodes[0].attrs.get("name")) # // # // event data (Read Trivial Catalog location) #// catalogQ = IMProvQuery("/site-local-config/site/event-data/catalog") catNodes = catalogQ(node) if len(catNodes) == 0: msg = "Unable to find catalog entry for event data in:\n"