def readTFC(filename): """ _readTFC_ Read the file provided and return a TrivialFileCatalog instance containing the details found in it """ if not os.path.exists(filename): msg = "TrivialFileCatalog not found: %s" % filename raise RuntimeError(msg) try: node = xmlFileToNode(filename) except Exception as ex: msg = "Error reading TrivialFileCatalog: %s\n" % filename msg += str(ex) raise RuntimeError(msg) parsedResult = nodeReader(node) tfcInstance = TrivialFileCatalog() for mapping in ["lfn-to-pfn", "pfn-to-lfn"]: for entry in parsedResult[mapping]: protocol = entry.get("protocol", None) match = entry.get("path-match", None) result = entry.get("result", None) chain = entry.get("chain", None) if True in (protocol, match == None): continue tfcInstance.addMapping(str(protocol), str(match), str(result), chain, mapping) return tfcInstance
def multiXmlToJobReport(reportInstance, multiReportFile, directory=None): """ _multiXmlToJobReport_ Util for reading a top level job report from a multi threaded CMSSW job report the multi report file is expected to contain a list of entries like: <ChildProcessFiles> <ChildProcessFile>FrameworkJobReport_0.xml</ChildProcessFile> <ChildProcessFile>FrameworkJobReport_1.xml</ChildProcessFile> <ChildProcessFile>FrameworkJobReport_2.xml</ChildProcessFile> </ChildProcessFiles> """ # read XML, build node structure jobRepNode = xmlFileToNode(multiReportFile) for repNode in childrenMatching(jobRepNode, "FrameworkJobReport"): for childProcFiles in childrenMatching(repNode, "ChildProcessFiles"): for childRep in childrenMatching(childProcFiles, "ChildProcessFile"): fileName = childRep.text if directory != None: fileName = "%s/%s" % (directory, fileName) if os.path.exists(fileName): xmlToJobReport(reportInstance, fileName) else: print "File %s not found" % fileName
def readTFC(filename): """ _readTFC_ Read the file provided and return a TrivialFileCatalog instance containing the details found in it """ if not os.path.exists(filename): msg = "TrivialFileCatalog not found: %s" % filename raise RuntimeError(msg) try: node = xmlFileToNode(filename) except Exception as ex: msg = "Error reading TrivialFileCatalog: %s\n" % filename msg += str(ex) raise RuntimeError(msg) parsedResult = nodeReader(node) tfcInstance = TrivialFileCatalog() for mapping in ['lfn-to-pfn', 'pfn-to-lfn']: for entry in parsedResult[mapping]: protocol = entry.get("protocol", None) match = entry.get("path-match", None) result = entry.get("result", None) chain = entry.get("chain", None) if True in (protocol, match == None): continue tfcInstance.addMapping(str(protocol), str(match), str(result), chain, mapping) return tfcInstance
def multiXmlToJobReport(reportInstance, multiReportFile, directory = None): """ _multiXmlToJobReport_ Util for reading a top level job report from a multi threaded CMSSW job report the multi report file is expected to contain a list of entries like: <ChildProcessFiles> <ChildProcessFile>FrameworkJobReport_0.xml</ChildProcessFile> <ChildProcessFile>FrameworkJobReport_1.xml</ChildProcessFile> <ChildProcessFile>FrameworkJobReport_2.xml</ChildProcessFile> </ChildProcessFiles> """ # read XML, build node structure jobRepNode = xmlFileToNode(multiReportFile) for repNode in childrenMatching(jobRepNode, "FrameworkJobReport"): for childProcFiles in childrenMatching(repNode, "ChildProcessFiles"): for childRep in childrenMatching(childProcFiles, "ChildProcessFile"): fileName = childRep.text if directory != None: fileName = "%s/%s" % (directory, fileName) if os.path.exists(fileName): xmlToJobReport(reportInstance, fileName) else: print "File %s not found" % fileName
def read(self): """ _read_ Load data from SiteLocal Config file and populate this object """ try: node = xmlFileToNode(self.siteConfigFile) except StandardError, ex: msg = "Unable to read SiteConfigFile: %s\n" % self.siteConfigFile msg += str(ex) raise SiteConfigError, msg
def xmlToJobReport(reportInstance, xmlFile): """ _xmlToJobReport_ parse the XML file and insert the information into the Report instance provided """ # read XML, build node structure node = xmlFileToNode(xmlFile) # // # // Set up coroutine pipeline # // fileDispatchers = { "Runs": runHandler(), "Branches": branchHandler(), "Inputs": inputAssocHandler(), } perfRepDispatchers = { "PerformanceSummary": perfSummaryHandler(), "CPU": perfCPUHandler(), "Memory": perfMemHandler(), "Storage": perfStoreHandler(), } dispatchers = { "File": fileHandler(fileDispatchers), "InputFile": inputFileHandler(fileDispatchers), "PerformanceReport": perfRepHandler(perfRepDispatchers), "AnalysisFile": analysisFileHandler(fileDispatchers), "FrameworkError": errorHandler(), "SkippedFile": skippedFileHandler(), "FallbackAttempt": fallbackAttemptHandler(), "SkippedEvent": skippedEventHandler(), } # // # // Feed pipeline with node structure and report result instance # // reportBuilder( node, reportInstance, reportDispatcher(dispatchers) ) return
def xmlToJobReport(reportInstance, xmlFile): """ _xmlToJobReport_ parse the XML file and insert the information into the Report instance provided """ # read XML, build node structure node = xmlFileToNode(xmlFile) # // # // Set up coroutine pipeline #// fileDispatchers = { "Runs" : runHandler(), "Branches" : branchHandler(), "Inputs" : inputAssocHandler(), } perfRepDispatchers = { "PerformanceSummary" : perfSummaryHandler(), "CPU" : perfCPUHandler(), "Memory" : perfMemHandler(), "Storage": perfStoreHandler(), } dispatchers = { "File" : fileHandler(fileDispatchers), "InputFile": inputFileHandler(fileDispatchers), "PerformanceReport" : perfRepHandler(perfRepDispatchers), "AnalysisFile" : analysisFileHandler(fileDispatchers), "FrameworkError" : errorHandler(), "SkippedFile" : skippedFileHandler(), "FallbackAttempt" : fallbackAttemptHandler(), "SkippedEvent" : skippedEventHandler(), } # // # // Feed pipeline with node structure and report result instance #// reportBuilder( node, reportInstance, reportDispatcher(dispatchers) ) return
def readTFC(filename): """ _readTFC_ Read the file provided and return a TrivialFileCatalog instance containing the details found in it """ if not os.path.exists(filename): msg = "TrivialFileCatalog not found: %s" % filename raise RuntimeError, msg try: node = xmlFileToNode(filename) except StandardError, ex: msg = "Error reading TrivialFileCatalog: %s\n" % filename msg += str(ex) raise RuntimeError, msg
def read(self): """ _read_ Load data from SiteLocal Config file and populate this object """ try: node = xmlFileToNode(self.siteConfigFile) except Exception as ex: msg = "Unable to read SiteConfigFile: %s\n" % self.siteConfigFile msg += str(ex) raise SiteConfigError(msg) nodeResult = nodeReader(node) if 'siteName' not in nodeResult: msg = "Unable to find site name in SiteConfigFile:\n" msg += self.siteConfigFile raise SiteConfigError(msg) if 'catalog' not in nodeResult: msg = "Unable to find catalog entry for event data in SiteConfigFile:\n" msg += self.siteConfigFile raise SiteConfigError(msg) if 'localStageOut' not in nodeResult: msg = "Error:Unable to find any local-stage-out" msg += "information in:\n" msg += self.siteConfigFile raise SiteConfigError(msg) self.siteName = nodeResult.get('siteName', None) self.eventData['catalog'] = nodeResult.get('catalog', None) self.localStageOut = nodeResult.get('localStageOut', []) self.fallbackStageOut = nodeResult.get('fallbackStageOut', []) self.frontierServers = nodeResult.get('frontierServers', []) self.frontierProxies = nodeResult.get('frontierProxies', []) return