def readTFC(filename):
"""
_readTFC_
Read the file provided and return a TrivialFileCatalog
instance containing the details found in it
"""
if not os.path.exists(filename):
msg = "TrivialFileCatalog not found: %s" % filename
raise RuntimeError(msg)
try:
node = xmlFileToNode(filename)
except Exception as ex:
msg = "Error reading TrivialFileCatalog: %s\n" % filename
msg += str(ex)
raise RuntimeError(msg)
parsedResult = nodeReader(node)
tfcInstance = TrivialFileCatalog()
for mapping in ["lfn-to-pfn", "pfn-to-lfn"]:
for entry in parsedResult[mapping]:
protocol = entry.get("protocol", None)
match = entry.get("path-match", None)
result = entry.get("result", None)
chain = entry.get("chain", None)
if True in (protocol, match == None):
continue
tfcInstance.addMapping(str(protocol), str(match), str(result), chain, mapping)
return tfcInstance
def multiXmlToJobReport(reportInstance, multiReportFile, directory=None):
"""
_multiXmlToJobReport_
Util for reading a top level job report from a multi threaded CMSSW job report
the multi report file is expected to contain a list of entries like:
<ChildProcessFiles>
<ChildProcessFile>FrameworkJobReport_0.xml</ChildProcessFile>
<ChildProcessFile>FrameworkJobReport_1.xml</ChildProcessFile>
<ChildProcessFile>FrameworkJobReport_2.xml</ChildProcessFile>
</ChildProcessFiles>
"""
# read XML, build node structure
jobRepNode = xmlFileToNode(multiReportFile)
for repNode in childrenMatching(jobRepNode, "FrameworkJobReport"):
for childProcFiles in childrenMatching(repNode, "ChildProcessFiles"):
for childRep in childrenMatching(childProcFiles, "ChildProcessFile"):
fileName = childRep.text
if directory != None:
fileName = "%s/%s" % (directory, fileName)
if os.path.exists(fileName):
xmlToJobReport(reportInstance, fileName)
else:
print "File %s not found" % fileName
def readTFC(filename):
"""
_readTFC_
Read the file provided and return a TrivialFileCatalog
instance containing the details found in it
"""
if not os.path.exists(filename):
msg = "TrivialFileCatalog not found: %s" % filename
raise RuntimeError(msg)
try:
node = xmlFileToNode(filename)
except Exception as ex:
msg = "Error reading TrivialFileCatalog: %s\n" % filename
msg += str(ex)
raise RuntimeError(msg)
parsedResult = nodeReader(node)
tfcInstance = TrivialFileCatalog()
for mapping in ['lfn-to-pfn', 'pfn-to-lfn']:
for entry in parsedResult[mapping]:
protocol = entry.get("protocol", None)
match = entry.get("path-match", None)
result = entry.get("result", None)
chain = entry.get("chain", None)
if True in (protocol, match == None):
continue
tfcInstance.addMapping(str(protocol), str(match), str(result),
chain, mapping)
return tfcInstance
def multiXmlToJobReport(reportInstance, multiReportFile, directory = None):
"""
_multiXmlToJobReport_
Util for reading a top level job report from a multi threaded CMSSW job report
the multi report file is expected to contain a list of entries like:
<ChildProcessFiles>
<ChildProcessFile>FrameworkJobReport_0.xml</ChildProcessFile>
<ChildProcessFile>FrameworkJobReport_1.xml</ChildProcessFile>
<ChildProcessFile>FrameworkJobReport_2.xml</ChildProcessFile>
</ChildProcessFiles>
"""
# read XML, build node structure
jobRepNode = xmlFileToNode(multiReportFile)
for repNode in childrenMatching(jobRepNode, "FrameworkJobReport"):
for childProcFiles in childrenMatching(repNode, "ChildProcessFiles"):
for childRep in childrenMatching(childProcFiles, "ChildProcessFile"):
fileName = childRep.text
if directory != None:
fileName = "%s/%s" % (directory, fileName)
if os.path.exists(fileName):
xmlToJobReport(reportInstance, fileName)
else:
print "File %s not found" % fileName
def read(self):
"""
_read_
Load data from SiteLocal Config file and populate this object
"""
try:
node = xmlFileToNode(self.siteConfigFile)
except StandardError, ex:
msg = "Unable to read SiteConfigFile: %s\n" % self.siteConfigFile
msg += str(ex)
raise SiteConfigError, msg
def read(self):
"""
_read_
Load data from SiteLocal Config file and populate this object
"""
try:
node = xmlFileToNode(self.siteConfigFile)
except StandardError, ex:
msg = "Unable to read SiteConfigFile: %s\n" % self.siteConfigFile
msg += str(ex)
raise SiteConfigError, msg
def xmlToJobReport(reportInstance, xmlFile):
"""
_xmlToJobReport_
parse the XML file and insert the information into the
Report instance provided
"""
# read XML, build node structure
node = xmlFileToNode(xmlFile)
# //
# // Set up coroutine pipeline
# //
fileDispatchers = {
"Runs": runHandler(),
"Branches": branchHandler(),
"Inputs": inputAssocHandler(),
}
perfRepDispatchers = {
"PerformanceSummary": perfSummaryHandler(),
"CPU": perfCPUHandler(),
"Memory": perfMemHandler(),
"Storage": perfStoreHandler(),
}
dispatchers = {
"File": fileHandler(fileDispatchers),
"InputFile": inputFileHandler(fileDispatchers),
"PerformanceReport": perfRepHandler(perfRepDispatchers),
"AnalysisFile": analysisFileHandler(fileDispatchers),
"FrameworkError": errorHandler(),
"SkippedFile": skippedFileHandler(),
"FallbackAttempt": fallbackAttemptHandler(),
"SkippedEvent": skippedEventHandler(),
}
# //
# // Feed pipeline with node structure and report result instance
# //
reportBuilder(
node, reportInstance,
reportDispatcher(dispatchers)
)
return
def xmlToJobReport(reportInstance, xmlFile):
"""
_xmlToJobReport_
parse the XML file and insert the information into the
Report instance provided
"""
# read XML, build node structure
node = xmlFileToNode(xmlFile)
# //
# // Set up coroutine pipeline
#//
fileDispatchers = {
"Runs" : runHandler(),
"Branches" : branchHandler(),
"Inputs" : inputAssocHandler(),
}
perfRepDispatchers = {
"PerformanceSummary" : perfSummaryHandler(),
"CPU" : perfCPUHandler(),
"Memory" : perfMemHandler(),
"Storage": perfStoreHandler(),
}
dispatchers = {
"File" : fileHandler(fileDispatchers),
"InputFile": inputFileHandler(fileDispatchers),
"PerformanceReport" : perfRepHandler(perfRepDispatchers),
"AnalysisFile" : analysisFileHandler(fileDispatchers),
"FrameworkError" : errorHandler(),
"SkippedFile" : skippedFileHandler(),
"FallbackAttempt" : fallbackAttemptHandler(),
"SkippedEvent" : skippedEventHandler(),
}
# //
# // Feed pipeline with node structure and report result instance
#//
reportBuilder(
node, reportInstance,
reportDispatcher(dispatchers)
)
return
def readTFC(filename):
"""
_readTFC_
Read the file provided and return a TrivialFileCatalog
instance containing the details found in it
"""
if not os.path.exists(filename):
msg = "TrivialFileCatalog not found: %s" % filename
raise RuntimeError, msg
try:
node = xmlFileToNode(filename)
except StandardError, ex:
msg = "Error reading TrivialFileCatalog: %s\n" % filename
msg += str(ex)
raise RuntimeError, msg
def readTFC(filename):
"""
_readTFC_
Read the file provided and return a TrivialFileCatalog
instance containing the details found in it
"""
if not os.path.exists(filename):
msg = "TrivialFileCatalog not found: %s" % filename
raise RuntimeError, msg
try:
node = xmlFileToNode(filename)
except StandardError, ex:
msg = "Error reading TrivialFileCatalog: %s\n" % filename
msg += str(ex)
raise RuntimeError, msg
def read(self):
"""
_read_
Load data from SiteLocal Config file and populate this object
"""
try:
node = xmlFileToNode(self.siteConfigFile)
except Exception as ex:
msg = "Unable to read SiteConfigFile: %s\n" % self.siteConfigFile
msg += str(ex)
raise SiteConfigError(msg)
nodeResult = nodeReader(node)
if 'siteName' not in nodeResult:
msg = "Unable to find site name in SiteConfigFile:\n"
msg += self.siteConfigFile
raise SiteConfigError(msg)
if 'catalog' not in nodeResult:
msg = "Unable to find catalog entry for event data in SiteConfigFile:\n"
msg += self.siteConfigFile
raise SiteConfigError(msg)
if 'localStageOut' not in nodeResult:
msg = "Error:Unable to find any local-stage-out"
msg += "information in:\n"
msg += self.siteConfigFile
raise SiteConfigError(msg)
self.siteName = nodeResult.get('siteName', None)
self.eventData['catalog'] = nodeResult.get('catalog', None)
self.localStageOut = nodeResult.get('localStageOut', [])
self.fallbackStageOut = nodeResult.get('fallbackStageOut', [])
self.frontierServers = nodeResult.get('frontierServers', [])
self.frontierProxies = nodeResult.get('frontierProxies', [])
return
def read(self):
"""
_read_
Load data from SiteLocal Config file and populate this object
"""
try:
node = xmlFileToNode(self.siteConfigFile)
except Exception as ex:
msg = "Unable to read SiteConfigFile: %s\n" % self.siteConfigFile
msg += str(ex)
raise SiteConfigError(msg)
nodeResult = nodeReader(node)
if 'siteName' not in nodeResult:
msg = "Unable to find site name in SiteConfigFile:\n"
msg += self.siteConfigFile
raise SiteConfigError(msg)
if 'catalog' not in nodeResult:
msg = "Unable to find catalog entry for event data in SiteConfigFile:\n"
msg += self.siteConfigFile
raise SiteConfigError(msg)
if 'localStageOut' not in nodeResult:
msg = "Error:Unable to find any local-stage-out"
msg += "information in:\n"
msg += self.siteConfigFile
raise SiteConfigError(msg)
self.siteName = nodeResult.get('siteName', None)
self.eventData['catalog'] = nodeResult.get('catalog', None)
self.localStageOut = nodeResult.get('localStageOut', [])
self.fallbackStageOut = nodeResult.get('fallbackStageOut', [])
self.frontierServers = nodeResult.get('frontierServers', [])
self.frontierProxies = nodeResult.get('frontierProxies', [])
return
