def getXMLSummaryScript(indent=''): '''Returns the necessary script to parse and make sense of the XMLSummary data''' import inspect from GangaLHCb.Lib.Applications.AppsBaseUtils import activeSummaryItems script = "###INDENT#### Parsed XMLSummary data extraction methods\n" for summaryItem in activeSummaryItems().values(): script += ''.join([ '###INDENT###' + line for line in inspect.getsourcelines(summaryItem)[0] ]) script += ''.join([ '###INDENT###' + line for line in inspect.getsourcelines(activeSummaryItems)[0] ]) import inspect script_location = os.path.join( os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))), 'XMLWorkerScript.py') from Ganga.GPIDev.Lib.File import FileUtils xml_script = FileUtils.loadScript(script_location, '###INDENT###') script += xml_script return script.replace('###INDENT###', indent)
def getXMLSummaryScript(indent=''): '''Returns the necessary script to parse and make sense of the XMLSummary data''' import inspect from GangaLHCb.Lib.Applications.AppsBaseUtils import activeSummaryItems script = "###INDENT#### Parsed XMLSummary data extraction methods\n" for summaryItem in activeSummaryItems().values(): script += ''.join(['###INDENT###' + line for line in inspect.getsourcelines(summaryItem)[0]]) script += ''.join(['###INDENT###' + line for line in inspect.getsourcelines(activeSummaryItems)[0]]) import inspect script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), 'XMLWorkerScript.py.template') from Ganga.GPIDev.Lib.File import FileUtils xml_script = FileUtils.loadScript(script_location, '###INDENT###') script += xml_script return script.replace('###INDENT###', indent)
def postprocess(self, logger): j = self.getJobObject() parsedXML = os.path.join(j.outputdir, '__parsedxmlsummary__') # use to avoid replacing 'lumi' etc as return value and not the method # pointer metadataItems = {} if os.path.exists(parsedXML): execfile(parsedXML, {}, metadataItems) # Combining subjobs XMLSummaries. if j.subjobs: env = self.getenv(self.is_prepared is None) if 'XMLSUMMARYBASEROOT' not in env: logger.warning( '"XMLSUMMARYBASEROOT" env var not defined so summary.xml files not merged for subjobs of job %s' % j.fqid) return summaries = [] for sj in j.subjobs: outputxml = os.path.join(sj.outputdir, 'summary.xml') if not os.path.exists(outputxml): logger.warning( "XMLSummary for job %s subjobs will not be merged as 'summary.xml' not present in job %s outputdir" % (j.fqid, sj.fqid)) return elif os.path.getsize(outputxml) == 0 or os.stat( outputxml).st_size == 0: logger.warning( "XMLSummary fro job %s subjobs will not be merged as %s appears to be an empty file" % (j.fqid, outputxml)) logger.warning( "Please try to recreate this file by either resubmitting your job or re-downloading the data from the backend" ) return summaries.append(outputxml) # Not needed now that we dont merge if ANY of subjobs have missing summary.xml #if not summaries: # logger.debug('None of the subjobs of job %s produced the output XML summary file "summary.xml". Merging will therefore not happen' % j.fqid) # return schemapath = os.path.join(env['XMLSUMMARYBASEROOT'], 'xml/XMLSummary.xsd') summarypath = os.path.join(env['XMLSUMMARYBASEROOT'], 'python/XMLSummaryBase') sys.path.append(summarypath) import summary try: XMLSummarydata = summary.Merge(summaries, schemapath) except Exception, err: logger.error('Problem while merging the subjobs XML summaries') raise for name, method in activeSummaryItems().iteritems(): try: metadataItems[name] = method(XMLSummarydata) except: metadataItems[name] = None logger.debug( 'Problem running "%s" method on merged xml output.' % name)
def postprocess(self, logger): j = self.getJobObject() parsedXML = os.path.join(j.outputdir, "__parsedxmlsummary__") # use to avoid replacing 'lumi' etc as return value and not the method # pointer metadataItems = {} if os.path.exists(parsedXML): execfile(parsedXML, {}, metadataItems) # Combining subjobs XMLSummaries. if j.subjobs: env = self.getenv(self.is_prepared is None) if "XMLSUMMARYBASEROOT" not in env: logger.warning( '"XMLSUMMARYBASEROOT" env var not defined so summary.xml files not merged for subjobs of job %s' % j.fqid ) return summaries = [] for sj in j.subjobs: outputxml = os.path.join(sj.outputdir, "summary.xml") if not os.path.exists(outputxml): logger.warning( "XMLSummary for job %s subjobs will not be merged as 'summary.xml' not present in job %s outputdir" % (j.fqid, sj.fqid) ) return elif os.path.getsize(outputxml) == 0 or os.stat(outputxml).st_size == 0: logger.warning( "XMLSummary fro job %s subjobs will not be merged as %s appears to be an empty file" % (j.fqid, outputxml) ) logger.warning( "Please try to recreate this file by either resubmitting your job or re-downloading the data from the backend" ) return summaries.append(outputxml) # Not needed now that we dont merge if ANY of subjobs have missing summary.xml # if not summaries: # logger.debug('None of the subjobs of job %s produced the output XML summary file "summary.xml". Merging will therefore not happen' % j.fqid) # return schemapath = os.path.join(env["XMLSUMMARYBASEROOT"], "xml/XMLSummary.xsd") summarypath = os.path.join(env["XMLSUMMARYBASEROOT"], "python/XMLSummaryBase") sys.path.append(summarypath) import summary try: XMLSummarydata = summary.Merge(summaries, schemapath) except Exception, err: logger.error("Problem while merging the subjobs XML summaries") raise for name, method in activeSummaryItems().iteritems(): try: metadataItems[name] = method(XMLSummarydata) except: metadataItems[name] = None logger.debug('Problem running "%s" method on merged xml output.' % name)
def GaudiExecPostProcess(self, logger): from GangaGaudi.Lib.Applications import GaudiXMLSummary j = self.getJobObject() metadataItems = {} #If this is a subjob then get the metadata from the summary.xml file if os.path.exists(os.path.join(j.outputdir, 'summary.xml')): sjSummary = GaudiXMLSummary(j, 'summary.xml').summary() sjMetadataItems = {} for name, method in activeSummaryItems().iteritems(): try: sjMetadataItems[name] = method(sjSummary) except: sjMetadataItems[name] = None logger.debug( 'Problem running "%s" method on merged xml output.' % name) for key, value in sjMetadataItems.iteritems(): if value is None: # Has to be explicit else empty list counts j.metadata[key] = 'Not Available.' else: j.metadata[key] = value # Combining subjobs XMLSummaries. if j.subjobs: env = self.getenv(self.is_prepared is None) if 'XMLSUMMARYBASEROOT' not in env: logger.warning( '"XMLSUMMARYBASEROOT" env var not defined so summary.xml files not merged for subjobs of job %s' % j.fqid) return summaries = [] for sj in j.subjobs: outputxml = os.path.join(sj.outputdir, 'summary.xml') if not os.path.exists(outputxml): logger.warning( "XMLSummary for job %s subjobs will not be merged as 'summary.xml' not present in job %s outputdir" % (j.fqid, sj.fqid)) return elif os.path.getsize(outputxml) == 0 or os.stat( outputxml).st_size == 0: logger.warning( "XMLSummary fro job %s subjobs will not be merged as %s appears to be an empty file" % (j.fqid, outputxml)) logger.warning( "Please try to recreate this file by either resubmitting your job or re-downloading the data from the backend" ) return summaries.append(outputxml) # Not needed now that we dont merge if ANY of subjobs have missing summary.xml #if not summaries: # logger.debug('None of the subjobs of job %s produced the output XML summary file "summary.xml". Merging will therefore not happen' % j.fqid) # return schemapath = os.path.join(env['XMLSUMMARYBASEROOT'], 'xml/XMLSummary.xsd') summarypath = os.path.join(env['XMLSUMMARYBASEROOT'], 'python/XMLSummaryBase') sys.path.append(summarypath) import summary try: XMLSummarydata = summary.Merge(summaries, schemapath) except Exception, err: logger.error('Problem while merging the subjobs XML summaries') raise for name, method in activeSummaryItems().iteritems(): try: metadataItems[name] = method(XMLSummarydata) except: metadataItems[name] = None logger.debug( 'Problem running "%s" method on merged xml output.' % name)