def __init__( self, doTimelines=False, doFiles=False, titles=[], accumulation=None, singleFile=False, progressTemplate=None, plotIterations=False, # plotIterations=True, startTime=None, endTime=None): """ :param titles: The titles of the data elements :param progressTemplate: Progress output to be reported :param plotIterations: plot iterations instead of the real time """ LogLineAnalyzer.__init__(self) self.doTimelines = doTimelines self.doFiles = doFiles self.singleFile = singleFile self.plotIterations = plotIterations if self.plotIterations: self.iterCounter = 0 self.files = None self.titles = titles self.setTitles(titles) accu = "first" if accumulation != None: accu = accumulation if self.doTimelines: self.lines = TimeLineCollection(accumulation=accu) else: self.lines = None self.startTime = startTime self.endTime = endTime self.master = None self.didProgress = False self.progressTemplate = progressTemplate
def __init__(self, doTimelines=False, doFiles=False, titles=[], accumulation=None, singleFile=False, progressTemplate=None, plotIterations=False, # plotIterations=True, startTime=None, endTime=None): """ :param titles: The titles of the data elements :param progressTemplate: Progress output to be reported :param plotIterations: plot iterations instead of the real time """ LogLineAnalyzer.__init__(self) self.doTimelines=doTimelines self.doFiles=doFiles self.singleFile=singleFile self.plotIterations=plotIterations if self.plotIterations: self.iterCounter=0 self.files=None self.titles=titles self.setTitles(titles) accu="first" if accumulation!=None: accu=accumulation if self.doTimelines: self.lines=TimeLineCollection(accumulation=accu) else: self.lines=None self.startTime=startTime self.endTime=endTime self.master=None self.didProgress=False self.progressTemplate=progressTemplate
class GeneralLineAnalyzer(LogLineAnalyzer): """Base class for analyzers that write data to files and store time-lines Combines the capabilities of TimeLineLineAnalyzer and FileLineAnalyzer""" # phase of the solver to distinguish similar results __phase = "" def __init__(self, doTimelines=False, doFiles=False, titles=[], accumulation=None, singleFile=False, progressTemplate=None, startTime=None, endTime=None): """ @param titles: The titles of the data elements @param progressTemplate: Progress output to be reported """ LogLineAnalyzer.__init__(self) self.doTimelines=doTimelines self.doFiles=doFiles self.singleFile=singleFile self.files=None self.titles=titles self.setTitles(titles) accu="first" if accumulation!=None: accu=accumulation if self.doTimelines: self.lines=TimeLineCollection(accumulation=accu) else: self.lines=None self.startTime=startTime self.endTime=endTime self.master=None self.didProgress=False self.progressTemplate=progressTemplate @staticmethod def setPhase(p=""): GeneralLineAnalyzer.__phase = p @staticmethod def fName(n): if GeneralLineAnalyzer.__phase=="": return n else: return n+"_"+GeneralLineAnalyzer.__phase def getCurrentData(self): if self.lines: return self.lines.getLatestData() else: return {} def setMaster(self,master): """Assign another line-analyzer that will do the actual data gathering""" self.master=master if self.lines and self.master.lines: self.master.lines.addSlave(self.lines) def setTitles(self,titles): """ Sets the titles anew @param titles: the new titles """ if self.doFiles: self.titles=titles if self.files!=None: self.files.setTitles(titles) def setDirectory(self,oDir): """Creates the OutFileCollection-object""" if self.doFiles: self.files=OutFileCollection(oDir, titles=self.titles, singleFile=self.singleFile) else: self.files=None def timeChanged(self): """Sets the current time in the timelines""" if self.doTimelines: try: time=float(self.getTime()) if (self.startTime==None or time>=self.startTime) and (self.endTime==None or time<=self.endTime): self.lines.setTime(self.getTime()) except ValueError: e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' warning("Problem with lines",e) raise e self.didProgress=False self.setPhase() def getTimeline(self,name): """@param name: Name of the timeline to return @return: the timeline as two list: the times and the values""" if self.doTimelines: return self.lines.getTimes(),self.lines.getValues(name) else: return [],[] def doAnalysis(self,line): """General analysis method. Derived classes should instead override callbacks""" m=self.exp.match(line) if m!=None: self.startAnalysis(m) if self.doTimelines: try: time=float(self.getTime()) if (self.startTime==None or time>=self.startTime) and (self.endTime==None or time<=self.endTime): self.addToTimelines(m) except ValueError: e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' warning("Problem doing timelines",e) if self.doFiles: self.addToFiles(m) self.endAnalysis(m) if not self.didProgress and self.progressTemplate: myProgress=self.progressTemplate for i,g in enumerate(m.groups()): myProgress=myProgress.replace("$%d" % i,g) self.writeProgress(myProgress) self.didProgress=False def startAnalysis(self,match): """Method at the start of a successfull match""" pass def endAnalysis(self,match): """Method at the end of a successfull match""" pass def addToTimelines(self,match): """Method that adds matched data to timelines @param match: data matched by a regular expression""" pass def addToFiles(self,match): """Method that adds matched data to files @param match: data matched by a regular expression""" pass def tearDown(self): """Closes files""" LogLineAnalyzer.tearDown(self) if self.files!=None: self.files.close()
class GeneralLineAnalyzer(LogLineAnalyzer): """Base class for analyzers that write data to files and store time-lines Combines the capabilities of TimeLineLineAnalyzer and FileLineAnalyzer""" # phase of the solver to distinguish similar results __phase = "" def __init__(self, doTimelines=False, doFiles=False, titles=[], accumulation=None, singleFile=False, progressTemplate=None, plotIterations=False, # plotIterations=True, startTime=None, endTime=None): """ :param titles: The titles of the data elements :param progressTemplate: Progress output to be reported :param plotIterations: plot iterations instead of the real time """ LogLineAnalyzer.__init__(self) self.doTimelines=doTimelines self.doFiles=doFiles self.singleFile=singleFile self.plotIterations=plotIterations if self.plotIterations: self.iterCounter=0 self.files=None self.titles=titles self.setTitles(titles) accu="first" if accumulation!=None: accu=accumulation if self.doTimelines: self.lines=TimeLineCollection(accumulation=accu) else: self.lines=None self.startTime=startTime self.endTime=endTime self.master=None self.didProgress=False self.progressTemplate=progressTemplate @staticmethod def setPhase(p=""): GeneralLineAnalyzer.__phase = p @staticmethod def fName(n): if GeneralLineAnalyzer.__phase=="": return n else: return n+"_"+GeneralLineAnalyzer.__phase def getCurrentData(self,structured=False): if self.lines: return self.lines.getLatestData(structured=structured) else: return {} def setMaster(self,master): """Assign another line-analyzer that will do the actual data gathering""" self.master=master if self.lines and self.master.lines: self.master.lines.addSlave(self.lines) def setTitles(self,titles): """ Sets the titles anew :param titles: the new titles """ if self.doFiles: self.titles=titles if self.files!=None: self.files.setTitles(titles) def setDirectory(self,oDir): """Creates the OutFileCollection-object""" if self.doFiles: self.files=OutFileCollection(oDir, titles=self.titles, singleFile=self.singleFile) else: self.files=None def timeChanged(self): """Sets the current time in the timelines""" if self.doTimelines and not self.plotIterations: try: time=float(self.getTime()) if (self.startTime==None or time>=self.startTime) and (self.endTime==None or time<=self.endTime): self.lines.setTime(self.getTime()) except ValueError: e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' warning("Problem with lines",e) raise e if self.plotIterations: self.lines.setTime(self.iterCounter) self.didProgress=False self.setPhase() def getTimeline(self,name): """:param name: Name of the timeline to return :return: the timeline as two list: the times and the values""" if self.doTimelines: return self.lines.getTimes(),self.lines.getValues(name) else: return [],[] def stringToMatch(self,line): """Returns string to match. To be overriden for multi-line expressions""" return line.strip() def doMatch(self,line): return self.exp.match(self.stringToMatch(line)) def doAnalysis(self,line): """General analysis method. Derived classes should instead override callbacks""" m=self.doMatch(line) if m!=None: self.startAnalysis(m) if self.doTimelines: if self.plotIterations: self.iterCounter+=1 self.lines.setTime(self.iterCounter) try: time=float(self.getTime()) try: if (self.startTime==None or time>=self.startTime) and (self.endTime==None or time<=self.endTime) or self.plotIterations: self.addToTimelines(m) except ValueError: e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e' warning("Problem doing timelines",e) except ValueError: # no valid time information yet pass if self.doFiles: self.addToFiles(m) self.endAnalysis(m) if not self.didProgress and self.progressTemplate: self.writeProgress(self.processProgressTemplate(m)) self.didProgress=False def processProgressTemplate(self,data): """Add progress information""" return "" def startAnalysis(self,match): """Method at the start of a successfull match""" pass def endAnalysis(self,match): """Method at the end of a successfull match""" pass def addToTimelines(self,match): """Method that adds matched data to timelines :param match: data matched by a regular expression""" pass def addToFiles(self,match): """Method that adds matched data to files :param match: data matched by a regular expression""" pass def tearDown(self): """Closes files""" LogLineAnalyzer.tearDown(self) if self.files!=None: self.files.close()
def run(self): if not self.opts.server and not self.opts.pickle: error("No mode selected") if self.opts.server and self.opts.pickle: error("Both modes selected") doPandas = self.opts.pandasData or self.opts.pandasSeries if self.opts.server: if len(self.parser.getArgs()) != 2: error("Need a server and a port to be specified") host = self.parser.getArgs()[0] port = int(self.parser.getArgs()[1]) try: self.server = ServerProxy("http://%s:%d" % (host, port)) methods = self.server.system.listMethods() except socket.error: reason = sys.exc_info()[ 1] # Needed because python 2.5 does not support 'as e' self.error("Socket error while connecting:", reason) except ProtocolError: reason = sys.exc_info()[ 1] # Needed because python 2.5 does not support 'as e' self.error("XMLRPC-problem", reason) plotInfo = self.executeCommand("getPlots()") lineInfo = self.executeCommand("getPlotData()") else: if len(self.parser.getArgs()) != 1: warning("Only the first parameter is used") fName = self.parser.getArgs()[0] unpick = pickle.Unpickler(open(fName)) lineInfo = unpick.load() plotInfo = unpick.load() if not self.quiet: print_("Found", len(plotInfo), "plots and", len(lineInfo), "data sets") registry = TimeLinesRegistry() for nr, line in iteritems(lineInfo): if not self.quiet: print_("Adding line", nr) TimeLineCollection(preloadData=line, registry=registry) registry.resolveSlaves() if (self.opts.csvFiles or self.opts.excelFiles or doPandas or self.opts.numpyData) and self.opts.rawLines: rawData = {} rawSeries = {} rawNumpy = {} for k, l in iteritems(registry.lines): name = str(k) if type(k) == int: name = "Line%d" % k csvName = self.opts.filePrefix + name + ".csv" if self.opts.csvFiles: if not self.quiet: print_("Writing", k, "to", csvName) l.getData().writeCSV(csvName) if self.opts.excelFiles: xlsName = self.opts.filePrefix + name + ".xls" if not self.quiet: print_("Writing", k, "to", xlsName) l.getData().getData().to_excel(xlsName) if self.opts.pandasData: rawData[k] = l.getData().getData() if self.opts.numpyData: rawNumpy[k] = l.getData().data.copy() if self.opts.pandasSeries: rawSeries[k] = l.getData().getSeries() if self.opts.numpyData: self.setData({"rawNumpy": rawNumpy}) if self.opts.pandasData: self.setData({"rawData": rawData}) if self.opts.pandasSeries: self.setData({"rawSeries": rawSeries}) if self.opts.csvFiles or self.opts.excelFiles: return pRegistry = PlotLinesRegistry() plotNumpy = {} plotData = {} plotSeries = {} for i, p in iteritems(plotInfo): theId = p["id"] if not self.quiet: print_("Plotting", i, ":", theId, end=" ") spec = CustomPlotInfo(raw=p["spec"]) if len(registry.get(p["data"]).getTimes()) > 0 and registry.get( p["data"]).getValueNames() > 0: if self.opts.csvFiles or self.opts.excelFiles or doPandas or self.opts.numpyData: dataSet = registry.get(p["data"]).getData() if self.opts.csvFiles: dataSet.writeCSV(self.opts.filePrefix + theId + ".csv") if self.opts.excelFiles: dataSet.getData().to_excel(self.opts.filePrefix + theId + ".xls") if self.opts.numpyData: plotNumpy[theId] = dataSet.data.copy() if self.opts.pandasData: plotData[theId] = dataSet.getData() if self.opts.pandasSeries: plotSeries[theId] = dataSet.getSeries() else: if self.opts.start or self.opts.end: # rewrite CustomPlotInfo one of these days if "start" in spec.getDict(): self.warning("Overriding plot start", spec["start"], "with", self.opts.start) spec.set("start", self.opts.start) if "end" in spec.getDict(): self.warning("Overriding plot end", spec["end"], "with", self.opts.end) spec.set("end", self.opts.end) mp = createPlotTimelines( registry.get(p["data"]), spec, implementation=self.opts.implementation, showWindow=self.opts.showWindow, registry=pRegistry) if self.opts.insertTitles: mp.actualSetTitle(p["spec"]["theTitle"]) if self.opts.writePictures: if mp.hasData(): mp.doHardcopy(self.opts.prefix + theId, "png") else: if not self.quiet: print_("has no data", end=" ") if not self.quiet: print_() else: if not self.quiet: print_("No data - skipping") if not (self.opts.csvFiles or doPandas): sleep(self.opts.sleepTime ) # there seems to be a timing issue with Gnuplot if self.opts.numpyData: self.setData({"plotNumpy": plotNumpy}) if self.opts.pandasData: self.setData({"plotData": plotData}) if self.opts.pandasSeries: self.setData({"plotSeries": plotSeries})