def __init__(
self,
doTimelines=False,
doFiles=False,
titles=[],
accumulation=None,
singleFile=False,
progressTemplate=None,
plotIterations=False,
# plotIterations=True,
startTime=None,
endTime=None):
"""
:param titles: The titles of the data elements
:param progressTemplate: Progress output to be reported
:param plotIterations: plot iterations instead of the real time
"""
LogLineAnalyzer.__init__(self)
self.doTimelines = doTimelines
self.doFiles = doFiles
self.singleFile = singleFile
self.plotIterations = plotIterations
if self.plotIterations:
self.iterCounter = 0
self.files = None
self.titles = titles
self.setTitles(titles)
accu = "first"
if accumulation != None:
accu = accumulation
if self.doTimelines:
self.lines = TimeLineCollection(accumulation=accu)
else:
self.lines = None
self.startTime = startTime
self.endTime = endTime
self.master = None
self.didProgress = False
self.progressTemplate = progressTemplate
def __init__(self,
doTimelines=False,
doFiles=False,
titles=[],
accumulation=None,
singleFile=False,
progressTemplate=None,
plotIterations=False,
# plotIterations=True,
startTime=None,
endTime=None):
"""
:param titles: The titles of the data elements
:param progressTemplate: Progress output to be reported
:param plotIterations: plot iterations instead of the real time
"""
LogLineAnalyzer.__init__(self)
self.doTimelines=doTimelines
self.doFiles=doFiles
self.singleFile=singleFile
self.plotIterations=plotIterations
if self.plotIterations:
self.iterCounter=0
self.files=None
self.titles=titles
self.setTitles(titles)
accu="first"
if accumulation!=None:
accu=accumulation
if self.doTimelines:
self.lines=TimeLineCollection(accumulation=accu)
else:
self.lines=None
self.startTime=startTime
self.endTime=endTime
self.master=None
self.didProgress=False
self.progressTemplate=progressTemplate
class GeneralLineAnalyzer(LogLineAnalyzer):
"""Base class for analyzers that write data to files and store time-lines
Combines the capabilities of TimeLineLineAnalyzer and FileLineAnalyzer"""
# phase of the solver to distinguish similar results
__phase = ""
def __init__(self,
doTimelines=False,
doFiles=False,
titles=[],
accumulation=None,
singleFile=False,
progressTemplate=None,
startTime=None,
endTime=None):
"""
@param titles: The titles of the data elements
@param progressTemplate: Progress output to be reported
"""
LogLineAnalyzer.__init__(self)
self.doTimelines=doTimelines
self.doFiles=doFiles
self.singleFile=singleFile
self.files=None
self.titles=titles
self.setTitles(titles)
accu="first"
if accumulation!=None:
accu=accumulation
if self.doTimelines:
self.lines=TimeLineCollection(accumulation=accu)
else:
self.lines=None
self.startTime=startTime
self.endTime=endTime
self.master=None
self.didProgress=False
self.progressTemplate=progressTemplate
@staticmethod
def setPhase(p=""):
GeneralLineAnalyzer.__phase = p
@staticmethod
def fName(n):
if GeneralLineAnalyzer.__phase=="":
return n
else:
return n+"_"+GeneralLineAnalyzer.__phase
def getCurrentData(self):
if self.lines:
return self.lines.getLatestData()
else:
return {}
def setMaster(self,master):
"""Assign another line-analyzer that will do the actual data gathering"""
self.master=master
if self.lines and self.master.lines:
self.master.lines.addSlave(self.lines)
def setTitles(self,titles):
"""
Sets the titles anew
@param titles: the new titles
"""
if self.doFiles:
self.titles=titles
if self.files!=None:
self.files.setTitles(titles)
def setDirectory(self,oDir):
"""Creates the OutFileCollection-object"""
if self.doFiles:
self.files=OutFileCollection(oDir,
titles=self.titles,
singleFile=self.singleFile)
else:
self.files=None
def timeChanged(self):
"""Sets the current time in the timelines"""
if self.doTimelines:
try:
time=float(self.getTime())
if (self.startTime==None or time>=self.startTime) and (self.endTime==None or time<=self.endTime):
self.lines.setTime(self.getTime())
except ValueError:
e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e'
warning("Problem with lines",e)
raise e
self.didProgress=False
self.setPhase()
def getTimeline(self,name):
"""@param name: Name of the timeline to return
@return: the timeline as two list: the times and the values"""
if self.doTimelines:
return self.lines.getTimes(),self.lines.getValues(name)
else:
return [],[]
def doAnalysis(self,line):
"""General analysis method. Derived classes should instead override callbacks"""
m=self.exp.match(line)
if m!=None:
self.startAnalysis(m)
if self.doTimelines:
try:
time=float(self.getTime())
if (self.startTime==None or time>=self.startTime) and (self.endTime==None or time<=self.endTime):
self.addToTimelines(m)
except ValueError:
e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e'
warning("Problem doing timelines",e)
if self.doFiles:
self.addToFiles(m)
self.endAnalysis(m)
if not self.didProgress and self.progressTemplate:
myProgress=self.progressTemplate
for i,g in enumerate(m.groups()):
myProgress=myProgress.replace("$%d" % i,g)
self.writeProgress(myProgress)
self.didProgress=False
def startAnalysis(self,match):
"""Method at the start of a successfull match"""
pass
def endAnalysis(self,match):
"""Method at the end of a successfull match"""
pass
def addToTimelines(self,match):
"""Method that adds matched data to timelines
@param match: data matched by a regular expression"""
pass
def addToFiles(self,match):
"""Method that adds matched data to files
@param match: data matched by a regular expression"""
pass
def tearDown(self):
"""Closes files"""
LogLineAnalyzer.tearDown(self)
if self.files!=None:
self.files.close()
class GeneralLineAnalyzer(LogLineAnalyzer):
"""Base class for analyzers that write data to files and store time-lines
Combines the capabilities of TimeLineLineAnalyzer and FileLineAnalyzer"""
# phase of the solver to distinguish similar results
__phase = ""
def __init__(self,
doTimelines=False,
doFiles=False,
titles=[],
accumulation=None,
singleFile=False,
progressTemplate=None,
plotIterations=False,
# plotIterations=True,
startTime=None,
endTime=None):
"""
:param titles: The titles of the data elements
:param progressTemplate: Progress output to be reported
:param plotIterations: plot iterations instead of the real time
"""
LogLineAnalyzer.__init__(self)
self.doTimelines=doTimelines
self.doFiles=doFiles
self.singleFile=singleFile
self.plotIterations=plotIterations
if self.plotIterations:
self.iterCounter=0
self.files=None
self.titles=titles
self.setTitles(titles)
accu="first"
if accumulation!=None:
accu=accumulation
if self.doTimelines:
self.lines=TimeLineCollection(accumulation=accu)
else:
self.lines=None
self.startTime=startTime
self.endTime=endTime
self.master=None
self.didProgress=False
self.progressTemplate=progressTemplate
@staticmethod
def setPhase(p=""):
GeneralLineAnalyzer.__phase = p
@staticmethod
def fName(n):
if GeneralLineAnalyzer.__phase=="":
return n
else:
return n+"_"+GeneralLineAnalyzer.__phase
def getCurrentData(self,structured=False):
if self.lines:
return self.lines.getLatestData(structured=structured)
else:
return {}
def setMaster(self,master):
"""Assign another line-analyzer that will do the actual data gathering"""
self.master=master
if self.lines and self.master.lines:
self.master.lines.addSlave(self.lines)
def setTitles(self,titles):
"""
Sets the titles anew
:param titles: the new titles
"""
if self.doFiles:
self.titles=titles
if self.files!=None:
self.files.setTitles(titles)
def setDirectory(self,oDir):
"""Creates the OutFileCollection-object"""
if self.doFiles:
self.files=OutFileCollection(oDir,
titles=self.titles,
singleFile=self.singleFile)
else:
self.files=None
def timeChanged(self):
"""Sets the current time in the timelines"""
if self.doTimelines and not self.plotIterations:
try:
time=float(self.getTime())
if (self.startTime==None or time>=self.startTime) and (self.endTime==None or time<=self.endTime):
self.lines.setTime(self.getTime())
except ValueError:
e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e'
warning("Problem with lines",e)
raise e
if self.plotIterations:
self.lines.setTime(self.iterCounter)
self.didProgress=False
self.setPhase()
def getTimeline(self,name):
""":param name: Name of the timeline to return
:return: the timeline as two list: the times and the values"""
if self.doTimelines:
return self.lines.getTimes(),self.lines.getValues(name)
else:
return [],[]
def stringToMatch(self,line):
"""Returns string to match. To be overriden for multi-line expressions"""
return line.strip()
def doMatch(self,line):
return self.exp.match(self.stringToMatch(line))
def doAnalysis(self,line):
"""General analysis method. Derived classes should instead override callbacks"""
m=self.doMatch(line)
if m!=None:
self.startAnalysis(m)
if self.doTimelines:
if self.plotIterations:
self.iterCounter+=1
self.lines.setTime(self.iterCounter)
try:
time=float(self.getTime())
try:
if (self.startTime==None or time>=self.startTime) and (self.endTime==None or time<=self.endTime) or self.plotIterations:
self.addToTimelines(m)
except ValueError:
e = sys.exc_info()[1] # Needed because python 2.5 does not support 'as e'
warning("Problem doing timelines",e)
except ValueError:
# no valid time information yet
pass
if self.doFiles:
self.addToFiles(m)
self.endAnalysis(m)
if not self.didProgress and self.progressTemplate:
self.writeProgress(self.processProgressTemplate(m))
self.didProgress=False
def processProgressTemplate(self,data):
"""Add progress information"""
return ""
def startAnalysis(self,match):
"""Method at the start of a successfull match"""
pass
def endAnalysis(self,match):
"""Method at the end of a successfull match"""
pass
def addToTimelines(self,match):
"""Method that adds matched data to timelines
:param match: data matched by a regular expression"""
pass
def addToFiles(self,match):
"""Method that adds matched data to files
:param match: data matched by a regular expression"""
pass
def tearDown(self):
"""Closes files"""
LogLineAnalyzer.tearDown(self)
if self.files!=None:
self.files.close()
def run(self):
if not self.opts.server and not self.opts.pickle:
error("No mode selected")
if self.opts.server and self.opts.pickle:
error("Both modes selected")
doPandas = self.opts.pandasData or self.opts.pandasSeries
if self.opts.server:
if len(self.parser.getArgs()) != 2:
error("Need a server and a port to be specified")
host = self.parser.getArgs()[0]
port = int(self.parser.getArgs()[1])
try:
self.server = ServerProxy("http://%s:%d" % (host, port))
methods = self.server.system.listMethods()
except socket.error:
reason = sys.exc_info()[
1] # Needed because python 2.5 does not support 'as e'
self.error("Socket error while connecting:", reason)
except ProtocolError:
reason = sys.exc_info()[
1] # Needed because python 2.5 does not support 'as e'
self.error("XMLRPC-problem", reason)
plotInfo = self.executeCommand("getPlots()")
lineInfo = self.executeCommand("getPlotData()")
else:
if len(self.parser.getArgs()) != 1:
warning("Only the first parameter is used")
fName = self.parser.getArgs()[0]
unpick = pickle.Unpickler(open(fName))
lineInfo = unpick.load()
plotInfo = unpick.load()
if not self.quiet:
print_("Found", len(plotInfo), "plots and", len(lineInfo),
"data sets")
registry = TimeLinesRegistry()
for nr, line in iteritems(lineInfo):
if not self.quiet:
print_("Adding line", nr)
TimeLineCollection(preloadData=line, registry=registry)
registry.resolveSlaves()
if (self.opts.csvFiles or self.opts.excelFiles or doPandas
or self.opts.numpyData) and self.opts.rawLines:
rawData = {}
rawSeries = {}
rawNumpy = {}
for k, l in iteritems(registry.lines):
name = str(k)
if type(k) == int:
name = "Line%d" % k
csvName = self.opts.filePrefix + name + ".csv"
if self.opts.csvFiles:
if not self.quiet:
print_("Writing", k, "to", csvName)
l.getData().writeCSV(csvName)
if self.opts.excelFiles:
xlsName = self.opts.filePrefix + name + ".xls"
if not self.quiet:
print_("Writing", k, "to", xlsName)
l.getData().getData().to_excel(xlsName)
if self.opts.pandasData:
rawData[k] = l.getData().getData()
if self.opts.numpyData:
rawNumpy[k] = l.getData().data.copy()
if self.opts.pandasSeries:
rawSeries[k] = l.getData().getSeries()
if self.opts.numpyData:
self.setData({"rawNumpy": rawNumpy})
if self.opts.pandasData:
self.setData({"rawData": rawData})
if self.opts.pandasSeries:
self.setData({"rawSeries": rawSeries})
if self.opts.csvFiles or self.opts.excelFiles:
return
pRegistry = PlotLinesRegistry()
plotNumpy = {}
plotData = {}
plotSeries = {}
for i, p in iteritems(plotInfo):
theId = p["id"]
if not self.quiet:
print_("Plotting", i, ":", theId, end=" ")
spec = CustomPlotInfo(raw=p["spec"])
if len(registry.get(p["data"]).getTimes()) > 0 and registry.get(
p["data"]).getValueNames() > 0:
if self.opts.csvFiles or self.opts.excelFiles or doPandas or self.opts.numpyData:
dataSet = registry.get(p["data"]).getData()
if self.opts.csvFiles:
dataSet.writeCSV(self.opts.filePrefix + theId + ".csv")
if self.opts.excelFiles:
dataSet.getData().to_excel(self.opts.filePrefix +
theId + ".xls")
if self.opts.numpyData:
plotNumpy[theId] = dataSet.data.copy()
if self.opts.pandasData:
plotData[theId] = dataSet.getData()
if self.opts.pandasSeries:
plotSeries[theId] = dataSet.getSeries()
else:
if self.opts.start or self.opts.end:
# rewrite CustomPlotInfo one of these days
if "start" in spec.getDict():
self.warning("Overriding plot start",
spec["start"], "with",
self.opts.start)
spec.set("start", self.opts.start)
if "end" in spec.getDict():
self.warning("Overriding plot end", spec["end"],
"with", self.opts.end)
spec.set("end", self.opts.end)
mp = createPlotTimelines(
registry.get(p["data"]),
spec,
implementation=self.opts.implementation,
showWindow=self.opts.showWindow,
registry=pRegistry)
if self.opts.insertTitles:
mp.actualSetTitle(p["spec"]["theTitle"])
if self.opts.writePictures:
if mp.hasData():
mp.doHardcopy(self.opts.prefix + theId, "png")
else:
if not self.quiet:
print_("has no data", end=" ")
if not self.quiet:
print_()
else:
if not self.quiet:
print_("No data - skipping")
if not (self.opts.csvFiles or doPandas):
sleep(self.opts.sleepTime
) # there seems to be a timing issue with Gnuplot
if self.opts.numpyData:
self.setData({"plotNumpy": plotNumpy})
if self.opts.pandasData:
self.setData({"plotData": plotData})
if self.opts.pandasSeries:
self.setData({"plotSeries": plotSeries})
