def verifyTrace(self, traceName): """ Verify that a trace is compatible with the loaded project @param traceName: Trace name """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) trace = self.analyzer.traces[traceName] TraceOperations.verify(self, self.analyzer.project, trace)
def testJoining(self): for traceFile in [testTrace1, testTrace2]: trace = self.loadTrace(traceFile) l = len(trace.events) a = TraceOperations.extract(trace, 0, l / 2) b = TraceOperations.extract(trace, l / 2, l) assert len(a.events) + len(b.events) == l newTrace = TraceOperations.join(a, b) self.assertEqualEventLists(newTrace.events, trace.events) for i, event in enumerate(newTrace.events): self.assertEqual(event.seq, i)
def extractTraceWithState(self, traceName, eventRange, traceNameOut=None): """ Extract a portion of a trace including preceding state to form a new trace. The command collects events prior to the first extracted event from the trace to set up the same state in the new trace. @param traceName: Name of the source trace @param eventRange: Event range to extract @param traceNameOut: Name under which the resulting trace is saved, or the original trace by default. """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) if traceNameOut is None: traceNameOut = traceName trace = self.analyzer.traces[traceName] firstEvent, lastEvent = self.analyzer.parseEventRange( trace, eventRange) trace = TraceOperations.extractWithState(self.analyzer.project, trace, firstEvent, lastEvent) self.analyzer.traces[traceNameOut] = trace self.reportInfo("%d events extracted from trace %s to trace %s." % (len(trace.events), traceName, traceNameOut)) return trace
def mergeTraces(self, traceName1, traceName2, traceNameOut=None, useTimeStamps=False): """ Merge two traces together to produce a third, sequentially coherent trace. @param traceName1: Name of the first trace @param traceName2: Name of the second trace @param traceNameOut: Name under which resulting trace is saved, or the first trace by default. """ if not traceName1 in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName1) if not traceName2 in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName2) if traceNameOut is None: traceNameOut = traceName1 useTimeStamps = self.analyzer.parseBoolean(useTimeStamps) trace = TraceOperations.merge(self.analyzer.traces[traceName1], self.analyzer.traces[traceName2], useTimeStamps=useTimeStamps) self.analyzer.traces[traceNameOut] = trace self.reportInfo("Traces %s and %s merged to trace %s." % (traceName1, traceName2, traceNameOut)) return trace
def createCallHistogramTable(self): t = Report.Table(["Call", "# of Calls"]) hist = TraceOperations.calculateCallHistogram(self.trace) values = [(count, name) for name, count in hist.items()] for i, value in enumerate(sorted(values, reverse = True)): count, name = value t.addRow(name, count) return t
def createCallHistogramTable(self): t = Report.Table(["Call", "# of Calls"]) hist = TraceOperations.calculateCallHistogram(self.trace) values = [(count, name) for name, count in hist.items()] for i, value in enumerate(sorted(values, reverse=True)): count, name = value t.addRow(name, count) return t
def histogram(trace): s = StringIO.StringIO() print >> s, "Call histogram:" hist = TraceOperations.calculateCallHistogram(trace) values = [(count, name) for name, count in hist.items()] for i, value in enumerate(sorted(values, reverse=True)): count, name = value print >> s, "%4d. %8d - %s" % (i + 1, count, name) print >> s return s.getvalue()
def histogram(trace): s = StringIO.StringIO() print >>s, "Call histogram:" hist = TraceOperations.calculateCallHistogram(trace) values = [(count, name) for name, count in hist.items()] for i, value in enumerate(sorted(values, reverse = True)): count, name = value print >>s, "%4d. %8d - %s" % (i + 1, count, name) print >>s return s.getvalue()
def playTrace(self, traceName, eventRange=None, profile=False, saveFrames=False, checkCoherence=True, synchronize=False): """ Play back a trace file using a generated trace player. @param traceName: Trace to play. @param eventRange: Range of events to play, or all events by default. @param profile: Should instrumentation data be collected or not. @param saveFrames: Should frame buffer snapshots be saved or not. @param checkCoherence: Verify that the trace is properly normalized before playing. @param sychronize: Attempt to match the original trace timings. """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) trace = self.analyzer.traces[traceName] traceFileName = self.analyzer.traceFiles.get(traceName, None) profile = self.analyzer.parseBoolean(profile) saveFrames = self.analyzer.parseBoolean(saveFrames) checkCoherence = self.analyzer.parseBoolean(checkCoherence) synchronize = self.analyzer.parseBoolean(synchronize) # Create a temporary trace that only contains the needed events if eventRange is not None: firstEvent, lastEvent = self.analyzer.parseEventRange( trace, eventRange) traceFileName = None trace = TraceOperations.extract(trace, firstEvent, lastEvent) # Create a player player = Player.createPlayer(self.analyzer) # Play it self.analyzer.reportInfo("Playing trace.") logFile = player.play(trace, traceFileName, profile=profile, saveFrames=saveFrames, checkCoherence=checkCoherence, synchronize=synchronize) # Load the instrumentation data if it exists if profile and logFile: Instrumentation.loadInstrumentationData(self.analyzer, trace, logFile) self.analyzer.reportInfo( "Instrumentation data loaded for trace %s." % traceName)
def calculateFrameStatistics(self, traceName): """ For each frame marker event, calculate the sum of the instrumentation data for the intermediate events. This operation will help to highlight frames that are especially resource intensive. @param traceName: Trace for which frame data should be calculated. """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) trace = self.analyzer.traces[traceName] events = TraceOperations.calculateFrameStatistics(self.analyzer.project, trace) self.analyzer.reportInfo("Data for %d frames collected."% len(events))
def testExtraction(self): for traceFile in [testTrace1, testTrace2]: trace = self.loadTrace(traceFile) newTrace = TraceOperations.extract(trace, 10, 20) assert len(newTrace.events) == 10 assert newTrace.events[0].time == 0 assert newTrace.events[0].seq == 0 assert newTrace.events[9].seq == 9 self.assertEqualEventLists(newTrace.events, trace.events[10:20]) newTrace.events[0].name = "foo" assert trace.events[0].name != "foo"
def calculateFrameStatistics(self, traceName): """ For each frame marker event, calculate the sum of the instrumentation data for the intermediate events. This operation will help to highlight frames that are especially resource intensive. @param traceName: Trace for which frame data should be calculated. """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) trace = self.analyzer.traces[traceName] events = TraceOperations.calculateFrameStatistics( self.analyzer.project, trace) self.analyzer.reportInfo("Data for %d frames collected." % len(events))
def loadTrace(self, fileName, traceName=None, format=None): """ Open a trace file. @param fileName: Trace file to open @param traceName: Resulting trace name @param format: Force a specific format to be used instead of autodetection. """ trace = Trace.Trace() for importer in self.analyzer.importPlugins: if format is not None and importer.formatName == format: break if importer.recognizeTraceFile(fileName): break else: if format is not None: self.analyzer.fail( "No such format. Available formats: %s." % (", ".join( [i.formatName for i in self.analyzer.importPlugins]))) self.analyzer.fail("Trace file format not recognized.") try: f = open(fileName, "rb") traces = importer.loadTrace(f) f.close() # Some decoders can load multiple trace files from one source if not isinstance(traces, list): traces = [traces] for trace in traces: traceName = traceName or "t%d" % len(self.analyzer.traces) self.analyzer.traces[traceName] = trace self.analyzer.traceFiles[traceName] = fileName if not TraceOperations.verify(self, self.analyzer.project, trace): self.reportWarning( "The loaded project probably does not match the trace file." ) self.reportInfo("Loaded trace from '%s' as %s." % (fileName, traceName)) traceName = None return traces[0] except IOError, e: self.analyzer.fail(e)
def showTraceState(self, traceName, eventNumber = None): """ Print out the computed API state at a particular trace event. @param traceName: Name of trace to examine @param eventNumber: Print out state at this event, or at the end of the trace by default """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) trace = self.analyzer.traces[traceName] if eventNumber is None: eventNumber = len(trace.events) - 1 else: eventNumber = self.analyzer.parseEventRange(trace, eventNumber)[0] state = TraceOperations.computeStateAtEvent(self.analyzer.project, trace, eventNumber) self.reportInfo(str(state))
def showCallHistogram(self, traceName): """ Show the function call frequency histogram for a trace. @param traceName: Name of the trace to examine """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) trace = self.analyzer.traces[traceName] hist = TraceOperations.calculateCallHistogram(trace) width = 40 largest = max(hist.values()) values = [(count, name) for name, count in hist.items()] for i, value in enumerate(sorted(values, reverse = True)): count, name = value self.reportInfo("%3d. %-25s %5d %s" % (i + 1, name, count, "#" * int(width * float(count) / largest)))
def showTraceState(self, traceName, eventNumber=None): """ Print out the computed API state at a particular trace event. @param traceName: Name of trace to examine @param eventNumber: Print out state at this event, or at the end of the trace by default """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) trace = self.analyzer.traces[traceName] if eventNumber is None: eventNumber = len(trace.events) - 1 else: eventNumber = self.analyzer.parseEventRange(trace, eventNumber)[0] state = TraceOperations.computeStateAtEvent(self.analyzer.project, trace, eventNumber) self.reportInfo(str(state))
def loadTrace(self, fileName, traceName = None, format = None): """ Open a trace file. @param fileName: Trace file to open @param traceName: Resulting trace name @param format: Force a specific format to be used instead of autodetection. """ trace = Trace.Trace() for importer in self.analyzer.importPlugins: if format is not None and importer.formatName == format: break if importer.recognizeTraceFile(fileName): break else: if format is not None: self.analyzer.fail("No such format. Available formats: %s." % (", ".join([i.formatName for i in self.analyzer.importPlugins]))) self.analyzer.fail("Trace file format not recognized.") try: f = open(fileName, "rb") traces = importer.loadTrace(f) f.close() # Some decoders can load multiple trace files from one source if not isinstance(traces, list): traces = [traces] for trace in traces: traceName = traceName or "t%d" % len(self.analyzer.traces) self.analyzer.traces[traceName] = trace self.analyzer.traceFiles[traceName] = fileName if not TraceOperations.verify(self, self.analyzer.project, trace): self.reportWarning("The loaded project probably does not match the trace file.") self.reportInfo("Loaded trace from '%s' as %s." % (fileName, traceName)) traceName = None return traces[0] except IOError, e: self.analyzer.fail(e)
def showCallHistogram(self, traceName): """ Show the function call frequency histogram for a trace. @param traceName: Name of the trace to examine """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) trace = self.analyzer.traces[traceName] hist = TraceOperations.calculateCallHistogram(trace) width = 40 largest = max(hist.values()) values = [(count, name) for name, count in hist.items()] for i, value in enumerate(sorted(values, reverse=True)): count, name = value self.reportInfo("%3d. %-25s %5d %s" % (i + 1, name, count, "#" * int(width * float(count) / largest)))
def extractTrace(self, traceName, eventRange, traceNameOut = None): """ Extract a portion of a trace to form a new trace. @param traceName: Name of the source trace @param eventRange: Event range to extract @param traceNameOut: Name under which the resulting trace is saved, or the original trace by default. """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) if traceNameOut is None: traceNameOut = traceName trace = self.analyzer.traces[traceName] firstEvent, lastEvent = self.analyzer.parseEventRange(trace, eventRange) trace = TraceOperations.extract(trace, firstEvent, lastEvent) self.analyzer.traces[traceNameOut] = trace self.reportInfo("%d events extracted from trace %s to trace %s." % (len(trace.events), traceName, traceNameOut)) return trace
def playTrace(self, traceName, eventRange = None, profile = False, saveFrames = False, checkCoherence = True, synchronize = False): """ Play back a trace file using a generated trace player. @param traceName: Trace to play. @param eventRange: Range of events to play, or all events by default. @param profile: Should instrumentation data be collected or not. @param saveFrames: Should frame buffer snapshots be saved or not. @param checkCoherence: Verify that the trace is properly normalized before playing. @param sychronize: Attempt to match the original trace timings. """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) trace = self.analyzer.traces[traceName] traceFileName = self.analyzer.traceFiles.get(traceName, None) profile = self.analyzer.parseBoolean(profile) saveFrames = self.analyzer.parseBoolean(saveFrames) checkCoherence = self.analyzer.parseBoolean(checkCoherence) synchronize = self.analyzer.parseBoolean(synchronize) # Create a temporary trace that only contains the needed events if eventRange is not None: firstEvent, lastEvent = self.analyzer.parseEventRange(trace, eventRange) traceFileName = None trace = TraceOperations.extract(trace, firstEvent, lastEvent) # Create a player player = Player.createPlayer(self.analyzer) # Play it self.analyzer.reportInfo("Playing trace.") logFile = player.play(trace, traceFileName, profile = profile, saveFrames = saveFrames, checkCoherence = checkCoherence, synchronize = synchronize) # Load the instrumentation data if it exists if profile and logFile: Instrumentation.loadInstrumentationData(self.analyzer, trace, logFile) self.analyzer.reportInfo("Instrumentation data loaded for trace %s." % traceName)
def joinTraces(self, traceName1, traceName2, traceNameOut = None): """ Join two traces together to produce a new third trace. @param traceName1: Name of the first trace @param traceName2: Name of the second trace @param traceNameOut: Name under which resulting trace is saved, or the first trace by default. """ if not traceName1 in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName1) if not traceName2 in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName2) if traceNameOut is None: traceNameOut = traceName1 trace = TraceOperations.join(self.analyzer.traces[traceName1], self.analyzer.traces[traceName2]) self.analyzer.traces[traceNameOut] = trace self.reportInfo("Traces %s and %s joined to trace %s." % (traceName1, traceName2, traceNameOut)) return trace
def extractTrace(self, traceName, eventRange, traceNameOut=None): """ Extract a portion of a trace to form a new trace. @param traceName: Name of the source trace @param eventRange: Event range to extract @param traceNameOut: Name under which the resulting trace is saved, or the original trace by default. """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) if traceNameOut is None: traceNameOut = traceName trace = self.analyzer.traces[traceName] firstEvent, lastEvent = self.analyzer.parseEventRange( trace, eventRange) trace = TraceOperations.extract(trace, firstEvent, lastEvent) self.analyzer.traces[traceNameOut] = trace self.reportInfo("%d events extracted from trace %s to trace %s." % (len(trace.events), traceName, traceNameOut)) return trace
def extractTraceWithState(self, traceName, eventRange, traceNameOut = None): """ Extract a portion of a trace including preceding state to form a new trace. The command collects events prior to the first extracted event from the trace to set up the same state in the new trace. @param traceName: Name of the source trace @param eventRange: Event range to extract @param traceNameOut: Name under which the resulting trace is saved, or the original trace by default. """ if not traceName in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName) if traceNameOut is None: traceNameOut = traceName trace = self.analyzer.traces[traceName] firstEvent, lastEvent = self.analyzer.parseEventRange(trace, eventRange) trace = TraceOperations.extractWithState(self.analyzer.project, trace, firstEvent, lastEvent) self.analyzer.traces[traceNameOut] = trace self.reportInfo("%d events extracted from trace %s to trace %s." % (len(trace.events), traceName, traceNameOut)) return trace
def mergeTraces(self, traceName1, traceName2, traceNameOut = None, useTimeStamps = False): """ Merge two traces together to produce a third, sequentially coherent trace. @param traceName1: Name of the first trace @param traceName2: Name of the second trace @param traceNameOut: Name under which resulting trace is saved, or the first trace by default. """ if not traceName1 in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName1) if not traceName2 in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName2) if traceNameOut is None: traceNameOut = traceName1 useTimeStamps = self.analyzer.parseBoolean(useTimeStamps) trace = TraceOperations.merge(self.analyzer.traces[traceName1], self.analyzer.traces[traceName2], useTimeStamps = useTimeStamps) self.analyzer.traces[traceNameOut] = trace self.reportInfo("Traces %s and %s merged to trace %s." % (traceName1, traceName2, traceNameOut)) return trace
def joinTraces(self, traceName1, traceName2, traceNameOut=None): """ Join two traces together to produce a new third trace. @param traceName1: Name of the first trace @param traceName2: Name of the second trace @param traceNameOut: Name under which resulting trace is saved, or the first trace by default. """ if not traceName1 in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName1) if not traceName2 in self.analyzer.traces: self.analyzer.fail("Trace not found: %s" % traceName2) if traceNameOut is None: traceNameOut = traceName1 trace = TraceOperations.join(self.analyzer.traces[traceName1], self.analyzer.traces[traceName2]) self.analyzer.traces[traceNameOut] = trace self.reportInfo("Traces %s and %s joined to trace %s." % (traceName1, traceName2, traceNameOut)) return trace
def calculateStatistics(self, frameSimilarityFunc = None, timeThreshold = .25, histogramThreshold = .96, frameCountThreshold = 20, monochromaticThreshold = .99): # Reset the list of frames self.frames = [] self.interestingFrames = [] # Combine frame level statistics TraceOperations.calculateFrameStatistics(self.project, self.trace) # Calculate some high level statistics events = [] for event in self.trace.events: func = self.library.functions[event.name] events.append(event) if func.isFrameMarker: frame = Frame() frame.startTime = events[0].time / 1e6 frame.endTime = (events[-1].time + events[-1].duration) / 1e6 frame.events = events frame.renderEvents = [e for e in events if self.library.functions[e.name].isRenderCall] frame.swapEvent = event frame.number = len(self.frames) # Calculate frame duration frame.duration = frame.endTime - frame.startTime if frame.duration < MIN_FRAME_DURATION: # If the frame has an essentially zero duration, the following event should give a proper time reading try: frame.duration = max(self.trace.events[self.trace.events.index(event) + 1].time / 1e6 - frame.startTime, MIN_FRAME_DURATION) except IndexError: frame.duration = MIN_FRAME_DURATION self.frames.append(frame) events = [] # Now prune the list of frames down to frames of special interest lastFrame = None histograms = {} def getFrameHistogram(frame): if not frame.swapEvent in histograms: try: image = player.Instrumentation.loadBuffer(frame.swapEvent) except: return if image: # Calculate a normalized histogram hist = image.histogram() f = 1.0 / (image.size[0] * image.size[1] * len(image.getbands())) hist = [h * f for h in hist] histograms[frame.swapEvent] = hist return histograms.get(frame.swapEvent) task = Task.startTask("choose-frames", "Choosing interesting frames", len(self.frames)) for frame in self.frames: task.step() if lastFrame and len(self.frames) > frameCountThreshold: # If this frame's duration is pretty much the same as the last one, skip it if abs(frame.duration - lastFrame.duration) <= timeThreshold * lastFrame.duration: continue # Get color buffer histograms for both images hist1 = getFrameHistogram(frame) hist2 = getFrameHistogram(lastFrame) if hist1 and hist2: # Calculate the Bhattacharyya distance, i.e. the cosine of the angle # between the two histograms dist = sum([math.sqrt(h2 * h1) for h1, h2 in zip(hist1, hist2)]) # If the distance is close to one, i.e. the frames are nearly identical, # skip this frame. if dist > histogramThreshold: continue # If the frame is mostly just one color, skip it if max(hist1) >= monochromaticThreshold: continue # If we have a similarity function, discard similar frames if frameSimilarityFunc and frameSimilarityFunc(lastFrame, frame): continue # Mark the new frame as interesting self.interestingFrames.append(frame) lastFrame = frame
def calculateStatistics(self, frameSimilarityFunc=None, timeThreshold=.25, histogramThreshold=.96, frameCountThreshold=20, monochromaticThreshold=.99): # Reset the list of frames self.frames = [] self.interestingFrames = [] # Combine frame level statistics TraceOperations.calculateFrameStatistics(self.project, self.trace) # Calculate some high level statistics events = [] for event in self.trace.events: func = self.library.functions[event.name] events.append(event) if func.isFrameMarker: frame = Frame() frame.startTime = events[0].time / 1e6 frame.endTime = (events[-1].time + events[-1].duration) / 1e6 frame.events = events frame.renderEvents = [ e for e in events if self.library.functions[e.name].isRenderCall ] frame.swapEvent = event frame.number = len(self.frames) # Calculate frame duration frame.duration = frame.endTime - frame.startTime if frame.duration < MIN_FRAME_DURATION: # If the frame has an essentially zero duration, the following event should give a proper time reading try: frame.duration = max( self.trace.events[self.trace.events.index(event) + 1].time / 1e6 - frame.startTime, MIN_FRAME_DURATION) except IndexError: frame.duration = MIN_FRAME_DURATION self.frames.append(frame) events = [] # Now prune the list of frames down to frames of special interest lastFrame = None histograms = {} def getFrameHistogram(frame): if not frame.swapEvent in histograms: try: image = player.Instrumentation.loadBuffer(frame.swapEvent) except: return if image: # Calculate a normalized histogram hist = image.histogram() f = 1.0 / (image.size[0] * image.size[1] * len(image.getbands())) hist = [h * f for h in hist] histograms[frame.swapEvent] = hist return histograms.get(frame.swapEvent) task = Task.startTask("choose-frames", "Choosing interesting frames", len(self.frames)) for frame in self.frames: task.step() if lastFrame and len(self.frames) > frameCountThreshold: # If this frame's duration is pretty much the same as the last one, skip it if abs(frame.duration - lastFrame.duration ) <= timeThreshold * lastFrame.duration: continue # Get color buffer histograms for both images hist1 = getFrameHistogram(frame) hist2 = getFrameHistogram(lastFrame) if hist1 and hist2: # Calculate the Bhattacharyya distance, i.e. the cosine of the angle # between the two histograms dist = sum( [math.sqrt(h2 * h1) for h1, h2 in zip(hist1, hist2)]) # If the distance is close to one, i.e. the frames are nearly identical, # skip this frame. if dist > histogramThreshold: continue # If the frame is mostly just one color, skip it if max(hist1) >= monochromaticThreshold: continue # If we have a similarity function, discard similar frames if frameSimilarityFunc and frameSimilarityFunc( lastFrame, frame): continue # Mark the new frame as interesting self.interestingFrames.append(frame) lastFrame = frame