def buildTransactionRepo(self, app, cpuInfo, probes, topdownCache,
topdownMetrics, events, benchmarkProbes,
benchmarkPaths):
"""
Builds a repository of transactions for current profile session and benchmarks
:param app: An instance of xpedite app, to interact with target application
:param cpuInfo: Cpu info of the host running target app
:param probes: List of probes enabled for the profile session
:param topdownCache: A cache of supported topdown hierarchies
:param topdownMetrics: Top down metrics to be computed
:param events: PMU events collected for the profiling session
:param benchmarkProbes: List of probes enabled for the benchmark session
:param benchmarkPaths: List of stored reports from previous runs, for benchmarking
"""
from xpedite.collector import Collector
from xpedite.benchmark import BenchmarksCollector
from xpedite.transactionLoader import ChaoticTransactionLoader, BoundedTransactionLoader
from xpedite.filter import TrivialCounterFilter
from xpedite.analytics import CURRENT_RUN
from xpedite.util import timeAction
counterFilter = TrivialCounterFilter()
collector = Collector(counterFilter, buildPrefix=self.buildPrefix)
if any(probe.canBeginTxn or probe.canEndTxn for probe in probes):
loaderType = BoundedTransactionLoader
else:
loaderType = ChaoticTransactionLoader
loader = loaderType(CURRENT_RUN, cpuInfo, probes, topdownMetrics,
events)
timeAction('gathering counters',
lambda: collector.gatherCounters(app, loader))
currentTransactions = loader.getData()
if not currentTransactions:
if loader.processedCounterCount:
msg = 'failed to load transactions. recheck routes specified in your profile info'
LOGGER.error(msg)
raise Exception(msg)
else:
msg = 'failed to load transactions. It appears the app hit any of the activated probes'
LOGGER.error(msg)
raise Exception(msg)
repo = TransactionRepo()
repo.addCurrent(currentTransactions)
if benchmarkPaths:
benchmarksCollector = BenchmarksCollector(benchmarkPaths)
benchmarksCollector.loadTransactions(
repo,
counterFilter,
benchmarksCollector.gatherBenchmarks(10),
loaderFactory=lambda benchmark: loaderFactory(
loaderType, benchmark, probes, benchmarkProbes,
topdownCache, topdownMetrics))
return repo
def buildElapsedTimeBundles(txnCollections, classifier):
"""
Builds elapsed timestamp counters for each of the categories in given transaction collections
:param repo: List of transaction collections from current profile session and benchmarks
:param classifier: Predicate to classify transactions into different categories
"""
elapsedTscBundles = {}
categorySet = set()
for i, txnCollection in enumerate(txnCollections):
txnSubCollection = txnCollection.getSubCollection()
probes = txnCollection.probes
elapsedTscMap = timeAction(
'aggregating time stamp counters per transaction',
lambda txnsc=txnSubCollection, txnCollection=txnCollection:
TxnAggregator.groupElapsedTime(
txnsc, txnCollection.cpuInfo, classifier=classifier))
if elapsedTscMap:
for category, elapsedTscList in elapsedTscMap.items():
if category in elapsedTscBundles:
elapsedTscBundles[category].append(elapsedTscList)
else:
if i == 0:
elapsedTscBundles.update(
{category: [elapsedTscList]})
else:
if category not in categorySet:
categorySet.add(category)
LOGGER.warn(
'current run missing trasactions for category "%s"',
category)
else:
scopeList = ', '.join([
probe.getCanonicalName() for probe in probes
if not probe.isAnonymous
])
errMsg = (
"""{}({}) doesn\'t have any transaction (#{} loaded) with probes\n\t[{}]
Did you generate any of these transactions ?""")
errMsg = errMsg.format(' benchmark ' if i > 0 else ' ',
txnCollection.name,
len(txnSubCollection), scopeList)
LOGGER.error(errMsg)
if i == 0:
raise Exception(
'report generation failed for current run. counters not available'
)
return elapsedTscBundles
def generateHistograms(self, repo, classifier, runId):
"""
Generates latency distribuion histograms for each category/route combination
:param repo: Repository of transaction collection
:type repo: xpedite.txn.repo.TxnRepo
:param classifier: Classifier to categorize transactions into various types
:param runId: Epoch time stamp to uniquely identify a profiling session
"""
histograms = {}
txnCollections = [repo.getCurrent()] + list(
repo.getBenchmarks().values())
if not txnCollections[0].isCurrent(
) or txnCollections[0].name != CURRENT_RUN:
from xpedite.types import InvariantViloation
raise InvariantViloation(
'expecing transactions for current run at index 0 in the repository. '
'instead found {}'.format(txnCollections[0].name))
elapsedTimeBundles = self.analytics.buildElapsedTimeBundles(
txnCollections, classifier)
for category, elaspsedTimeBundle in elapsedTimeBundles.items():
buckets = buildBuckets(elaspsedTimeBundle[0], 35)
if not buckets:
LOGGER.debug(
'category %s has not enough data points to generate histogram',
category)
continue
LOGGER.debug('Buckets:\n%s', buckets)
yaxis = []
conflatedCounts = []
LOGGER.debug('Bucket values:')
for i, elapsedTimeList in enumerate(elaspsedTimeBundle):
bucketValues, conflatedCountersCount = timeAction(
'building counter distribution',
lambda bkts=buckets, etl=elapsedTimeList:
buildDistribution(bkts, etl))
conflatedCounts.append(conflatedCountersCount)
LOGGER.debug('%s', bucketValues)
title = txnCollections[i].name
legend = formatLegend(title, min(elapsedTimeList),
max(elapsedTimeList),
numpy.mean(elapsedTimeList),
numpy.median(elapsedTimeList),
numpy.percentile(elapsedTimeList, 95),
numpy.percentile(elapsedTimeList, 99))
yaxis.append((legend, bucketValues))
benchmarkConflatedCounts = sum(conflatedCounts, 1)
if conflatedCounts[0] + benchmarkConflatedCounts > 0:
LOGGER.debug(
'conflation - due to narrow bucket range [%s to %s] - (%d) in current run and (%d) in all '
'bencmark counter values are conflated', buckets[0],
buckets[len(buckets) - 1], conflatedCounts[0],
benchmarkConflatedCounts)
buckets = formatBuckets(buckets)
options, data = buildHistograms(buckets, yaxis, False)
title = '{} - latency distribution benchmark'.format(category)
description = 'Latency distribution (current run ID #{} vs chosen benchmarks)'.format(
runId)
histograms.update(
{category: Histogram(title, description, data, options)})
return histograms
