def gatherCounters(self, app, loader):
"""
Gathers time and pmu counters from sample files for the current profile session
:param app: Handle to the instance of the xpedite app
:type app: xpedite.profiler.environment.XpediteApp
:param loader: Loader to build transactions out of the counters
"""
pattern = app.sampleFilePattern()
LOGGER.info('scanning for samples files matching - %s', pattern)
filePaths = app.gatherFiles(pattern)
samplePath = makeLogPath('{}/{}'.format(app.name, app.runId))
dataSource = DataSource(app.appInfoPath, samplePath)
loader.beginCollection(dataSource)
for filePath in filePaths:
(threadId, tlsAddr) = self.extractThreadInfo(filePath)
if not threadId or not tlsAddr:
raise Exception('failed to extract thread info for file {}'.format(filePath))
LOGGER.info('loading counters for thread %s from file %s -> ', threadId, filePath)
iterBegin = begin = time.time()
loader.beginLoad(threadId, tlsAddr)
inflateFd = self.openInflateFile(samplePath, threadId, tlsAddr)
extractor = subprocess.Popen([self.samplesLoader, filePath],
bufsize=2*1024*1024, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
recordCount = 0
while True:
record = extractor.stdout.readline()
if record.strip() == '':
if extractor.poll() is not None:
errmsg = extractor.stderr.read()
if errmsg:
raise Exception('failed to load {} - {}'.format(filePath, errmsg))
break
if inflateFd:
inflateFd.write(record)
if recordCount > 0:
self.loadCounter(threadId, loader, app.probes, record)
elapsed = time.time() - iterBegin
if elapsed >= 5:
LOGGER.completed('\n\tprocessed %d counters | ', recordCount-1)
iterBegin = time.time()
recordCount += 1
loader.endLoad()
if inflateFd:
inflateFd.close()
elapsed = time.time() - begin
self.logCounterFilterReport()
if self.orphanedRecords:
LOGGER.warn('detected mismatch in binary vs app info - %d counters ignored', len(self.orphanedRecords))
LOGGER.completed('%d records | %d txns loaded in %0.2f sec.', recordCount-1, loader.getCount(), elapsed)
if loader.isCompromised() or loader.getTxnCount() <= 0:
LOGGER.warn(loader.report())
elif loader.isNotAccounted():
LOGGER.debug(loader.report())
loader.endCollection()
def gatherDataSources(path):
"""
Gathers the list of profile data files for a benchmark
:param path: path to benchmark directory
"""
dataSources = []
for runId in os.listdir(path):
reportPath = os.path.join(path, runId)
appInfoPath = os.path.join(path, BENCHMARK_APPINFO_FILE_NAME)
if os.path.isdir(reportPath) and os.path.isfile(appInfoPath):
dataSources.append(DataSource(appInfoPath, reportPath))
return dataSources
def gatherDataSource(path):
"""
Gathers appinfo and samples dir to build a data source
:param path: path to directory with txn data
"""
appInfoPath = os.path.join(path, APPINFO_FILE_NAME)
if not os.path.isfile(appInfoPath):
LOGGER.error(
'skipping data source %s - detected missing appinfo file %s',
path, APPINFO_FILE_NAME)
return None
directories = next(os.walk(path))[1]
if len(directories) > 1:
LOGGER.error(
'skipping data source %s - detected more than one (%d) directories',
path, len(directories))
return None
return DataSource(appInfoPath, os.path.join(path, directories[0]))