def gatherCounters(self, app, loader): """ Gathers time and pmu counters from sample files for the current profile session :param app: Handle to the instance of the xpedite app :type app: xpedite.profiler.environment.XpediteApp :param loader: Loader to build transactions out of the counters """ pattern = app.sampleFilePattern() LOGGER.info('scanning for samples files matching - %s', pattern) filePaths = app.gatherFiles(pattern) samplePath = makeLogPath('{}/{}'.format(app.name, app.runId)) dataSource = DataSource(app.appInfoPath, samplePath) loader.beginCollection(dataSource) for filePath in filePaths: (threadId, tlsAddr) = self.extractThreadInfo(filePath) if not threadId or not tlsAddr: raise Exception('failed to extract thread info for file {}'.format(filePath)) LOGGER.info('loading counters for thread %s from file %s -> ', threadId, filePath) iterBegin = begin = time.time() loader.beginLoad(threadId, tlsAddr) inflateFd = self.openInflateFile(samplePath, threadId, tlsAddr) extractor = subprocess.Popen([self.samplesLoader, filePath], bufsize=2*1024*1024, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) recordCount = 0 while True: record = extractor.stdout.readline() if record.strip() == '': if extractor.poll() is not None: errmsg = extractor.stderr.read() if errmsg: raise Exception('failed to load {} - {}'.format(filePath, errmsg)) break if inflateFd: inflateFd.write(record) if recordCount > 0: self.loadCounter(threadId, loader, app.probes, record) elapsed = time.time() - iterBegin if elapsed >= 5: LOGGER.completed('\n\tprocessed %d counters | ', recordCount-1) iterBegin = time.time() recordCount += 1 loader.endLoad() if inflateFd: inflateFd.close() elapsed = time.time() - begin self.logCounterFilterReport() if self.orphanedRecords: LOGGER.warn('detected mismatch in binary vs app info - %d counters ignored', len(self.orphanedRecords)) LOGGER.completed('%d records | %d txns loaded in %0.2f sec.', recordCount-1, loader.getCount(), elapsed) if loader.isCompromised() or loader.getTxnCount() <= 0: LOGGER.warn(loader.report()) elif loader.isNotAccounted(): LOGGER.debug(loader.report()) loader.endCollection()
def gatherDataSources(path): """ Gathers the list of profile data files for a benchmark :param path: path to benchmark directory """ dataSources = [] for runId in os.listdir(path): reportPath = os.path.join(path, runId) appInfoPath = os.path.join(path, BENCHMARK_APPINFO_FILE_NAME) if os.path.isdir(reportPath) and os.path.isfile(appInfoPath): dataSources.append(DataSource(appInfoPath, reportPath)) return dataSources
def gatherDataSource(path): """ Gathers appinfo and samples dir to build a data source :param path: path to directory with txn data """ appInfoPath = os.path.join(path, APPINFO_FILE_NAME) if not os.path.isfile(appInfoPath): LOGGER.error( 'skipping data source %s - detected missing appinfo file %s', path, APPINFO_FILE_NAME) return None directories = next(os.walk(path))[1] if len(directories) > 1: LOGGER.error( 'skipping data source %s - detected more than one (%d) directories', path, len(directories)) return None return DataSource(appInfoPath, os.path.join(path, directories[0]))