Beispiel #1
0
  def gatherCounters(self, app, loader):
    """
    Gathers time and pmu counters from sample files for the current profile session

    :param app: Handle to the instance of the xpedite app
    :type app: xpedite.profiler.environment.XpediteApp
    :param loader: Loader to build transactions out of the counters

    """
    pattern = app.sampleFilePattern()
    LOGGER.info('scanning for samples files matching - %s', pattern)
    filePaths = app.gatherFiles(pattern)

    samplePath = makeLogPath('{}/{}'.format(app.name, app.runId))
    dataSource = DataSource(app.appInfoPath, samplePath)
    loader.beginCollection(dataSource)

    for filePath in filePaths:
      (threadId, tlsAddr) = self.extractThreadInfo(filePath)
      if not threadId or not tlsAddr:
        raise Exception('failed to extract thread info for file {}'.format(filePath))
      LOGGER.info('loading counters for thread %s from file %s -> ', threadId, filePath)

      iterBegin = begin = time.time()
      loader.beginLoad(threadId, tlsAddr)
      inflateFd = self.openInflateFile(samplePath, threadId, tlsAddr)
      extractor = subprocess.Popen([self.samplesLoader, filePath],
        bufsize=2*1024*1024, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
      recordCount = 0
      while True:
        record = extractor.stdout.readline()
        if record.strip() == '':
          if extractor.poll() is not None:
            errmsg = extractor.stderr.read()
            if errmsg:
              raise Exception('failed to load {} - {}'.format(filePath, errmsg))
          break
        if inflateFd:
          inflateFd.write(record)
        if recordCount > 0:
          self.loadCounter(threadId, loader, app.probes, record)
          elapsed = time.time() - iterBegin
          if elapsed >= 5:
            LOGGER.completed('\n\tprocessed %d counters | ', recordCount-1)
            iterBegin = time.time()
        recordCount += 1
      loader.endLoad()
      if inflateFd:
        inflateFd.close()
      elapsed = time.time() - begin
      self.logCounterFilterReport()
      if self.orphanedRecords:
        LOGGER.warn('detected mismatch in binary vs app info - %d counters ignored', len(self.orphanedRecords))
      LOGGER.completed('%d records | %d txns loaded in %0.2f sec.', recordCount-1, loader.getCount(), elapsed)
    if loader.isCompromised() or loader.getTxnCount() <= 0:
      LOGGER.warn(loader.report())
    elif loader.isNotAccounted():
      LOGGER.debug(loader.report())
    loader.endCollection()
Beispiel #2
0
    def gatherDataSources(path):
        """
    Gathers the list of profile data files for a benchmark

    :param path: path to benchmark directory

    """
        dataSources = []
        for runId in os.listdir(path):
            reportPath = os.path.join(path, runId)
            appInfoPath = os.path.join(path, BENCHMARK_APPINFO_FILE_NAME)
            if os.path.isdir(reportPath) and os.path.isfile(appInfoPath):
                dataSources.append(DataSource(appInfoPath, reportPath))
        return dataSources
Beispiel #3
0
    def gatherDataSource(path):
        """
    Gathers appinfo and samples dir to build a data source

    :param path: path to directory with txn data

    """
        appInfoPath = os.path.join(path, APPINFO_FILE_NAME)
        if not os.path.isfile(appInfoPath):
            LOGGER.error(
                'skipping data source %s - detected missing appinfo file %s',
                path, APPINFO_FILE_NAME)
            return None

        directories = next(os.walk(path))[1]
        if len(directories) > 1:
            LOGGER.error(
                'skipping data source %s - detected more than one (%d) directories',
                path, len(directories))
            return None
        return DataSource(appInfoPath, os.path.join(path, directories[0]))