def getCpuInfo(): """Loads cpu info for localhost""" from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load(Package.PyCpuInfo) from cpuinfo import cpuinfo fullCpuInfo = cpuinfo.get_cpu_info() return fullCpuInfo
def __init__(self, app, probes, pmc=None, cpuSet=None, pollInterval=4, benchmarkProbes=None): """ Creates a new profiler runtime Construction of the runtime will execute the following steps 1. Starts the xpedite app to attach to profiling target 2. Queries and resolves location of probes in profile info 3. Load events and topdown database for the target cpu's micro architecture 4. Resolves pmu events and topdown metrics from events database/topdown hierarchy 5. Opens xpedite device driver to program pmu events and enable userspace pmc collection 6. Activates resolved probes and begins sample collection in the target process :param app: an instance of xpedite app, to interact with target application :type app: xpedite.profiler.app.XpediteApp :param probes: List of probes to be enabled for the current profile session :param pmc: PMU events to be enabled for the current profile session :param cpuSet: List of cpu, where the userspace pmu collection will be enabled :type cpuSet: int :param pollInterval: Sample collection period in milli seconds :type pollInterval: int :param benchmarkProbes: optional map to override probes used for benchmarks, defaults to active probes of the current profile session """ from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load(Package.Numpy, Package.FuncTools) if len(probes) < 2: raise Exception( 'invalid request - profiling needs at least two named probes to be enabled. Found only {}' .format(probes)) try: AbstractRuntime.__init__(self, app, probes) self.benchmarkProbes = benchmarkProbes self.cpuInfo = app.getCpuInfo() eventsDb = self.eventsDbCache.get( self.cpuInfo.cpuId) if pmc else None if pmc: LOGGER.debug('detected %s', eventsDb.uarchSpec) self.initTopdown(pmc) pmc = self.aggregatePmc(pmc) if not self.app.dryRun: if pmc: self.eventState = self.app.enablePMU(eventsDb, cpuSet, pmc) anchoredProbes = self.resolveProbes(probes) self.enableProbes(anchoredProbes) self.app.beginProfile(pollInterval) else: if pmc: self.eventState = self.resolveEvents(eventsDb, cpuSet, pmc) LOGGER.warn('DRY Run selected - xpedite won\'t enable probes') except Exception as ex: LOGGER.exception('failed to start profiling') raise ex
def format(self, record): """Format log messages with level name and color""" from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load(Package.Termcolor) from termcolor import colored msg = super(ConsoleFormatter, self).format(record) prefix = '\n' if ConsoleFormatter.lineCount > 0 else '' if record.levelname == 'ERROR' or record.levelname == 'WARNING': msg = '{}[{}]: {}'.format(prefix, record.levelname, msg) elif record.levelname != 'COMPLETED': msg = prefix + msg ConsoleFormatter.lineCount += 1 if record.levelname in self.textColor: return colored(msg, self.textColor[record.levelname]) else: return msg
def disable(self): """Disaables profiling""" self.cprofile.disable() strIO = StringIO.StringIO() sortby = 'cumulative' ps = pstats.Stats(self.cprofile, stream=strIO).sort_stats(sortby) LOGGER.info('Profiler disabled, data written to %s', self.path) from xpedite.dependencies import Package, DEPENDENCY_LOADER if DEPENDENCY_LOADER.load(Package.PyProf2Calltree): # pylint: disable=no-member from pyprof2calltree import convert # pylint: disable=import-error convert(ps, self.path) LOGGER.info( 'Open the report in KCachegrind to see the profile report')
""" This module provides markup, css selectors, javascripts for generation of HTML reports Author: Manikandan Dhamodharan, Morgan Stanley """ import os from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load(Package.HTML, Package.Pygments) from html import HTML # pylint: disable=wrong-import-position def loadFile(path): """ Loads contents of the given file :param path: Path of the file to load """ with open(path) as fileHandle: return fileHandle.read() def formatList(inputList): """ Formats a list of items to html unordered list :param inputList: list of items to be formatted """
""" This module provides markup, css selectors, javascripts for generation of HTML reports Author: Manikandan Dhamodharan, Morgan Stanley """ import os from thirdParty.html import HTML from xpedite.util import loadTextFile from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load(Package.Pygments, Package.Six) def formatList(inputList): """ Formats a list of items to html unordered list :param inputList: list of items to be formatted """ report = HTML() htmlList = report.ul for val in inputList: val = str(val) htmlList.li(val) return report TABLE_ENV = 'tableEnv tablesorter' TABLE_SUMMARY = 'tableSummary tablesorter' TABLE_REPORT_CONTAINER = 'tableReportContainer'
1. The key press by the user. 2. Termination of the target application 3. Expiry of profiling duration Author: Manikandan Dhamodharan, Morgan Stanley """ import sys import os import logging import xpedite from xpedite.profiler.profileInfo import loadProfileInfo from xpedite.profiler.app import XpediteApp, XpediteDormantApp, pingApp from logger import enableVerboseLogging from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load(Package.Six) LOGGER = logging.getLogger(__name__) def buildReportName(appName, reportName): """Constructs report name from app + user supplied report name""" import re from datetime import datetime reportName = reportName if reportName else '{}-{}'.format(appName, datetime.now().strftime('%Y-%m-%d-%H:%M:%S')) reportName = re.sub(r'[^\w\-:_\. ]', '_', reportName) return reportName def validateBenchmarkPath(path): """Validate the given path for write access""" if path: if os.path.exists(path):
#!/usr/bin/env python2.7 """ Topdown hierarchy builder This module is used to build, topdown hierarchy for all supported micro architectures. The module also supports resolution of pmc events for nodes in the hierarchy. Author: Manikandan Dhamodharan, Morgan Stanley """ import types from collections import defaultdict, Counter from xpedite.pmu.uarchEvent import GenericCoreEvent, FixedCoreEvent, OffCoreEvent from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load(Package.Termcolor) from termcolor import colored # pylint: disable=wrong-import-position class Root(object): """Root of the topdown hierarchy tree""" name = 'Root' domain = 'Slots' area = 'FE+BE' desc = """ Root of top down micro architecture analysis hierarchy. """ level = 0 htoff = False sample = [] errcount = 0
1. Establish tcp connection with target application 2. Support running python code in a remote box using ssh and rpyc 3. Frame a stream of data to construct datagrams 4. Accumulator to buffer datagrams, to send in one shot Author: Manikandan Dhamodharan, Morgan Stanley """ import os import sys import socket import logging from xpedite.util import promptUser from xpedite.transport.client import Client from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load(Package.Netifaces, Package.Rpyc) LOGGER = logging.getLogger(__name__) def encode(msg): """ Encodes length prefixed message :param msg: Message to be encoded """ payload = '{0:0>8}'.format(len(msg)) + msg return payload
1. An application that allocates memory 2. An application to exercise txn carrying different units of data 3. A multithreaded application 4. An application to parse FIX messages Each application is used in 2 scenarios: 1. A regular scenario 2. A scenario using Xpedite's benchmarks Author: Brooke Elizabeth Cantwell, Morgan Stanley """ import os import json from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load(Package.Enum) DEPENDENCY_LOADER.load(Package.Rpyc) from enum import Enum import rpyc from test_xpedite import ( REPORT_CMD_BASELINE_PATH, PROFILE_INFO_PATH, DATA_FILE_EXT, BASELINE_CPU_INFO_PATH, PROBE_CMD_BASELINE_PATH, GENERATE_CMD_BASELINE_PATH, XPEDITE_APP_INFO_PATH, XPEDITE_APP_INFO_PARAMETER_PATH, PARAMETERS_DATA_DIR, DIR_PATH, SRC_DIR_PATH, LOCALHOST, BINARY_PATH, loadProfileInfo ) from test_xpedite.test_profiler.profile import validateBenchmarks
""" This package provides modules and drivers to support real time analytics on Xpedite profile data This modules include 1. Driver to start a jupyter instance 2. Modules to collect and serialize profiles and html reports 3. Logic to generate and provide code snippets 4. Logic to integrate Xpedite commands and visualizations in jupyter shell Author: Manikandan Dhamodharan, Morgan Stanley """ from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load( Package.Enum, Package.Futures, Package.FuncTools, Package.Jupyter, Package.Six, ) DATA_DIR = 'xpData' DATA_FILE_EXT = '.xpd' NOTEBOOK_EXT = '.ipynb' ARCHIVE_FILE_EXT = '.tar{}'.format(DATA_FILE_EXT) EXPORT_PREFIX = 'xpediteExport' SHELL_PREFIX = 'xpediteShell' TEMP_PREFIX = 'xpedite' PROFILES_KEY = 'profiles'
""" Util class to deal with nuances in PyCpuInfo API Author: Manikandan Dhamodharan, Morgan Stanley """ import json from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load(Package.PyCpuInfo) class CpuInfo(object): """ Builds a map of /proc/cpuinfo to derive id and advertised frequecy for a cpu """ def __init__(self, info=None): """Loads cpu info from localhost""" from cpuinfo import cpuinfo self.info = info if info else cpuinfo.get_cpu_info() self.cpuId = self._loadId() self.advertisedHz = self._loadAdvertisedHz() def _loadId(self): """Returns the cpu identifier from vendor, family, model and stepping""" vendorId = self.info.get('vendor_id') vendorId = vendorId if vendorId else self.info.get('vendor_id_raw') if vendorId: return '{}-{}-{:02X}-{}'.format(vendorId, self.info['family'], self.info['model'], self.info['stepping'])
This package includes 1. Logic for grouping data from related probes, to build transactions 2. Classify and aggreate transactions based on route (control flow) 3. Build timelines and duration series from aggregated transactions 4. Logic to conflate transactions from mulitiple profiles Author: Manikandan Dhamodharan, Morgan Stanley """ import sys import time import logging from xpedite.util import timeAction from xpedite.types.containers import ProbeMap from xpedite.dependencies import Package, DEPENDENCY_LOADER DEPENDENCY_LOADER.load(Package.Numpy) from xpedite.analytics.aggregator import TxnAggregator, RouteAggregator, RouteConflatingAggregator # pylint: disable=wrong-import-position from xpedite.analytics.timeline import buildTimelineStats # pylint: disable=wrong-import-position from xpedite.analytics.treeCollections import TreeCollectionFactory # pylint: disable=wrong-import-position LOGGER = logging.getLogger(__name__) CURRENT_RUN = 'current run' class Analytics(object): """Analytics logic to build transactions for current profile session and bechmarks""" @staticmethod def buildElapsedTimeBundles(txnCollections, classifier): """ Builds elapsed timestamp counters for each of the categories in given transaction collections