def getCpuInfo():
"""Loads cpu info for localhost"""
from xpedite.dependencies import Package, DEPENDENCY_LOADER
DEPENDENCY_LOADER.load(Package.PyCpuInfo)
from cpuinfo import cpuinfo
fullCpuInfo = cpuinfo.get_cpu_info()
return fullCpuInfo
def __init__(self,
app,
probes,
pmc=None,
cpuSet=None,
pollInterval=4,
benchmarkProbes=None):
"""
Creates a new profiler runtime
Construction of the runtime will execute the following steps
1. Starts the xpedite app to attach to profiling target
2. Queries and resolves location of probes in profile info
3. Load events and topdown database for the target cpu's micro architecture
4. Resolves pmu events and topdown metrics from events database/topdown hierarchy
5. Opens xpedite device driver to program pmu events and enable userspace pmc collection
6. Activates resolved probes and begins sample collection in the target process
:param app: an instance of xpedite app, to interact with target application
:type app: xpedite.profiler.app.XpediteApp
:param probes: List of probes to be enabled for the current profile session
:param pmc: PMU events to be enabled for the current profile session
:param cpuSet: List of cpu, where the userspace pmu collection will be enabled
:type cpuSet: int
:param pollInterval: Sample collection period in milli seconds
:type pollInterval: int
:param benchmarkProbes: optional map to override probes used for benchmarks,
defaults to active probes of the current profile session
"""
from xpedite.dependencies import Package, DEPENDENCY_LOADER
DEPENDENCY_LOADER.load(Package.Numpy, Package.FuncTools)
if len(probes) < 2:
raise Exception(
'invalid request - profiling needs at least two named probes to be enabled. Found only {}'
.format(probes))
try:
AbstractRuntime.__init__(self, app, probes)
self.benchmarkProbes = benchmarkProbes
self.cpuInfo = app.getCpuInfo()
eventsDb = self.eventsDbCache.get(
self.cpuInfo.cpuId) if pmc else None
if pmc:
LOGGER.debug('detected %s', eventsDb.uarchSpec)
self.initTopdown(pmc)
pmc = self.aggregatePmc(pmc)
if not self.app.dryRun:
if pmc:
self.eventState = self.app.enablePMU(eventsDb, cpuSet, pmc)
anchoredProbes = self.resolveProbes(probes)
self.enableProbes(anchoredProbes)
self.app.beginProfile(pollInterval)
else:
if pmc:
self.eventState = self.resolveEvents(eventsDb, cpuSet, pmc)
LOGGER.warn('DRY Run selected - xpedite won\'t enable probes')
except Exception as ex:
LOGGER.exception('failed to start profiling')
raise ex
def format(self, record):
"""Format log messages with level name and color"""
from xpedite.dependencies import Package, DEPENDENCY_LOADER
DEPENDENCY_LOADER.load(Package.Termcolor)
from termcolor import colored
msg = super(ConsoleFormatter, self).format(record)
prefix = '\n' if ConsoleFormatter.lineCount > 0 else ''
if record.levelname == 'ERROR' or record.levelname == 'WARNING':
msg = '{}[{}]: {}'.format(prefix, record.levelname, msg)
elif record.levelname != 'COMPLETED':
msg = prefix + msg
ConsoleFormatter.lineCount += 1
if record.levelname in self.textColor:
return colored(msg, self.textColor[record.levelname])
else:
return msg
def disable(self):
"""Disaables profiling"""
self.cprofile.disable()
strIO = StringIO.StringIO()
sortby = 'cumulative'
ps = pstats.Stats(self.cprofile, stream=strIO).sort_stats(sortby)
LOGGER.info('Profiler disabled, data written to %s', self.path)
from xpedite.dependencies import Package, DEPENDENCY_LOADER
if DEPENDENCY_LOADER.load(Package.PyProf2Calltree): # pylint: disable=no-member
from pyprof2calltree import convert # pylint: disable=import-error
convert(ps, self.path)
LOGGER.info(
'Open the report in KCachegrind to see the profile report')
"""
This module provides markup, css selectors, javascripts for generation of HTML reports
Author: Manikandan Dhamodharan, Morgan Stanley
"""
import os
from xpedite.dependencies import Package, DEPENDENCY_LOADER
DEPENDENCY_LOADER.load(Package.HTML, Package.Pygments)
from html import HTML # pylint: disable=wrong-import-position
def loadFile(path):
"""
Loads contents of the given file
:param path: Path of the file to load
"""
with open(path) as fileHandle:
return fileHandle.read()
def formatList(inputList):
"""
Formats a list of items to html unordered list
:param inputList: list of items to be formatted
"""
"""
This module provides markup, css selectors, javascripts for generation of HTML reports
Author: Manikandan Dhamodharan, Morgan Stanley
"""
import os
from thirdParty.html import HTML
from xpedite.util import loadTextFile
from xpedite.dependencies import Package, DEPENDENCY_LOADER
DEPENDENCY_LOADER.load(Package.Pygments, Package.Six)
def formatList(inputList):
"""
Formats a list of items to html unordered list
:param inputList: list of items to be formatted
"""
report = HTML()
htmlList = report.ul
for val in inputList:
val = str(val)
htmlList.li(val)
return report
TABLE_ENV = 'tableEnv tablesorter'
TABLE_SUMMARY = 'tableSummary tablesorter'
TABLE_REPORT_CONTAINER = 'tableReportContainer'
1. The key press by the user.
2. Termination of the target application
3. Expiry of profiling duration
Author: Manikandan Dhamodharan, Morgan Stanley
"""
import sys
import os
import logging
import xpedite
from xpedite.profiler.profileInfo import loadProfileInfo
from xpedite.profiler.app import XpediteApp, XpediteDormantApp, pingApp
from logger import enableVerboseLogging
from xpedite.dependencies import Package, DEPENDENCY_LOADER
DEPENDENCY_LOADER.load(Package.Six)
LOGGER = logging.getLogger(__name__)
def buildReportName(appName, reportName):
"""Constructs report name from app + user supplied report name"""
import re
from datetime import datetime
reportName = reportName if reportName else '{}-{}'.format(appName, datetime.now().strftime('%Y-%m-%d-%H:%M:%S'))
reportName = re.sub(r'[^\w\-:_\. ]', '_', reportName)
return reportName
def validateBenchmarkPath(path):
"""Validate the given path for write access"""
if path:
if os.path.exists(path):
#!/usr/bin/env python2.7
"""
Topdown hierarchy builder
This module is used to build, topdown hierarchy for all supported micro architectures.
The module also supports resolution of pmc events for nodes in the hierarchy.
Author: Manikandan Dhamodharan, Morgan Stanley
"""
import types
from collections import defaultdict, Counter
from xpedite.pmu.uarchEvent import GenericCoreEvent, FixedCoreEvent, OffCoreEvent
from xpedite.dependencies import Package, DEPENDENCY_LOADER
DEPENDENCY_LOADER.load(Package.Termcolor)
from termcolor import colored # pylint: disable=wrong-import-position
class Root(object):
"""Root of the topdown hierarchy tree"""
name = 'Root'
domain = 'Slots'
area = 'FE+BE'
desc = """
Root of top down micro architecture analysis hierarchy.
"""
level = 0
htoff = False
sample = []
errcount = 0
1. Establish tcp connection with target application
2. Support running python code in a remote box using ssh and rpyc
3. Frame a stream of data to construct datagrams
4. Accumulator to buffer datagrams, to send in one shot
Author: Manikandan Dhamodharan, Morgan Stanley
"""
import os
import sys
import socket
import logging
from xpedite.util import promptUser
from xpedite.transport.client import Client
from xpedite.dependencies import Package, DEPENDENCY_LOADER
DEPENDENCY_LOADER.load(Package.Netifaces, Package.Rpyc)
LOGGER = logging.getLogger(__name__)
def encode(msg):
"""
Encodes length prefixed message
:param msg: Message to be encoded
"""
payload = '{0:0>8}'.format(len(msg)) + msg
return payload
1. An application that allocates memory
2. An application to exercise txn carrying different units of data
3. A multithreaded application
4. An application to parse FIX messages
Each application is used in 2 scenarios:
1. A regular scenario
2. A scenario using Xpedite's benchmarks
Author: Brooke Elizabeth Cantwell, Morgan Stanley
"""
import os
import json
from xpedite.dependencies import Package, DEPENDENCY_LOADER
DEPENDENCY_LOADER.load(Package.Enum)
DEPENDENCY_LOADER.load(Package.Rpyc)
from enum import Enum
import rpyc
from test_xpedite import (
REPORT_CMD_BASELINE_PATH,
PROFILE_INFO_PATH,
DATA_FILE_EXT, BASELINE_CPU_INFO_PATH,
PROBE_CMD_BASELINE_PATH,
GENERATE_CMD_BASELINE_PATH,
XPEDITE_APP_INFO_PATH,
XPEDITE_APP_INFO_PARAMETER_PATH,
PARAMETERS_DATA_DIR, DIR_PATH, SRC_DIR_PATH,
LOCALHOST, BINARY_PATH, loadProfileInfo
)
from test_xpedite.test_profiler.profile import validateBenchmarks
"""
This package provides modules and drivers to support real time analytics
on Xpedite profile data
This modules include
1. Driver to start a jupyter instance
2. Modules to collect and serialize profiles and html reports
3. Logic to generate and provide code snippets
4. Logic to integrate Xpedite commands and visualizations in jupyter shell
Author: Manikandan Dhamodharan, Morgan Stanley
"""
from xpedite.dependencies import Package, DEPENDENCY_LOADER
DEPENDENCY_LOADER.load(
Package.Enum,
Package.Futures,
Package.FuncTools,
Package.Jupyter,
Package.Six,
)
DATA_DIR = 'xpData'
DATA_FILE_EXT = '.xpd'
NOTEBOOK_EXT = '.ipynb'
ARCHIVE_FILE_EXT = '.tar{}'.format(DATA_FILE_EXT)
EXPORT_PREFIX = 'xpediteExport'
SHELL_PREFIX = 'xpediteShell'
TEMP_PREFIX = 'xpedite'
PROFILES_KEY = 'profiles'
"""
Util class to deal with nuances in PyCpuInfo API
Author: Manikandan Dhamodharan, Morgan Stanley
"""
import json
from xpedite.dependencies import Package, DEPENDENCY_LOADER
DEPENDENCY_LOADER.load(Package.PyCpuInfo)
class CpuInfo(object):
"""
Builds a map of /proc/cpuinfo to derive id and advertised frequecy for a cpu
"""
def __init__(self, info=None):
"""Loads cpu info from localhost"""
from cpuinfo import cpuinfo
self.info = info if info else cpuinfo.get_cpu_info()
self.cpuId = self._loadId()
self.advertisedHz = self._loadAdvertisedHz()
def _loadId(self):
"""Returns the cpu identifier from vendor, family, model and stepping"""
vendorId = self.info.get('vendor_id')
vendorId = vendorId if vendorId else self.info.get('vendor_id_raw')
if vendorId:
return '{}-{}-{:02X}-{}'.format(vendorId, self.info['family'],
self.info['model'],
self.info['stepping'])
This package includes
1. Logic for grouping data from related probes, to build transactions
2. Classify and aggreate transactions based on route (control flow)
3. Build timelines and duration series from aggregated transactions
4. Logic to conflate transactions from mulitiple profiles
Author: Manikandan Dhamodharan, Morgan Stanley
"""
import sys
import time
import logging
from xpedite.util import timeAction
from xpedite.types.containers import ProbeMap
from xpedite.dependencies import Package, DEPENDENCY_LOADER
DEPENDENCY_LOADER.load(Package.Numpy)
from xpedite.analytics.aggregator import TxnAggregator, RouteAggregator, RouteConflatingAggregator # pylint: disable=wrong-import-position
from xpedite.analytics.timeline import buildTimelineStats # pylint: disable=wrong-import-position
from xpedite.analytics.treeCollections import TreeCollectionFactory # pylint: disable=wrong-import-position
LOGGER = logging.getLogger(__name__)
CURRENT_RUN = 'current run'
class Analytics(object):
"""Analytics logic to build transactions for current profile session and bechmarks"""
@staticmethod
def buildElapsedTimeBundles(txnCollections, classifier):
"""
Builds elapsed timestamp counters for each of the categories in given transaction collections