def loadDataSource(self, loader, dataSource):
"""
Loads counters for a profile session from csv sample files
:param loader: Loader to build transactions out of the counters
:param dataSource: Sample files for a thread with counter data in csv format
:type dataSource: xpedite.types.DataSource
"""
appInfo = AppInfo(dataSource.appInfoPath)
appInfo.load()
recordCount = 0
reportDirs = list(sorted(os.listdir(dataSource.path)))
for threadInfo in reportDirs:
fields = threadInfo.split('-')
if len(fields) < 2:
raise Exception(
'Datasource {} missing tls storage info {}\n'.format(
reportDirs, threadInfo))
threadId = fields[0]
dirPath = os.path.join(dataSource.path, threadInfo)
if os.path.isdir(dirPath):
loader.beginLoad(threadId, fields[1])
for filePath in self.listSampleFiles(dirPath):
recordCount += self.loadCounters(threadId, loader,
appInfo.probes, filePath)
loader.endLoad()
self.logCounterFilterReport()
return recordCount
def loadDataSource(self, dataSource, loader):
"""
Loads counters for the given dataSource
:param dataSource: List of data sources with profile data
:param loader: Loader implementation to build transactions from counters
"""
from xpedite.profiler.appInfo import AppInfo
loader.beginCollection(dataSource)
appInfo = AppInfo(dataSource.appInfoPath)
appInfo.load()
self.loadSamples(loader, appInfo.probes, dataSource)
loader.endCollection()
def __enter__(self):
"""
Instantiates a tcp client and connects to target application
The methods executes the following actions
1. Builds a rpyc connection to the target host
2. Copies appInfo file from remote host to a temporary filesystem path in localhost
3. Loads AppInfo object from the copied file
4. Establishes a tcp connection to the target application
"""
from xpedite.transport.remote import deliver
self.remote.__enter__()
self.proxy = deliver(self.remote.connection, self.proxy)
result = self.gatherFiles(self.appInfo.path)
if len(result) != 1:
errmsg = 'Failed to gather app info file {} from remote host {} - got {}'.format(
self.appInfo.path, self.ip, result)
LOGGER.error(errmsg)
raise Exception(errmsg)
self.appInfo = AppInfo(result[0])
LOGGER.debug('Copied appinfo files from remote host to %s',
self.appInfo.path)
Environment.__enter__(self)
LOGGER.debug(
'initializing remote environment - delivered proxy to %s ',
self.ip)
return self
def __init__(self, ip, appInfoPath, dryRun):
self.ip = ip
self.appInfo = AppInfo(appInfoPath)
self.proxy = ProxyEnvironment()
self.dryRun = dryRun
self.client = None
class Environment(object):
"""Provides logic to interact with target process running in the same host"""
def __init__(self, ip, appInfoPath, dryRun):
self.ip = ip
self.appInfo = AppInfo(appInfoPath)
self.proxy = ProxyEnvironment()
self.dryRun = dryRun
self.client = None
@property
def appInfoPath(self):
"""Path to the app info file"""
return self.appInfo.path
@property
def pid(self):
"""Pid of the target process"""
return self.appInfo.pid
@property
def port(self):
"""Port number of the xpedite lister in the target process"""
return self.appInfo.port
@property
def binaryPath(self):
"""Path to the executable file of the target process"""
return self.appInfo.binaryPath
@property
def binaryName(self):
"""Name of the executable file of the target process"""
return self.appInfo.binaryName
@property
def tscHz(self):
"""Returns estimated frequency of cpu time stamp counter"""
return self.appInfo.tscHz
@property
def probes(self):
"""List of probes instrumented in the target process"""
return self.appInfo.probes
def gatherFiles(self, pattern):
"""
Gathers files matching the given pattern
:param pattern: Wild card pattern for files to be collected
"""
return self.proxy.gatherFiles(pattern)
def getCpuClockFrequencyRaw(self):
"""Returns the raw cpu clock frequency for localhost"""
rawHz = self.getFullCpuInfo()['hz_advertised_raw'][0]
if self.tscHz / 10**9 != rawHz / 10**9:
LOGGER.error(
'Detected mismatch in TSC (%s) and raw frequency (%s)',
self.tscHz, rawHz)
return rawHz
def getCpuInfo(self):
"""Returns cpu info for localhost"""
return CpuInfo(self.getCpuId(), self.getCpuClockFrequencyRaw())
def getFullCpuInfo(self):
"""Returns full cpu info of localhost"""
return self.proxy.getFullCpuInfo()
def getCpuId(self):
"""Returns cpu model of localhost"""
return self.proxy.getCpuId()
def getOsUname(self):
"""Returns kernel version of operating system running in target host"""
return self.proxy.getOsUname()
def getBootParam(self):
"""Returns kernel boot paramters of localhost"""
return self.proxy.getBootParam()
def enablePMU(self, eventsDb, cpuSet, events):
"""
Enables PMU counters for the given cpu set in target host
:param eventsDb: A database of programmable performance counters
:type eventsDb: xpedite.pmu.eventsDb.EventsDb
:param cpuSet: A list of cpu cores to enable pmc collection
:param events: A user supplied list of pmc events to be programmed
"""
return self.proxy.enablePMU(eventsDb, cpuSet, events)
def disablePMU(self):
"""Disables user space pmc collection and restores cpu core to orginal state"""
return self.proxy.disablePMU()
def getVmStats(self):
"""
Collects virtual memory usage statistics for a target process
:param pid: pid for the target process
"""
return self.proxy.getVmStats(self.pid)
def getBinaryPath(self):
"""Returns path to the executable file of the target process"""
return self.binaryPath
def getBinaryName(self):
"""Returns name of the executable file of the target process"""
return self.binaryName
def keepAlive(self):
"""Initiates a method call to keep the rpyc connection alive for long profiling sessions"""
return self.proxy.keepAlive()
def admin(self, cmd, timeout=10):
"""
Sends command to enable/disable/query probe status
:param cmd: Command to execute in target application
:param timeout: Maximum time to await a response from app (Default value = 10 seconds)
"""
self.client.send(cmd)
return self.client.readFrame(timeout)
def estimateTscHz(self, timeout=10):
"""Sends request to estimate frequency of cpu time stamp counter"""
return self.admin('tscHz', timeout)
def __enter__(self):
"""Instantiates a tcp client and connects to the target application"""
if self.client:
from xpedite.types import InvariantViloation
raise InvariantViloation('environment already in use')
self.loadAppInfo()
if not self.dryRun:
self.client = DatagramClient(self.ip, self.port)
self.client.connect()
return self
def loadAppInfo(self):
"""Loads application info of the target process"""
self.appInfo.load()
LOGGER.debug('resolved target pid to %s', self.pid)
LOGGER.debug('resolved target port to %s', self.port)
LOGGER.debug('resolved binary path to %s', self.binaryPath)
def __exit__(self, objType, value, traceback):
"""Disconnects tcp connection to the target app"""
if not self.dryRun:
if self.client:
self.client.close()
self.client = None
class Environment(object):
"""Provides logic to interact with target process running in the same host"""
def __init__(self, ip, appInfoPath, dryRun, workspace=None):
self.ip = ip
self.appInfo = AppInfo(appInfoPath, workspace)
self.proxy = ProxyEnvironment()
self.dryRun = dryRun
self.client = None
@property
def appInfoPath(self):
"""Path to the app info file"""
return self.appInfo.path
@property
def pid(self):
"""Pid of the target process"""
return self.appInfo.pid
@property
def port(self):
"""Port number of the xpedite lister in the target process"""
return self.appInfo.port
@property
def executablePath(self):
"""Path to the executable file of the target process"""
return self.appInfo.executablePath
@property
def executableName(self):
"""Name of the executable file of the target process"""
return self.appInfo.executableName
@property
def tscHz(self):
"""Returns estimated frequency of cpu time stamp counter"""
return self.appInfo.tscHz
@property
def probes(self):
"""List of probes instrumented in the target process"""
return self.appInfo.probes
def gatherFiles(self, pattern):
"""
Gathers files matching the given pattern
:param pattern: Wild card pattern for files to be collected
"""
return self.proxy.gatherFiles(pattern)
def getCpuClockFrequencyRaw(self):
"""Returns the raw cpu clock frequency for localhost"""
rawHz = self.getFullCpuInfo().advertisedHz
if int(self.tscHz / 10**9) != int(rawHz / 10**9):
LOGGER.error(
'Detected mismatch in estimated TSC (%s) vs raw frequency (%s)',
self.tscHz, rawHz)
return rawHz
def getCpuInfo(self):
"""Returns cpu info for localhost"""
return CpuInfo(self.getCpuId(), self.getCpuClockFrequencyRaw())
def getFullCpuInfo(self):
"""Returns full cpu info of localhost"""
return self.proxy.getFullCpuInfo()
def getCpuId(self):
"""Returns cpu model of localhost"""
return self.proxy.getCpuId()
def getOsUname(self):
"""Returns kernel version of operating system running in target host"""
return self.proxy.getOsUname()
def getBootParam(self):
"""Returns kernel boot paramters of localhost"""
return self.proxy.getBootParam()
def isDriverLoaded(self):
"""Returns status of xpedite device driver"""
return self.proxy.isDriverLoaded()
def enablePMU(self, eventsDb, cpuSet, events):
"""
Enables PMU counters for the given cpu set in target host
:param eventsDb: A database of programmable performance counters
:type eventsDb: xpedite.pmu.eventsDb.EventsDb
:param cpuSet: A list of cpu cores to enable pmc collection
:param events: A user supplied list of pmc events to be programmed
"""
if not self.isDriverLoaded():
(eventSet,
request) = PMUCtrl.buildPerfEventsRequest(eventsDb, events)
if eventSet and request:
LOGGER.warn(
'xpedite device driver not loaded - falling back to perf events api'
)
LOGGER.debug('sending request (%d bytes) to xpedite [%s]',
len(request), request)
rc = self.admin('ActivatePerfEvents --data {}'.format(request))
if rc:
raise Exception(rc)
return eventSet
return self.proxy.enablePMU(eventsDb, cpuSet, events)
def disablePMU(self):
"""Disables user space pmc collection and restores cpu core to orginal state"""
return self.proxy.disablePMU()
def getVmStats(self):
"""
Collects virtual memory usage statistics for a target process
:param pid: pid for the target process
"""
return self.proxy.getVmStats(self.pid)
def keepAlive(self):
"""Initiates a method call to keep the rpyc connection alive for long profiling sessions"""
return self.proxy.keepAlive()
def admin(self, cmd, timeout=10):
"""
Sends command to enable/disable/query probe status
:param cmd: Command to execute in target application
:param timeout: Maximum time to await a response from app (Default value = 10 seconds)
"""
self.client.send(cmd)
pdu = self.client.readFrame(timeout)
if len(pdu) < 5 or pdu[4] != '|':
raise InvariantViloation(
'Invalid response - pdu not in expected format \n{}\n'.format(
pdu))
status = pdu[3]
if not str.isdigit(status):
raise InvariantViloation(
'Invalid response - status code not in expected format \n{}\n'.
format(pdu))
result = pdu[5:] if len(pdu) > 5 else ''
if int(status):
raise Exception('Failed to execute request - {}'.format(result))
return result
def estimateTscHz(self, timeout=10):
"""Sends request to estimate frequency of cpu time stamp counter"""
return self.admin('TscHz', timeout)
def __enter__(self):
"""Instantiates a tcp client and connects to the target application"""
if self.client:
raise InvariantViloation('environment already in use')
self.loadAppInfo()
if not self.dryRun:
self.client = DatagramClient(self.ip, self.port)
self.client.connect()
return self
def loadAppInfo(self):
"""Loads application info of the target process"""
self.appInfo.load()
LOGGER.debug('resolved target pid to %s', self.pid)
LOGGER.debug('resolved target port to %s', self.port)
LOGGER.debug('resolved binary path to %s', self.executablePath)
def __exit__(self, objType, value, traceback):
"""Disconnects tcp connection to the target app"""
if not self.dryRun:
if self.client:
self.client.close()
self.client = None
