def loadDataSource(self, loader, dataSource): """ Loads counters for a profile session from csv sample files :param loader: Loader to build transactions out of the counters :param dataSource: Sample files for a thread with counter data in csv format :type dataSource: xpedite.types.DataSource """ appInfo = AppInfo(dataSource.appInfoPath) appInfo.load() recordCount = 0 reportDirs = list(sorted(os.listdir(dataSource.path))) for threadInfo in reportDirs: fields = threadInfo.split('-') if len(fields) < 2: raise Exception( 'Datasource {} missing tls storage info {}\n'.format( reportDirs, threadInfo)) threadId = fields[0] dirPath = os.path.join(dataSource.path, threadInfo) if os.path.isdir(dirPath): loader.beginLoad(threadId, fields[1]) for filePath in self.listSampleFiles(dirPath): recordCount += self.loadCounters(threadId, loader, appInfo.probes, filePath) loader.endLoad() self.logCounterFilterReport() return recordCount
def loadDataSource(self, dataSource, loader): """ Loads counters for the given dataSource :param dataSource: List of data sources with profile data :param loader: Loader implementation to build transactions from counters """ from xpedite.profiler.appInfo import AppInfo loader.beginCollection(dataSource) appInfo = AppInfo(dataSource.appInfoPath) appInfo.load() self.loadSamples(loader, appInfo.probes, dataSource) loader.endCollection()
class Environment(object): """Provides logic to interact with target process running in the same host""" def __init__(self, ip, appInfoPath, dryRun): self.ip = ip self.appInfo = AppInfo(appInfoPath) self.proxy = ProxyEnvironment() self.dryRun = dryRun self.client = None @property def appInfoPath(self): """Path to the app info file""" return self.appInfo.path @property def pid(self): """Pid of the target process""" return self.appInfo.pid @property def port(self): """Port number of the xpedite lister in the target process""" return self.appInfo.port @property def binaryPath(self): """Path to the executable file of the target process""" return self.appInfo.binaryPath @property def binaryName(self): """Name of the executable file of the target process""" return self.appInfo.binaryName @property def tscHz(self): """Returns estimated frequency of cpu time stamp counter""" return self.appInfo.tscHz @property def probes(self): """List of probes instrumented in the target process""" return self.appInfo.probes def gatherFiles(self, pattern): """ Gathers files matching the given pattern :param pattern: Wild card pattern for files to be collected """ return self.proxy.gatherFiles(pattern) def getCpuClockFrequencyRaw(self): """Returns the raw cpu clock frequency for localhost""" rawHz = self.getFullCpuInfo()['hz_advertised_raw'][0] if self.tscHz / 10**9 != rawHz / 10**9: LOGGER.error( 'Detected mismatch in TSC (%s) and raw frequency (%s)', self.tscHz, rawHz) return rawHz def getCpuInfo(self): """Returns cpu info for localhost""" return CpuInfo(self.getCpuId(), self.getCpuClockFrequencyRaw()) def getFullCpuInfo(self): """Returns full cpu info of localhost""" return self.proxy.getFullCpuInfo() def getCpuId(self): """Returns cpu model of localhost""" return self.proxy.getCpuId() def getOsUname(self): """Returns kernel version of operating system running in target host""" return self.proxy.getOsUname() def getBootParam(self): """Returns kernel boot paramters of localhost""" return self.proxy.getBootParam() def enablePMU(self, eventsDb, cpuSet, events): """ Enables PMU counters for the given cpu set in target host :param eventsDb: A database of programmable performance counters :type eventsDb: xpedite.pmu.eventsDb.EventsDb :param cpuSet: A list of cpu cores to enable pmc collection :param events: A user supplied list of pmc events to be programmed """ return self.proxy.enablePMU(eventsDb, cpuSet, events) def disablePMU(self): """Disables user space pmc collection and restores cpu core to orginal state""" return self.proxy.disablePMU() def getVmStats(self): """ Collects virtual memory usage statistics for a target process :param pid: pid for the target process """ return self.proxy.getVmStats(self.pid) def getBinaryPath(self): """Returns path to the executable file of the target process""" return self.binaryPath def getBinaryName(self): """Returns name of the executable file of the target process""" return self.binaryName def keepAlive(self): """Initiates a method call to keep the rpyc connection alive for long profiling sessions""" return self.proxy.keepAlive() def admin(self, cmd, timeout=10): """ Sends command to enable/disable/query probe status :param cmd: Command to execute in target application :param timeout: Maximum time to await a response from app (Default value = 10 seconds) """ self.client.send(cmd) return self.client.readFrame(timeout) def estimateTscHz(self, timeout=10): """Sends request to estimate frequency of cpu time stamp counter""" return self.admin('tscHz', timeout) def __enter__(self): """Instantiates a tcp client and connects to the target application""" if self.client: from xpedite.types import InvariantViloation raise InvariantViloation('environment already in use') self.loadAppInfo() if not self.dryRun: self.client = DatagramClient(self.ip, self.port) self.client.connect() return self def loadAppInfo(self): """Loads application info of the target process""" self.appInfo.load() LOGGER.debug('resolved target pid to %s', self.pid) LOGGER.debug('resolved target port to %s', self.port) LOGGER.debug('resolved binary path to %s', self.binaryPath) def __exit__(self, objType, value, traceback): """Disconnects tcp connection to the target app""" if not self.dryRun: if self.client: self.client.close() self.client = None
class Environment(object): """Provides logic to interact with target process running in the same host""" def __init__(self, ip, appInfoPath, dryRun, workspace=None): self.ip = ip self.appInfo = AppInfo(appInfoPath, workspace) self.proxy = ProxyEnvironment() self.dryRun = dryRun self.client = None @property def appInfoPath(self): """Path to the app info file""" return self.appInfo.path @property def pid(self): """Pid of the target process""" return self.appInfo.pid @property def port(self): """Port number of the xpedite lister in the target process""" return self.appInfo.port @property def executablePath(self): """Path to the executable file of the target process""" return self.appInfo.executablePath @property def executableName(self): """Name of the executable file of the target process""" return self.appInfo.executableName @property def tscHz(self): """Returns estimated frequency of cpu time stamp counter""" return self.appInfo.tscHz @property def probes(self): """List of probes instrumented in the target process""" return self.appInfo.probes def gatherFiles(self, pattern): """ Gathers files matching the given pattern :param pattern: Wild card pattern for files to be collected """ return self.proxy.gatherFiles(pattern) def getCpuClockFrequencyRaw(self): """Returns the raw cpu clock frequency for localhost""" rawHz = self.getFullCpuInfo().advertisedHz if int(self.tscHz / 10**9) != int(rawHz / 10**9): LOGGER.error( 'Detected mismatch in estimated TSC (%s) vs raw frequency (%s)', self.tscHz, rawHz) return rawHz def getCpuInfo(self): """Returns cpu info for localhost""" return CpuInfo(self.getCpuId(), self.getCpuClockFrequencyRaw()) def getFullCpuInfo(self): """Returns full cpu info of localhost""" return self.proxy.getFullCpuInfo() def getCpuId(self): """Returns cpu model of localhost""" return self.proxy.getCpuId() def getOsUname(self): """Returns kernel version of operating system running in target host""" return self.proxy.getOsUname() def getBootParam(self): """Returns kernel boot paramters of localhost""" return self.proxy.getBootParam() def isDriverLoaded(self): """Returns status of xpedite device driver""" return self.proxy.isDriverLoaded() def enablePMU(self, eventsDb, cpuSet, events): """ Enables PMU counters for the given cpu set in target host :param eventsDb: A database of programmable performance counters :type eventsDb: xpedite.pmu.eventsDb.EventsDb :param cpuSet: A list of cpu cores to enable pmc collection :param events: A user supplied list of pmc events to be programmed """ if not self.isDriverLoaded(): (eventSet, request) = PMUCtrl.buildPerfEventsRequest(eventsDb, events) if eventSet and request: LOGGER.warn( 'xpedite device driver not loaded - falling back to perf events api' ) LOGGER.debug('sending request (%d bytes) to xpedite [%s]', len(request), request) rc = self.admin('ActivatePerfEvents --data {}'.format(request)) if rc: raise Exception(rc) return eventSet return self.proxy.enablePMU(eventsDb, cpuSet, events) def disablePMU(self): """Disables user space pmc collection and restores cpu core to orginal state""" return self.proxy.disablePMU() def getVmStats(self): """ Collects virtual memory usage statistics for a target process :param pid: pid for the target process """ return self.proxy.getVmStats(self.pid) def keepAlive(self): """Initiates a method call to keep the rpyc connection alive for long profiling sessions""" return self.proxy.keepAlive() def admin(self, cmd, timeout=10): """ Sends command to enable/disable/query probe status :param cmd: Command to execute in target application :param timeout: Maximum time to await a response from app (Default value = 10 seconds) """ self.client.send(cmd) pdu = self.client.readFrame(timeout) if len(pdu) < 5 or pdu[4] != '|': raise InvariantViloation( 'Invalid response - pdu not in expected format \n{}\n'.format( pdu)) status = pdu[3] if not str.isdigit(status): raise InvariantViloation( 'Invalid response - status code not in expected format \n{}\n'. format(pdu)) result = pdu[5:] if len(pdu) > 5 else '' if int(status): raise Exception('Failed to execute request - {}'.format(result)) return result def estimateTscHz(self, timeout=10): """Sends request to estimate frequency of cpu time stamp counter""" return self.admin('TscHz', timeout) def __enter__(self): """Instantiates a tcp client and connects to the target application""" if self.client: raise InvariantViloation('environment already in use') self.loadAppInfo() if not self.dryRun: self.client = DatagramClient(self.ip, self.port) self.client.connect() return self def loadAppInfo(self): """Loads application info of the target process""" self.appInfo.load() LOGGER.debug('resolved target pid to %s', self.pid) LOGGER.debug('resolved target port to %s', self.port) LOGGER.debug('resolved binary path to %s', self.executablePath) def __exit__(self, objType, value, traceback): """Disconnects tcp connection to the target app""" if not self.dryRun: if self.client: self.client.close() self.client = None