def test_probe_states(capsys): """ Test xpedite probes and probes state for an application's against baseline probe states for the application """ import cPickle as pickle from xpedite.types.probe import compareProbes profileInfo = loadProfileInfo(DATA_DIR, os.path.join(DATA_DIR, 'profileInfo.py'), REMOTE) probes = [] probeMap = {} baselineProbeMap = {} with capsys.disabled(): probes = loadProbes(FIX_DECODER_BINARY, profileInfo, TXN_COUNT, THREAD_COUNT, REMOTE, WORKSPACE) for probe in probes: probeMap[probe.sysName] = probe with open(os.path.join(DATA_DIR, 'probeCmdBaseline.pkl'), 'r') as probeFileHandle: baselineProbes = pickle.load(probeFileHandle) for probe in baselineProbes: baselineProbeMap[probe.sysName] = probe assert len(probes) == len(baselineProbes) findDiff(probeMap, baselineProbeMap) for probe in probeMap.keys(): assert compareProbes(probeMap[probe], baselineProbeMap[probe])
def compareAgainstBaseline(profileInfoPath, baselinePath, dataDir, workspace=None): """ Compare profiles with benchmarks and profiles without benchmarks against existing profiles """ import json profileInfo = loadProfileInfo(dataDir, profileInfoPath) profileInfo.appInfo = os.path.join(dataDir, 'xpedite-appinfo.txt') runId = collectDataFiles(dataDir) baselineProfiles = loadBaseline(dataDir, baselinePath) with open(os.path.join(dataDir, 'baselineCpuInfo.json')) as fileHandle: fullCpuInfo = json.load(fileHandle) sampleFilePath = '{}/xpedite-*-{}-[0-9]*.data'.format(dataDir, runId) result = runXpediteReport(runId, profileInfo, sampleFilePath=sampleFilePath, cpuInfo=fullCpuInfo, workspace=workspace) reportProfiles = result.profiles reportProfiles.transactionRepo = None # transactionRepo not stored in .xpd files reportProfiles.cpuInfo.cpuId = baselineProfiles.cpuInfo.cpuId if profileInfo.benchmarkPaths: validateBenchmarks(len(profileInfo.benchmarkPaths), reportProfiles) validateBenchmarks(len(profileInfo.benchmarkPaths), baselineProfiles) findDiff(reportProfiles.__dict__, baselineProfiles.__dict__) assert reportProfiles == baselineProfiles
def test_record_vs_report(capsys, scenarioName): """ Run xpedite record and xpedite report to compare profiles """ checkPmcSupport(scenarioName) with SCENARIO_LOADER[scenarioName] as scenarios: with capsys.disabled(): currentReport, _, _ = runXpediteRecord(CONTEXT, scenarios) report = runXpediteReport(currentReport.runId, CONTEXT, scenarios) findDiff(report.profiles.__dict__, currentReport.profiles.__dict__) assert report.profiles == currentReport.profiles
def test_report_against_baseline(): """ Run xpedite report on a data file in the test directory, return profiles and compare the previously generated profiles from the same xpedite run """ from test_xpedite.test_profiler.comparator import findDiff profileInfo = loadProfileInfo('profileInfo.py') for runId, dataFilePath in collectDataFiles().iteritems(): reportProfiles, _ = runXpediteReport(runId, profileInfo, dataFilePath) reportProfiles.transactionRepo = None # transactionRepo not stored in .xpd files baselineProfiles = loadBaseline() findDiff(reportProfiles.__dict__, baselineProfiles.__dict__) assert reportProfiles == baselineProfiles
def test_generate_cmd_vs_baseline(scenarioName): """ Test xpedite generate by generating a new profileInfo.py file and comparing to baseline profileInfo.py in the test data directory """ from test_xpedite.test_profiler.app import TargetLauncher from xpedite.profiler.probeAdmin import ProbeAdmin with SCENARIO_LOADER[scenarioName] as scenarios: with TargetLauncher(CONTEXT, scenarios) as app: probes = ProbeAdmin.loadProbes(app.xpediteApp) profileInfoFile = generateProfileInfoFile(app.xpediteApp, probes) generatedProfileInfo = loadProfileInfo('', profileInfoFile) (generatedProfileInfo.benchmarkPaths, generatedProfileInfo.cpuSet, generatedProfileInfo.pmc) = (None, None, None) findDiff(generatedProfileInfo.__dict__, scenarios.baselineProfileInfo.__dict__) assert generatedProfileInfo == scenarios.baselineProfileInfo
def test_probe_states(capsys, scenarioName): """ Test xpedite probes and probes state for an application's against baseline probe states for the application """ from xpedite.types.probe import compareProbes with SCENARIO_LOADER[scenarioName] as scenarios: probeMap = {} with capsys.disabled(): probes = loadProbes(CONTEXT, scenarios) for probe in probes: probeMap[probe.sysName] = probe assert len(probes) == len(scenarios.baselineProbeMap) findDiff(probeMap, scenarios.baselineProbeMap) for probe in probeMap: assert compareProbes(probeMap[probe], scenarios.baselineProbeMap[probe])
def test_record_against_report(capsys, profileInfoPath=None): """ Run xpedite record and xpedite report to compare profiles @param capsys: A pytest fixture allowing disabling of I/O capturing @param profileInfoPath: Override default profileInfo.py file from the test data directory """ from test_xpedite.test_profiler.comparator import findDiff profileInfoPath = profileInfoPath if profileInfoPath else 'profileInfo.py' profileInfo = loadProfileInfo(profileInfoPath, remote) with capsys.disabled(): app, recordProfiles, _ = runXpediteRecord(DEMO_BINARY, profileInfo, remote) profileInfo = loadProfileInfo(profileInfoPath, remote) profileInfo.appInfo = os.path.join(app.tempDir, 'xpedite-appinfo.txt') reportProfiles, _ = runXpediteReport(app.xpediteApp.runId, profileInfo) findDiff(reportProfiles.__dict__, recordProfiles.__dict__) assert recordProfiles == reportProfiles
def compareVsBaseline(context, scenario): """ Compare profiles with benchmarks and profiles without benchmarks against existing profiles """ runId = scenario.discoverRunId() sampleFilePath = SAMPLE_FILE_PATH.format(dataDir=scenario.dataDir, runId=runId) report = runXpediteReport(runId, context, scenario, sampleFilePath=sampleFilePath, cpuInfoOverride=True) reportProfiles = report.profiles reportProfiles.transactionRepo = None reportProfiles.cpuInfo.cpuId = scenario.cpuId if scenario.benchmarkPaths: benchmarkCount = len(scenario.benchmarkPaths) validateBenchmarks(reportProfiles, benchmarkCount) findDiff(reportProfiles.__dict__, scenario.baselineProfiles.__dict__) assert reportProfiles == scenario.baselineProfiles
def test_generate_against_baseline(): """ Test xpedite generate by generating a new profileInfo.py file and comparing to baseline profileInfo.py in the test data directory """ profileInfoBaseline = os.path.join(DATA_DIR, 'generateCmdBaseline.py') baselineProfileInfo = loadProfileInfo(DATA_DIR, profileInfoBaseline, REMOTE) tempDir = tempfile.mkdtemp() os.chdir(tempDir) profileInfo = loadProfileInfo(DATA_DIR, 'profileInfo.py', REMOTE) tempProfilePath = generateProfileInfo(FIX_DECODER_BINARY, profileInfo, TXN_COUNT, THREAD_COUNT, REMOTE, WORKSPACE) tempProfileInfo = loadProfileInfo(DATA_DIR, tempProfilePath, REMOTE) tempProfileInfo.appInfo = os.path.basename(tempProfileInfo.appInfo) (tempProfileInfo.appHost, baselineProfileInfo.appHost) = ('localhost', 'localhost') findDiff(tempProfileInfo.__dict__, baselineProfileInfo.__dict__) assert tempProfileInfo == baselineProfileInfo
def test_record_against_report(capsys, profileInfoPath=None): """ Run xpedite record and xpedite report to compare profiles @param capsys: A pytest fixture allowing disabling of I/O capturing @param profileInfoPath: Override default profileInfo.py file from the test data directory """ profileInfoPath = profileInfoPath if profileInfoPath else 'profileInfo.py' profileInfo = loadProfileInfo(DATA_DIR, profileInfoPath, REMOTE) with capsys.disabled(): app, result = runXpediteRecord(FIX_DECODER_BINARY, profileInfo, TXN_COUNT, THREAD_COUNT, REMOTE, WORKSPACE) recordProfiles = result.profiles profileInfo = loadProfileInfo(DATA_DIR, profileInfoPath, REMOTE) profileInfo.appInfo = os.path.join(app.tempDir, 'xpedite-appinfo.txt') result = runXpediteReport(app.xpediteApp.runId, profileInfo, workspace=WORKSPACE) reportProfiles = result.profiles findDiff(reportProfiles.__dict__, recordProfiles.__dict__) assert reportProfiles == recordProfiles