def test_probe_states(capsys):
"""
Test xpedite probes and probes state for an application's against baseline probe states
for the application
"""
import cPickle as pickle
from xpedite.types.probe import compareProbes
profileInfo = loadProfileInfo(DATA_DIR,
os.path.join(DATA_DIR, 'profileInfo.py'),
REMOTE)
probes = []
probeMap = {}
baselineProbeMap = {}
with capsys.disabled():
probes = loadProbes(FIX_DECODER_BINARY, profileInfo, TXN_COUNT,
THREAD_COUNT, REMOTE, WORKSPACE)
for probe in probes:
probeMap[probe.sysName] = probe
with open(os.path.join(DATA_DIR, 'probeCmdBaseline.pkl'),
'r') as probeFileHandle:
baselineProbes = pickle.load(probeFileHandle)
for probe in baselineProbes:
baselineProbeMap[probe.sysName] = probe
assert len(probes) == len(baselineProbes)
findDiff(probeMap, baselineProbeMap)
for probe in probeMap.keys():
assert compareProbes(probeMap[probe], baselineProbeMap[probe])
def compareAgainstBaseline(profileInfoPath,
baselinePath,
dataDir,
workspace=None):
"""
Compare profiles with benchmarks and profiles without benchmarks against existing profiles
"""
import json
profileInfo = loadProfileInfo(dataDir, profileInfoPath)
profileInfo.appInfo = os.path.join(dataDir, 'xpedite-appinfo.txt')
runId = collectDataFiles(dataDir)
baselineProfiles = loadBaseline(dataDir, baselinePath)
with open(os.path.join(dataDir, 'baselineCpuInfo.json')) as fileHandle:
fullCpuInfo = json.load(fileHandle)
sampleFilePath = '{}/xpedite-*-{}-[0-9]*.data'.format(dataDir, runId)
result = runXpediteReport(runId,
profileInfo,
sampleFilePath=sampleFilePath,
cpuInfo=fullCpuInfo,
workspace=workspace)
reportProfiles = result.profiles
reportProfiles.transactionRepo = None # transactionRepo not stored in .xpd files
reportProfiles.cpuInfo.cpuId = baselineProfiles.cpuInfo.cpuId
if profileInfo.benchmarkPaths:
validateBenchmarks(len(profileInfo.benchmarkPaths), reportProfiles)
validateBenchmarks(len(profileInfo.benchmarkPaths), baselineProfiles)
findDiff(reportProfiles.__dict__, baselineProfiles.__dict__)
assert reportProfiles == baselineProfiles
def test_record_vs_report(capsys, scenarioName):
"""
Run xpedite record and xpedite report to compare profiles
"""
checkPmcSupport(scenarioName)
with SCENARIO_LOADER[scenarioName] as scenarios:
with capsys.disabled():
currentReport, _, _ = runXpediteRecord(CONTEXT, scenarios)
report = runXpediteReport(currentReport.runId, CONTEXT, scenarios)
findDiff(report.profiles.__dict__, currentReport.profiles.__dict__)
assert report.profiles == currentReport.profiles
def test_report_against_baseline():
"""
Run xpedite report on a data file in the test directory, return profiles and compare
the previously generated profiles from the same xpedite run
"""
from test_xpedite.test_profiler.comparator import findDiff
profileInfo = loadProfileInfo('profileInfo.py')
for runId, dataFilePath in collectDataFiles().iteritems():
reportProfiles, _ = runXpediteReport(runId, profileInfo, dataFilePath)
reportProfiles.transactionRepo = None # transactionRepo not stored in .xpd files
baselineProfiles = loadBaseline()
findDiff(reportProfiles.__dict__, baselineProfiles.__dict__)
assert reportProfiles == baselineProfiles
def test_generate_cmd_vs_baseline(scenarioName):
"""
Test xpedite generate by generating a new profileInfo.py file and comparing to baseline
profileInfo.py in the test data directory
"""
from test_xpedite.test_profiler.app import TargetLauncher
from xpedite.profiler.probeAdmin import ProbeAdmin
with SCENARIO_LOADER[scenarioName] as scenarios:
with TargetLauncher(CONTEXT, scenarios) as app:
probes = ProbeAdmin.loadProbes(app.xpediteApp)
profileInfoFile = generateProfileInfoFile(app.xpediteApp, probes)
generatedProfileInfo = loadProfileInfo('', profileInfoFile)
(generatedProfileInfo.benchmarkPaths, generatedProfileInfo.cpuSet, generatedProfileInfo.pmc) = (None, None, None)
findDiff(generatedProfileInfo.__dict__, scenarios.baselineProfileInfo.__dict__)
assert generatedProfileInfo == scenarios.baselineProfileInfo
def test_probe_states(capsys, scenarioName):
"""
Test xpedite probes and probes state for an application's against baseline probe states
for the application
"""
from xpedite.types.probe import compareProbes
with SCENARIO_LOADER[scenarioName] as scenarios:
probeMap = {}
with capsys.disabled():
probes = loadProbes(CONTEXT, scenarios)
for probe in probes:
probeMap[probe.sysName] = probe
assert len(probes) == len(scenarios.baselineProbeMap)
findDiff(probeMap, scenarios.baselineProbeMap)
for probe in probeMap:
assert compareProbes(probeMap[probe], scenarios.baselineProbeMap[probe])
def test_record_against_report(capsys, profileInfoPath=None):
"""
Run xpedite record and xpedite report to compare profiles
@param capsys: A pytest fixture allowing disabling of I/O capturing
@param profileInfoPath: Override default profileInfo.py file from the test data directory
"""
from test_xpedite.test_profiler.comparator import findDiff
profileInfoPath = profileInfoPath if profileInfoPath else 'profileInfo.py'
profileInfo = loadProfileInfo(profileInfoPath, remote)
with capsys.disabled():
app, recordProfiles, _ = runXpediteRecord(DEMO_BINARY, profileInfo,
remote)
profileInfo = loadProfileInfo(profileInfoPath, remote)
profileInfo.appInfo = os.path.join(app.tempDir, 'xpedite-appinfo.txt')
reportProfiles, _ = runXpediteReport(app.xpediteApp.runId, profileInfo)
findDiff(reportProfiles.__dict__, recordProfiles.__dict__)
assert recordProfiles == reportProfiles
def compareVsBaseline(context, scenario):
"""
Compare profiles with benchmarks and profiles without benchmarks against existing profiles
"""
runId = scenario.discoverRunId()
sampleFilePath = SAMPLE_FILE_PATH.format(dataDir=scenario.dataDir,
runId=runId)
report = runXpediteReport(runId,
context,
scenario,
sampleFilePath=sampleFilePath,
cpuInfoOverride=True)
reportProfiles = report.profiles
reportProfiles.transactionRepo = None
reportProfiles.cpuInfo.cpuId = scenario.cpuId
if scenario.benchmarkPaths:
benchmarkCount = len(scenario.benchmarkPaths)
validateBenchmarks(reportProfiles, benchmarkCount)
findDiff(reportProfiles.__dict__, scenario.baselineProfiles.__dict__)
assert reportProfiles == scenario.baselineProfiles
def test_generate_against_baseline():
"""
Test xpedite generate by generating a new profileInfo.py file and comparing to baseline
profileInfo.py in the test data directory
"""
profileInfoBaseline = os.path.join(DATA_DIR, 'generateCmdBaseline.py')
baselineProfileInfo = loadProfileInfo(DATA_DIR, profileInfoBaseline,
REMOTE)
tempDir = tempfile.mkdtemp()
os.chdir(tempDir)
profileInfo = loadProfileInfo(DATA_DIR, 'profileInfo.py', REMOTE)
tempProfilePath = generateProfileInfo(FIX_DECODER_BINARY, profileInfo,
TXN_COUNT, THREAD_COUNT, REMOTE,
WORKSPACE)
tempProfileInfo = loadProfileInfo(DATA_DIR, tempProfilePath, REMOTE)
tempProfileInfo.appInfo = os.path.basename(tempProfileInfo.appInfo)
(tempProfileInfo.appHost, baselineProfileInfo.appHost) = ('localhost',
'localhost')
findDiff(tempProfileInfo.__dict__, baselineProfileInfo.__dict__)
assert tempProfileInfo == baselineProfileInfo
def test_record_against_report(capsys, profileInfoPath=None):
"""
Run xpedite record and xpedite report to compare profiles
@param capsys: A pytest fixture allowing disabling of I/O capturing
@param profileInfoPath: Override default profileInfo.py file from the test data directory
"""
profileInfoPath = profileInfoPath if profileInfoPath else 'profileInfo.py'
profileInfo = loadProfileInfo(DATA_DIR, profileInfoPath, REMOTE)
with capsys.disabled():
app, result = runXpediteRecord(FIX_DECODER_BINARY, profileInfo,
TXN_COUNT, THREAD_COUNT, REMOTE,
WORKSPACE)
recordProfiles = result.profiles
profileInfo = loadProfileInfo(DATA_DIR, profileInfoPath, REMOTE)
profileInfo.appInfo = os.path.join(app.tempDir, 'xpedite-appinfo.txt')
result = runXpediteReport(app.xpediteApp.runId,
profileInfo,
workspace=WORKSPACE)
reportProfiles = result.profiles
findDiff(reportProfiles.__dict__, recordProfiles.__dict__)
assert reportProfiles == recordProfiles
