def CompareHistograms(test_ctx):
CheckConfig(test_ctx.simple_config)
config = test_ctx.simple_config.config
v2_metric = config['v2_metric']
v3_metric = config['v3_metric']
v2_histograms = test_ctx.RunTBMv2(v2_metric)
v3_histograms = test_ctx.RunTBMv3(v3_metric)
metric_precision = config['float_precision']
for v2_hist_name, v3_hist_info in config['histogram_mappings'].items():
if isinstance(v3_hist_info, str):
v3_hist_name = v3_hist_info
precision = metric_precision
elif isinstance(v3_hist_info, tuple):
v3_hist_name = v3_hist_info[0]
precision = v3_hist_info[1]
else:
raise Exception('v3_histogram must be either string of v3_histogram '
' name of (v3_hist_name, precision) tuple.')
v2_hist = GetHistogram(v2_histograms, v2_hist_name, v2_metric, 'v2')
v3_hist = GetHistogram(v3_histograms, v3_hist_name, v3_metric, 'v3')
try:
utils.AssertHistogramStatsAlmostEqual(test_ctx, v2_hist, v3_hist,
precision)
utils.AssertHistogramSamplesAlmostEqual(test_ctx, v2_hist, v3_hist,
precision)
except AssertionError as err:
message = (
'Error comparing TBMv2 histogram %s with TBMv3 histogram %s: %s' %
(v2_hist.name, v3_hist.name, err.message))
raise AssertionError, message, sys.exc_info()[2]
def CompareHistograms(test_ctx):
v2_histograms = test_ctx.RunTBMv2('renderingMetric')
v3_histograms = test_ctx.RunTBMv3('frame_times')
v2_hist = v2_histograms.GetHistogramNamed('frame_times')
v3_hist = v3_histograms.GetHistogramNamed('frame_times::frame_time')
utils.AssertHistogramStatsAlmostEqual(test_ctx, v2_hist, v3_hist)
def CompareHistograms(test_ctx):
CheckConfig(test_ctx.simple_config)
config = test_ctx.simple_config.config
v2_metric = config['v2_metric']
v3_metric = config['v3_metric']
v2_histograms = test_ctx.RunTBMv2(v2_metric)
v3_histograms = test_ctx.RunTBMv3(v3_metric)
metric_precision = config['float_precision']
for v2_hist_name, v3_hist_info in config['histogram_mappings'].items():
if isinstance(v3_hist_info, str):
v3_hist_name = v3_hist_info
precision = metric_precision
elif isinstance(v3_hist_info, tuple):
v3_hist_name = v3_hist_info[0]
precision = v3_hist_info[1]
else:
raise Exception(
'v3_histogram must be either string of v3_histogram '
' name of (v3_hist_name, precision) tuple.')
v2_hist = OptionalGetHistogram(v2_histograms, v2_hist_name, v2_metric,
'v2')
v3_hist = OptionalGetHistogram(v3_histograms, v3_hist_name, v3_metric,
'v3')
if (v2_hist is None) or (v2_hist.num_values == 0):
if (v3_hist is not None) and (v3_hist.num_values > 0):
raise Exception(
'v3 metric produced non-empty histogram %s, but '
'equivalent histogram %s is not present or empty '
'in v2 metric' % (v3_hist_name, v2_hist_name))
continue
if v3_hist is None:
msg = ('List of histograms produced by v3 metric %s:\n' %
(v3_metric))
msg += '\n'.join([h.name for h in v3_histograms])
raise Exception('Histogram %s not produced by v3 metric\n%s' %
(v3_hist_name, msg))
try:
utils.AssertHistogramStatsAlmostEqual(test_ctx, v2_hist, v3_hist,
precision)
utils.AssertHistogramSamplesAlmostEqual(test_ctx, v2_hist, v3_hist,
precision)
except AssertionError as err:
message = (
'Error comparing TBMv2 histogram %s with TBMv3 histogram %s: %s'
% (v2_hist.name, v3_hist.name, err.message))
raise AssertionError, message, sys.exc_info()[2]