def ProcessTestResult(test_result, upload_bucket, results_label,
run_identifier, test_suite_start, should_compute_metrics,
max_num_values, test_path_format, trace_processor_path,
enable_tbmv3, fetch_power_profile):
ConvertProtoTraces(test_result, trace_processor_path)
AggregateTBMv2Traces(test_result)
if enable_tbmv3:
AggregateTBMv3Traces(test_result)
if upload_bucket is not None:
UploadArtifacts(test_result, upload_bucket, run_identifier)
if should_compute_metrics:
test_result['_histograms'] = histogram_set.HistogramSet()
compute_metrics.ComputeTBMv2Metrics(test_result)
if enable_tbmv3:
compute_metrics.ComputeTBMv3Metrics(test_result,
trace_processor_path,
fetch_power_profile)
ExtractMeasurements(test_result)
num_values = len(test_result['_histograms'])
if max_num_values is not None and num_values > max_num_values:
logging.error('%s produced %d values, but only %d are allowed.',
test_result['testPath'], num_values, max_num_values)
util.SetUnexpectedFailure(test_result)
del test_result['_histograms']
else:
AddDiagnosticsToHistograms(test_result, test_suite_start,
results_label, test_path_format)
def testComputeTBMv2Metrics(self):
in_results = testing.IntermediateResults([
testing.TestResult(
'benchmark/story1',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace1.html', 'gs://trace1.html')
},
tags=['tbmv2:metric1'],
),
testing.TestResult(
'benchmark/story2',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace2.html', 'gs://trace2.html')
},
tags=['tbmv2:metric2'],
),
])
test_dict = histogram.Histogram('a', 'unitless').AsDict()
metrics_result = mre_result.MreResult()
metrics_result.AddPair('histograms', [test_dict])
with mock.patch(GETSIZE_METHOD) as getsize_mock:
with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock:
getsize_mock.return_value = 1000
run_metrics_mock.return_value = metrics_result
histogram_dicts = compute_metrics.ComputeTBMv2Metrics(
in_results)
self.assertEqual(histogram_dicts, [test_dict, test_dict])
self.assertEqual(in_results['testResults'][0]['status'], 'PASS')
self.assertEqual(in_results['testResults'][1]['status'], 'PASS')
def testComputeTBMv2MetricsSkipped(self):
test_result = testing.TestResult(
'benchmark/story1',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace1.html', 'gs://trace1.html')
},
tags=['tbmv2:metric1'],
status='SKIP',
)
test_result['_histograms'] = histogram_set.HistogramSet()
with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock:
compute_metrics.ComputeTBMv2Metrics(test_result)
self.assertEqual(run_metrics_mock.call_count, 0)
histogram_dicts = test_result['_histograms'].AsDicts()
self.assertEqual(histogram_dicts, [])
self.assertEqual(test_result['status'], 'SKIP')
def testComputeTBMv2MetricsSkipped(self):
in_results = testing.IntermediateResults([
testing.TestResult(
'benchmark/story1',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace1.html', 'gs://trace1.html')
},
tags=['tbmv2:metric1'],
status='SKIP',
),
])
with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock:
histogram_dicts = compute_metrics.ComputeTBMv2Metrics(in_results)
self.assertEqual(run_metrics_mock.call_count, 0)
self.assertEqual(histogram_dicts, [])
self.assertEqual(in_results['testResults'][0]['status'], 'SKIP')
def testComputeTBMv2MetricsTraceTooBig(self):
test_result = testing.TestResult(
'benchmark/story1',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace1.html', 'gs://trace1.html')
},
tags=['tbmv2:metric1'],
)
test_result['_histograms'] = histogram_set.HistogramSet()
with mock.patch(GETSIZE_METHOD) as getsize_mock:
with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock:
getsize_mock.return_value = 1e9
compute_metrics.ComputeTBMv2Metrics(test_result)
self.assertEqual(run_metrics_mock.call_count, 0)
histogram_dicts = test_result['_histograms'].AsDicts()
self.assertEqual(histogram_dicts, [])
self.assertEqual(test_result['status'], 'FAIL')
self.assertFalse(test_result['expected'])
def testComputeTBMv2MetricsTraceTooBig(self):
in_results = testing.IntermediateResults([
testing.TestResult(
'benchmark/story1',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace1.html', 'gs://trace1.html')
},
tags=['tbmv2:metric1'],
),
])
with mock.patch(GETSIZE_METHOD) as getsize_mock:
with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock:
getsize_mock.return_value = 1e9
histogram_dicts = compute_metrics.ComputeTBMv2Metrics(
in_results)
self.assertEqual(run_metrics_mock.call_count, 0)
self.assertEqual(histogram_dicts, [])
self.assertEqual(in_results['testResults'][0]['status'], 'FAIL')
def testComputeTBMv2MetricsFailure(self):
test_result = testing.TestResult(
'benchmark/story1',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace1.html', 'gs://trace1.html')
},
tags=['tbmv2:metric1'],
)
test_result['_histograms'] = histogram_set.HistogramSet()
metrics_result = mre_result.MreResult()
metrics_result.AddFailure(failure.Failure(job.Job(0), 0, 0, 0, 0, 0))
with mock.patch(GETSIZE_METHOD) as getsize_mock:
with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock:
getsize_mock.return_value = 100
run_metrics_mock.return_value = metrics_result
compute_metrics.ComputeTBMv2Metrics(test_result)
histogram_dicts = test_result['_histograms'].AsDicts()
self.assertEqual(histogram_dicts, [])
self.assertEqual(test_result['status'], 'FAIL')
self.assertFalse(test_result['expected'])
def testComputeTBMv2Metrics(self):
test_result = testing.TestResult(
'benchmark/story1',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace1.html', 'gs://trace1.html')
},
tags=['tbmv2:metric1'],
)
test_result['_histograms'] = histogram_set.HistogramSet()
test_dict = histogram.Histogram('a', 'unitless').AsDict()
metrics_result = mre_result.MreResult()
metrics_result.AddPair('histograms', [test_dict])
with mock.patch(GETSIZE_METHOD) as getsize_mock:
with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock:
getsize_mock.return_value = 1000
run_metrics_mock.return_value = metrics_result
compute_metrics.ComputeTBMv2Metrics(test_result)
histogram_dicts = test_result['_histograms'].AsDicts()
self.assertEqual(histogram_dicts, [test_dict])
self.assertEqual(test_result['status'], 'PASS')
def testComputeTBMv2MetricsFailure(self):
in_results = testing.IntermediateResults([
testing.TestResult(
'benchmark/story1',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace1.html', 'gs://trace1.html')
},
tags=['tbmv2:metric1'],
),
])
metrics_result = mre_result.MreResult()
metrics_result.AddFailure(failure.Failure(job.Job(0), 0, 0, 0, 0, 0))
with mock.patch(GETSIZE_METHOD) as getsize_mock:
with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock:
getsize_mock.return_value = 100
run_metrics_mock.return_value = metrics_result
histogram_dicts = compute_metrics.ComputeTBMv2Metrics(
in_results)
self.assertEqual(histogram_dicts, [])
self.assertEqual(in_results['testResults'][0]['status'], 'FAIL')
def _ComputeMetrics(intermediate_results, results_label):
histogram_dicts = compute_metrics.ComputeTBMv2Metrics(intermediate_results)
histogram_dicts += ExtractMeasurements(intermediate_results)
histogram_dicts = AddDiagnosticsToHistograms(
histogram_dicts, intermediate_results, results_label)
return histogram_dicts
