def ProcessTestResult(test_result, upload_bucket, results_label, run_identifier, test_suite_start, should_compute_metrics, max_num_values, test_path_format, trace_processor_path, enable_tbmv3, fetch_power_profile): ConvertProtoTraces(test_result, trace_processor_path) AggregateTBMv2Traces(test_result) if enable_tbmv3: AggregateTBMv3Traces(test_result) if upload_bucket is not None: UploadArtifacts(test_result, upload_bucket, run_identifier) if should_compute_metrics: test_result['_histograms'] = histogram_set.HistogramSet() compute_metrics.ComputeTBMv2Metrics(test_result) if enable_tbmv3: compute_metrics.ComputeTBMv3Metrics(test_result, trace_processor_path, fetch_power_profile) ExtractMeasurements(test_result) num_values = len(test_result['_histograms']) if max_num_values is not None and num_values > max_num_values: logging.error('%s produced %d values, but only %d are allowed.', test_result['testPath'], num_values, max_num_values) util.SetUnexpectedFailure(test_result) del test_result['_histograms'] else: AddDiagnosticsToHistograms(test_result, test_suite_start, results_label, test_path_format)
def testComputeTBMv2Metrics(self): in_results = testing.IntermediateResults([ testing.TestResult( 'benchmark/story1', output_artifacts={ compute_metrics.HTML_TRACE_NAME: testing.Artifact('/trace1.html', 'gs://trace1.html') }, tags=['tbmv2:metric1'], ), testing.TestResult( 'benchmark/story2', output_artifacts={ compute_metrics.HTML_TRACE_NAME: testing.Artifact('/trace2.html', 'gs://trace2.html') }, tags=['tbmv2:metric2'], ), ]) test_dict = histogram.Histogram('a', 'unitless').AsDict() metrics_result = mre_result.MreResult() metrics_result.AddPair('histograms', [test_dict]) with mock.patch(GETSIZE_METHOD) as getsize_mock: with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock: getsize_mock.return_value = 1000 run_metrics_mock.return_value = metrics_result histogram_dicts = compute_metrics.ComputeTBMv2Metrics( in_results) self.assertEqual(histogram_dicts, [test_dict, test_dict]) self.assertEqual(in_results['testResults'][0]['status'], 'PASS') self.assertEqual(in_results['testResults'][1]['status'], 'PASS')
def testComputeTBMv2MetricsSkipped(self): test_result = testing.TestResult( 'benchmark/story1', output_artifacts={ compute_metrics.HTML_TRACE_NAME: testing.Artifact('/trace1.html', 'gs://trace1.html') }, tags=['tbmv2:metric1'], status='SKIP', ) test_result['_histograms'] = histogram_set.HistogramSet() with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock: compute_metrics.ComputeTBMv2Metrics(test_result) self.assertEqual(run_metrics_mock.call_count, 0) histogram_dicts = test_result['_histograms'].AsDicts() self.assertEqual(histogram_dicts, []) self.assertEqual(test_result['status'], 'SKIP')
def testComputeTBMv2MetricsSkipped(self): in_results = testing.IntermediateResults([ testing.TestResult( 'benchmark/story1', output_artifacts={ compute_metrics.HTML_TRACE_NAME: testing.Artifact('/trace1.html', 'gs://trace1.html') }, tags=['tbmv2:metric1'], status='SKIP', ), ]) with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock: histogram_dicts = compute_metrics.ComputeTBMv2Metrics(in_results) self.assertEqual(run_metrics_mock.call_count, 0) self.assertEqual(histogram_dicts, []) self.assertEqual(in_results['testResults'][0]['status'], 'SKIP')
def testComputeTBMv2MetricsTraceTooBig(self): test_result = testing.TestResult( 'benchmark/story1', output_artifacts={ compute_metrics.HTML_TRACE_NAME: testing.Artifact('/trace1.html', 'gs://trace1.html') }, tags=['tbmv2:metric1'], ) test_result['_histograms'] = histogram_set.HistogramSet() with mock.patch(GETSIZE_METHOD) as getsize_mock: with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock: getsize_mock.return_value = 1e9 compute_metrics.ComputeTBMv2Metrics(test_result) self.assertEqual(run_metrics_mock.call_count, 0) histogram_dicts = test_result['_histograms'].AsDicts() self.assertEqual(histogram_dicts, []) self.assertEqual(test_result['status'], 'FAIL') self.assertFalse(test_result['expected'])
def testComputeTBMv2MetricsTraceTooBig(self): in_results = testing.IntermediateResults([ testing.TestResult( 'benchmark/story1', output_artifacts={ compute_metrics.HTML_TRACE_NAME: testing.Artifact('/trace1.html', 'gs://trace1.html') }, tags=['tbmv2:metric1'], ), ]) with mock.patch(GETSIZE_METHOD) as getsize_mock: with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock: getsize_mock.return_value = 1e9 histogram_dicts = compute_metrics.ComputeTBMv2Metrics( in_results) self.assertEqual(run_metrics_mock.call_count, 0) self.assertEqual(histogram_dicts, []) self.assertEqual(in_results['testResults'][0]['status'], 'FAIL')
def testComputeTBMv2MetricsFailure(self): test_result = testing.TestResult( 'benchmark/story1', output_artifacts={ compute_metrics.HTML_TRACE_NAME: testing.Artifact('/trace1.html', 'gs://trace1.html') }, tags=['tbmv2:metric1'], ) test_result['_histograms'] = histogram_set.HistogramSet() metrics_result = mre_result.MreResult() metrics_result.AddFailure(failure.Failure(job.Job(0), 0, 0, 0, 0, 0)) with mock.patch(GETSIZE_METHOD) as getsize_mock: with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock: getsize_mock.return_value = 100 run_metrics_mock.return_value = metrics_result compute_metrics.ComputeTBMv2Metrics(test_result) histogram_dicts = test_result['_histograms'].AsDicts() self.assertEqual(histogram_dicts, []) self.assertEqual(test_result['status'], 'FAIL') self.assertFalse(test_result['expected'])
def testComputeTBMv2Metrics(self): test_result = testing.TestResult( 'benchmark/story1', output_artifacts={ compute_metrics.HTML_TRACE_NAME: testing.Artifact('/trace1.html', 'gs://trace1.html') }, tags=['tbmv2:metric1'], ) test_result['_histograms'] = histogram_set.HistogramSet() test_dict = histogram.Histogram('a', 'unitless').AsDict() metrics_result = mre_result.MreResult() metrics_result.AddPair('histograms', [test_dict]) with mock.patch(GETSIZE_METHOD) as getsize_mock: with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock: getsize_mock.return_value = 1000 run_metrics_mock.return_value = metrics_result compute_metrics.ComputeTBMv2Metrics(test_result) histogram_dicts = test_result['_histograms'].AsDicts() self.assertEqual(histogram_dicts, [test_dict]) self.assertEqual(test_result['status'], 'PASS')
def testComputeTBMv2MetricsFailure(self): in_results = testing.IntermediateResults([ testing.TestResult( 'benchmark/story1', output_artifacts={ compute_metrics.HTML_TRACE_NAME: testing.Artifact('/trace1.html', 'gs://trace1.html') }, tags=['tbmv2:metric1'], ), ]) metrics_result = mre_result.MreResult() metrics_result.AddFailure(failure.Failure(job.Job(0), 0, 0, 0, 0, 0)) with mock.patch(GETSIZE_METHOD) as getsize_mock: with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock: getsize_mock.return_value = 100 run_metrics_mock.return_value = metrics_result histogram_dicts = compute_metrics.ComputeTBMv2Metrics( in_results) self.assertEqual(histogram_dicts, []) self.assertEqual(in_results['testResults'][0]['status'], 'FAIL')
def _ComputeMetrics(intermediate_results, results_label): histogram_dicts = compute_metrics.ComputeTBMv2Metrics(intermediate_results) histogram_dicts += ExtractMeasurements(intermediate_results) histogram_dicts = AddDiagnosticsToHistograms( histogram_dicts, intermediate_results, results_label) return histogram_dicts