def testUploadArtifacts(self):
in_results = testing.IntermediateResults(test_results=[
testing.TestResult(
'benchmark/story',
output_artifacts={'log': testing.Artifact('/log.log')},
),
testing.TestResult(
'benchmark/story',
output_artifacts={
'trace.html': testing.Artifact('/trace.html'),
'screenshot': testing.Artifact('/screenshot.png'),
},
),
], )
with mock.patch('py_utils.cloud_storage.Insert') as cloud_patch:
cloud_patch.return_value = 'gs://url'
processor.UploadArtifacts(in_results, 'bucket', None)
cloud_patch.assert_has_calls(
[
mock.call('bucket', mock.ANY, '/log.log'),
mock.call('bucket', mock.ANY, '/trace.html'),
mock.call('bucket', mock.ANY, '/screenshot.png'),
],
any_order=True,
)
for result in in_results['testResults']:
for artifact in result['outputArtifacts'].itervalues():
self.assertEqual(artifact['remoteUrl'], 'gs://url')
def testComputeTBMv2Metrics(self):
in_results = testing.IntermediateResults([
testing.TestResult(
'benchmark/story1',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace1.html', 'gs://trace1.html')
},
tags=['tbmv2:metric1'],
),
testing.TestResult(
'benchmark/story2',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace2.html', 'gs://trace2.html')
},
tags=['tbmv2:metric2'],
),
])
test_dict = histogram.Histogram('a', 'unitless').AsDict()
metrics_result = mre_result.MreResult()
metrics_result.AddPair('histograms', [test_dict])
with mock.patch(GETSIZE_METHOD) as getsize_mock:
with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock:
getsize_mock.return_value = 1000
run_metrics_mock.return_value = metrics_result
histogram_dicts = compute_metrics.ComputeTBMv2Metrics(
in_results)
self.assertEqual(histogram_dicts, [test_dict, test_dict])
self.assertEqual(in_results['testResults'][0]['status'], 'PASS')
self.assertEqual(in_results['testResults'][1]['status'], 'PASS')
def testConvertDiagnostics(self):
hist_file = os.path.join(self.output_dir,
histograms_output.HISTOGRAM_DICTS_NAME)
with open(hist_file, 'w') as f:
json.dump([histogram.Histogram('a', 'unitless').AsDict()], f)
in_results = testing.IntermediateResults(
test_results=[],
diagnostics={
'benchmarks': ['benchmark'],
'osNames': ['linux'],
'documentationUrls': [['documentation', 'url']],
},
)
histogram_dicts = histograms_output.Convert(in_results,
results_label='label')
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(histogram_dicts)
diag_values = [list(v) for v in out_histograms.shared_diagnostics]
self.assertEqual(len(diag_values), 4)
self.assertIn(['benchmark'], diag_values)
self.assertIn(['linux'], diag_values)
self.assertIn([['documentation', 'url']], diag_values)
self.assertIn(['label'], diag_values)
def Convert(self, test_results, **kwargs):
base_dir = kwargs.pop('base_dir', self.base_dir)
original_results = testing.IntermediateResults(test_results, **kwargs)
intermediate_results = copy.deepcopy(original_results)
results = json3_output.Convert(intermediate_results, base_dir)
# Convert should not modify the original intermediate results.
self.assertEqual(intermediate_results, original_results)
return results
def testAggregateTraces(self):
in_results = testing.IntermediateResults(test_results=[
testing.TestResult(
'benchmark/story1',
output_artifacts={
'trace/1.json':
testing.Artifact(
os.path.join('test_run', 'story1', 'trace', '1.json')),
},
),
testing.TestResult(
'benchmark/story2',
output_artifacts={
'trace/1.json':
testing.Artifact(
os.path.join('test_run', 'story2', 'trace', '1.json')),
'trace/2.json':
testing.Artifact(
os.path.join('test_run', 'story2', 'trace', '2.json')),
},
),
], )
with mock.patch(
'tracing.trace_data.trace_data.SerializeAsHtml') as patch:
processor.AggregateTraces(in_results)
call_list = [list(call[0]) for call in patch.call_args_list]
self.assertEqual(len(call_list), 2)
for call in call_list:
call[0] = set(call[0])
self.assertIn([
set([os.path.join('test_run', 'story1', 'trace', '1.json')]),
os.path.join('test_run', 'story1', 'trace', 'trace.html'),
], call_list)
self.assertIn([
set([
os.path.join('test_run', 'story2', 'trace', '1.json'),
os.path.join('test_run', 'story2', 'trace', '2.json'),
]),
os.path.join('test_run', 'story2', 'trace', 'trace.html'),
], call_list)
for result in in_results['testResults']:
artifacts = result['outputArtifacts']
self.assertEqual(len(artifacts), 1)
self.assertEqual(artifacts.keys()[0], 'trace.html')
def testComputeTBMv2MetricsSkipped(self):
in_results = testing.IntermediateResults([
testing.TestResult(
'benchmark/story1',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace1.html', 'gs://trace1.html')
},
tags=['tbmv2:metric1'],
status='SKIP',
),
])
with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock:
histogram_dicts = compute_metrics.ComputeTBMv2Metrics(in_results)
self.assertEqual(run_metrics_mock.call_count, 0)
self.assertEqual(histogram_dicts, [])
self.assertEqual(in_results['testResults'][0]['status'], 'SKIP')
def testConvertTwoStories(self):
hist_file = os.path.join(self.output_dir,
histograms_output.HISTOGRAM_DICTS_NAME)
with open(hist_file, 'w') as f:
json.dump([histogram.Histogram('a', 'unitless').AsDict()], f)
in_results = testing.IntermediateResults(test_results=[
testing.TestResult(
'benchmark/story1',
artifacts={
'histogram_dicts.json': testing.Artifact(hist_file)
},
),
testing.TestResult(
'benchmark/story2',
artifacts={
'histogram_dicts.json': testing.Artifact(hist_file)
},
),
testing.TestResult(
'benchmark/story1',
artifacts={
'histogram_dicts.json': testing.Artifact(hist_file)
},
),
testing.TestResult(
'benchmark/story2',
artifacts={
'histogram_dicts.json': testing.Artifact(hist_file)
},
),
], )
histogram_dicts = histograms_output.Convert(in_results,
results_label='label')
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(histogram_dicts)
self.assertEqual(len(out_histograms), 4)
hist = out_histograms.GetFirstHistogram()
self.assertEqual(hist.name, 'a')
self.assertEqual(hist.unit, 'unitless')
self.assertEqual(list(hist.diagnostics['labels']), ['label'])
def testComputeTBMv2MetricsTraceTooBig(self):
in_results = testing.IntermediateResults([
testing.TestResult(
'benchmark/story1',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace1.html', 'gs://trace1.html')
},
tags=['tbmv2:metric1'],
),
])
with mock.patch(GETSIZE_METHOD) as getsize_mock:
with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock:
getsize_mock.return_value = 1e9
histogram_dicts = compute_metrics.ComputeTBMv2Metrics(
in_results)
self.assertEqual(run_metrics_mock.call_count, 0)
self.assertEqual(histogram_dicts, [])
self.assertEqual(in_results['testResults'][0]['status'], 'FAIL')
def testUploadArtifacts_CheckRemoteUrl(self):
in_results = testing.IntermediateResults(
test_results=[
testing.TestResult(
'benchmark/story',
output_artifacts={
'trace.html': testing.Artifact('/trace.html')
},
),
],
start_time='2019-10-01T12:00:00.123456Z',
)
with mock.patch('py_utils.cloud_storage.Insert') as cloud_patch:
with mock.patch('random.randint') as randint_patch:
randint_patch.return_value = 54321
processor.UploadArtifacts(in_results, 'bucket',
'src@abc + 123')
cloud_patch.assert_called_once_with(
'bucket',
'src_abc_123_20191001T120000_54321/benchmark/story/trace.html',
'/trace.html')
def testAddDiagnosticsToHistograms(self):
histogram_dicts = [histogram.Histogram('a', 'unitless').AsDict()]
in_results = testing.IntermediateResults(
test_results=[],
diagnostics={
'benchmarks': ['benchmark'],
'osNames': ['linux'],
'documentationUrls': [['documentation', 'url']],
},
)
histograms_with_diagnostics = processor.AddDiagnosticsToHistograms(
histogram_dicts, in_results, results_label='label')
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(histograms_with_diagnostics)
diag_values = [list(v) for v in out_histograms.shared_diagnostics]
self.assertEqual(len(diag_values), 4)
self.assertIn(['benchmark'], diag_values)
self.assertIn(['linux'], diag_values)
self.assertIn([['documentation', 'url']], diag_values)
self.assertIn(['label'], diag_values)
def testComputeTBMv2MetricsFailure(self):
in_results = testing.IntermediateResults([
testing.TestResult(
'benchmark/story1',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact('/trace1.html', 'gs://trace1.html')
},
tags=['tbmv2:metric1'],
),
])
metrics_result = mre_result.MreResult()
metrics_result.AddFailure(failure.Failure(job.Job(0), 0, 0, 0, 0, 0))
with mock.patch(GETSIZE_METHOD) as getsize_mock:
with mock.patch(RUN_METRICS_METHOD) as run_metrics_mock:
getsize_mock.return_value = 100
run_metrics_mock.return_value = metrics_result
histogram_dicts = compute_metrics.ComputeTBMv2Metrics(
in_results)
self.assertEqual(histogram_dicts, [])
self.assertEqual(in_results['testResults'][0]['status'], 'FAIL')
def SerializeIntermediateResults(self, *args, **kwargs):
in_results = testing.IntermediateResults(*args, **kwargs)
testing.SerializeIntermediateResults(in_results, os.path.join(
self.intermediate_dir, processor.TELEMETRY_RESULTS))