def testJson3Output(self):
self.SerializeIntermediateResults(
testing.TestResult('benchmark/story',
run_duration='1.1s',
tags=['shard:7'],
start_time='2009-02-13T23:31:30.987000Z'),
testing.TestResult('benchmark/story',
run_duration='1.2s',
tags=['shard:7']),
)
processor.main([
'--output-format', 'json-test-results', '--output-dir',
self.output_dir, '--intermediate-dir', self.intermediate_dir
])
with open(os.path.join(self.output_dir,
json3_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
self.assertFalse(results['interrupted'])
self.assertEqual(results['num_failures_by_type'], {'PASS': 2})
self.assertEqual(results['seconds_since_epoch'], 1234567890.987)
self.assertEqual(results['version'], 3)
self.assertIn('benchmark', results['tests'])
self.assertIn('story', results['tests']['benchmark'])
test_result = results['tests']['benchmark']['story']
self.assertEqual(test_result['actual'], 'PASS')
self.assertEqual(test_result['expected'], 'PASS')
self.assertEqual(test_result['times'], [1.1, 1.2])
self.assertEqual(test_result['time'], 1.1)
self.assertEqual(test_result['shard'], 7)
def testCsvOutputAppendResults(self):
hist_file = os.path.join(self.output_dir,
histograms_output.HISTOGRAM_DICTS_NAME)
with open(hist_file, 'w') as f:
json.dump([histogram.Histogram('a', 'unitless').AsDict()], f)
self.SerializeIntermediateResults(
test_results=[
testing.TestResult(
'benchmark/story',
artifacts={'histogram_dicts.json': testing.Artifact(hist_file)},
),
],
)
processor.main([
'--output-format', 'csv',
'--output-dir', self.output_dir,
'--intermediate-dir', self.intermediate_dir,
'--results-label', 'label1',
])
processor.main([
'--output-format', 'csv',
'--output-dir', self.output_dir,
'--intermediate-dir', self.intermediate_dir,
'--results-label', 'label2',
])
with open(os.path.join(self.output_dir, csv_output.OUTPUT_FILENAME)) as f:
lines = [line for line in f]
self.assertEqual(len(lines), 3)
self.assertIn('label2', lines[1])
self.assertIn('label1', lines[2])
def testJson3OutputWithArtifacts(self):
self.SerializeIntermediateResults(
testing.TestResult('benchmark/story',
output_artifacts={
'logs':
testing.Artifact('/logs.txt',
'gs://logs.txt'),
'trace/telemetry':
testing.Artifact('/telemetry.json'),
'trace.html':
testing.Artifact('/trace.html',
'gs://trace.html'),
}), )
processor.main([
'--output-format', 'json-test-results', '--output-dir',
self.output_dir, '--intermediate-dir', self.intermediate_dir
])
with open(os.path.join(self.output_dir,
json3_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
self.assertIn('benchmark', results['tests'])
self.assertIn('story', results['tests']['benchmark'])
self.assertIn('artifacts', results['tests']['benchmark']['story'])
artifacts = results['tests']['benchmark']['story']['artifacts']
self.assertEqual(len(artifacts), 2)
self.assertEqual(artifacts['logs'], ['gs://logs.txt'])
self.assertEqual(artifacts['trace.html'], ['gs://trace.html'])
def testHtmlOutputAppendResults(self):
self.SerializeIntermediateResults([])
processor.main([
'--output-format', 'html',
'--output-dir', self.output_dir,
'--intermediate-dir', self.intermediate_dir,
'--results-label', 'label1',
])
processor.main([
'--output-format', 'html',
'--output-dir', self.output_dir,
'--intermediate-dir', self.intermediate_dir,
'--results-label', 'label2',
])
with open(os.path.join(
self.output_dir, html_output.OUTPUT_FILENAME)) as f:
results = render_histograms_viewer.ReadExistingResults(f.read())
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
diag_values = [list(v) for v in out_histograms.shared_diagnostics]
self.assertIn(['label1'], diag_values)
self.assertIn(['label2'], diag_values)
def testCsvOutputAppendResults(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHtmlTraceArtifact(),
],
tags=['tbmv2:sampleMetric'],
),
)
processor.main([
'--output-format', 'csv',
'--output-dir', self.output_dir,
'--intermediate-dir', self.intermediate_dir,
'--results-label', 'label1',
])
processor.main([
'--output-format', 'csv',
'--output-dir', self.output_dir,
'--intermediate-dir', self.intermediate_dir,
'--results-label', 'label2',
])
sample_rows = self.ReadSampleHistogramsFromCsv()
self.assertEqual(len(sample_rows), 2)
self.assertEqual(sample_rows[0]['displayLabel'], 'label2')
self.assertEqual(sample_rows[1]['displayLabel'], 'label1')
def testHistogramsOutputMeasurements(self):
measurements = {
'a': {'unit': 'ms', 'samples': [4, 6], 'description': 'desc_a'},
'b': {'unit': 'ms', 'samples': [5], 'description': 'desc_b'},
}
start_ts = 1500000000
start_iso = datetime.datetime.utcfromtimestamp(start_ts).isoformat() + 'Z'
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateMeasurementsArtifact(measurements),
],
tags=['story_tag:test'],
start_time=start_iso,
),
)
processor.main([
'--output-format', 'histograms',
'--output-dir', self.output_dir,
'--intermediate-dir', self.intermediate_dir,
])
with open(os.path.join(
self.output_dir, histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
self.assertEqual(len(out_histograms), 2)
hist = out_histograms.GetHistogramNamed('a')
self.assertEqual(hist.name, 'a')
self.assertEqual(hist.unit, 'ms_smallerIsBetter')
self.assertEqual(hist.sample_values, [4, 6])
self.assertEqual(hist.description, 'desc_a')
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['stories'],
generic_set.GenericSet(['story']))
self.assertEqual(hist.diagnostics['storyTags'],
generic_set.GenericSet(['test']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(start_ts * 1e3))
hist = out_histograms.GetHistogramNamed('b')
self.assertEqual(hist.name, 'b')
self.assertEqual(hist.unit, 'ms_smallerIsBetter')
self.assertEqual(hist.sample_values, [5])
self.assertEqual(hist.description, 'desc_b')
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['stories'],
generic_set.GenericSet(['story']))
self.assertEqual(hist.diagnostics['storyTags'],
generic_set.GenericSet(['test']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(start_ts * 1e3))
def testHistogramsOutputNoAggregatedTrace(self):
json_trace = os.path.join(self.output_dir, 'trace.json')
with open(json_trace, 'w') as f:
json.dump({'traceEvents': []}, f)
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts={'trace/json': testing.Artifact(json_trace)},
tags=['tbmv2:sampleMetric'],
), )
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
# sampleMetric records a histogram with the name 'foo'.
hist = out_histograms.GetHistogramNamed('foo')
self.assertIsNotNone(hist)
self.assertIn('traceUrls', hist.diagnostics)
def testHistogramsOutput(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHtmlTraceArtifact(),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']])
],
tags=['tbmv2:sampleMetric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
with mock.patch('py_utils.cloud_storage.Upload') as cloud_patch:
cloud_patch.return_value = processor.cloud_storage.CloudFilepath(
bucket='bucket', remote_path='trace.html')
processor.main([
'--is-unittest',
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
'--upload-results',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME)
self.assertEqual(hist.unit, SAMPLE_HISTOGRAM_UNIT)
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['osNames'],
generic_set.GenericSet(['linux']))
self.assertEqual(hist.diagnostics['documentationUrls'],
generic_set.GenericSet([['documentation', 'url']]))
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(1234567890987))
self.assertEqual(
hist.diagnostics['traceUrls'],
generic_set.GenericSet([
'https://console.developers.google.com'
'/m/cloudstorage/b/bucket/o/trace.html'
]))
def testCsvOutput(self):
hist_file = os.path.join(self.output_dir,
compute_metrics.HISTOGRAM_DICTS_FILE)
test_hist = histogram.Histogram('a', 'ms')
test_hist.AddSample(3000)
with open(hist_file, 'w') as f:
json.dump([test_hist.AsDict()], f)
self.SerializeIntermediateResults(
test_results=[
testing.TestResult(
'benchmark/story',
output_artifacts={
'histogram_dicts.json': testing.Artifact(hist_file)
},
),
],
diagnostics={
'benchmarks': ['benchmark'],
'osNames': ['linux'],
'documentationUrls': [['documentation', 'url']],
},
)
processor.main([
'--output-format',
'csv',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
])
with open(os.path.join(self.output_dir,
csv_output.OUTPUT_FILENAME)) as f:
lines = [line for line in f]
actual = list(zip(*csv.reader(lines)))
expected = [('name', 'a'), ('unit', 'ms'), ('avg', '3000'),
('count', '1'), ('max', '3000'), ('min', '3000'),
('std', '0'), ('sum', '3000'), ('architectures', ''),
('benchmarks', 'benchmark'), ('benchmarkStart', ''),
('bots', ''), ('builds', ''), ('deviceIds', ''),
('displayLabel', 'label'), ('masters', ''),
('memoryAmounts', ''), ('osNames', 'linux'),
('osVersions', ''),
('productVersions', ''), ('stories', ''),
('storysetRepeats', ''), ('traceStart', ''),
('traceUrls', '')]
self.assertEqual(actual, expected)
def testHistogramsOutput(self):
hist_file = os.path.join(self.output_dir,
compute_metrics.HISTOGRAM_DICTS_FILE)
with open(hist_file, 'w') as f:
json.dump([histogram.Histogram('a', 'unitless').AsDict()], f)
self.SerializeIntermediateResults(
test_results=[
testing.TestResult(
'benchmark/story',
output_artifacts={
'histogram_dicts.json': testing.Artifact(hist_file)
},
),
],
diagnostics={
'benchmarks': ['benchmark'],
'osNames': ['linux'],
'documentationUrls': [['documentation', 'url']],
},
start_time='2009-02-13T23:31:30.987000Z',
)
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
self.assertEqual(len(out_histograms), 1)
self.assertEqual(out_histograms.GetFirstHistogram().name, 'a')
self.assertEqual(out_histograms.GetFirstHistogram().unit, 'unitless')
diag_values = [list(v) for v in out_histograms.shared_diagnostics]
self.assertEqual(len(diag_values), 4)
self.assertIn(['benchmark'], diag_values)
self.assertIn(['linux'], diag_values)
self.assertIn([['documentation', 'url']], diag_values)
self.assertIn(['label'], diag_values)
def testHistogramsOutput_TBMv3(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateProtoTraceArtifact(),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']])
],
tags=['tbmv3:dummy_metric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--is-unittest',
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
'--experimental-tbmv3-metrics',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
# For testing the TBMv3 workflow we use dummy_metric defined in
# tools/perf/core/tbmv3/metrics/dummy_metric_*.
hist = out_histograms.GetHistogramNamed('dummy::simple_field')
self.assertEqual(hist.unit, 'count_smallerIsBetter')
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['osNames'],
generic_set.GenericSet(['linux']))
self.assertEqual(hist.diagnostics['documentationUrls'],
generic_set.GenericSet([['documentation', 'url']]))
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(1234567890987))
def testHistogramsOutputAppendResults(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHtmlTraceArtifact(),
],
tags=['tbmv2:sampleMetric'],
), )
processor.main([
'--is-unittest',
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label1',
])
processor.main([
'--is-unittest',
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label2',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
sample_histograms = out_histograms.GetHistogramsNamed(
SAMPLE_HISTOGRAM_NAME)
self.assertEqual(len(sample_histograms), 2)
expected_labels = set(['label1', 'label2'])
observed_labels = set(label for hist in sample_histograms
for label in hist.diagnostics['labels'])
self.assertEqual(observed_labels, expected_labels)
def testHistogramsOutput(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHistogramsArtifact(
histogram.Histogram('a', 'unitless')),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']])
],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
self.assertEqual(len(out_histograms), 1)
hist = out_histograms.GetFirstHistogram()
self.assertEqual(hist.name, 'a')
self.assertEqual(hist.unit, 'unitless')
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['osNames'],
generic_set.GenericSet(['linux']))
self.assertEqual(hist.diagnostics['documentationUrls'],
generic_set.GenericSet([['documentation', 'url']]))
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(1234567890987))
def testHistogramsOutputResetResults(self):
hist_file = os.path.join(self.output_dir,
compute_metrics.HISTOGRAM_DICTS_FILE)
with open(hist_file, 'w') as f:
json.dump([histogram.Histogram('a', 'unitless').AsDict()], f)
self.SerializeIntermediateResults(test_results=[
testing.TestResult(
'benchmark/story',
output_artifacts={
'histogram_dicts.json': testing.Artifact(hist_file)
},
),
], )
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label1',
])
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label2',
'--reset-results',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
self.assertEqual(len(out_histograms), 1)
diag_values = [list(v) for v in out_histograms.shared_diagnostics]
self.assertNotIn(['label1'], diag_values)
self.assertIn(['label2'], diag_values)
def testMaxValuesPerTestCase(self):
def SomeMeasurements(num):
return self.CreateMeasurementsArtifact({
'n%d' % i: {'unit': 'count', 'samples': [i]}
for i in range(num)})
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story1', status='PASS',
output_artifacts=[SomeMeasurements(3)]),
testing.TestResult(
'benchmark/story2', status='PASS',
output_artifacts=[SomeMeasurements(7)]),
)
exit_code = processor.main([
'--output-format', 'json-test-results',
'--output-format', 'histograms',
'--output-dir', self.output_dir,
'--intermediate-dir', self.intermediate_dir,
'--max-values-per-test-case', '5'
])
self.assertEqual(exit_code, 1)
with open(os.path.join(
self.output_dir, json3_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
self.assertEqual(results['tests']['benchmark']['story1']['actual'], 'PASS')
self.assertEqual(results['tests']['benchmark']['story2']['actual'], 'FAIL')
self.assertTrue(results['tests']['benchmark']['story2']['is_unexpected'])
def testHtmlOutput(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHtmlTraceArtifact(),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']]),
],
tags=['tbmv2:sampleMetric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--is-unittest',
'--output-format',
'html',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
])
with open(os.path.join(self.output_dir,
html_output.OUTPUT_FILENAME)) as f:
results = render_histograms_viewer.ReadExistingResults(f.read())
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME)
self.assertEqual(hist.unit, SAMPLE_HISTOGRAM_UNIT)
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['osNames'],
generic_set.GenericSet(['linux']))
self.assertEqual(hist.diagnostics['documentationUrls'],
generic_set.GenericSet([['documentation', 'url']]))
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(1234567890987))
def testHistogramsOutput_TBMv3(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateProtoTraceArtifact(),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']])
],
tags=['tbmv3:dummy_metric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
hist = out_histograms.GetHistogramNamed(DUMMY_HISTOGRAM_NAME)
self.assertEqual(hist.unit, DUMMY_HISTOGRAM_UNIT)
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['osNames'],
generic_set.GenericSet(['linux']))
self.assertEqual(hist.diagnostics['documentationUrls'],
generic_set.GenericSet([['documentation', 'url']]))
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(1234567890987))
def testMultipleTBMv3Metrics(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateProtoTraceArtifact(),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']])
],
tags=['tbmv3:dummy_metric', 'tbmv3:test_chrome_metric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--is-unittest',
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
'--experimental-tbmv3-metrics',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
# We use two metrics for testing here. The dummy_metric is defined in
# tools/perf/core/tbmv3/metrics/dummy_metric_*.
# The test_chrome_metric is built into trace_processor, see source in
# third_party/perfetto/src/trace_processor/metrics/chrome/test_chrome_metric.sql.
hist1 = out_histograms.GetHistogramNamed('dummy::simple_field')
self.assertEqual(hist1.sample_values, [42])
hist2 = out_histograms.GetHistogramNamed('test_chrome::test_value')
self.assertEqual(hist2.sample_values, [1])
def testHtmlOutputResetResults(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHtmlTraceArtifact(),
],
tags=['tbmv2:sampleMetric'],
), )
processor.main([
'--is-unittest',
'--output-format',
'html',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label1',
])
processor.main([
'--is-unittest',
'--output-format',
'html',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label2',
'--reset-results',
])
with open(os.path.join(self.output_dir,
html_output.OUTPUT_FILENAME)) as f:
results = render_histograms_viewer.ReadExistingResults(f.read())
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME)
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label2']))
def testHistogramsOutputAppendResults(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHistogramsArtifact(
histogram.Histogram('a', 'unitless')),
],
), )
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label1',
])
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label2',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
self.assertEqual(len(out_histograms), 2)
expected_labels = set(['label1', 'label2'])
observed_labels = set(label for hist in out_histograms
for label in hist.diagnostics['labels'])
self.assertEqual(observed_labels, expected_labels)
def testCsvOutput(self):
test_hist = histogram.Histogram('a', 'ms')
test_hist.AddSample(3000)
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHistogramsArtifact(test_hist),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']]),
],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--output-format',
'csv',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
])
with open(os.path.join(self.output_dir,
csv_output.OUTPUT_FILENAME)) as f:
lines = [line for line in f]
actual = list(zip(*csv.reader(lines)))
expected = [('name', 'a'), ('unit', 'ms'), ('avg', '3000'),
('count', '1'), ('max', '3000'), ('min', '3000'),
('std', '0'), ('sum', '3000'), ('architectures', ''),
('benchmarks', 'benchmark'),
('benchmarkStart', '2009-02-13 23:31:30'), ('bots', ''),
('builds', ''), ('deviceIds', ''),
('displayLabel', 'label'), ('masters', ''),
('memoryAmounts', ''), ('osNames', 'linux'),
('osVersions', ''),
('productVersions', ''), ('stories', ''),
('storysetRepeats', ''), ('traceStart', ''),
('traceUrls', '')]
self.assertEqual(actual, expected)
def testHistogramsOutputResetResults(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHistogramsArtifact(
histogram.Histogram('a', 'unitless')),
],
), )
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label1',
])
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label2',
'--reset-results',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
self.assertEqual(len(out_histograms), 1)
hist = out_histograms.GetFirstHistogram()
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label2']))
def testCsvOutput(self):
test_hist = histogram.Histogram('a', 'ms')
test_hist.AddSample(3000)
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHtmlTraceArtifact(),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']]),
],
tags=['tbmv2:sampleMetric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--is-unittest',
'--output-format',
'csv',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
])
sample_rows = self.ReadSampleHistogramsFromCsv()
self.assertEqual(len(sample_rows), 1)
actual = sample_rows[0]
self.assertEqual(actual['name'], SAMPLE_HISTOGRAM_NAME)
self.assertEqual(actual['unit'], 'B')
self.assertEqual(actual['avg'], '50')
self.assertEqual(actual['count'], '2')
self.assertEqual(actual['benchmarks'], 'benchmark')
self.assertEqual(actual['benchmarkStart'], '2009-02-13 23:31:30')
self.assertEqual(actual['displayLabel'], 'label')
self.assertEqual(actual['osNames'], 'linux')
self.assertEqual(actual['traceStart'], '2009-02-13 23:31:30')
def testExitCodeSomeSkipped(self):
self.SerializeIntermediateResults(
testing.TestResult('benchmark/story', status='SKIP'),
testing.TestResult('benchmark/story', status='PASS'),
)
exit_code = processor.main([
'--output-format', 'json-test-results', '--output-dir',
self.output_dir, '--intermediate-dir', self.intermediate_dir
])
self.assertEqual(exit_code, 0)
def testExitCodeHasFailures(self):
self.SerializeIntermediateResults([
testing.TestResult('benchmark/story', status='PASS'),
testing.TestResult('benchmark/story', status='FAIL'),
])
exit_code = processor.main([
'--output-format', 'json-test-results', '--output-dir',
self.output_dir, '--intermediate-dir', self.intermediate_dir
])
self.assertEqual(exit_code, 1)
def testExtraMetrics(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHtmlTraceArtifact(),
self.CreateProtoTraceArtifact(),
],
tags=['tbmv2:sampleMetric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--is-unittest',
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
'--experimental-tbmv3-metrics',
'--extra-metric',
'tbmv3:dummy_metric',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
# Both sampleMetric and dummy_metric should have been computed.
hist1 = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME)
self.assertEqual(hist1.unit, SAMPLE_HISTOGRAM_UNIT)
hist2 = out_histograms.GetHistogramNamed('dummy::simple_field')
self.assertEqual(hist2.unit, 'count_smallerIsBetter')
def testHistogramsOutputNoMetricsFromTelemetry(self):
trace_file = os.path.join(self.output_dir,
compute_metrics.HTML_TRACE_NAME)
with open(trace_file, 'w') as f:
pass
self.SerializeIntermediateResults(test_results=[
testing.TestResult(
'benchmark/story',
output_artifacts={
compute_metrics.HTML_TRACE_NAME:
testing.Artifact(trace_file, 'gs://trace.html')
},
tags=['tbmv2:sampleMetric'],
),
], )
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
# sampleMetric records a histogram with the name 'foo'.
hist = out_histograms.GetHistogramNamed('foo')
self.assertIsNotNone(hist)
self.assertEqual(hist.diagnostics['traceUrls'],
generic_set.GenericSet(['gs://trace.html']))
def testCsvOutputAppendResults(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHistogramsArtifact(
histogram.Histogram('a', 'unitless')),
],
), )
processor.main([
'--output-format',
'csv',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label1',
])
processor.main([
'--output-format',
'csv',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label2',
])
with open(os.path.join(self.output_dir,
csv_output.OUTPUT_FILENAME)) as f:
lines = [line for line in f]
self.assertEqual(len(lines), 3)
self.assertIn('label2', lines[1])
self.assertIn('label1', lines[2])
def testComplexMetricOutput_TBMv3(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateProtoTraceArtifact(),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']])
],
tags=['tbmv3:dummy_metric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
'--experimental-tbmv3-metrics',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
# For testing the TBMv3 workflow we use dummy_metric defined in
# tools/perf/core/tbmv3/metrics/dummy_metric_*.
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
simple_field = out_histograms.GetHistogramNamed("dummy::simple_field")
self.assertEqual(simple_field.unit, "count_smallerIsBetter")
self.assertEqual((simple_field.num_values, simple_field.average),
(1, 42))
repeated_field = out_histograms.GetHistogramNamed(
"dummy::repeated_field")
self.assertEqual(repeated_field.unit, "ms_biggerIsBetter")
self.assertEqual(repeated_field.num_values, 3)
self.assertEqual(repeated_field.sample_values, [1, 2, 3])
# Unannotated fields should not be included in final histogram output.
simple_nested_unannotated = out_histograms.GetHistogramsNamed(
"dummy::simple_nested:unannotated_field")
self.assertEqual(len(simple_nested_unannotated), 0)
repeated_nested_unannotated = out_histograms.GetHistogramsNamed(
"dummy::repeated_nested:unannotated_field")
self.assertEqual(len(repeated_nested_unannotated), 0)
simple_nested_annotated = out_histograms.GetHistogramNamed(
"dummy::simple_nested:annotated_field")
self.assertEqual(simple_nested_annotated.unit, "ms_smallerIsBetter")
self.assertEqual(simple_nested_annotated.num_values, 1)
self.assertEqual(simple_nested_annotated.average, 44)
repeated_nested_annotated = out_histograms.GetHistogramNamed(
"dummy::repeated_nested:annotated_field")
self.assertEqual(repeated_nested_annotated.unit, "ms_smallerIsBetter")
self.assertEqual(repeated_nested_annotated.num_values, 2)
self.assertEqual(repeated_nested_annotated.sample_values, [2, 4])
