def testJson3Output(self): self.SerializeIntermediateResults( testing.TestResult('benchmark/story', run_duration='1.1s', tags=['shard:7'], start_time='2009-02-13T23:31:30.987000Z'), testing.TestResult('benchmark/story', run_duration='1.2s', tags=['shard:7']), ) processor.main([ '--output-format', 'json-test-results', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir ]) with open(os.path.join(self.output_dir, json3_output.OUTPUT_FILENAME)) as f: results = json.load(f) self.assertFalse(results['interrupted']) self.assertEqual(results['num_failures_by_type'], {'PASS': 2}) self.assertEqual(results['seconds_since_epoch'], 1234567890.987) self.assertEqual(results['version'], 3) self.assertIn('benchmark', results['tests']) self.assertIn('story', results['tests']['benchmark']) test_result = results['tests']['benchmark']['story'] self.assertEqual(test_result['actual'], 'PASS') self.assertEqual(test_result['expected'], 'PASS') self.assertEqual(test_result['times'], [1.1, 1.2]) self.assertEqual(test_result['time'], 1.1) self.assertEqual(test_result['shard'], 7)
def testCsvOutputAppendResults(self): hist_file = os.path.join(self.output_dir, histograms_output.HISTOGRAM_DICTS_NAME) with open(hist_file, 'w') as f: json.dump([histogram.Histogram('a', 'unitless').AsDict()], f) self.SerializeIntermediateResults( test_results=[ testing.TestResult( 'benchmark/story', artifacts={'histogram_dicts.json': testing.Artifact(hist_file)}, ), ], ) processor.main([ '--output-format', 'csv', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label1', ]) processor.main([ '--output-format', 'csv', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label2', ]) with open(os.path.join(self.output_dir, csv_output.OUTPUT_FILENAME)) as f: lines = [line for line in f] self.assertEqual(len(lines), 3) self.assertIn('label2', lines[1]) self.assertIn('label1', lines[2])
def testJson3OutputWithArtifacts(self): self.SerializeIntermediateResults( testing.TestResult('benchmark/story', output_artifacts={ 'logs': testing.Artifact('/logs.txt', 'gs://logs.txt'), 'trace/telemetry': testing.Artifact('/telemetry.json'), 'trace.html': testing.Artifact('/trace.html', 'gs://trace.html'), }), ) processor.main([ '--output-format', 'json-test-results', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir ]) with open(os.path.join(self.output_dir, json3_output.OUTPUT_FILENAME)) as f: results = json.load(f) self.assertIn('benchmark', results['tests']) self.assertIn('story', results['tests']['benchmark']) self.assertIn('artifacts', results['tests']['benchmark']['story']) artifacts = results['tests']['benchmark']['story']['artifacts'] self.assertEqual(len(artifacts), 2) self.assertEqual(artifacts['logs'], ['gs://logs.txt']) self.assertEqual(artifacts['trace.html'], ['gs://trace.html'])
def testHtmlOutputAppendResults(self): self.SerializeIntermediateResults([]) processor.main([ '--output-format', 'html', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label1', ]) processor.main([ '--output-format', 'html', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label2', ]) with open(os.path.join( self.output_dir, html_output.OUTPUT_FILENAME)) as f: results = render_histograms_viewer.ReadExistingResults(f.read()) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) diag_values = [list(v) for v in out_histograms.shared_diagnostics] self.assertIn(['label1'], diag_values) self.assertIn(['label2'], diag_values)
def testCsvOutputAppendResults(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHtmlTraceArtifact(), ], tags=['tbmv2:sampleMetric'], ), ) processor.main([ '--output-format', 'csv', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label1', ]) processor.main([ '--output-format', 'csv', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label2', ]) sample_rows = self.ReadSampleHistogramsFromCsv() self.assertEqual(len(sample_rows), 2) self.assertEqual(sample_rows[0]['displayLabel'], 'label2') self.assertEqual(sample_rows[1]['displayLabel'], 'label1')
def testHistogramsOutputMeasurements(self): measurements = { 'a': {'unit': 'ms', 'samples': [4, 6], 'description': 'desc_a'}, 'b': {'unit': 'ms', 'samples': [5], 'description': 'desc_b'}, } start_ts = 1500000000 start_iso = datetime.datetime.utcfromtimestamp(start_ts).isoformat() + 'Z' self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateMeasurementsArtifact(measurements), ], tags=['story_tag:test'], start_time=start_iso, ), ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, ]) with open(os.path.join( self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) self.assertEqual(len(out_histograms), 2) hist = out_histograms.GetHistogramNamed('a') self.assertEqual(hist.name, 'a') self.assertEqual(hist.unit, 'ms_smallerIsBetter') self.assertEqual(hist.sample_values, [4, 6]) self.assertEqual(hist.description, 'desc_a') self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['stories'], generic_set.GenericSet(['story'])) self.assertEqual(hist.diagnostics['storyTags'], generic_set.GenericSet(['test'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(start_ts * 1e3)) hist = out_histograms.GetHistogramNamed('b') self.assertEqual(hist.name, 'b') self.assertEqual(hist.unit, 'ms_smallerIsBetter') self.assertEqual(hist.sample_values, [5]) self.assertEqual(hist.description, 'desc_b') self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['stories'], generic_set.GenericSet(['story'])) self.assertEqual(hist.diagnostics['storyTags'], generic_set.GenericSet(['test'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(start_ts * 1e3))
def testHistogramsOutputNoAggregatedTrace(self): json_trace = os.path.join(self.output_dir, 'trace.json') with open(json_trace, 'w') as f: json.dump({'traceEvents': []}, f) self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts={'trace/json': testing.Artifact(json_trace)}, tags=['tbmv2:sampleMetric'], ), ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) # sampleMetric records a histogram with the name 'foo'. hist = out_histograms.GetHistogramNamed('foo') self.assertIsNotNone(hist) self.assertIn('traceUrls', hist.diagnostics)
def testHistogramsOutput(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHtmlTraceArtifact(), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]) ], tags=['tbmv2:sampleMetric'], start_time='2009-02-13T23:31:30.987000Z', ), ) with mock.patch('py_utils.cloud_storage.Upload') as cloud_patch: cloud_patch.return_value = processor.cloud_storage.CloudFilepath( bucket='bucket', remote_path='trace.html') processor.main([ '--is-unittest', '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', '--upload-results', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME) self.assertEqual(hist.unit, SAMPLE_HISTOGRAM_UNIT) self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['osNames'], generic_set.GenericSet(['linux'])) self.assertEqual(hist.diagnostics['documentationUrls'], generic_set.GenericSet([['documentation', 'url']])) self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(1234567890987)) self.assertEqual( hist.diagnostics['traceUrls'], generic_set.GenericSet([ 'https://console.developers.google.com' '/m/cloudstorage/b/bucket/o/trace.html' ]))
def testCsvOutput(self): hist_file = os.path.join(self.output_dir, compute_metrics.HISTOGRAM_DICTS_FILE) test_hist = histogram.Histogram('a', 'ms') test_hist.AddSample(3000) with open(hist_file, 'w') as f: json.dump([test_hist.AsDict()], f) self.SerializeIntermediateResults( test_results=[ testing.TestResult( 'benchmark/story', output_artifacts={ 'histogram_dicts.json': testing.Artifact(hist_file) }, ), ], diagnostics={ 'benchmarks': ['benchmark'], 'osNames': ['linux'], 'documentationUrls': [['documentation', 'url']], }, ) processor.main([ '--output-format', 'csv', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', ]) with open(os.path.join(self.output_dir, csv_output.OUTPUT_FILENAME)) as f: lines = [line for line in f] actual = list(zip(*csv.reader(lines))) expected = [('name', 'a'), ('unit', 'ms'), ('avg', '3000'), ('count', '1'), ('max', '3000'), ('min', '3000'), ('std', '0'), ('sum', '3000'), ('architectures', ''), ('benchmarks', 'benchmark'), ('benchmarkStart', ''), ('bots', ''), ('builds', ''), ('deviceIds', ''), ('displayLabel', 'label'), ('masters', ''), ('memoryAmounts', ''), ('osNames', 'linux'), ('osVersions', ''), ('productVersions', ''), ('stories', ''), ('storysetRepeats', ''), ('traceStart', ''), ('traceUrls', '')] self.assertEqual(actual, expected)
def testHistogramsOutput(self): hist_file = os.path.join(self.output_dir, compute_metrics.HISTOGRAM_DICTS_FILE) with open(hist_file, 'w') as f: json.dump([histogram.Histogram('a', 'unitless').AsDict()], f) self.SerializeIntermediateResults( test_results=[ testing.TestResult( 'benchmark/story', output_artifacts={ 'histogram_dicts.json': testing.Artifact(hist_file) }, ), ], diagnostics={ 'benchmarks': ['benchmark'], 'osNames': ['linux'], 'documentationUrls': [['documentation', 'url']], }, start_time='2009-02-13T23:31:30.987000Z', ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) self.assertEqual(len(out_histograms), 1) self.assertEqual(out_histograms.GetFirstHistogram().name, 'a') self.assertEqual(out_histograms.GetFirstHistogram().unit, 'unitless') diag_values = [list(v) for v in out_histograms.shared_diagnostics] self.assertEqual(len(diag_values), 4) self.assertIn(['benchmark'], diag_values) self.assertIn(['linux'], diag_values) self.assertIn([['documentation', 'url']], diag_values) self.assertIn(['label'], diag_values)
def testHistogramsOutput_TBMv3(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateProtoTraceArtifact(), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]) ], tags=['tbmv3:dummy_metric'], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--is-unittest', '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', '--experimental-tbmv3-metrics', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) # For testing the TBMv3 workflow we use dummy_metric defined in # tools/perf/core/tbmv3/metrics/dummy_metric_*. hist = out_histograms.GetHistogramNamed('dummy::simple_field') self.assertEqual(hist.unit, 'count_smallerIsBetter') self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['osNames'], generic_set.GenericSet(['linux'])) self.assertEqual(hist.diagnostics['documentationUrls'], generic_set.GenericSet([['documentation', 'url']])) self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(1234567890987))
def testHistogramsOutputAppendResults(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHtmlTraceArtifact(), ], tags=['tbmv2:sampleMetric'], ), ) processor.main([ '--is-unittest', '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label1', ]) processor.main([ '--is-unittest', '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label2', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) sample_histograms = out_histograms.GetHistogramsNamed( SAMPLE_HISTOGRAM_NAME) self.assertEqual(len(sample_histograms), 2) expected_labels = set(['label1', 'label2']) observed_labels = set(label for hist in sample_histograms for label in hist.diagnostics['labels']) self.assertEqual(observed_labels, expected_labels)
def testHistogramsOutput(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHistogramsArtifact( histogram.Histogram('a', 'unitless')), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]) ], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) self.assertEqual(len(out_histograms), 1) hist = out_histograms.GetFirstHistogram() self.assertEqual(hist.name, 'a') self.assertEqual(hist.unit, 'unitless') self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['osNames'], generic_set.GenericSet(['linux'])) self.assertEqual(hist.diagnostics['documentationUrls'], generic_set.GenericSet([['documentation', 'url']])) self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(1234567890987))
def testHistogramsOutputResetResults(self): hist_file = os.path.join(self.output_dir, compute_metrics.HISTOGRAM_DICTS_FILE) with open(hist_file, 'w') as f: json.dump([histogram.Histogram('a', 'unitless').AsDict()], f) self.SerializeIntermediateResults(test_results=[ testing.TestResult( 'benchmark/story', output_artifacts={ 'histogram_dicts.json': testing.Artifact(hist_file) }, ), ], ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label1', ]) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label2', '--reset-results', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) self.assertEqual(len(out_histograms), 1) diag_values = [list(v) for v in out_histograms.shared_diagnostics] self.assertNotIn(['label1'], diag_values) self.assertIn(['label2'], diag_values)
def testMaxValuesPerTestCase(self): def SomeMeasurements(num): return self.CreateMeasurementsArtifact({ 'n%d' % i: {'unit': 'count', 'samples': [i]} for i in range(num)}) self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story1', status='PASS', output_artifacts=[SomeMeasurements(3)]), testing.TestResult( 'benchmark/story2', status='PASS', output_artifacts=[SomeMeasurements(7)]), ) exit_code = processor.main([ '--output-format', 'json-test-results', '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--max-values-per-test-case', '5' ]) self.assertEqual(exit_code, 1) with open(os.path.join( self.output_dir, json3_output.OUTPUT_FILENAME)) as f: results = json.load(f) self.assertEqual(results['tests']['benchmark']['story1']['actual'], 'PASS') self.assertEqual(results['tests']['benchmark']['story2']['actual'], 'FAIL') self.assertTrue(results['tests']['benchmark']['story2']['is_unexpected'])
def testHtmlOutput(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHtmlTraceArtifact(), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]), ], tags=['tbmv2:sampleMetric'], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--is-unittest', '--output-format', 'html', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', ]) with open(os.path.join(self.output_dir, html_output.OUTPUT_FILENAME)) as f: results = render_histograms_viewer.ReadExistingResults(f.read()) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME) self.assertEqual(hist.unit, SAMPLE_HISTOGRAM_UNIT) self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['osNames'], generic_set.GenericSet(['linux'])) self.assertEqual(hist.diagnostics['documentationUrls'], generic_set.GenericSet([['documentation', 'url']])) self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(1234567890987))
def testHistogramsOutput_TBMv3(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateProtoTraceArtifact(), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]) ], tags=['tbmv3:dummy_metric'], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) hist = out_histograms.GetHistogramNamed(DUMMY_HISTOGRAM_NAME) self.assertEqual(hist.unit, DUMMY_HISTOGRAM_UNIT) self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['osNames'], generic_set.GenericSet(['linux'])) self.assertEqual(hist.diagnostics['documentationUrls'], generic_set.GenericSet([['documentation', 'url']])) self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(1234567890987))
def testMultipleTBMv3Metrics(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateProtoTraceArtifact(), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]) ], tags=['tbmv3:dummy_metric', 'tbmv3:test_chrome_metric'], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--is-unittest', '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', '--experimental-tbmv3-metrics', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) # We use two metrics for testing here. The dummy_metric is defined in # tools/perf/core/tbmv3/metrics/dummy_metric_*. # The test_chrome_metric is built into trace_processor, see source in # third_party/perfetto/src/trace_processor/metrics/chrome/test_chrome_metric.sql. hist1 = out_histograms.GetHistogramNamed('dummy::simple_field') self.assertEqual(hist1.sample_values, [42]) hist2 = out_histograms.GetHistogramNamed('test_chrome::test_value') self.assertEqual(hist2.sample_values, [1])
def testHtmlOutputResetResults(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHtmlTraceArtifact(), ], tags=['tbmv2:sampleMetric'], ), ) processor.main([ '--is-unittest', '--output-format', 'html', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label1', ]) processor.main([ '--is-unittest', '--output-format', 'html', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label2', '--reset-results', ]) with open(os.path.join(self.output_dir, html_output.OUTPUT_FILENAME)) as f: results = render_histograms_viewer.ReadExistingResults(f.read()) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME) self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label2']))
def testHistogramsOutputAppendResults(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHistogramsArtifact( histogram.Histogram('a', 'unitless')), ], ), ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label1', ]) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label2', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) self.assertEqual(len(out_histograms), 2) expected_labels = set(['label1', 'label2']) observed_labels = set(label for hist in out_histograms for label in hist.diagnostics['labels']) self.assertEqual(observed_labels, expected_labels)
def testCsvOutput(self): test_hist = histogram.Histogram('a', 'ms') test_hist.AddSample(3000) self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHistogramsArtifact(test_hist), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]), ], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--output-format', 'csv', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', ]) with open(os.path.join(self.output_dir, csv_output.OUTPUT_FILENAME)) as f: lines = [line for line in f] actual = list(zip(*csv.reader(lines))) expected = [('name', 'a'), ('unit', 'ms'), ('avg', '3000'), ('count', '1'), ('max', '3000'), ('min', '3000'), ('std', '0'), ('sum', '3000'), ('architectures', ''), ('benchmarks', 'benchmark'), ('benchmarkStart', '2009-02-13 23:31:30'), ('bots', ''), ('builds', ''), ('deviceIds', ''), ('displayLabel', 'label'), ('masters', ''), ('memoryAmounts', ''), ('osNames', 'linux'), ('osVersions', ''), ('productVersions', ''), ('stories', ''), ('storysetRepeats', ''), ('traceStart', ''), ('traceUrls', '')] self.assertEqual(actual, expected)
def testHistogramsOutputResetResults(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHistogramsArtifact( histogram.Histogram('a', 'unitless')), ], ), ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label1', ]) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label2', '--reset-results', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) self.assertEqual(len(out_histograms), 1) hist = out_histograms.GetFirstHistogram() self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label2']))
def testCsvOutput(self): test_hist = histogram.Histogram('a', 'ms') test_hist.AddSample(3000) self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHtmlTraceArtifact(), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]), ], tags=['tbmv2:sampleMetric'], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--is-unittest', '--output-format', 'csv', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', ]) sample_rows = self.ReadSampleHistogramsFromCsv() self.assertEqual(len(sample_rows), 1) actual = sample_rows[0] self.assertEqual(actual['name'], SAMPLE_HISTOGRAM_NAME) self.assertEqual(actual['unit'], 'B') self.assertEqual(actual['avg'], '50') self.assertEqual(actual['count'], '2') self.assertEqual(actual['benchmarks'], 'benchmark') self.assertEqual(actual['benchmarkStart'], '2009-02-13 23:31:30') self.assertEqual(actual['displayLabel'], 'label') self.assertEqual(actual['osNames'], 'linux') self.assertEqual(actual['traceStart'], '2009-02-13 23:31:30')
def testExitCodeSomeSkipped(self): self.SerializeIntermediateResults( testing.TestResult('benchmark/story', status='SKIP'), testing.TestResult('benchmark/story', status='PASS'), ) exit_code = processor.main([ '--output-format', 'json-test-results', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir ]) self.assertEqual(exit_code, 0)
def testExitCodeHasFailures(self): self.SerializeIntermediateResults([ testing.TestResult('benchmark/story', status='PASS'), testing.TestResult('benchmark/story', status='FAIL'), ]) exit_code = processor.main([ '--output-format', 'json-test-results', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir ]) self.assertEqual(exit_code, 1)
def testExtraMetrics(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHtmlTraceArtifact(), self.CreateProtoTraceArtifact(), ], tags=['tbmv2:sampleMetric'], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--is-unittest', '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', '--experimental-tbmv3-metrics', '--extra-metric', 'tbmv3:dummy_metric', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) # Both sampleMetric and dummy_metric should have been computed. hist1 = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME) self.assertEqual(hist1.unit, SAMPLE_HISTOGRAM_UNIT) hist2 = out_histograms.GetHistogramNamed('dummy::simple_field') self.assertEqual(hist2.unit, 'count_smallerIsBetter')
def testHistogramsOutputNoMetricsFromTelemetry(self): trace_file = os.path.join(self.output_dir, compute_metrics.HTML_TRACE_NAME) with open(trace_file, 'w') as f: pass self.SerializeIntermediateResults(test_results=[ testing.TestResult( 'benchmark/story', output_artifacts={ compute_metrics.HTML_TRACE_NAME: testing.Artifact(trace_file, 'gs://trace.html') }, tags=['tbmv2:sampleMetric'], ), ], ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) # sampleMetric records a histogram with the name 'foo'. hist = out_histograms.GetHistogramNamed('foo') self.assertIsNotNone(hist) self.assertEqual(hist.diagnostics['traceUrls'], generic_set.GenericSet(['gs://trace.html']))
def testCsvOutputAppendResults(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHistogramsArtifact( histogram.Histogram('a', 'unitless')), ], ), ) processor.main([ '--output-format', 'csv', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label1', ]) processor.main([ '--output-format', 'csv', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label2', ]) with open(os.path.join(self.output_dir, csv_output.OUTPUT_FILENAME)) as f: lines = [line for line in f] self.assertEqual(len(lines), 3) self.assertIn('label2', lines[1]) self.assertIn('label1', lines[2])
def testComplexMetricOutput_TBMv3(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateProtoTraceArtifact(), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]) ], tags=['tbmv3:dummy_metric'], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', '--experimental-tbmv3-metrics', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) # For testing the TBMv3 workflow we use dummy_metric defined in # tools/perf/core/tbmv3/metrics/dummy_metric_*. out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) simple_field = out_histograms.GetHistogramNamed("dummy::simple_field") self.assertEqual(simple_field.unit, "count_smallerIsBetter") self.assertEqual((simple_field.num_values, simple_field.average), (1, 42)) repeated_field = out_histograms.GetHistogramNamed( "dummy::repeated_field") self.assertEqual(repeated_field.unit, "ms_biggerIsBetter") self.assertEqual(repeated_field.num_values, 3) self.assertEqual(repeated_field.sample_values, [1, 2, 3]) # Unannotated fields should not be included in final histogram output. simple_nested_unannotated = out_histograms.GetHistogramsNamed( "dummy::simple_nested:unannotated_field") self.assertEqual(len(simple_nested_unannotated), 0) repeated_nested_unannotated = out_histograms.GetHistogramsNamed( "dummy::repeated_nested:unannotated_field") self.assertEqual(len(repeated_nested_unannotated), 0) simple_nested_annotated = out_histograms.GetHistogramNamed( "dummy::simple_nested:annotated_field") self.assertEqual(simple_nested_annotated.unit, "ms_smallerIsBetter") self.assertEqual(simple_nested_annotated.num_values, 1) self.assertEqual(simple_nested_annotated.average, 44) repeated_nested_annotated = out_histograms.GetHistogramNamed( "dummy::repeated_nested:annotated_field") self.assertEqual(repeated_nested_annotated.unit, "ms_smallerIsBetter") self.assertEqual(repeated_nested_annotated.num_values, 2) self.assertEqual(repeated_nested_annotated.sample_values, [2, 4])