def testHistogramsOutputMeasurements(self): measurements = { 'a': {'unit': 'ms', 'samples': [4, 6], 'description': 'desc_a'}, 'b': {'unit': 'ms', 'samples': [5], 'description': 'desc_b'}, } start_ts = 1500000000 start_iso = datetime.datetime.utcfromtimestamp(start_ts).isoformat() + 'Z' self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateMeasurementsArtifact(measurements), ], tags=['story_tag:test'], start_time=start_iso, ), ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, ]) with open(os.path.join( self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) self.assertEqual(len(out_histograms), 2) hist = out_histograms.GetHistogramNamed('a') self.assertEqual(hist.name, 'a') self.assertEqual(hist.unit, 'ms_smallerIsBetter') self.assertEqual(hist.sample_values, [4, 6]) self.assertEqual(hist.description, 'desc_a') self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['stories'], generic_set.GenericSet(['story'])) self.assertEqual(hist.diagnostics['storyTags'], generic_set.GenericSet(['test'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(start_ts * 1e3)) hist = out_histograms.GetHistogramNamed('b') self.assertEqual(hist.name, 'b') self.assertEqual(hist.unit, 'ms_smallerIsBetter') self.assertEqual(hist.sample_values, [5]) self.assertEqual(hist.description, 'desc_b') self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['stories'], generic_set.GenericSet(['story'])) self.assertEqual(hist.diagnostics['storyTags'], generic_set.GenericSet(['test'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(start_ts * 1e3))
def testRoundtrip(self): dr = date_range.DateRange(1496693745000) dr.AddDiagnostic(date_range.DateRange(1496693746000)) self.assertEqual(calendar.timegm(dr.min_date.timetuple()), 1496693745) self.assertEqual(calendar.timegm(dr.max_date.timetuple()), 1496693746) clone = diagnostic.Diagnostic.FromDict(dr.AsDict()) self.assertEqual(clone.min_date, dr.min_date) self.assertEqual(clone.max_date, dr.max_date)
def testDisallowReservedNames(self): diagnostics = histogram.DiagnosticMap() with self.assertRaises(TypeError): diagnostics[None] = generic_set.GenericSet(()) with self.assertRaises(TypeError): diagnostics['generic'] = None diagnostics[reserved_infos.TRACE_URLS.name] = date_range.DateRange(0) diagnostics.DisallowReservedNames() diagnostics[reserved_infos.TRACE_URLS.name] = generic_set.GenericSet(()) with self.assertRaises(TypeError): diagnostics[reserved_infos.TRACE_URLS.name] = date_range.DateRange(0)
def AddDiagnosticsToHistograms(test_result, test_suite_start, results_label): """Add diagnostics to all histograms of a test run. Reads diagnostics from the test artifact and adds them to all histograms. This overwrites the corresponding diagnostics previously set by e.g. run_metrics. """ artifacts = test_result.get('outputArtifacts', {}) if DIAGNOSTICS_NAME in artifacts: with open(artifacts[DIAGNOSTICS_NAME]['filePath']) as f: diagnostics = json.load(f)['diagnostics'] for name, diag in diagnostics.items(): # For now, we only support GenericSet diagnostics that are serialized # as lists of values. assert isinstance(diag, list) test_result['_histograms'].AddSharedDiagnosticToAllHistograms( name, generic_set.GenericSet(diag)) timestamp_ms = util.IsoTimestampToEpoch(test_suite_start) * 1e3 test_result['_histograms'].AddSharedDiagnosticToAllHistograms( reserved_infos.BENCHMARK_START.name, date_range.DateRange(timestamp_ms)) if results_label is not None: test_result['_histograms'].AddSharedDiagnosticToAllHistograms( reserved_infos.LABELS.name, generic_set.GenericSet([results_label]))
def DiagnosticValue(value): if isinstance(value, (int, long)): return date_range.DateRange(value) elif isinstance(value, list): return generic_set.GenericSet(value) else: raise NotImplementedError(type(value))
def testHistogramsOutput(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHtmlTraceArtifact(), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]) ], tags=['tbmv2:sampleMetric'], start_time='2009-02-13T23:31:30.987000Z', ), ) with mock.patch('py_utils.cloud_storage.Upload') as cloud_patch: cloud_patch.return_value = processor.cloud_storage.CloudFilepath( bucket='bucket', remote_path='trace.html') processor.main([ '--is-unittest', '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', '--upload-results', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME) self.assertEqual(hist.unit, SAMPLE_HISTOGRAM_UNIT) self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['osNames'], generic_set.GenericSet(['linux'])) self.assertEqual(hist.diagnostics['documentationUrls'], generic_set.GenericSet([['documentation', 'url']])) self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(1234567890987)) self.assertEqual( hist.diagnostics['traceUrls'], generic_set.GenericSet([ 'https://console.developers.google.com' '/m/cloudstorage/b/bucket/o/trace.html' ]))
def testGetSharedDiagnosticsOfType(self): d0 = generic_set.GenericSet(['foo']) d1 = date_range.DateRange(0) hs = histogram_set.HistogramSet() hs.AddSharedDiagnosticToAllHistograms('generic', d0) hs.AddSharedDiagnosticToAllHistograms('generic', d1) diagnostics = hs.GetSharedDiagnosticsOfType(generic_set.GenericSet) self.assertEqual(len(diagnostics), 1) self.assertIsInstance(diagnostics[0], generic_set.GenericSet)
def _GlobalDiagnostics(benchmark_run): """Extract diagnostics information about the whole benchmark run. These diagnostics will be added to ad-hoc measurements recorded by benchmarks. """ timestamp_ms = util.IsoTimestampToEpoch(benchmark_run['startTime']) * 1e3 return { reserved_infos.BENCHMARK_START.name: date_range.DateRange(timestamp_ms), }
def testAddDiagnosticsToHistograms(self): start_ts = 1500000000 start_iso = datetime.datetime.utcfromtimestamp( start_ts).isoformat() + 'Z' test_result = testing.TestResult( 'benchmark/story', output_artifacts={ 'trace.html': testing.Artifact('/trace.html', 'gs://trace.html'), }, start_time=start_iso, tags=['story_tag:test'], result_id='3', ) test_result['_histograms'] = histogram_set.HistogramSet() test_result['_histograms'].CreateHistogram('a', 'unitless', [0]) processor.AddDiagnosticsToHistograms(test_result, test_suite_start=start_iso, results_label='label', test_path_format='telemetry') hist = test_result['_histograms'].GetFirstHistogram() self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label'])) self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(start_ts * 1e3)) self.assertEqual(hist.diagnostics['traceStart'], date_range.DateRange(start_ts * 1e3)) self.assertEqual(hist.diagnostics['stories'], generic_set.GenericSet(['story'])) self.assertEqual(hist.diagnostics['storyTags'], generic_set.GenericSet(['test'])) self.assertEqual(hist.diagnostics['storysetRepeats'], generic_set.GenericSet([3])) self.assertEqual(hist.diagnostics['traceUrls'], generic_set.GenericSet(['gs://trace.html']))
def testHistogramsOutput_TBMv3(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateProtoTraceArtifact(), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]) ], tags=['tbmv3:dummy_metric'], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--is-unittest', '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', '--experimental-tbmv3-metrics', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) # For testing the TBMv3 workflow we use dummy_metric defined in # tools/perf/core/tbmv3/metrics/dummy_metric_*. hist = out_histograms.GetHistogramNamed('dummy::simple_field') self.assertEqual(hist.unit, 'count_smallerIsBetter') self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['osNames'], generic_set.GenericSet(['linux'])) self.assertEqual(hist.diagnostics['documentationUrls'], generic_set.GenericSet([['documentation', 'url']])) self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(1234567890987))
def testHistogramsOutput(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHistogramsArtifact( histogram.Histogram('a', 'unitless')), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]) ], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) self.assertEqual(len(out_histograms), 1) hist = out_histograms.GetFirstHistogram() self.assertEqual(hist.name, 'a') self.assertEqual(hist.unit, 'unitless') self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['osNames'], generic_set.GenericSet(['linux'])) self.assertEqual(hist.diagnostics['documentationUrls'], generic_set.GenericSet([['documentation', 'url']])) self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(1234567890987))
def testHtmlOutput(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateHtmlTraceArtifact(), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]), ], tags=['tbmv2:sampleMetric'], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--is-unittest', '--output-format', 'html', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', ]) with open(os.path.join(self.output_dir, html_output.OUTPUT_FILENAME)) as f: results = render_histograms_viewer.ReadExistingResults(f.read()) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME) self.assertEqual(hist.unit, SAMPLE_HISTOGRAM_UNIT) self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['osNames'], generic_set.GenericSet(['linux'])) self.assertEqual(hist.diagnostics['documentationUrls'], generic_set.GenericSet([['documentation', 'url']])) self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(1234567890987))
def testHistogramsOutput_TBMv3(self): self.SerializeIntermediateResults( testing.TestResult( 'benchmark/story', output_artifacts=[ self.CreateProtoTraceArtifact(), self.CreateDiagnosticsArtifact( benchmarks=['benchmark'], osNames=['linux'], documentationUrls=[['documentation', 'url']]) ], tags=['tbmv3:dummy_metric'], start_time='2009-02-13T23:31:30.987000Z', ), ) processor.main([ '--output-format', 'histograms', '--output-dir', self.output_dir, '--intermediate-dir', self.intermediate_dir, '--results-label', 'label', ]) with open( os.path.join(self.output_dir, histograms_output.OUTPUT_FILENAME)) as f: results = json.load(f) out_histograms = histogram_set.HistogramSet() out_histograms.ImportDicts(results) hist = out_histograms.GetHistogramNamed(DUMMY_HISTOGRAM_NAME) self.assertEqual(hist.unit, DUMMY_HISTOGRAM_UNIT) self.assertEqual(hist.diagnostics['benchmarks'], generic_set.GenericSet(['benchmark'])) self.assertEqual(hist.diagnostics['osNames'], generic_set.GenericSet(['linux'])) self.assertEqual(hist.diagnostics['documentationUrls'], generic_set.GenericSet([['documentation', 'url']])) self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(1234567890987))
def testAddDiagnosticsToHistograms(self): test_result = testing.TestResult('benchmark/story') test_result['_histograms'] = histogram_set.HistogramSet() test_result['_histograms'].CreateHistogram('a', 'unitless', [0]) start_ts = 1500000000 start_iso = datetime.datetime.utcfromtimestamp( start_ts).isoformat() + 'Z' processor.AddDiagnosticsToHistograms(test_result, test_suite_start=start_iso, results_label='label') hist = test_result['_histograms'].GetFirstHistogram() self.assertEqual(hist.diagnostics['labels'], generic_set.GenericSet(['label'])) self.assertEqual(hist.diagnostics['benchmarkStart'], date_range.DateRange(start_ts * 1e3))
def Profile(self, root): self._histograms = histogram_set.HistogramSet() total_hist = self._GetOrCreateHistogram('heap') total_hist.diagnostics['types'] = related_name_map.RelatedNameMap() total_breakdown = breakdown.Breakdown() total_size = self._Recurse(root, total_hist.diagnostics['types'], total_breakdown) builtins_size = total_size - sum(subsize for _, subsize in total_breakdown) if builtins_size: total_breakdown.Set('(builtin types)', builtins_size) total_hist.AddSample(total_size, dict(types=total_breakdown)) self._histograms.AddSharedDiagnosticToAllHistograms( reserved_infos.TRACE_START.name, date_range.DateRange(time.time() * 1000)) return self._histograms
def testSerialize(self): dr = date_range.DateRange(123) self.assertEqual(dr.Serialize(None), 123) dr.AddDiagnostic(date_range.DateRange(100)) self.assertEqual(dr.Serialize(None), [100, 123])
def testDateRange(self): grouping = histogram_grouping.DateRangeGrouping('foo') hist = histogram.Histogram('', 'count') self.assertEqual(grouping.callback(hist), '') hist.diagnostics['foo'] = date_range.DateRange(15e11) self.assertEqual(grouping.callback(hist), str(hist.diagnostics['foo']))
def testMaxTimestamp(self): dr = date_range.DateRange(1496693745123) dr.AddDiagnostic(date_range.DateRange(1496693746123)) self.assertEqual(dr.max_timestamp, 1496693746123)
def testDeduplicateDiagnostics(self): generic_a = generic_set.GenericSet(['A']) generic_b = generic_set.GenericSet(['B']) date_a = date_range.DateRange(42) date_b = date_range.DateRange(57) a_hist = histogram.Histogram('a', 'unitless') generic0 = generic_set.GenericSet.FromDict(generic_a.AsDict()) generic0.AddDiagnostic(generic_b) a_hist.diagnostics['generic'] = generic0 date0 = date_range.DateRange.FromDict(date_a.AsDict()) date0.AddDiagnostic(date_b) a_hist.diagnostics['date'] = date0 b_hist = histogram.Histogram('b', 'unitless') generic1 = generic_set.GenericSet.FromDict(generic_a.AsDict()) generic1.AddDiagnostic(generic_b) b_hist.diagnostics['generic'] = generic1 date1 = date_range.DateRange.FromDict(date_a.AsDict()) date1.AddDiagnostic(date_b) b_hist.diagnostics['date'] = date1 c_hist = histogram.Histogram('c', 'unitless') c_hist.diagnostics['generic'] = generic1 histograms = histogram_set.HistogramSet([a_hist, b_hist, c_hist]) self.assertNotEqual(a_hist.diagnostics['generic'].guid, b_hist.diagnostics['generic'].guid) self.assertEqual(b_hist.diagnostics['generic'].guid, c_hist.diagnostics['generic'].guid) self.assertEqual(a_hist.diagnostics['generic'], b_hist.diagnostics['generic']) self.assertNotEqual(a_hist.diagnostics['date'].guid, b_hist.diagnostics['date'].guid) self.assertEqual(a_hist.diagnostics['date'], b_hist.diagnostics['date']) histograms.DeduplicateDiagnostics() self.assertEqual(a_hist.diagnostics['generic'].guid, b_hist.diagnostics['generic'].guid) self.assertEqual(b_hist.diagnostics['generic'].guid, c_hist.diagnostics['generic'].guid) self.assertEqual(a_hist.diagnostics['generic'], b_hist.diagnostics['generic']) self.assertEqual(a_hist.diagnostics['date'].guid, b_hist.diagnostics['date'].guid) self.assertEqual(a_hist.diagnostics['date'], b_hist.diagnostics['date']) histogram_dicts = histograms.AsDicts() # All diagnostics should have been serialized as DiagnosticRefs. for d in histogram_dicts: if 'type' not in d: for diagnostic_dict in d['diagnostics'].values(): self.assertIsInstance(diagnostic_dict, str) histograms2 = histogram_set.HistogramSet() histograms2.ImportDicts(histograms.AsDicts()) a_hists = histograms2.GetHistogramsNamed('a') self.assertEqual(len(a_hists), 1) a_hist2 = a_hists[0] b_hists = histograms2.GetHistogramsNamed('b') self.assertEqual(len(b_hists), 1) b_hist2 = b_hists[0] self.assertEqual(a_hist2.diagnostics['generic'].guid, b_hist2.diagnostics['generic'].guid) self.assertEqual(a_hist2.diagnostics['generic'], b_hist2.diagnostics['generic']) self.assertEqual(a_hist2.diagnostics['date'].guid, b_hist2.diagnostics['date'].guid) self.assertEqual(a_hist2.diagnostics['date'], b_hist2.diagnostics['date'])