Exemple #1
0
  def testHistogramsOutputMeasurements(self):
    measurements = {
        'a': {'unit': 'ms', 'samples': [4, 6], 'description': 'desc_a'},
        'b': {'unit': 'ms', 'samples': [5], 'description': 'desc_b'},
    }
    start_ts = 1500000000
    start_iso = datetime.datetime.utcfromtimestamp(start_ts).isoformat() + 'Z'

    self.SerializeIntermediateResults(
        testing.TestResult(
            'benchmark/story',
            output_artifacts=[
                self.CreateMeasurementsArtifact(measurements),
            ],
            tags=['story_tag:test'],
            start_time=start_iso,
        ),
    )

    processor.main([
        '--output-format', 'histograms',
        '--output-dir', self.output_dir,
        '--intermediate-dir', self.intermediate_dir,
    ])

    with open(os.path.join(
        self.output_dir, histograms_output.OUTPUT_FILENAME)) as f:
      results = json.load(f)

    out_histograms = histogram_set.HistogramSet()
    out_histograms.ImportDicts(results)
    self.assertEqual(len(out_histograms), 2)

    hist = out_histograms.GetHistogramNamed('a')
    self.assertEqual(hist.name, 'a')
    self.assertEqual(hist.unit, 'ms_smallerIsBetter')
    self.assertEqual(hist.sample_values, [4, 6])
    self.assertEqual(hist.description, 'desc_a')
    self.assertEqual(hist.diagnostics['benchmarks'],
                     generic_set.GenericSet(['benchmark']))
    self.assertEqual(hist.diagnostics['stories'],
                     generic_set.GenericSet(['story']))
    self.assertEqual(hist.diagnostics['storyTags'],
                     generic_set.GenericSet(['test']))
    self.assertEqual(hist.diagnostics['benchmarkStart'],
                     date_range.DateRange(start_ts * 1e3))

    hist = out_histograms.GetHistogramNamed('b')
    self.assertEqual(hist.name, 'b')
    self.assertEqual(hist.unit, 'ms_smallerIsBetter')
    self.assertEqual(hist.sample_values, [5])
    self.assertEqual(hist.description, 'desc_b')
    self.assertEqual(hist.diagnostics['benchmarks'],
                     generic_set.GenericSet(['benchmark']))
    self.assertEqual(hist.diagnostics['stories'],
                     generic_set.GenericSet(['story']))
    self.assertEqual(hist.diagnostics['storyTags'],
                     generic_set.GenericSet(['test']))
    self.assertEqual(hist.diagnostics['benchmarkStart'],
                     date_range.DateRange(start_ts * 1e3))
 def testRoundtrip(self):
     dr = date_range.DateRange(1496693745000)
     dr.AddDiagnostic(date_range.DateRange(1496693746000))
     self.assertEqual(calendar.timegm(dr.min_date.timetuple()), 1496693745)
     self.assertEqual(calendar.timegm(dr.max_date.timetuple()), 1496693746)
     clone = diagnostic.Diagnostic.FromDict(dr.AsDict())
     self.assertEqual(clone.min_date, dr.min_date)
     self.assertEqual(clone.max_date, dr.max_date)
 def testDisallowReservedNames(self):
   diagnostics = histogram.DiagnosticMap()
   with self.assertRaises(TypeError):
     diagnostics[None] = generic_set.GenericSet(())
   with self.assertRaises(TypeError):
     diagnostics['generic'] = None
   diagnostics[reserved_infos.TRACE_URLS.name] = date_range.DateRange(0)
   diagnostics.DisallowReservedNames()
   diagnostics[reserved_infos.TRACE_URLS.name] = generic_set.GenericSet(())
   with self.assertRaises(TypeError):
     diagnostics[reserved_infos.TRACE_URLS.name] = date_range.DateRange(0)
Exemple #4
0
def AddDiagnosticsToHistograms(test_result, test_suite_start, results_label):
    """Add diagnostics to all histograms of a test run.

  Reads diagnostics from the test artifact and adds them to all histograms.
  This overwrites the corresponding diagnostics previously set by e.g.
  run_metrics.
  """
    artifacts = test_result.get('outputArtifacts', {})
    if DIAGNOSTICS_NAME in artifacts:
        with open(artifacts[DIAGNOSTICS_NAME]['filePath']) as f:
            diagnostics = json.load(f)['diagnostics']
        for name, diag in diagnostics.items():
            # For now, we only support GenericSet diagnostics that are serialized
            # as lists of values.
            assert isinstance(diag, list)
            test_result['_histograms'].AddSharedDiagnosticToAllHistograms(
                name, generic_set.GenericSet(diag))

    timestamp_ms = util.IsoTimestampToEpoch(test_suite_start) * 1e3
    test_result['_histograms'].AddSharedDiagnosticToAllHistograms(
        reserved_infos.BENCHMARK_START.name,
        date_range.DateRange(timestamp_ms))

    if results_label is not None:
        test_result['_histograms'].AddSharedDiagnosticToAllHistograms(
            reserved_infos.LABELS.name,
            generic_set.GenericSet([results_label]))
Exemple #5
0
 def DiagnosticValue(value):
   if isinstance(value, (int, long)):
     return date_range.DateRange(value)
   elif isinstance(value, list):
     return generic_set.GenericSet(value)
   else:
     raise NotImplementedError(type(value))
    def testHistogramsOutput(self):
        self.SerializeIntermediateResults(
            testing.TestResult(
                'benchmark/story',
                output_artifacts=[
                    self.CreateHtmlTraceArtifact(),
                    self.CreateDiagnosticsArtifact(
                        benchmarks=['benchmark'],
                        osNames=['linux'],
                        documentationUrls=[['documentation', 'url']])
                ],
                tags=['tbmv2:sampleMetric'],
                start_time='2009-02-13T23:31:30.987000Z',
            ), )

        with mock.patch('py_utils.cloud_storage.Upload') as cloud_patch:
            cloud_patch.return_value = processor.cloud_storage.CloudFilepath(
                bucket='bucket', remote_path='trace.html')
            processor.main([
                '--is-unittest',
                '--output-format',
                'histograms',
                '--output-dir',
                self.output_dir,
                '--intermediate-dir',
                self.intermediate_dir,
                '--results-label',
                'label',
                '--upload-results',
            ])

        with open(
                os.path.join(self.output_dir,
                             histograms_output.OUTPUT_FILENAME)) as f:
            results = json.load(f)

        out_histograms = histogram_set.HistogramSet()
        out_histograms.ImportDicts(results)

        hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME)
        self.assertEqual(hist.unit, SAMPLE_HISTOGRAM_UNIT)

        self.assertEqual(hist.diagnostics['benchmarks'],
                         generic_set.GenericSet(['benchmark']))
        self.assertEqual(hist.diagnostics['osNames'],
                         generic_set.GenericSet(['linux']))
        self.assertEqual(hist.diagnostics['documentationUrls'],
                         generic_set.GenericSet([['documentation', 'url']]))
        self.assertEqual(hist.diagnostics['labels'],
                         generic_set.GenericSet(['label']))
        self.assertEqual(hist.diagnostics['benchmarkStart'],
                         date_range.DateRange(1234567890987))
        self.assertEqual(
            hist.diagnostics['traceUrls'],
            generic_set.GenericSet([
                'https://console.developers.google.com'
                '/m/cloudstorage/b/bucket/o/trace.html'
            ]))
 def testGetSharedDiagnosticsOfType(self):
     d0 = generic_set.GenericSet(['foo'])
     d1 = date_range.DateRange(0)
     hs = histogram_set.HistogramSet()
     hs.AddSharedDiagnosticToAllHistograms('generic', d0)
     hs.AddSharedDiagnosticToAllHistograms('generic', d1)
     diagnostics = hs.GetSharedDiagnosticsOfType(generic_set.GenericSet)
     self.assertEqual(len(diagnostics), 1)
     self.assertIsInstance(diagnostics[0], generic_set.GenericSet)
Exemple #8
0
def _GlobalDiagnostics(benchmark_run):
  """Extract diagnostics information about the whole benchmark run.

  These diagnostics will be added to ad-hoc measurements recorded by
  benchmarks.
  """
  timestamp_ms = util.IsoTimestampToEpoch(benchmark_run['startTime']) * 1e3
  return {
    reserved_infos.BENCHMARK_START.name: date_range.DateRange(timestamp_ms),
  }
    def testAddDiagnosticsToHistograms(self):
        start_ts = 1500000000
        start_iso = datetime.datetime.utcfromtimestamp(
            start_ts).isoformat() + 'Z'

        test_result = testing.TestResult(
            'benchmark/story',
            output_artifacts={
                'trace.html': testing.Artifact('/trace.html',
                                               'gs://trace.html'),
            },
            start_time=start_iso,
            tags=['story_tag:test'],
            result_id='3',
        )
        test_result['_histograms'] = histogram_set.HistogramSet()
        test_result['_histograms'].CreateHistogram('a', 'unitless', [0])

        processor.AddDiagnosticsToHistograms(test_result,
                                             test_suite_start=start_iso,
                                             results_label='label',
                                             test_path_format='telemetry')

        hist = test_result['_histograms'].GetFirstHistogram()
        self.assertEqual(hist.diagnostics['labels'],
                         generic_set.GenericSet(['label']))
        self.assertEqual(hist.diagnostics['benchmarks'],
                         generic_set.GenericSet(['benchmark']))
        self.assertEqual(hist.diagnostics['benchmarkStart'],
                         date_range.DateRange(start_ts * 1e3))
        self.assertEqual(hist.diagnostics['traceStart'],
                         date_range.DateRange(start_ts * 1e3))
        self.assertEqual(hist.diagnostics['stories'],
                         generic_set.GenericSet(['story']))
        self.assertEqual(hist.diagnostics['storyTags'],
                         generic_set.GenericSet(['test']))
        self.assertEqual(hist.diagnostics['storysetRepeats'],
                         generic_set.GenericSet([3]))
        self.assertEqual(hist.diagnostics['traceUrls'],
                         generic_set.GenericSet(['gs://trace.html']))
    def testHistogramsOutput_TBMv3(self):
        self.SerializeIntermediateResults(
            testing.TestResult(
                'benchmark/story',
                output_artifacts=[
                    self.CreateProtoTraceArtifact(),
                    self.CreateDiagnosticsArtifact(
                        benchmarks=['benchmark'],
                        osNames=['linux'],
                        documentationUrls=[['documentation', 'url']])
                ],
                tags=['tbmv3:dummy_metric'],
                start_time='2009-02-13T23:31:30.987000Z',
            ), )

        processor.main([
            '--is-unittest',
            '--output-format',
            'histograms',
            '--output-dir',
            self.output_dir,
            '--intermediate-dir',
            self.intermediate_dir,
            '--results-label',
            'label',
            '--experimental-tbmv3-metrics',
        ])

        with open(
                os.path.join(self.output_dir,
                             histograms_output.OUTPUT_FILENAME)) as f:
            results = json.load(f)

        out_histograms = histogram_set.HistogramSet()
        out_histograms.ImportDicts(results)

        # For testing the TBMv3 workflow we use dummy_metric defined in
        # tools/perf/core/tbmv3/metrics/dummy_metric_*.
        hist = out_histograms.GetHistogramNamed('dummy::simple_field')
        self.assertEqual(hist.unit, 'count_smallerIsBetter')

        self.assertEqual(hist.diagnostics['benchmarks'],
                         generic_set.GenericSet(['benchmark']))
        self.assertEqual(hist.diagnostics['osNames'],
                         generic_set.GenericSet(['linux']))
        self.assertEqual(hist.diagnostics['documentationUrls'],
                         generic_set.GenericSet([['documentation', 'url']]))
        self.assertEqual(hist.diagnostics['labels'],
                         generic_set.GenericSet(['label']))
        self.assertEqual(hist.diagnostics['benchmarkStart'],
                         date_range.DateRange(1234567890987))
Exemple #11
0
    def testHistogramsOutput(self):
        self.SerializeIntermediateResults(
            testing.TestResult(
                'benchmark/story',
                output_artifacts=[
                    self.CreateHistogramsArtifact(
                        histogram.Histogram('a', 'unitless')),
                    self.CreateDiagnosticsArtifact(
                        benchmarks=['benchmark'],
                        osNames=['linux'],
                        documentationUrls=[['documentation', 'url']])
                ],
                start_time='2009-02-13T23:31:30.987000Z',
            ), )

        processor.main([
            '--output-format',
            'histograms',
            '--output-dir',
            self.output_dir,
            '--intermediate-dir',
            self.intermediate_dir,
            '--results-label',
            'label',
        ])

        with open(
                os.path.join(self.output_dir,
                             histograms_output.OUTPUT_FILENAME)) as f:
            results = json.load(f)

        out_histograms = histogram_set.HistogramSet()
        out_histograms.ImportDicts(results)
        self.assertEqual(len(out_histograms), 1)

        hist = out_histograms.GetFirstHistogram()
        self.assertEqual(hist.name, 'a')
        self.assertEqual(hist.unit, 'unitless')

        self.assertEqual(hist.diagnostics['benchmarks'],
                         generic_set.GenericSet(['benchmark']))
        self.assertEqual(hist.diagnostics['osNames'],
                         generic_set.GenericSet(['linux']))
        self.assertEqual(hist.diagnostics['documentationUrls'],
                         generic_set.GenericSet([['documentation', 'url']]))
        self.assertEqual(hist.diagnostics['labels'],
                         generic_set.GenericSet(['label']))
        self.assertEqual(hist.diagnostics['benchmarkStart'],
                         date_range.DateRange(1234567890987))
    def testHtmlOutput(self):
        self.SerializeIntermediateResults(
            testing.TestResult(
                'benchmark/story',
                output_artifacts=[
                    self.CreateHtmlTraceArtifact(),
                    self.CreateDiagnosticsArtifact(
                        benchmarks=['benchmark'],
                        osNames=['linux'],
                        documentationUrls=[['documentation', 'url']]),
                ],
                tags=['tbmv2:sampleMetric'],
                start_time='2009-02-13T23:31:30.987000Z',
            ), )

        processor.main([
            '--is-unittest',
            '--output-format',
            'html',
            '--output-dir',
            self.output_dir,
            '--intermediate-dir',
            self.intermediate_dir,
            '--results-label',
            'label',
        ])

        with open(os.path.join(self.output_dir,
                               html_output.OUTPUT_FILENAME)) as f:
            results = render_histograms_viewer.ReadExistingResults(f.read())

        out_histograms = histogram_set.HistogramSet()
        out_histograms.ImportDicts(results)

        hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME)
        self.assertEqual(hist.unit, SAMPLE_HISTOGRAM_UNIT)

        self.assertEqual(hist.diagnostics['benchmarks'],
                         generic_set.GenericSet(['benchmark']))
        self.assertEqual(hist.diagnostics['osNames'],
                         generic_set.GenericSet(['linux']))
        self.assertEqual(hist.diagnostics['documentationUrls'],
                         generic_set.GenericSet([['documentation', 'url']]))
        self.assertEqual(hist.diagnostics['labels'],
                         generic_set.GenericSet(['label']))
        self.assertEqual(hist.diagnostics['benchmarkStart'],
                         date_range.DateRange(1234567890987))
    def testHistogramsOutput_TBMv3(self):
        self.SerializeIntermediateResults(
            testing.TestResult(
                'benchmark/story',
                output_artifacts=[
                    self.CreateProtoTraceArtifact(),
                    self.CreateDiagnosticsArtifact(
                        benchmarks=['benchmark'],
                        osNames=['linux'],
                        documentationUrls=[['documentation', 'url']])
                ],
                tags=['tbmv3:dummy_metric'],
                start_time='2009-02-13T23:31:30.987000Z',
            ), )

        processor.main([
            '--output-format',
            'histograms',
            '--output-dir',
            self.output_dir,
            '--intermediate-dir',
            self.intermediate_dir,
            '--results-label',
            'label',
        ])

        with open(
                os.path.join(self.output_dir,
                             histograms_output.OUTPUT_FILENAME)) as f:
            results = json.load(f)

        out_histograms = histogram_set.HistogramSet()
        out_histograms.ImportDicts(results)

        hist = out_histograms.GetHistogramNamed(DUMMY_HISTOGRAM_NAME)
        self.assertEqual(hist.unit, DUMMY_HISTOGRAM_UNIT)

        self.assertEqual(hist.diagnostics['benchmarks'],
                         generic_set.GenericSet(['benchmark']))
        self.assertEqual(hist.diagnostics['osNames'],
                         generic_set.GenericSet(['linux']))
        self.assertEqual(hist.diagnostics['documentationUrls'],
                         generic_set.GenericSet([['documentation', 'url']]))
        self.assertEqual(hist.diagnostics['labels'],
                         generic_set.GenericSet(['label']))
        self.assertEqual(hist.diagnostics['benchmarkStart'],
                         date_range.DateRange(1234567890987))
Exemple #14
0
    def testAddDiagnosticsToHistograms(self):
        test_result = testing.TestResult('benchmark/story')
        test_result['_histograms'] = histogram_set.HistogramSet()
        test_result['_histograms'].CreateHistogram('a', 'unitless', [0])

        start_ts = 1500000000
        start_iso = datetime.datetime.utcfromtimestamp(
            start_ts).isoformat() + 'Z'

        processor.AddDiagnosticsToHistograms(test_result,
                                             test_suite_start=start_iso,
                                             results_label='label')

        hist = test_result['_histograms'].GetFirstHistogram()
        self.assertEqual(hist.diagnostics['labels'],
                         generic_set.GenericSet(['label']))
        self.assertEqual(hist.diagnostics['benchmarkStart'],
                         date_range.DateRange(start_ts * 1e3))
    def Profile(self, root):
        self._histograms = histogram_set.HistogramSet()
        total_hist = self._GetOrCreateHistogram('heap')
        total_hist.diagnostics['types'] = related_name_map.RelatedNameMap()
        total_breakdown = breakdown.Breakdown()
        total_size = self._Recurse(root, total_hist.diagnostics['types'],
                                   total_breakdown)
        builtins_size = total_size - sum(subsize
                                         for _, subsize in total_breakdown)

        if builtins_size:
            total_breakdown.Set('(builtin types)', builtins_size)
        total_hist.AddSample(total_size, dict(types=total_breakdown))

        self._histograms.AddSharedDiagnosticToAllHistograms(
            reserved_infos.TRACE_START.name,
            date_range.DateRange(time.time() * 1000))

        return self._histograms
Exemple #16
0
 def testSerialize(self):
     dr = date_range.DateRange(123)
     self.assertEqual(dr.Serialize(None), 123)
     dr.AddDiagnostic(date_range.DateRange(100))
     self.assertEqual(dr.Serialize(None), [100, 123])
Exemple #17
0
 def testDateRange(self):
     grouping = histogram_grouping.DateRangeGrouping('foo')
     hist = histogram.Histogram('', 'count')
     self.assertEqual(grouping.callback(hist), '')
     hist.diagnostics['foo'] = date_range.DateRange(15e11)
     self.assertEqual(grouping.callback(hist), str(hist.diagnostics['foo']))
 def testMaxTimestamp(self):
     dr = date_range.DateRange(1496693745123)
     dr.AddDiagnostic(date_range.DateRange(1496693746123))
     self.assertEqual(dr.max_timestamp, 1496693746123)
    def testDeduplicateDiagnostics(self):
        generic_a = generic_set.GenericSet(['A'])
        generic_b = generic_set.GenericSet(['B'])
        date_a = date_range.DateRange(42)
        date_b = date_range.DateRange(57)

        a_hist = histogram.Histogram('a', 'unitless')
        generic0 = generic_set.GenericSet.FromDict(generic_a.AsDict())
        generic0.AddDiagnostic(generic_b)
        a_hist.diagnostics['generic'] = generic0
        date0 = date_range.DateRange.FromDict(date_a.AsDict())
        date0.AddDiagnostic(date_b)
        a_hist.diagnostics['date'] = date0

        b_hist = histogram.Histogram('b', 'unitless')
        generic1 = generic_set.GenericSet.FromDict(generic_a.AsDict())
        generic1.AddDiagnostic(generic_b)
        b_hist.diagnostics['generic'] = generic1
        date1 = date_range.DateRange.FromDict(date_a.AsDict())
        date1.AddDiagnostic(date_b)
        b_hist.diagnostics['date'] = date1

        c_hist = histogram.Histogram('c', 'unitless')
        c_hist.diagnostics['generic'] = generic1

        histograms = histogram_set.HistogramSet([a_hist, b_hist, c_hist])
        self.assertNotEqual(a_hist.diagnostics['generic'].guid,
                            b_hist.diagnostics['generic'].guid)
        self.assertEqual(b_hist.diagnostics['generic'].guid,
                         c_hist.diagnostics['generic'].guid)
        self.assertEqual(a_hist.diagnostics['generic'],
                         b_hist.diagnostics['generic'])
        self.assertNotEqual(a_hist.diagnostics['date'].guid,
                            b_hist.diagnostics['date'].guid)
        self.assertEqual(a_hist.diagnostics['date'],
                         b_hist.diagnostics['date'])

        histograms.DeduplicateDiagnostics()

        self.assertEqual(a_hist.diagnostics['generic'].guid,
                         b_hist.diagnostics['generic'].guid)
        self.assertEqual(b_hist.diagnostics['generic'].guid,
                         c_hist.diagnostics['generic'].guid)
        self.assertEqual(a_hist.diagnostics['generic'],
                         b_hist.diagnostics['generic'])
        self.assertEqual(a_hist.diagnostics['date'].guid,
                         b_hist.diagnostics['date'].guid)
        self.assertEqual(a_hist.diagnostics['date'],
                         b_hist.diagnostics['date'])

        histogram_dicts = histograms.AsDicts()

        # All diagnostics should have been serialized as DiagnosticRefs.
        for d in histogram_dicts:
            if 'type' not in d:
                for diagnostic_dict in d['diagnostics'].values():
                    self.assertIsInstance(diagnostic_dict, str)

        histograms2 = histogram_set.HistogramSet()
        histograms2.ImportDicts(histograms.AsDicts())
        a_hists = histograms2.GetHistogramsNamed('a')
        self.assertEqual(len(a_hists), 1)
        a_hist2 = a_hists[0]
        b_hists = histograms2.GetHistogramsNamed('b')
        self.assertEqual(len(b_hists), 1)
        b_hist2 = b_hists[0]

        self.assertEqual(a_hist2.diagnostics['generic'].guid,
                         b_hist2.diagnostics['generic'].guid)
        self.assertEqual(a_hist2.diagnostics['generic'],
                         b_hist2.diagnostics['generic'])
        self.assertEqual(a_hist2.diagnostics['date'].guid,
                         b_hist2.diagnostics['date'].guid)
        self.assertEqual(a_hist2.diagnostics['date'],
                         b_hist2.diagnostics['date'])