def testSharedDiagnostic(self): hist = histogram.Histogram('', 'unitless') hists = histogram.HistogramSet([hist]) diag = histogram.Generic('shared') hists.AddSharedDiagnostic('generic', diag) # Serializing a single Histogram with a single shared diagnostic should # produce 2 dicts. ds = hists.AsDicts() self.assertEqual(len(ds), 2) self.assertDeepEqual(diag.AsDict(), ds[0]) # The serialized Histogram should refer to the shared diagnostic by its # guid. self.assertEqual(ds[1]['diagnostics']['generic'], diag.guid) # Deserialize ds. hists2 = histogram.HistogramSet() hists2.ImportDicts(ds) self.assertEqual(len(hists2), 1) hist2 = [h for h in hists2][0] # The diagnostic reference should be deserialized as a DiagnosticRef until # resolveRelatedHistograms is called. self.assertIsInstance(hist2.diagnostics.get('generic'), histogram.DiagnosticRef) hists2.ResolveRelatedHistograms() self.assertIsInstance(hist2.diagnostics.get('generic'), histogram.Generic) self.assertEqual(diag.value, hist2.diagnostics.get('generic').value)
def testRelatedHistogramSet(self): a = histogram.Histogram('a', 'unitless') b = histogram.Histogram('b', 'unitless') c = histogram.Histogram('c', 'unitless') a.diagnostics['rhs'] = histogram.RelatedHistogramSet([b, c]) # Don't serialize c yet. hists = histogram.HistogramSet([a, b]) hists2 = histogram.HistogramSet() hists2.ImportDicts(hists.AsDicts()) hists2.ResolveRelatedHistograms() a2 = hists2.GetHistogramsNamed('a') self.assertEqual(len(a2), 1) a2 = a2[0] self.assertEqual(a2.guid, a.guid) self.assertIsInstance(a2, histogram.Histogram) self.assertIsNot(a2, a) b2 = hists2.GetHistogramsNamed('b') self.assertEqual(len(b2), 1) b2 = b2[0] self.assertEqual(b2.guid, b.guid) self.assertIsInstance(b2, histogram.Histogram) self.assertIsNot(b2, b) rhs2 = a2.diagnostics['rhs'] self.assertIsInstance(rhs2, histogram.RelatedHistogramSet) self.assertEqual(len(rhs2), 2) # Assert that b and c are in a2's RelatedHistogramSet, rhs2. rhs2hs = list(rhs2) rhs2guids = [h.guid for h in rhs2hs] b2i = rhs2guids.index(b.guid) self.assertIs(rhs2hs[b2i], b2) c2i = rhs2guids.index(c.guid) self.assertIsInstance(rhs2hs[c2i], histogram.HistogramRef) # Now serialize c and add it to hists2. hists2.ImportDicts([c.AsDict()]) hists2.ResolveRelatedHistograms() c2 = hists2.GetHistogramsNamed('c') self.assertEqual(len(c2), 1) c2 = c2[0] self.assertEqual(c2.guid, c.guid) self.assertIsNot(c2, c) rhs2hs = list(rhs2) rhs2guids = [h.guid for h in rhs2hs] b2i = rhs2guids.index(b.guid) c2i = rhs2guids.index(c.guid) self.assertIs(b2, rhs2hs[b2i]) self.assertIs(c2, rhs2hs[c2i])
def testComputeRevision(self): histogram = histogram_module.Histogram('hist', 'count') histograms = histogram_module.HistogramSet([histogram]) revision_info = histogram_module.RevisionInfo( {'chromiumCommitPosition': 424242}) histograms.AddSharedDiagnostic('revisions', revision_info) self.assertEqual(424242, add_histograms.ComputeRevision(histograms))
def ProcessHistogramSet(histogram_dicts): if not isinstance(histogram_dicts, list): raise BadRequestError('HistogramSet JSON much be a list of dicts') histograms = histogram_module.HistogramSet() histograms.ImportDicts(histogram_dicts) histograms.ResolveRelatedHistograms() InlineDenseSharedDiagnostics(histograms) revision = ComputeRevision(histograms) task_list = [] suite_path = ComputeSuitePath(histograms) for diagnostic in histograms.shared_diagnostics: # We'll skip the histogram-level sparse diagnostics because we need to # handle those with the histograms, below, so that we can properly assign # test paths. if type(diagnostic) in SUITE_LEVEL_SPARSE_DIAGNOSTIC_TYPES: task_list.append(_MakeTask(diagnostic, suite_path, revision)) for histogram in histograms: guid = histogram.guid objects = FindHistogramLevelSparseDiagnostics(guid, histograms) test_path = ComputeTestPath(guid, histograms) # We need to queue the histogram in addition to its dense shared # diagnostics. objects.append(histogram) # TODO(eakuefner): Batch these better than one per task. for obj in objects: task_list.append(_MakeTask(obj, test_path, revision)) queue = taskqueue.Queue(TASK_QUEUE_NAME) queue.add(task_list)
def testRelatedHistogramMap(self): a = histogram.Histogram('a', 'unitless') b = histogram.Histogram('b', 'unitless') c = histogram.Histogram('c', 'unitless') rhm = histogram.RelatedHistogramMap() rhm.Set('y', b) rhm.Set('z', c) a.diagnostics['rhm'] = rhm # Don't serialize c yet. hists = histogram.HistogramSet([a, b]) hists2 = histogram.HistogramSet() hists2.ImportDicts(hists.AsDicts()) hists2.ResolveRelatedHistograms() a2 = hists2.GetHistogramsNamed('a') self.assertEqual(len(a2), 1) a2 = a2[0] self.assertEqual(a2.guid, a.guid) self.assertIsInstance(a2, histogram.Histogram) self.assertIsNot(a2, a) b2 = hists2.GetHistogramsNamed('b') self.assertEqual(len(b2), 1) b2 = b2[0] self.assertEqual(b2.guid, b.guid) self.assertIsInstance(b2, histogram.Histogram) self.assertIsNot(b2, b) rhm2 = a2.diagnostics['rhm'] self.assertIsInstance(rhm2, histogram.RelatedHistogramMap) self.assertEqual(len(rhm2), 2) # Assert that b and c are in a2's RelatedHistogramMap, rhm2. self.assertIs(b2, rhm2.Get('y')) self.assertIsInstance(rhm2.Get('z'), histogram.HistogramRef) # Now serialize c and add it to hists2. hists2.ImportDicts([c.AsDict()]) hists2.ResolveRelatedHistograms() c2 = hists2.GetHistogramsNamed('c') self.assertEqual(len(c2), 1) c2 = c2[0] self.assertEqual(c2.guid, c.guid) self.assertIsNot(c2, c) self.assertIs(b2, rhm2.Get('y')) self.assertIs(c2, rhm2.Get('z'))
def testAddHistogramRaises(self): hist = histogram.Histogram('', 'unitless') hists = histogram.HistogramSet([hist]) with self.assertRaises(Exception): hists.AddHistogram(hist) hist2 = histogram.Histogram('', 'unitless') # Do not ever do this in real code: hist2.guid = hist.guid with self.assertRaises(Exception): hists.AddHistogram(hist2)
def testFindHistogramLevelSparseDiagnostics(self): histogram = histogram_module.Histogram('hist', 'count') histograms = histogram_module.HistogramSet([histogram]) histograms.AddSharedDiagnostic('foo', histogram_module.Generic('bar')) histograms.AddSharedDiagnostic( 'telemetry', histogram_module.TelemetryInfo()) diagnostics = add_histograms.FindHistogramLevelSparseDiagnostics( histogram.guid, histograms) self.assertEqual(1, len(diagnostics)) self.assertIsInstance(diagnostics[0], histogram_module.TelemetryInfo)
def __init__(self, output_formatters=None, progress_reporter=None, trace_tag='', output_dir=None, value_can_be_added_predicate=lambda v, is_first: True, benchmark_enabled=True): """ Args: output_formatters: A list of output formatters. The output formatters are typically used to format the test results, such as CsvPivotTableOutputFormatter, which output the test results as CSV. progress_reporter: An instance of progress_reporter.ProgressReporter, to be used to output test status/results progressively. trace_tag: A string to append to the buildbot trace name. Currently only used for buildbot. output_dir: A string specified the directory where to store the test artifacts, e.g: trace, videos,... value_can_be_added_predicate: A function that takes two arguments: a value.Value instance (except failure.FailureValue, skip.SkipValue or trace.TraceValue) and a boolean (True when the value is part of the first result for the story). It returns True if the value can be added to the test results and False otherwise. """ # TODO(chrishenry): Figure out if trace_tag is still necessary. super(PageTestResults, self).__init__() self._progress_reporter = (progress_reporter if progress_reporter is not None else reporter_module.ProgressReporter()) self._output_formatters = (output_formatters if output_formatters is not None else []) self._trace_tag = trace_tag self._output_dir = output_dir self._value_can_be_added_predicate = value_can_be_added_predicate self._current_page_run = None self._all_page_runs = [] self._all_stories = set() self._representative_value_for_each_value_name = {} self._all_summary_values = [] self._serialized_trace_file_ids_to_paths = {} self._pages_to_profiling_files = collections.defaultdict(list) self._pages_to_profiling_files_cloud_url = collections.defaultdict( list) self._histograms = histogram.HistogramSet() self._telemetry_info = TelemetryInfo() # State of the benchmark this set of results represents. self._benchmark_enabled = benchmark_enabled
def testSparseDiagnosticsAreNotInlined(self): histogram = histogram_module.Histogram('hist', 'count') histograms = histogram_module.HistogramSet([histogram]) histograms.AddSharedDiagnostic('foo', histogram_module.BuildbotInfo({ 'displayMasterName': 'dmn', 'displayBotName': 'dbn', 'buildbotMasterName': 'bbmn', 'buildbotName': 'bbn', 'buildNumber': 42, 'logUri': 'uri', })) add_histograms.InlineDenseSharedDiagnostics(histograms) self.assertFalse(histogram.diagnostics['foo'].is_inline)
def testReplaceSharedDiagnostic_NonRefAddsToMap(self): hist = histogram.Histogram('', 'unitless') hists = histogram.HistogramSet([hist]) diag0 = histogram.Generic('shared0') diag1 = histogram.Generic('shared1') hists.AddSharedDiagnostic('generic0', diag0) guid0 = diag0.guid guid1 = diag1.guid hists.ReplaceSharedDiagnostic(guid0, diag1) self.assertIsNotNone(hists.LookupDiagnostic(guid1))
def testComputeTestPathWithoutStory(self): histogram = histogram_module.Histogram('hist', 'count') histograms = histogram_module.HistogramSet([histogram]) telemetry_info = histogram_module.TelemetryInfo() telemetry_info.AddInfo({'benchmarkName': 'benchmark'}) histograms.AddSharedDiagnostic('telemetry', telemetry_info) buildbot_info = histogram_module.BuildbotInfo({ 'displayMasterName': 'master', 'displayBotName': 'bot' }) histograms.AddSharedDiagnostic('buildbot', buildbot_info) histogram = histograms.GetFirstHistogram() test_path = add_histograms.ComputeTestPath(histogram.guid, histograms) self.assertEqual('master/bot/benchmark/hist', test_path)
def testReplaceSharedDiagnostic(self): hist = histogram.Histogram('', 'unitless') hists = histogram.HistogramSet([hist]) diag0 = histogram.Generic('shared0') diag1 = histogram.Generic('shared1') hists.AddSharedDiagnostic('generic0', diag0) hists.AddSharedDiagnostic('generic1', diag1) guid0 = diag0.guid guid1 = diag1.guid hists.ReplaceSharedDiagnostic(guid0, histogram.DiagnosticRef('fakeGuid')) self.assertEqual(hist.diagnostics['generic0'].guid, 'fakeGuid') self.assertEqual(hist.diagnostics['generic1'].guid, guid1)
def testPopulateHistogramSet_UsesHistogramSetData(self): original_diagnostic = histogram_module.TelemetryInfo() results = page_test_results.PageTestResults() results.WillRunPage(self.pages[0]) results.histograms.AddHistogram(histogram_module.Histogram('foo', 'count')) results.histograms.AddSharedDiagnostic( histogram_module.TelemetryInfo.NAME, original_diagnostic) results.DidRunPage(self.pages[0]) results.CleanUp() benchmark_metadata = benchmark.BenchmarkMetadata( 'benchmark_name', 'benchmark_description') results.PopulateHistogramSet(benchmark_metadata) histogram_dicts = results.AsHistogramDicts() self.assertEquals(2, len(histogram_dicts)) hs = histogram_module.HistogramSet() hs.ImportDicts(histogram_dicts) diagnostic = hs.LookupDiagnostic(original_diagnostic.guid) self.assertIsInstance(diagnostic, histogram_module.TelemetryInfo)
def testImportDicts(self): hist = histogram.Histogram('', 'unitless') hists = histogram.HistogramSet([hist]) hists2 = histogram.HistogramSet() hists2.ImportDicts(hists.AsDicts()) self.assertEqual(len(hists), len(hists2))