def testSharedDiagnostic(self):
hist = histogram.Histogram('', 'unitless')
hists = histogram.HistogramSet([hist])
diag = histogram.Generic('shared')
hists.AddSharedDiagnostic('generic', diag)
# Serializing a single Histogram with a single shared diagnostic should
# produce 2 dicts.
ds = hists.AsDicts()
self.assertEqual(len(ds), 2)
self.assertDeepEqual(diag.AsDict(), ds[0])
# The serialized Histogram should refer to the shared diagnostic by its
# guid.
self.assertEqual(ds[1]['diagnostics']['generic'], diag.guid)
# Deserialize ds.
hists2 = histogram.HistogramSet()
hists2.ImportDicts(ds)
self.assertEqual(len(hists2), 1)
hist2 = [h for h in hists2][0]
# The diagnostic reference should be deserialized as a DiagnosticRef until
# resolveRelatedHistograms is called.
self.assertIsInstance(hist2.diagnostics.get('generic'),
histogram.DiagnosticRef)
hists2.ResolveRelatedHistograms()
self.assertIsInstance(hist2.diagnostics.get('generic'),
histogram.Generic)
self.assertEqual(diag.value, hist2.diagnostics.get('generic').value)
def testRelatedHistogramSet(self):
a = histogram.Histogram('a', 'unitless')
b = histogram.Histogram('b', 'unitless')
c = histogram.Histogram('c', 'unitless')
a.diagnostics['rhs'] = histogram.RelatedHistogramSet([b, c])
# Don't serialize c yet.
hists = histogram.HistogramSet([a, b])
hists2 = histogram.HistogramSet()
hists2.ImportDicts(hists.AsDicts())
hists2.ResolveRelatedHistograms()
a2 = hists2.GetHistogramsNamed('a')
self.assertEqual(len(a2), 1)
a2 = a2[0]
self.assertEqual(a2.guid, a.guid)
self.assertIsInstance(a2, histogram.Histogram)
self.assertIsNot(a2, a)
b2 = hists2.GetHistogramsNamed('b')
self.assertEqual(len(b2), 1)
b2 = b2[0]
self.assertEqual(b2.guid, b.guid)
self.assertIsInstance(b2, histogram.Histogram)
self.assertIsNot(b2, b)
rhs2 = a2.diagnostics['rhs']
self.assertIsInstance(rhs2, histogram.RelatedHistogramSet)
self.assertEqual(len(rhs2), 2)
# Assert that b and c are in a2's RelatedHistogramSet, rhs2.
rhs2hs = list(rhs2)
rhs2guids = [h.guid for h in rhs2hs]
b2i = rhs2guids.index(b.guid)
self.assertIs(rhs2hs[b2i], b2)
c2i = rhs2guids.index(c.guid)
self.assertIsInstance(rhs2hs[c2i], histogram.HistogramRef)
# Now serialize c and add it to hists2.
hists2.ImportDicts([c.AsDict()])
hists2.ResolveRelatedHistograms()
c2 = hists2.GetHistogramsNamed('c')
self.assertEqual(len(c2), 1)
c2 = c2[0]
self.assertEqual(c2.guid, c.guid)
self.assertIsNot(c2, c)
rhs2hs = list(rhs2)
rhs2guids = [h.guid for h in rhs2hs]
b2i = rhs2guids.index(b.guid)
c2i = rhs2guids.index(c.guid)
self.assertIs(b2, rhs2hs[b2i])
self.assertIs(c2, rhs2hs[c2i])
def testComputeRevision(self):
histogram = histogram_module.Histogram('hist', 'count')
histograms = histogram_module.HistogramSet([histogram])
revision_info = histogram_module.RevisionInfo(
{'chromiumCommitPosition': 424242})
histograms.AddSharedDiagnostic('revisions', revision_info)
self.assertEqual(424242, add_histograms.ComputeRevision(histograms))
def ProcessHistogramSet(histogram_dicts):
if not isinstance(histogram_dicts, list):
raise BadRequestError('HistogramSet JSON much be a list of dicts')
histograms = histogram_module.HistogramSet()
histograms.ImportDicts(histogram_dicts)
histograms.ResolveRelatedHistograms()
InlineDenseSharedDiagnostics(histograms)
revision = ComputeRevision(histograms)
task_list = []
suite_path = ComputeSuitePath(histograms)
for diagnostic in histograms.shared_diagnostics:
# We'll skip the histogram-level sparse diagnostics because we need to
# handle those with the histograms, below, so that we can properly assign
# test paths.
if type(diagnostic) in SUITE_LEVEL_SPARSE_DIAGNOSTIC_TYPES:
task_list.append(_MakeTask(diagnostic, suite_path, revision))
for histogram in histograms:
guid = histogram.guid
objects = FindHistogramLevelSparseDiagnostics(guid, histograms)
test_path = ComputeTestPath(guid, histograms)
# We need to queue the histogram in addition to its dense shared
# diagnostics.
objects.append(histogram)
# TODO(eakuefner): Batch these better than one per task.
for obj in objects:
task_list.append(_MakeTask(obj, test_path, revision))
queue = taskqueue.Queue(TASK_QUEUE_NAME)
queue.add(task_list)
def testRelatedHistogramMap(self):
a = histogram.Histogram('a', 'unitless')
b = histogram.Histogram('b', 'unitless')
c = histogram.Histogram('c', 'unitless')
rhm = histogram.RelatedHistogramMap()
rhm.Set('y', b)
rhm.Set('z', c)
a.diagnostics['rhm'] = rhm
# Don't serialize c yet.
hists = histogram.HistogramSet([a, b])
hists2 = histogram.HistogramSet()
hists2.ImportDicts(hists.AsDicts())
hists2.ResolveRelatedHistograms()
a2 = hists2.GetHistogramsNamed('a')
self.assertEqual(len(a2), 1)
a2 = a2[0]
self.assertEqual(a2.guid, a.guid)
self.assertIsInstance(a2, histogram.Histogram)
self.assertIsNot(a2, a)
b2 = hists2.GetHistogramsNamed('b')
self.assertEqual(len(b2), 1)
b2 = b2[0]
self.assertEqual(b2.guid, b.guid)
self.assertIsInstance(b2, histogram.Histogram)
self.assertIsNot(b2, b)
rhm2 = a2.diagnostics['rhm']
self.assertIsInstance(rhm2, histogram.RelatedHistogramMap)
self.assertEqual(len(rhm2), 2)
# Assert that b and c are in a2's RelatedHistogramMap, rhm2.
self.assertIs(b2, rhm2.Get('y'))
self.assertIsInstance(rhm2.Get('z'), histogram.HistogramRef)
# Now serialize c and add it to hists2.
hists2.ImportDicts([c.AsDict()])
hists2.ResolveRelatedHistograms()
c2 = hists2.GetHistogramsNamed('c')
self.assertEqual(len(c2), 1)
c2 = c2[0]
self.assertEqual(c2.guid, c.guid)
self.assertIsNot(c2, c)
self.assertIs(b2, rhm2.Get('y'))
self.assertIs(c2, rhm2.Get('z'))
def testAddHistogramRaises(self):
hist = histogram.Histogram('', 'unitless')
hists = histogram.HistogramSet([hist])
with self.assertRaises(Exception):
hists.AddHistogram(hist)
hist2 = histogram.Histogram('', 'unitless')
# Do not ever do this in real code:
hist2.guid = hist.guid
with self.assertRaises(Exception):
hists.AddHistogram(hist2)
def testFindHistogramLevelSparseDiagnostics(self):
histogram = histogram_module.Histogram('hist', 'count')
histograms = histogram_module.HistogramSet([histogram])
histograms.AddSharedDiagnostic('foo', histogram_module.Generic('bar'))
histograms.AddSharedDiagnostic(
'telemetry', histogram_module.TelemetryInfo())
diagnostics = add_histograms.FindHistogramLevelSparseDiagnostics(
histogram.guid, histograms)
self.assertEqual(1, len(diagnostics))
self.assertIsInstance(diagnostics[0], histogram_module.TelemetryInfo)
def __init__(self,
output_formatters=None,
progress_reporter=None,
trace_tag='',
output_dir=None,
value_can_be_added_predicate=lambda v, is_first: True,
benchmark_enabled=True):
"""
Args:
output_formatters: A list of output formatters. The output
formatters are typically used to format the test results, such
as CsvPivotTableOutputFormatter, which output the test results as CSV.
progress_reporter: An instance of progress_reporter.ProgressReporter,
to be used to output test status/results progressively.
trace_tag: A string to append to the buildbot trace name. Currently only
used for buildbot.
output_dir: A string specified the directory where to store the test
artifacts, e.g: trace, videos,...
value_can_be_added_predicate: A function that takes two arguments:
a value.Value instance (except failure.FailureValue, skip.SkipValue
or trace.TraceValue) and a boolean (True when the value is part of
the first result for the story). It returns True if the value
can be added to the test results and False otherwise.
"""
# TODO(chrishenry): Figure out if trace_tag is still necessary.
super(PageTestResults, self).__init__()
self._progress_reporter = (progress_reporter
if progress_reporter is not None else
reporter_module.ProgressReporter())
self._output_formatters = (output_formatters
if output_formatters is not None else [])
self._trace_tag = trace_tag
self._output_dir = output_dir
self._value_can_be_added_predicate = value_can_be_added_predicate
self._current_page_run = None
self._all_page_runs = []
self._all_stories = set()
self._representative_value_for_each_value_name = {}
self._all_summary_values = []
self._serialized_trace_file_ids_to_paths = {}
self._pages_to_profiling_files = collections.defaultdict(list)
self._pages_to_profiling_files_cloud_url = collections.defaultdict(
list)
self._histograms = histogram.HistogramSet()
self._telemetry_info = TelemetryInfo()
# State of the benchmark this set of results represents.
self._benchmark_enabled = benchmark_enabled
def testSparseDiagnosticsAreNotInlined(self):
histogram = histogram_module.Histogram('hist', 'count')
histograms = histogram_module.HistogramSet([histogram])
histograms.AddSharedDiagnostic('foo', histogram_module.BuildbotInfo({
'displayMasterName': 'dmn',
'displayBotName': 'dbn',
'buildbotMasterName': 'bbmn',
'buildbotName': 'bbn',
'buildNumber': 42,
'logUri': 'uri',
}))
add_histograms.InlineDenseSharedDiagnostics(histograms)
self.assertFalse(histogram.diagnostics['foo'].is_inline)
def testReplaceSharedDiagnostic_NonRefAddsToMap(self):
hist = histogram.Histogram('', 'unitless')
hists = histogram.HistogramSet([hist])
diag0 = histogram.Generic('shared0')
diag1 = histogram.Generic('shared1')
hists.AddSharedDiagnostic('generic0', diag0)
guid0 = diag0.guid
guid1 = diag1.guid
hists.ReplaceSharedDiagnostic(guid0, diag1)
self.assertIsNotNone(hists.LookupDiagnostic(guid1))
def testComputeTestPathWithoutStory(self):
histogram = histogram_module.Histogram('hist', 'count')
histograms = histogram_module.HistogramSet([histogram])
telemetry_info = histogram_module.TelemetryInfo()
telemetry_info.AddInfo({'benchmarkName': 'benchmark'})
histograms.AddSharedDiagnostic('telemetry', telemetry_info)
buildbot_info = histogram_module.BuildbotInfo({
'displayMasterName': 'master',
'displayBotName': 'bot'
})
histograms.AddSharedDiagnostic('buildbot', buildbot_info)
histogram = histograms.GetFirstHistogram()
test_path = add_histograms.ComputeTestPath(histogram.guid, histograms)
self.assertEqual('master/bot/benchmark/hist', test_path)
def testReplaceSharedDiagnostic(self):
hist = histogram.Histogram('', 'unitless')
hists = histogram.HistogramSet([hist])
diag0 = histogram.Generic('shared0')
diag1 = histogram.Generic('shared1')
hists.AddSharedDiagnostic('generic0', diag0)
hists.AddSharedDiagnostic('generic1', diag1)
guid0 = diag0.guid
guid1 = diag1.guid
hists.ReplaceSharedDiagnostic(guid0,
histogram.DiagnosticRef('fakeGuid'))
self.assertEqual(hist.diagnostics['generic0'].guid, 'fakeGuid')
self.assertEqual(hist.diagnostics['generic1'].guid, guid1)
def testPopulateHistogramSet_UsesHistogramSetData(self):
original_diagnostic = histogram_module.TelemetryInfo()
results = page_test_results.PageTestResults()
results.WillRunPage(self.pages[0])
results.histograms.AddHistogram(histogram_module.Histogram('foo', 'count'))
results.histograms.AddSharedDiagnostic(
histogram_module.TelemetryInfo.NAME, original_diagnostic)
results.DidRunPage(self.pages[0])
results.CleanUp()
benchmark_metadata = benchmark.BenchmarkMetadata(
'benchmark_name', 'benchmark_description')
results.PopulateHistogramSet(benchmark_metadata)
histogram_dicts = results.AsHistogramDicts()
self.assertEquals(2, len(histogram_dicts))
hs = histogram_module.HistogramSet()
hs.ImportDicts(histogram_dicts)
diagnostic = hs.LookupDiagnostic(original_diagnostic.guid)
self.assertIsInstance(diagnostic, histogram_module.TelemetryInfo)
def testImportDicts(self):
hist = histogram.Histogram('', 'unitless')
hists = histogram.HistogramSet([hist])
hists2 = histogram.HistogramSet()
hists2.ImportDicts(hists.AsDicts())
self.assertEqual(len(hists), len(hists2))
