def _CreateHistogramSet(
master, bot, benchmark, commit_position, histograms):
histograms = histogram_set.HistogramSet(histograms)
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.MASTERS.name,
generic_set.GenericSet([master]))
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.BOTS.name,
generic_set.GenericSet([bot]))
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.CHROMIUM_COMMIT_POSITIONS.name,
generic_set.GenericSet([commit_position]))
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.BENCHMARKS.name,
generic_set.GenericSet([benchmark]))
return histograms
def GetOwners(self):
"""Returns a Generic Diagnostic containing the benchmark's owners' emails
in a list.
Returns:
Diagnostic with a list of the benchmark's owners' emails
"""
return generic_set.GenericSet(decorators.GetEmails(self) or [])
def AddDurationHistogram(self, duration_in_milliseconds):
hist = histogram.Histogram(
'benchmark_total_duration', 'ms_smallerIsBetter')
hist.AddSample(duration_in_milliseconds)
# TODO(#4244): Do this generally.
if self.telemetry_info.label:
hist.diagnostics[reserved_infos.LABELS.name] = generic_set.GenericSet(
[self.telemetry_info.label])
hist.diagnostics[reserved_infos.BENCHMARKS.name] = generic_set.GenericSet(
[self.telemetry_info.benchmark_name])
hist.diagnostics[reserved_infos.BENCHMARK_START.name] = histogram.DateRange(
self.telemetry_info.benchmark_start_epoch * 1000)
if self.telemetry_info.benchmark_descriptions:
hist.diagnostics[
reserved_infos.BENCHMARK_DESCRIPTIONS.name] = generic_set.GenericSet([
self.telemetry_info.benchmark_descriptions])
self._histograms.AddHistogram(hist)
def testRemoveOrphanedDiagnostics(self):
da = generic_set.GenericSet(['a'])
db = generic_set.GenericSet(['b'])
a = histogram.Histogram('a', 'unitless')
b = histogram.Histogram('b', 'unitless')
hs = histogram_set.HistogramSet([a])
hs.AddSharedDiagnosticToAllHistograms('a', da)
hs.AddHistogram(b)
hs.AddSharedDiagnosticToAllHistograms('b', db)
hs.FilterHistograms(lambda h: h.name == 'a')
dicts = hs.AsDicts()
self.assertEqual(3, len(dicts))
hs.RemoveOrphanedDiagnostics()
dicts = hs.AsDicts()
self.assertEqual(2, len(dicts))
def testSparseDiagnosticsAreNotInlined(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnostic(
reserved_infos.BENCHMARKS.name,
generic_set.GenericSet(['benchmark']))
add_histograms.InlineDenseSharedDiagnostics(histograms)
self.assertTrue(hist.diagnostics[reserved_infos.BENCHMARKS.name].has_guid)
def testMerge(self):
hs1 = histogram_set.HistogramSet([histogram.Histogram('a', 'unitless')])
hs1.AddSharedDiagnosticToAllHistograms('name',
generic_set.GenericSet(['diag1']))
hs2 = histogram_set.HistogramSet([histogram.Histogram('b', 'unitless')])
hs2.AddSharedDiagnosticToAllHistograms('name',
generic_set.GenericSet(['diag2']))
hs1.Merge(hs2)
self.assertEqual(len(hs1), 2)
self.assertEqual(len(hs1.shared_diagnostics), 2)
self.assertEqual(hs1.GetHistogramNamed('a').diagnostics['name'],
generic_set.GenericSet(['diag1']))
self.assertEqual(hs1.GetHistogramNamed('b').diagnostics['name'],
generic_set.GenericSet(['diag2']))
def testLogDebugInfo_Succeeds(self, mock_log):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnostic(
reserved_infos.LOG_URLS.name,
generic_set.GenericSet(['http://foo']))
add_histograms._LogDebugInfo(histograms)
mock_log.assert_called_once_with('Buildbot URL: %s', "['http://foo']")
def testMerge(self):
t0 = histogram.TagMap({
'tagsToStoryNames': {
'press': ['story0', 'story1'],
'desktop': ['story0', 'story1', 'story2']
}
})
t1 = histogram.TagMap({
'tagsToStoryNames': {
'press': ['story3', 'story4'],
'android': ['story3', 'story4', 'story5']
}
})
self.assertFalse(t0.CanAddDiagnostic(generic_set.GenericSet([])))
self.assertTrue(t0.CanAddDiagnostic(t1))
m0 = diagnostic.Diagnostic.FromDict(t0.AsDict())
self.assertTrue(isinstance(m0, histogram.TagMap))
self.assertFalse(m0.CanAddDiagnostic(generic_set.GenericSet([])))
self.assertTrue(m0.CanAddDiagnostic(t1))
m0.AddDiagnostic(t1)
m1 = diagnostic.Diagnostic.FromDict(t1.AsDict())
m1.AddDiagnostic(t0)
self.assertDictEqual(m0.AsDict(), m1.AsDict())
m2 = diagnostic.Diagnostic.FromDict(t1.AsDict())
self.assertNotEqual(m2.AsDict(), m0.AsDict())
# Test round-tripping of merged diagnostic
clone = diagnostic.Diagnostic.FromDict(m0.AsDict())
self.assertSetEqual(set(clone.tags_to_story_names.keys()),
set(['press', 'desktop', 'android']))
self.assertSetEqual(clone.tags_to_story_names.get('press'),
set(['story0', 'story1', 'story3', 'story4']))
self.assertSetEqual(clone.tags_to_story_names.get('desktop'),
set(['story0', 'story1', 'story2']))
self.assertSetEqual(clone.tags_to_story_names.get('android'),
set(['story3', 'story4', 'story5']))
def testComputeRevision_RaisesOnError(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
chromium_commit = generic_set.GenericSet([424242, 0])
histograms.AddSharedDiagnostic(
reserved_infos.CHROMIUM_COMMIT_POSITIONS.name, chromium_commit)
with self.assertRaises(api_request_handler.BadRequestError):
add_histograms.ComputeRevision(histograms)
def testGetGroupingLabelFromHistogram_NoValidTags_ReturnsEmpty(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.STORY_TAGS.name,
generic_set.GenericSet(['foo', 'bar']))
self.assertEqual('',
histogram_helpers.GetGroupingLabelFromHistogram(hist))
def testHtmlOutput(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHtmlTraceArtifact(),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']]),
],
tags=['tbmv2:sampleMetric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--output-format',
'html',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
])
with open(os.path.join(self.output_dir,
html_output.OUTPUT_FILENAME)) as f:
results = render_histograms_viewer.ReadExistingResults(f.read())
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME)
self.assertEqual(hist.unit, SAMPLE_HISTOGRAM_UNIT)
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['osNames'],
generic_set.GenericSet(['linux']))
self.assertEqual(hist.diagnostics['documentationUrls'],
generic_set.GenericSet([['documentation', 'url']]))
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(1234567890987))
def testReadHistogramsTraceUrls(self):
hist = histogram_module.Histogram('hist', 'count')
hist.AddSample(0)
hist.diagnostics[reserved_infos.TRACE_URLS.name] = (
generic_set.GenericSet(['trace_url1', 'trace_url2']))
hist2 = histogram_module.Histogram('hist2', 'count')
hist2.diagnostics[reserved_infos.TRACE_URLS.name] = (
generic_set.GenericSet(['trace_url3']))
hist3 = histogram_module.Histogram('hist3', 'count')
hist3.diagnostics[reserved_infos.TRACE_URLS.name] = (
generic_set.GenericSet(['trace_url2']))
histograms = histogram_set.HistogramSet([hist, hist2, hist3])
self.SetOutputFileContents(histograms.AsDicts())
quest = read_value.ReadHistogramsJsonValue('chartjson-output.json',
hist_name=hist.name)
execution = quest.Start(None, 'server', 'output hash')
execution.Poll()
self.assertReadValueSuccess(execution)
self.assertEqual(execution.result_values, (0, ))
self.assertEqual(
{
'completed':
True,
'exception':
None,
'details': [
{
'key': 'trace',
'value': 'trace_url1',
'url': 'trace_url1',
},
{
'key': 'trace',
'value': 'trace_url2',
'url': 'trace_url2',
},
{
'key': 'trace',
'value': 'trace_url3',
'url': 'trace_url3',
},
],
}, execution.AsDict())
self.assertRetrievedOutputJson()
def AddReservedDiagnostics(histogram_dicts, names_to_values):
# We need to generate summary statistics for anything that had a story, so
# filter out every histogram with no stories, then merge. If you keep the
# histograms with no story, you end up with duplicates.
hs_with_stories = _LoadHistogramSet(histogram_dicts)
hs_with_stories.FilterHistograms(
lambda h: not h.diagnostics.get(reserved_infos.STORIES.name, []))
hs_with_no_stories = _LoadHistogramSet(histogram_dicts)
hs_with_no_stories.FilterHistograms(
lambda h: h.diagnostics.get(reserved_infos.STORIES.name, []))
# TODO(#3987): Refactor recipes to call merge_histograms separately.
with TempFile() as temp:
temp.write(json.dumps(hs_with_stories.AsDicts()))
temp.close()
# This call combines all repetitions of a metric for a given story into a
# single histogram.
dicts_across_repeats = merge_histograms.MergeHistograms(temp.name, (
'name', 'stories'))
# This call creates summary metrics across each set of stories.
dicts_across_stories = merge_histograms.MergeHistograms(temp.name, (
'name',))
# Now load everything into one histogram set. First we load the summary
# histograms, since we need to mark them with IS_SUMMARY.
# After that we load the rest, and then apply all the diagnostics specified
# on the command line. Finally, since we end up with a lot of diagnostics
# that no histograms refer to, we make sure to prune those.
histograms = histogram_set.HistogramSet()
histograms.ImportDicts(dicts_across_stories)
for h in histograms:
h.diagnostics[
reserved_infos.IS_SUMMARY.name] = generic_set.GenericSet([True])
histograms.ImportDicts(dicts_across_repeats)
histograms.ImportDicts(hs_with_no_stories.AsDicts())
histograms.DeduplicateDiagnostics()
for name, value in names_to_values.iteritems():
assert name in ALL_NAMES
histograms.AddSharedDiagnostic(name, generic_set.GenericSet([value]))
histograms.RemoveOrphanedDiagnostics()
return json.dumps(histograms.AsDicts())
def testReadHistogramsDiagnosticRefSkipTraceUrls(self):
hist = histogram_module.Histogram('hist', 'count')
hist.AddSample(0)
hist.diagnostics[reserved_infos.TRACE_URLS.name] = (
generic_set.GenericSet(['trace_url1', 'trace_url2']))
hist2 = histogram_module.Histogram('hist2', 'count')
hist2.diagnostics[reserved_infos.TRACE_URLS.name] = (
generic_set.GenericSet(['trace_url3']))
hist2.diagnostics[reserved_infos.TRACE_URLS.name].guid = 'foo'
histograms = histogram_set.HistogramSet([hist, hist2])
self.SetOutputFileContents(histograms.AsDicts())
quest = read_value.ReadHistogramsJsonValue(hist.name, None, None)
execution = quest.Start(None, 'output hash')
execution.Poll()
self.assertTrue(execution.completed)
self.assertFalse(execution.failed)
self.assertEqual(execution.result_values, (0, ))
self.assertEqual(
{
'result_values': (0, ),
'completed': True,
'exception': None,
'result_arguments': {},
'details': {
'traces': [
{
'url': 'trace_url1',
'name': 'trace_url1'
},
{
'url': 'trace_url2',
'name': 'trace_url2'
},
]
}
}, execution.AsDict())
self.assertEqual(execution.result_arguments, {})
expected_calls = [
mock.call('output hash'),
mock.call('output json hash')
]
self.assertEqual(self._retrieve.mock_calls, expected_calls)
def _CreateHistogram(revision, device, owner):
hists = [histogram_module.Histogram('hist', 'count')]
histograms = histogram_set.HistogramSet(hists)
histograms.AddSharedDiagnostic(
reserved_infos.MASTERS.name,
generic_set.GenericSet(['master']))
histograms.AddSharedDiagnostic(
reserved_infos.BOTS.name,
generic_set.GenericSet(['bot']))
histograms.AddSharedDiagnostic(
reserved_infos.CHROMIUM_COMMIT_POSITIONS.name,
generic_set.GenericSet([revision]))
histograms.AddSharedDiagnostic(
reserved_infos.BENCHMARKS.name,
generic_set.GenericSet(['benchmark']))
histograms.AddSharedDiagnostic(
reserved_infos.OWNERS.name,
generic_set.GenericSet([owner]))
histograms.AddSharedDiagnostic(
reserved_infos.DEVICE_IDS.name,
generic_set.GenericSet([device]))
histograms.AddSharedDiagnostic(
reserved_infos.STORIES.name,
generic_set.GenericSet(['story1', 'story2']))
return histograms
def _CreateHistogram(name, unit, story=None, summary_options=None):
h = histogram_module.Histogram(name, unit)
if summary_options:
h.CustomizeSummaryOptions(summary_options)
if story:
h.diagnostics[reserved_infos.STORIES.name] = (
generic_set.GenericSet([story]))
return h
def testGetSharedDiagnosticsOfType(self):
d0 = generic_set.GenericSet(['foo'])
d1 = histogram.DateRange(0)
hs = histogram_set.HistogramSet()
hs.AddSharedDiagnostic('generic', d0)
hs.AddSharedDiagnostic('generic', d1)
diagnostics = hs.GetSharedDiagnosticsOfType(generic_set.GenericSet)
self.assertEqual(len(diagnostics), 1)
self.assertIsInstance(diagnostics[0], generic_set.GenericSet)
def testReadHistogramsJsonValueStatisticNoSamples(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnostic(
reserved_infos.STORY_TAGS.name,
generic_set.GenericSet(['group:tir_label']))
histograms.AddSharedDiagnostic(
reserved_infos.STORIES.name,
generic_set.GenericSet(['story']))
self.SetOutputFileContents(histograms.AsDicts())
quest = read_value.ReadHistogramsJsonValue(
'chartjson-output.json', hist.name,
'tir_label', 'story', statistic='avg')
execution = quest.Start(None, 'server', 'output hash')
execution.Poll()
self.assertReadValueError(execution)
def AddDurationHistogram(self, duration_in_milliseconds):
hist = histogram.Histogram('benchmark_total_duration',
'ms_smallerIsBetter')
hist.AddSample(duration_in_milliseconds)
# TODO(eakuefner): Add his diagnostic in AddHistogram().
# https://github.com/catapult-project/catapult/issues/4244
hist.diagnostics[reserved_infos.LABELS.name] = generic_set.GenericSet(
[self.telemetry_info.label])
self._histograms.AddHistogram(hist)
def testComputeTestPathWithStory(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.STORIES.name,
generic_set.GenericSet(['http://story']))
hist = histograms.GetFirstHistogram()
test_path = histogram_helpers.ComputeTestPath(hist)
self.assertEqual('hist/http___story', test_path)
def testComputeTestPathWithIsRefWithoutStory(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.IS_REFERENCE_BUILD.name,
generic_set.GenericSet([True]))
hist = histograms.GetFirstHistogram()
test_path = histogram_helpers.ComputeTestPath(hist)
self.assertEqual('hist/ref', test_path)
def testGetTIRLabelFromHistogram_ValidTags_SortsByKey(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.STORY_TAGS.name,
generic_set.GenericSet(
['z:last', 'ignore', 'a:first', 'me', 'm:middle']))
self.assertEqual(
'first_middle_last', histogram_helpers.GetTIRLabelFromHistogram(hist))
def testSerialize(self):
hist = histogram.Histogram('aaa', 'count_biggerIsBetter')
hist.description = 'lorem ipsum'
hist.diagnostics['bbb'] = related_name_map.RelatedNameMap({
'ccc': 'a:c',
'ddd': 'a:d',
})
hist.diagnostics['hhh'] = generic_set.GenericSet(['ggg'])
hist.AddSample(0, {
'bbb': breakdown.Breakdown.FromEntries({
'ccc': 11,
'ddd': 31,
}),
'eee': related_event_set.RelatedEventSet([{
'stableId': 'fff',
'title': 'ggg',
'start': 3,
'duration': 4,
}]),
})
data = histogram_serializer.Serialize([hist])
self.assertEqual(data, [
[
'aaa',
[1, [1, 1000.0, 20]],
'',
'ccc',
'ddd',
[3, 4],
'ggg',
'avg', 'count', 'max', 'min', 'std', 'sum',
'lorem ipsum',
'a:c',
'a:d',
],
{
'Breakdown': {'bbb': {5: [2, 5, 11, 31]}},
'GenericSet': {
'hhh': {0: 6},
'statisticsNames': {1: [7, 8, 9, 10, 11, 12]},
'description': {3: 13},
},
'RelatedEventSet': {'eee': {4: [['fff', 6, 3, 4]]}},
'RelatedNameMap': {'bbb': {2: [5, 14, 15]}},
},
[
0,
'count+',
1,
[0, 1, 2, 3],
[1, 0, None, 0, 0, 0, 0],
{0: [1, [None, 4, 5]]},
0,
]
])
def testReadHistogramsJsonValueStatisticNoSamples(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.STORY_TAGS.name,
generic_set.GenericSet(['group:label']))
histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.STORIES.name, generic_set.GenericSet(['story']))
self.SetOutputFileContents(histograms.AsDicts())
quest = read_value.ReadValue(results_filename='chartjson-output.json',
results_path=['chartjson-output.json'],
metric=hist.name,
grouping_label='label',
trace_or_story='story',
statistic='avg')
execution = quest.Start(None, 'server', 'output hash')
execution.Poll()
self.assertReadValueError(execution, 'ReadValueNoValues')
def testEquals(self):
a_names = histogram.RelatedNameMap()
a_names.Set('a', 'A')
self.assertNotEqual(a_names, generic_set.GenericSet([]))
b_names = histogram.RelatedNameMap()
self.assertNotEqual(a_names, b_names)
b_names.Set('a', 'B')
self.assertNotEqual(a_names, b_names)
b_names.Set('a', 'A')
self.assertEqual(a_names, b_names)
def GetDocumentationLink(self):
"""Returns a Generic Diagnostic containing the benchmark's documentation
link in a string.
Returns:
Diagnostic with the link (string) to the benchmark documentation.
"""
pair = ['Benchmark documentation link',
decorators.GetDocumentationLink(self)]
return generic_set.GenericSet([pair])
def Serialize(self, serializer):
nan_bin = self.num_nans
if self.nan_diagnostic_maps:
nan_bin = [nan_bin] + [[None] + dm.Serialize(serializer)
for dm in self.nan_diagnostic_maps]
self.diagnostics[reserved_infos.STATISTICS_NAMES.name] = \
generic_set.GenericSet(sorted(self.statistics_names))
self.diagnostics[reserved_infos.DESCRIPTION.name] = \
generic_set.GenericSet([self._description])
return [
serializer.GetOrAllocateId(self.name),
self.unit.replace('_biggerIsBetter',
'+').replace('_smallerIsBetter', '-'),
serializer.GetOrAllocateId(self._bin_boundaries_dict),
self.diagnostics.Serialize(serializer),
self._running.AsDict() if self._running else 0,
self._SerializeBins(serializer),
nan_bin,
]
def testMerge(self):
a_set = generic_set.GenericSet([
None,
True,
False,
0,
1,
42,
[],
{},
[0, False],
{
'a': 1,
'b': True
},
])
b_set = generic_set.GenericSet([
{
'b': True,
'a': 1
},
[0, False],
{},
[],
42,
1,
0,
False,
True,
None,
])
self.assertTrue(a_set.CanAddDiagnostic(b_set))
self.assertTrue(b_set.CanAddDiagnostic(a_set))
a_set.AddDiagnostic(b_set)
self.assertEqual(a_set, b_set)
b_set.AddDiagnostic(a_set)
self.assertEqual(a_set, b_set)
c_dict = {'a': 1, 'b': 1}
c_set = generic_set.GenericSet([c_dict])
a_set.AddDiagnostic(c_set)
self.assertEqual(len(a_set), 1 + len(b_set))
self.assertIn(c_dict, a_set)
def testComputeTestPathWithStory(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnostic(
reserved_infos.MASTERS.name,
generic_set.GenericSet(['master']))
histograms.AddSharedDiagnostic(
reserved_infos.BOTS.name,
generic_set.GenericSet(['bot']))
histograms.AddSharedDiagnostic(
reserved_infos.BENCHMARKS.name,
generic_set.GenericSet(['benchmark']))
histograms.AddSharedDiagnostic(
reserved_infos.STORIES.name,
generic_set.GenericSet(['http://story']))
hist = histograms.GetFirstHistogram()
test_path = add_histograms.ComputeTestPath(
'master/bot/benchmark', hist.guid, histograms)
self.assertEqual('master/bot/benchmark/hist/http___story', test_path)
def testComputeTestPathWithIsRefWithoutStory(self):
hist = histogram_module.Histogram('hist', 'count')
histograms = histogram_set.HistogramSet([hist])
histograms.AddSharedDiagnostic(
reserved_infos.MASTERS.name,
generic_set.GenericSet(['master']))
histograms.AddSharedDiagnostic(
reserved_infos.BOTS.name,
generic_set.GenericSet(['bot']))
histograms.AddSharedDiagnostic(
reserved_infos.BENCHMARKS.name,
generic_set.GenericSet(['benchmark']))
histograms.AddSharedDiagnostic(
reserved_infos.IS_REFERENCE_BUILD.name,
generic_set.GenericSet([True]))
hist = histograms.GetFirstHistogram()
test_path = add_histograms.ComputeTestPath(
'master/bot/benchmark', hist.guid, histograms)
self.assertEqual('master/bot/benchmark/hist/ref', test_path)
