def testHistogramsOutputMeasurements(self):
measurements = {
'a': {'unit': 'ms', 'samples': [4, 6], 'description': 'desc_a'},
'b': {'unit': 'ms', 'samples': [5], 'description': 'desc_b'},
}
start_ts = 1500000000
start_iso = datetime.datetime.utcfromtimestamp(start_ts).isoformat() + 'Z'
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateMeasurementsArtifact(measurements),
],
tags=['story_tag:test'],
start_time=start_iso,
),
)
processor.main([
'--output-format', 'histograms',
'--output-dir', self.output_dir,
'--intermediate-dir', self.intermediate_dir,
])
with open(os.path.join(
self.output_dir, histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
self.assertEqual(len(out_histograms), 2)
hist = out_histograms.GetHistogramNamed('a')
self.assertEqual(hist.name, 'a')
self.assertEqual(hist.unit, 'ms_smallerIsBetter')
self.assertEqual(hist.sample_values, [4, 6])
self.assertEqual(hist.description, 'desc_a')
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['stories'],
generic_set.GenericSet(['story']))
self.assertEqual(hist.diagnostics['storyTags'],
generic_set.GenericSet(['test']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(start_ts * 1e3))
hist = out_histograms.GetHistogramNamed('b')
self.assertEqual(hist.name, 'b')
self.assertEqual(hist.unit, 'ms_smallerIsBetter')
self.assertEqual(hist.sample_values, [5])
self.assertEqual(hist.description, 'desc_b')
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['stories'],
generic_set.GenericSet(['story']))
self.assertEqual(hist.diagnostics['storyTags'],
generic_set.GenericSet(['test']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(start_ts * 1e3))
def testRoundtrip(self):
dr = date_range.DateRange(1496693745000)
dr.AddDiagnostic(date_range.DateRange(1496693746000))
self.assertEqual(calendar.timegm(dr.min_date.timetuple()), 1496693745)
self.assertEqual(calendar.timegm(dr.max_date.timetuple()), 1496693746)
clone = diagnostic.Diagnostic.FromDict(dr.AsDict())
self.assertEqual(clone.min_date, dr.min_date)
self.assertEqual(clone.max_date, dr.max_date)
def testDisallowReservedNames(self):
diagnostics = histogram.DiagnosticMap()
with self.assertRaises(TypeError):
diagnostics[None] = generic_set.GenericSet(())
with self.assertRaises(TypeError):
diagnostics['generic'] = None
diagnostics[reserved_infos.TRACE_URLS.name] = date_range.DateRange(0)
diagnostics.DisallowReservedNames()
diagnostics[reserved_infos.TRACE_URLS.name] = generic_set.GenericSet(())
with self.assertRaises(TypeError):
diagnostics[reserved_infos.TRACE_URLS.name] = date_range.DateRange(0)
def AddDiagnosticsToHistograms(test_result, test_suite_start, results_label):
"""Add diagnostics to all histograms of a test run.
Reads diagnostics from the test artifact and adds them to all histograms.
This overwrites the corresponding diagnostics previously set by e.g.
run_metrics.
"""
artifacts = test_result.get('outputArtifacts', {})
if DIAGNOSTICS_NAME in artifacts:
with open(artifacts[DIAGNOSTICS_NAME]['filePath']) as f:
diagnostics = json.load(f)['diagnostics']
for name, diag in diagnostics.items():
# For now, we only support GenericSet diagnostics that are serialized
# as lists of values.
assert isinstance(diag, list)
test_result['_histograms'].AddSharedDiagnosticToAllHistograms(
name, generic_set.GenericSet(diag))
timestamp_ms = util.IsoTimestampToEpoch(test_suite_start) * 1e3
test_result['_histograms'].AddSharedDiagnosticToAllHistograms(
reserved_infos.BENCHMARK_START.name,
date_range.DateRange(timestamp_ms))
if results_label is not None:
test_result['_histograms'].AddSharedDiagnosticToAllHistograms(
reserved_infos.LABELS.name,
generic_set.GenericSet([results_label]))
def DiagnosticValue(value):
if isinstance(value, (int, long)):
return date_range.DateRange(value)
elif isinstance(value, list):
return generic_set.GenericSet(value)
else:
raise NotImplementedError(type(value))
def testHistogramsOutput(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHtmlTraceArtifact(),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']])
],
tags=['tbmv2:sampleMetric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
with mock.patch('py_utils.cloud_storage.Upload') as cloud_patch:
cloud_patch.return_value = processor.cloud_storage.CloudFilepath(
bucket='bucket', remote_path='trace.html')
processor.main([
'--is-unittest',
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
'--upload-results',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME)
self.assertEqual(hist.unit, SAMPLE_HISTOGRAM_UNIT)
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['osNames'],
generic_set.GenericSet(['linux']))
self.assertEqual(hist.diagnostics['documentationUrls'],
generic_set.GenericSet([['documentation', 'url']]))
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(1234567890987))
self.assertEqual(
hist.diagnostics['traceUrls'],
generic_set.GenericSet([
'https://console.developers.google.com'
'/m/cloudstorage/b/bucket/o/trace.html'
]))
def testGetSharedDiagnosticsOfType(self):
d0 = generic_set.GenericSet(['foo'])
d1 = date_range.DateRange(0)
hs = histogram_set.HistogramSet()
hs.AddSharedDiagnosticToAllHistograms('generic', d0)
hs.AddSharedDiagnosticToAllHistograms('generic', d1)
diagnostics = hs.GetSharedDiagnosticsOfType(generic_set.GenericSet)
self.assertEqual(len(diagnostics), 1)
self.assertIsInstance(diagnostics[0], generic_set.GenericSet)
def _GlobalDiagnostics(benchmark_run):
"""Extract diagnostics information about the whole benchmark run.
These diagnostics will be added to ad-hoc measurements recorded by
benchmarks.
"""
timestamp_ms = util.IsoTimestampToEpoch(benchmark_run['startTime']) * 1e3
return {
reserved_infos.BENCHMARK_START.name: date_range.DateRange(timestamp_ms),
}
def testAddDiagnosticsToHistograms(self):
start_ts = 1500000000
start_iso = datetime.datetime.utcfromtimestamp(
start_ts).isoformat() + 'Z'
test_result = testing.TestResult(
'benchmark/story',
output_artifacts={
'trace.html': testing.Artifact('/trace.html',
'gs://trace.html'),
},
start_time=start_iso,
tags=['story_tag:test'],
result_id='3',
)
test_result['_histograms'] = histogram_set.HistogramSet()
test_result['_histograms'].CreateHistogram('a', 'unitless', [0])
processor.AddDiagnosticsToHistograms(test_result,
test_suite_start=start_iso,
results_label='label',
test_path_format='telemetry')
hist = test_result['_histograms'].GetFirstHistogram()
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(start_ts * 1e3))
self.assertEqual(hist.diagnostics['traceStart'],
date_range.DateRange(start_ts * 1e3))
self.assertEqual(hist.diagnostics['stories'],
generic_set.GenericSet(['story']))
self.assertEqual(hist.diagnostics['storyTags'],
generic_set.GenericSet(['test']))
self.assertEqual(hist.diagnostics['storysetRepeats'],
generic_set.GenericSet([3]))
self.assertEqual(hist.diagnostics['traceUrls'],
generic_set.GenericSet(['gs://trace.html']))
def testHistogramsOutput_TBMv3(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateProtoTraceArtifact(),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']])
],
tags=['tbmv3:dummy_metric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--is-unittest',
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
'--experimental-tbmv3-metrics',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
# For testing the TBMv3 workflow we use dummy_metric defined in
# tools/perf/core/tbmv3/metrics/dummy_metric_*.
hist = out_histograms.GetHistogramNamed('dummy::simple_field')
self.assertEqual(hist.unit, 'count_smallerIsBetter')
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['osNames'],
generic_set.GenericSet(['linux']))
self.assertEqual(hist.diagnostics['documentationUrls'],
generic_set.GenericSet([['documentation', 'url']]))
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(1234567890987))
def testHistogramsOutput(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHistogramsArtifact(
histogram.Histogram('a', 'unitless')),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']])
],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
self.assertEqual(len(out_histograms), 1)
hist = out_histograms.GetFirstHistogram()
self.assertEqual(hist.name, 'a')
self.assertEqual(hist.unit, 'unitless')
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['osNames'],
generic_set.GenericSet(['linux']))
self.assertEqual(hist.diagnostics['documentationUrls'],
generic_set.GenericSet([['documentation', 'url']]))
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(1234567890987))
def testHtmlOutput(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateHtmlTraceArtifact(),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']]),
],
tags=['tbmv2:sampleMetric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--is-unittest',
'--output-format',
'html',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
])
with open(os.path.join(self.output_dir,
html_output.OUTPUT_FILENAME)) as f:
results = render_histograms_viewer.ReadExistingResults(f.read())
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
hist = out_histograms.GetHistogramNamed(SAMPLE_HISTOGRAM_NAME)
self.assertEqual(hist.unit, SAMPLE_HISTOGRAM_UNIT)
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['osNames'],
generic_set.GenericSet(['linux']))
self.assertEqual(hist.diagnostics['documentationUrls'],
generic_set.GenericSet([['documentation', 'url']]))
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(1234567890987))
def testHistogramsOutput_TBMv3(self):
self.SerializeIntermediateResults(
testing.TestResult(
'benchmark/story',
output_artifacts=[
self.CreateProtoTraceArtifact(),
self.CreateDiagnosticsArtifact(
benchmarks=['benchmark'],
osNames=['linux'],
documentationUrls=[['documentation', 'url']])
],
tags=['tbmv3:dummy_metric'],
start_time='2009-02-13T23:31:30.987000Z',
), )
processor.main([
'--output-format',
'histograms',
'--output-dir',
self.output_dir,
'--intermediate-dir',
self.intermediate_dir,
'--results-label',
'label',
])
with open(
os.path.join(self.output_dir,
histograms_output.OUTPUT_FILENAME)) as f:
results = json.load(f)
out_histograms = histogram_set.HistogramSet()
out_histograms.ImportDicts(results)
hist = out_histograms.GetHistogramNamed(DUMMY_HISTOGRAM_NAME)
self.assertEqual(hist.unit, DUMMY_HISTOGRAM_UNIT)
self.assertEqual(hist.diagnostics['benchmarks'],
generic_set.GenericSet(['benchmark']))
self.assertEqual(hist.diagnostics['osNames'],
generic_set.GenericSet(['linux']))
self.assertEqual(hist.diagnostics['documentationUrls'],
generic_set.GenericSet([['documentation', 'url']]))
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(1234567890987))
def testAddDiagnosticsToHistograms(self):
test_result = testing.TestResult('benchmark/story')
test_result['_histograms'] = histogram_set.HistogramSet()
test_result['_histograms'].CreateHistogram('a', 'unitless', [0])
start_ts = 1500000000
start_iso = datetime.datetime.utcfromtimestamp(
start_ts).isoformat() + 'Z'
processor.AddDiagnosticsToHistograms(test_result,
test_suite_start=start_iso,
results_label='label')
hist = test_result['_histograms'].GetFirstHistogram()
self.assertEqual(hist.diagnostics['labels'],
generic_set.GenericSet(['label']))
self.assertEqual(hist.diagnostics['benchmarkStart'],
date_range.DateRange(start_ts * 1e3))
def Profile(self, root):
self._histograms = histogram_set.HistogramSet()
total_hist = self._GetOrCreateHistogram('heap')
total_hist.diagnostics['types'] = related_name_map.RelatedNameMap()
total_breakdown = breakdown.Breakdown()
total_size = self._Recurse(root, total_hist.diagnostics['types'],
total_breakdown)
builtins_size = total_size - sum(subsize
for _, subsize in total_breakdown)
if builtins_size:
total_breakdown.Set('(builtin types)', builtins_size)
total_hist.AddSample(total_size, dict(types=total_breakdown))
self._histograms.AddSharedDiagnosticToAllHistograms(
reserved_infos.TRACE_START.name,
date_range.DateRange(time.time() * 1000))
return self._histograms
def testSerialize(self):
dr = date_range.DateRange(123)
self.assertEqual(dr.Serialize(None), 123)
dr.AddDiagnostic(date_range.DateRange(100))
self.assertEqual(dr.Serialize(None), [100, 123])
def testDateRange(self):
grouping = histogram_grouping.DateRangeGrouping('foo')
hist = histogram.Histogram('', 'count')
self.assertEqual(grouping.callback(hist), '')
hist.diagnostics['foo'] = date_range.DateRange(15e11)
self.assertEqual(grouping.callback(hist), str(hist.diagnostics['foo']))
def testMaxTimestamp(self):
dr = date_range.DateRange(1496693745123)
dr.AddDiagnostic(date_range.DateRange(1496693746123))
self.assertEqual(dr.max_timestamp, 1496693746123)
def testDeduplicateDiagnostics(self):
generic_a = generic_set.GenericSet(['A'])
generic_b = generic_set.GenericSet(['B'])
date_a = date_range.DateRange(42)
date_b = date_range.DateRange(57)
a_hist = histogram.Histogram('a', 'unitless')
generic0 = generic_set.GenericSet.FromDict(generic_a.AsDict())
generic0.AddDiagnostic(generic_b)
a_hist.diagnostics['generic'] = generic0
date0 = date_range.DateRange.FromDict(date_a.AsDict())
date0.AddDiagnostic(date_b)
a_hist.diagnostics['date'] = date0
b_hist = histogram.Histogram('b', 'unitless')
generic1 = generic_set.GenericSet.FromDict(generic_a.AsDict())
generic1.AddDiagnostic(generic_b)
b_hist.diagnostics['generic'] = generic1
date1 = date_range.DateRange.FromDict(date_a.AsDict())
date1.AddDiagnostic(date_b)
b_hist.diagnostics['date'] = date1
c_hist = histogram.Histogram('c', 'unitless')
c_hist.diagnostics['generic'] = generic1
histograms = histogram_set.HistogramSet([a_hist, b_hist, c_hist])
self.assertNotEqual(a_hist.diagnostics['generic'].guid,
b_hist.diagnostics['generic'].guid)
self.assertEqual(b_hist.diagnostics['generic'].guid,
c_hist.diagnostics['generic'].guid)
self.assertEqual(a_hist.diagnostics['generic'],
b_hist.diagnostics['generic'])
self.assertNotEqual(a_hist.diagnostics['date'].guid,
b_hist.diagnostics['date'].guid)
self.assertEqual(a_hist.diagnostics['date'],
b_hist.diagnostics['date'])
histograms.DeduplicateDiagnostics()
self.assertEqual(a_hist.diagnostics['generic'].guid,
b_hist.diagnostics['generic'].guid)
self.assertEqual(b_hist.diagnostics['generic'].guid,
c_hist.diagnostics['generic'].guid)
self.assertEqual(a_hist.diagnostics['generic'],
b_hist.diagnostics['generic'])
self.assertEqual(a_hist.diagnostics['date'].guid,
b_hist.diagnostics['date'].guid)
self.assertEqual(a_hist.diagnostics['date'],
b_hist.diagnostics['date'])
histogram_dicts = histograms.AsDicts()
# All diagnostics should have been serialized as DiagnosticRefs.
for d in histogram_dicts:
if 'type' not in d:
for diagnostic_dict in d['diagnostics'].values():
self.assertIsInstance(diagnostic_dict, str)
histograms2 = histogram_set.HistogramSet()
histograms2.ImportDicts(histograms.AsDicts())
a_hists = histograms2.GetHistogramsNamed('a')
self.assertEqual(len(a_hists), 1)
a_hist2 = a_hists[0]
b_hists = histograms2.GetHistogramsNamed('b')
self.assertEqual(len(b_hists), 1)
b_hist2 = b_hists[0]
self.assertEqual(a_hist2.diagnostics['generic'].guid,
b_hist2.diagnostics['generic'].guid)
self.assertEqual(a_hist2.diagnostics['generic'],
b_hist2.diagnostics['generic'])
self.assertEqual(a_hist2.diagnostics['date'].guid,
b_hist2.diagnostics['date'].guid)
self.assertEqual(a_hist2.diagnostics['date'],
b_hist2.diagnostics['date'])