def testAsChartDictValueSmokeTest(self):
v0 = list_of_scalar_values.ListOfScalarValues(
None,
'foo.bar',
'seconds', [3, 4],
improvement_direction=improvement_direction.DOWN)
with _MakePageTestResults() as results:
results.AddSummaryValue(v0)
d = chart_json_output_formatter.ResultsAsChartDict(results)
self.assertEquals(d['charts']['foo']['bar']['values'], [3, 4])
def testAsChartDictPageSpecificValuesSamePageWithoutGroupingLabel(self):
with _MakePageTestResults() as results:
results.WillRunPage(self._story_set[0])
results.AddMeasurement('foo', 'seconds', 3)
results.AddMeasurement('foo', 'seconds', 4)
results.DidRunPage(self._story_set[0])
d = chart_json_output_formatter.ResultsAsChartDict(results)
self.assertIn('foo', d['charts'])
self.assertIn('http://www.foo.com/', d['charts']['foo'])
self.assertTrue(d['enabled'])
def testAsChartDictSummaryValueWithoutTraceName(self):
v0 = list_of_scalar_values.ListOfScalarValues(
None, 'foo', 'seconds', [3, 4],
improvement_direction=improvement_direction.DOWN)
results = _MakePageTestResults()
results.AddSummaryValue(v0)
d = chart_json_output_formatter.ResultsAsChartDict(results)
self.assertIn('summary', d['charts']['foo'])
self.assertTrue(d['enabled'])
def testAsChartDictPageSpecificValuesSamePage(self):
v0 = scalar.ScalarValue(self._page_set[0], 'foo', 'seconds', 3)
v1 = scalar.ScalarValue(self._page_set[0], 'foo', 'seconds', 4)
page_specific_values = [v0, v1]
summary_values = []
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata, page_specific_values, summary_values)
self.assertTrue('foo' in d['charts'])
self.assertTrue('http://www.foo.com/' in d['charts']['foo'])
def testAsChartDictBaseKeys(self):
page_specific_values = []
summary_values = []
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata,
page_specific_values,
summary_values)
self.assertEquals(d['format_version'], '0.1')
self.assertEquals(d['benchmark_name'], 'benchmark_name')
self.assertEquals(d['benchmark_description'], 'benchmark_description')
def testAsChartDictValueSmokeTest(self):
v0 = list_of_scalar_values.ListOfScalarValues(None, 'foo.bar', 'seconds',
[3, 4])
page_specific_values = []
summary_values = [v0]
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata,
page_specific_values,
summary_values)
self.assertEquals(d['charts']['foo']['bar']['values'], [3, 4])
def testAsChartDictBaseKeys(self):
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata, _MakePageTestResults())
self.assertEquals(d['format_version'], '0.1')
self.assertEquals(d['next_version'], '0.2')
self.assertEquals(d['benchmark_metadata']['name'], 'benchmark_name')
self.assertEquals(d['benchmark_metadata']['description'],
'benchmark_description')
self.assertEquals(d['benchmark_metadata']['type'],
'telemetry_benchmark')
self.assertTrue(d['enabled'])
def testAsChartDictWithTraceValuesThatHasTirLabel(self):
v = trace.TraceValue(self._story_set[0], trace_data.TraceData({'test': 1}))
v.tir_label = 'background'
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata,
page_specific_values=[v],
summary_values=[v])
self.assertTrue('trace' in d['charts'])
self.assertTrue('http://www.foo.com/' in d['charts']['trace'],
msg=d['charts']['trace'])
def testAsChartDictSummaryValueWithoutTraceName(self):
v0 = list_of_scalar_values.ListOfScalarValues(None, 'foo', 'seconds',
[3, 4])
page_specific_values = []
summary_values = [v0]
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata,
page_specific_values,
summary_values)
self.assertTrue('summary' in d['charts']['foo'])
def testAsChartDictSerializable(self):
v0 = scalar.ScalarValue(
self._story_set[0],
'foo',
'seconds',
3,
improvement_direction=improvement_direction.DOWN)
page_specific_values = [v0]
summary_values = []
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata, page_specific_values, summary_values)
json.dumps(d)
def testAsChartDictWithTracesInArtifacts(self):
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
with _MakePageTestResults(output_dir=tempdir) as results:
results.WillRunPage(self._story_set[0])
with results.CreateArtifact(results_processor.HTML_TRACE_NAME):
pass
results.DidRunPage(self._story_set[0])
d = chart_json_output_formatter.ResultsAsChartDict(results)
self.assertIn('trace', d['charts'])
self.assertIn('http://www.foo.com/', d['charts']['trace'])
self.assertTrue(d['enabled'])
def testAsChartDictSerializable(self):
v0 = scalar.ScalarValue(
self._story_set[0],
'foo',
'seconds',
3,
improvement_direction=improvement_direction.DOWN)
results = _MakePageTestResults()
results.WillRunPage(self._story_set[0])
results.AddValue(v0)
results.DidRunPage(self._story_set[0])
d = chart_json_output_formatter.ResultsAsChartDict(results)
json.dumps(d)
def testAsChartDictSummaryValueWithTraceName(self):
v0 = list_of_scalar_values.ListOfScalarValues(
None,
'foo.bar',
'seconds', [3, 4],
improvement_direction=improvement_direction.DOWN)
results = page_test_results.PageTestResults()
results.AddSummaryValue(v0)
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata, results)
self.assertTrue('bar' in d['charts']['foo'])
self.assertTrue(d['enabled'])
def testAsChartDictSummaryValueWithTraceName(self):
v0 = list_of_scalar_values.ListOfScalarValues(
None, 'foo.bar', 'seconds', [3, 4],
improvement_direction=improvement_direction.DOWN)
page_specific_values = []
summary_values = [v0]
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata,
page_specific_values,
summary_values)
self.assertTrue('bar' in d['charts']['foo'])
self.assertTrue(d['enabled'])
def testAsChartDictWithTraceValues(self):
results = _MakePageTestResults()
try:
results.WillRunPage(self._story_set[0])
results.AddTraces(trace_data.CreateTestTrace())
results.DidRunPage(self._story_set[0])
d = chart_json_output_formatter.ResultsAsChartDict(results)
self.assertTrue('trace' in d['charts'])
self.assertTrue('http://www.foo.com/' in d['charts']['trace'],
msg=d['charts']['trace'])
self.assertTrue(d['enabled'])
finally:
results.CleanUp()
def testAsChartDictPageSpecificValuesSamePageWithoutInteractionRecord(self):
v0 = scalar.ScalarValue(self._story_set[0], 'foo', 'seconds', 3,
improvement_direction=improvement_direction.DOWN)
v1 = scalar.ScalarValue(self._story_set[0], 'foo', 'seconds', 4,
improvement_direction=improvement_direction.DOWN)
page_specific_values = [v0, v1]
summary_values = []
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata,
page_specific_values,
summary_values)
self.assertTrue('foo' in d['charts'])
self.assertTrue('http://www.foo.com/' in d['charts']['foo'])
def testAsChartDictPageSpecificValuesAndComputedSummaryWithTraceName(self):
with _MakePageTestResults() as results:
results.WillRunPage(self._story_set[0])
results.AddMeasurement('foo.bar', 'seconds', 3)
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
results.AddMeasurement('foo.bar', 'seconds', 4)
results.DidRunPage(self._story_set[1])
d = chart_json_output_formatter.ResultsAsChartDict(results)
self.assertIn('foo', d['charts'])
self.assertIn('http://www.foo.com/', d['charts']['foo'])
self.assertIn('http://www.bar.com/', d['charts']['foo'])
self.assertIn('bar', d['charts']['foo'])
self.assertTrue(d['enabled'])
def testAsChartDictBaseKeys(self):
page_specific_values = []
summary_values = []
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata, page_specific_values, summary_values)
self.assertEquals(d['format_version'], '0.1')
self.assertEquals(d['next_version'], '0.2')
self.assertEquals(d['benchmark_metadata']['name'], 'benchmark_name')
self.assertEquals(d['benchmark_metadata']['description'],
'benchmark_description')
self.assertEquals(d['benchmark_metadata']['type'],
'telemetry_benchmark')
self.assertTrue(d['enabled'])
def testAsChartDictPageSpecificValuesSamePageWithoutGroupingLabel(self):
v0 = scalar.ScalarValue(self._story_set[0], 'foo', 'seconds', 3,
improvement_direction=improvement_direction.DOWN)
v1 = scalar.ScalarValue(self._story_set[0], 'foo', 'seconds', 4,
improvement_direction=improvement_direction.DOWN)
results = _MakePageTestResults()
results.WillRunPage(self._story_set[0])
results.AddValue(v0)
results.AddValue(v1)
results.DidRunPage(self._story_set[0])
d = chart_json_output_formatter.ResultsAsChartDict(results)
self.assertIn('foo', d['charts'])
self.assertIn('http://www.foo.com/', d['charts']['foo'])
self.assertTrue(d['enabled'])
def Format(self, page_test_results):
if page_test_results.value_set:
html2_formatter = html2_output_formatter.Html2OutputFormatter(
self._output_stream, self._reset_results, self._upload_results)
html2_formatter.Format(page_test_results)
return
chart_json_dict = chart_json_output_formatter.ResultsAsChartDict(
self._metadata, page_test_results.all_page_specific_values,
page_test_results.all_summary_values)
self._TranslateChartJson(chart_json_dict)
self._PrintPerfResult('telemetry_page_measurement_results',
'num_failed', [len(page_test_results.failures)],
'count', 'unimportant')
self._combined_results = self._ReadExistingResults(self._output_stream)
self._combined_results.append(self._result)
html = self._GetHtmlTemplate()
html = html.replace('%json_results%',
json.dumps(self.GetCombinedResults()))
html = html.replace('%json_units%', self._GetUnitJson())
html = html.replace('%plugins%', self._GetPlugins())
self._SaveResults(html)
if self._upload_results:
file_path = os.path.abspath(self._output_stream.name)
file_name = 'html-results/results-%s' % datetime.datetime.now(
).strftime('%Y-%m-%d_%H-%M-%S')
try:
cloud_storage.Insert(cloud_storage.PUBLIC_BUCKET, file_name,
file_path)
print
print(
'View online at '
'http://storage.googleapis.com/chromium-telemetry/%s' %
file_name)
except cloud_storage.PermissionError as e:
logging.error(
'Cannot upload profiling files to cloud storage due to '
' permission error: %s' % e.message)
print
print 'View result at file://%s' % os.path.abspath(
self._output_stream.name)
def Format(self, page_test_results):
chart_json_dict = chart_json_output_formatter.ResultsAsChartDict(
self._metadata, page_test_results.all_page_specific_values,
page_test_results.all_summary_values)
self._TranslateChartJson(chart_json_dict)
self._PrintPerfResult('telemetry_page_measurement_results', 'num_failed',
[len(page_test_results.failures)], 'count',
'unimportant')
self._combined_results = self._ReadExistingResults(self._output_stream)
self._combined_results.append(self._result)
html = self._GetHtmlTemplate()
html = html.replace('%json_results%', json.dumps(self.GetCombinedResults()))
html = html.replace('%json_units%', self._GetUnitJson())
html = html.replace('%plugins%', self._GetPlugins())
self._SaveResults(html)
def testAsChartDictPageSpecificValuesAndComputedSummaryWithTraceName(self):
v0 = scalar.ScalarValue(self._story_set[0], 'foo.bar', 'seconds', 3,
improvement_direction=improvement_direction.DOWN)
v1 = scalar.ScalarValue(self._story_set[1], 'foo.bar', 'seconds', 4,
improvement_direction=improvement_direction.DOWN)
page_specific_values = [v0, v1]
summary_values = []
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata,
page_specific_values,
summary_values)
self.assertTrue('foo' in d['charts'])
self.assertTrue('http://www.foo.com/' in d['charts']['foo'])
self.assertTrue('http://www.bar.com/' in d['charts']['foo'])
self.assertTrue('bar' in d['charts']['foo'])
self.assertTrue(d['enabled'])
def testAsChartDictWithTraceValuesThatHasTirLabel(self):
v = trace.TraceValue(
self._story_set[0],
trace_data.CreateTraceDataFromRawData([{
'test': 1
}]))
v.tir_label = 'background'
results = page_test_results.PageTestResults()
results.WillRunPage(self._story_set[0])
results.AddValue(v)
results.DidRunPage(self._story_set[0])
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata, results)
self.assertTrue('trace' in d['charts'])
self.assertTrue('http://www.foo.com/' in d['charts']['trace'],
msg=d['charts']['trace'])
self.assertTrue(d['enabled'])
def testAsChartDictWithTraceValuesThatHasTirLabel(self):
results = _MakePageTestResults()
try:
results.WillRunPage(self._story_set[0])
v = trace.TraceValue(self._story_set[0],
trace_data.CreateTestTrace())
v.SerializeTraceData()
v.tir_label = 'background'
results.AddValue(v)
results.DidRunPage(self._story_set[0])
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata, results)
self.assertTrue('trace' in d['charts'])
self.assertTrue('http://www.foo.com/' in d['charts']['trace'],
msg=d['charts']['trace'])
self.assertTrue(d['enabled'])
finally:
results.CleanUp()
def _ConvertChartJson(self, page_test_results):
chart_json = chart_json_output_formatter.ResultsAsChartDict(
self._metadata, page_test_results.all_page_specific_values,
page_test_results.all_summary_values)
info = page_test_results.telemetry_info
chart_json['label'] = info.label
chart_json['benchmarkStartMs'] = info.benchmark_start_epoch * 1000.0
file_descriptor, chart_json_path = tempfile.mkstemp()
os.close(file_descriptor)
json.dump(chart_json, file(chart_json_path, 'w'))
vinn_result = convert_chart_json.ConvertChartJson(chart_json_path)
os.remove(chart_json_path)
if vinn_result.returncode != 0:
logging.error('Error converting chart json to Histograms:\n' +
vinn_result.stdout)
return []
return json.loads(vinn_result.stdout)
def PopulateHistogramSet(self):
if len(self._histograms):
return
chart_json = chart_json_output_formatter.ResultsAsChartDict(self)
info = self.telemetry_info
chart_json['label'] = info.label
chart_json['benchmarkStartMs'] = info.benchmark_start_us / 1000.0
file_descriptor, chart_json_path = tempfile.mkstemp()
os.close(file_descriptor)
json.dump(chart_json, file(chart_json_path, 'w'))
vinn_result = convert_chart_json.ConvertChartJson(chart_json_path)
os.remove(chart_json_path)
if vinn_result.returncode != 0:
logging.error('Error converting chart json to Histograms:\n' +
vinn_result.stdout)
return []
self._histograms.ImportDicts(json.loads(vinn_result.stdout))
self._histograms.ImportDicts(self._histogram_dicts_to_add)
def PopulateHistogramSet(self):
if len(self._histograms):
return
# We ensure that html traces are serialized and uploaded if necessary
results_processor.SerializeAndUploadHtmlTraces(self)
chart_json = chart_json_output_formatter.ResultsAsChartDict(self)
chart_json['label'] = self.label
chart_json['benchmarkStartMs'] = self.benchmark_start_us / 1000.0
file_descriptor, chart_json_path = tempfile.mkstemp()
os.close(file_descriptor)
json.dump(chart_json, file(chart_json_path, 'w'))
vinn_result = convert_chart_json.ConvertChartJson(chart_json_path)
os.remove(chart_json_path)
if vinn_result.returncode != 0:
logging.error('Error converting chart json to Histograms:\n' +
vinn_result.stdout)
return []
self._histograms.ImportDicts(json.loads(vinn_result.stdout))
def PopulateHistogramSet(self, benchmark_metadata):
if self.histograms:
return
chart_json = chart_json_output_formatter.ResultsAsChartDict(
benchmark_metadata, self.all_page_specific_values,
self.all_summary_values)
info = self.telemetry_info
chart_json['label'] = info.label
chart_json['benchmarkStartMs'] = info.benchmark_start_ms
file_descriptor, chart_json_path = tempfile.mkstemp()
os.close(file_descriptor)
json.dump(chart_json, file(chart_json_path, 'w'))
vinn_result = convert_chart_json.ConvertChartJson(chart_json_path)
os.remove(chart_json_path)
if vinn_result.returncode != 0:
logging.error('Error converting chart json to Histograms:\n' +
vinn_result.stdout)
return []
self.histograms.ImportDicts(json.loads(vinn_result.stdout))
def testAsChartDictNoDescription(self):
d = chart_json_output_formatter.ResultsAsChartDict(
benchmark.BenchmarkMetadata('benchmark_name', ''),
_MakePageTestResults())
self.assertEquals('', d['benchmark_metadata']['description'])
def testAsChartDictNoDescription(self):
with _MakePageTestResults(description=None) as results:
d = chart_json_output_formatter.ResultsAsChartDict(results)
self.assertEquals('', d['benchmark_metadata']['description'])
