def testAddTracesForSameStory(self):
with self.CreateResults() as results:
with results.CreateStoryRun(self.stories[0]):
results.AddTraces(trace_data.CreateTestTrace(1))
results.AddTraces(trace_data.CreateTestTrace(2))
runs = list(results.IterRunsWithTraces())
self.assertEqual(1, len(runs))
def testAddTracesForSamePage(self):
with self.CreateResults() as results:
results.WillRunPage(self.pages[0])
results.AddTraces(trace_data.CreateTestTrace(1))
results.AddTraces(trace_data.CreateTestTrace(2))
results.DidRunPage(self.pages[0])
runs = list(results.IterRunsWithTraces())
self.assertEqual(1, len(runs))
def testAddTracesForSamePage(self):
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
results = page_test_results.PageTestResults(output_dir=tempdir)
results.WillRunPage(self.pages[0])
results.AddTraces(trace_data.CreateTestTrace(1))
results.AddTraces(trace_data.CreateTestTrace(2))
results.DidRunPage(self.pages[0])
runs = list(results.IterRunsWithTraces())
self.assertEquals(1, len(runs))
def testAddTracesForSameStory(self):
with self.CreateResults() as results:
with results.CreateStoryRun(self.stories[0]):
results.AddTraces(trace_data.CreateTestTrace(1))
results.AddTraces(trace_data.CreateTestTrace(2))
test_results = self.ReadTestResults()
self.assertEqual(len(test_results), 1)
for test_result in test_results:
trace_names = [
name for name in test_result['outputArtifacts']
if name.startswith('trace/')
]
self.assertTrue(len(trace_names), 2)
def testNoTracesLeftAfterCleanUp(self):
results = self.getPageTestResults()
try:
results.WillRunPage(self.pages[0])
results.AddTraces(trace_data.CreateTestTrace(1))
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.AddTraces(trace_data.CreateTestTrace(2))
results.DidRunPage(self.pages[1])
finally:
results.CleanUp()
self.assertFalse(results.FindAllTraceValues())
def testRepr(self):
with trace.TraceValue(page.Page('http://www.bar.com/',
name='load:story:bar'),
trace_data.CreateTestTrace(),
important=True,
description='desc') as v:
self.assertEquals("TraceValue('load:story:bar', 'trace')", str(v))
def testMultiplePagesAndValues(self):
self.SimulateBenchmarkRun([
(self._story_set[0], [
scalar.ScalarValue(
None, 'foo', 'seconds', 4,
improvement_direction=improvement_direction.DOWN)]),
(self._story_set[1], [
scalar.ScalarValue(
None, 'foo', 'seconds', 3.4,
improvement_direction=improvement_direction.DOWN),
trace_data.CreateTestTrace(),
scalar.ScalarValue(
None, 'bar', 'km', 10,
improvement_direction=improvement_direction.DOWN),
scalar.ScalarValue(
None, 'baz', 'count', 5,
improvement_direction=improvement_direction.DOWN)])])
# Parse CSV output into list of lists.
csv_string = self.Format()
lines = csv_string.split('\r\n')
values = [s.split(',') for s in lines[1:-1]]
values.sort()
self.assertEquals(len(values), 4)
self.assertEquals(len(set((v[1] for v in values))), 2) # 2 pages.
self.assertEquals(len(set((v[2] for v in values))), 4) # 4 value names.
sample_row = values[2]
trace_url = sample_row.pop()
self.assertEquals(sample_row, [
'foo', 'ms', '3400', '1', '3400', '3400', '0', '3400', '', 'benchmark',
'2017-07-14 02:40:00', '', '', '', 'benchmark 2017-07-14 02:40:00', '',
'', '', '', '', 'http://www.bar.com/', '', ''])
self.assertRegexpMatches(trace_url, r'^https://.*fake_bucket')
def testMultiplePagesAndValues(self, cloud_storage_insert_patch):
cloud_storage_insert_patch.return_value = 'fake_url'
self._results.WillRunPage(self._story_set[0])
self._results.AddMeasurement('foo', 'seconds', 4)
self._results.DidRunPage(self._story_set[0])
self._results.WillRunPage(self._story_set[1])
self._results.AddMeasurement('foo', 'seconds', 3.4)
self._results.AddTraces(trace_data.CreateTestTrace())
self._results.AddMeasurement('bar', 'km', 10)
self._results.AddMeasurement('baz', 'count', 5)
self._results.DidRunPage(self._story_set[1])
# Parse CSV output into list of lists.
values = list(csv.reader(self.Format().splitlines()))[1:]
values.sort()
self.assertEquals(len(values), 4)
self.assertEquals(len(set((v[1] for v in values))), 2) # 2 pages.
self.assertEquals(len(set((v[2] for v in values))),
4) # 4 value names.
sample_row = values[2]
self.assertEquals(sample_row, [
'foo', 'ms', '3400', '1', '3400', '3400', '0', '3400', '',
'benchmark', '2017-07-14 02:40:00', '', '', '',
'benchmark 2017-07-14 02:40:00', '', '', '', '', '',
'http://www.bar.com/', '', '', 'fake_url'
])
def testTraceValue(self):
results = self.getPageTestResults()
try:
results.WillRunPage(self.pages[0])
results.AddTraces(trace_data.CreateTestTrace(1))
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.AddTraces(trace_data.CreateTestTrace(2))
results.DidRunPage(self.pages[1])
results.PrintSummary()
values = results.FindAllTraceValues()
self.assertEquals(2, len(values))
finally:
results.CleanUp()
def testNoLeakedTempFiles(self):
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
with mock.patch('tempfile.tempdir', new=tempdir):
with trace.TraceValue(None, trace_data.CreateTestTrace()) as v:
v.SerializeTraceData()
self.assertTrue(os.path.exists(tempdir))
self.assertFalse(os.listdir(tempdir))
def testMultiplePagesAndValues(self, cs_insert_mock):
cs_insert_mock.return_value = 'https://cloud_storage_url/foo'
trace_value = trace.TraceValue(None,
trace_data.CreateTestTrace(),
remote_path='rp',
upload_bucket='foo',
cloud_url='http://google.com')
trace_value.SerializeTraceData()
trace_value.UploadToCloud()
self.SimulateBenchmarkRun([
(self._story_set[0], [
scalar.ScalarValue(
None,
'foo',
'seconds',
4,
improvement_direction=improvement_direction.DOWN)
]),
(self._story_set[1], [
scalar.ScalarValue(
None,
'foo',
'seconds',
3.4,
improvement_direction=improvement_direction.DOWN),
trace_value,
scalar.ScalarValue(
None,
'bar',
'km',
10,
improvement_direction=improvement_direction.DOWN),
scalar.ScalarValue(
None,
'baz',
'count',
5,
improvement_direction=improvement_direction.DOWN)
])
])
# Parse CSV output into list of lists.
csv_string = self.Format()
lines = csv_string.split('\r\n')
values = [s.split(',') for s in lines[1:-1]]
values.sort()
self.assertEquals(len(values), 4)
self.assertEquals(len(set((v[1] for v in values))), 2) # 2 pages.
self.assertEquals(len(set((v[2] for v in values))),
4) # 4 value names.
self.assertEquals(values[2], [
'foo', 'ms', '3400', '1', '3400', '3400', '0', '3400', '',
'benchmark', '2017-07-14 02:40:00', '', '', '',
'benchmark 2017-07-14 02:40:00', '', '', '', '', '',
'http://www.bar.com/', '', '', 'http://google.com'
])
def testCleanUpCleansUpTraceValues(self):
results = self.getPageTestResults()
try:
v0 = trace.TraceValue(None, trace_data.CreateTestTrace(1))
v1 = trace.TraceValue(None, trace_data.CreateTestTrace(2))
results.WillRunPage(self.pages[0])
results.AddValue(v0)
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.AddValue(v1)
results.DidRunPage(self.pages[1])
finally:
results.CleanUp()
self.assertTrue(v0.cleaned_up)
self.assertTrue(v1.cleaned_up)
def testAsChartDictWithTraceValues(self):
with _MakePageTestResults() as results:
results.WillRunPage(self._story_set[0])
results.AddTraces(trace_data.CreateTestTrace())
results.DidRunPage(self._story_set[0])
d = chart_json_output_formatter.ResultsAsChartDict(results)
self.assertIn('trace', d['charts'])
self.assertIn('http://www.foo.com/', d['charts']['trace'])
self.assertTrue(d['enabled'])
def testAsDictWhenTraceIsNotSerializedAndUploaded(self, insert_mock):
with trace.TraceValue(None,
trace_data.CreateTestTrace(),
upload_bucket=trace.cloud_storage.PUBLIC_BUCKET,
remote_path='a.html',
cloud_url='http://example.com/a.html') as v:
v.SerializeTraceData()
cloud_url = v.UploadToCloud()
d = v.AsDict()
self.assertEqual(d['cloud_url'], cloud_url)
insert_mock.assert_called_with(trace.cloud_storage.PUBLIC_BUCKET,
'a.html', v.filename)
def testMultiplePagesAndValues(self, cloud_storage_insert_patch):
cloud_storage_insert_patch.return_value = 'fake_url'
self.SimulateBenchmarkRun([
(self._story_set[0], [
scalar.ScalarValue(
None,
'foo',
'seconds',
4,
improvement_direction=improvement_direction.DOWN)
]),
(self._story_set[1], [
scalar.ScalarValue(
None,
'foo',
'seconds',
3.4,
improvement_direction=improvement_direction.DOWN),
trace_data.CreateTestTrace(),
scalar.ScalarValue(
None,
'bar',
'km',
10,
improvement_direction=improvement_direction.DOWN),
scalar.ScalarValue(
None,
'baz',
'count',
5,
improvement_direction=improvement_direction.DOWN)
])
])
# Parse CSV output into list of lists.
values = list(csv.reader(self.Format().splitlines()))[1:]
values.sort()
self.assertEquals(len(values), 4)
self.assertEquals(len(set((v[1] for v in values))), 2) # 2 pages.
self.assertEquals(len(set((v[2] for v in values))),
4) # 4 value names.
sample_row = values[2]
self.assertEquals(sample_row, [
'foo', 'ms', '3400', '1', '3400', '3400', '0', '3400', '',
'benchmark', '2017-07-14 02:40:00', '', '', '',
'benchmark 2017-07-14 02:40:00', '', '', '', '', '',
'http://www.bar.com/', '', '', 'fake_url'
])
def testAsChartDictWithTraceValues(self):
results = _MakePageTestResults()
try:
results.WillRunPage(self._story_set[0])
results.AddTraces(trace_data.CreateTestTrace())
results.DidRunPage(self._story_set[0])
d = chart_json_output_formatter.ResultsAsChartDict(results)
self.assertTrue('trace' in d['charts'])
self.assertTrue('http://www.foo.com/' in d['charts']['trace'],
msg=d['charts']['trace'])
self.assertTrue(d['enabled'])
finally:
results.CleanUp()
def testAsDictWhenTraceSerializedAndUploaded(self, insert_mock):
with tempfile_ext.TemporaryFileName('test.html') as file_path:
with trace.TraceValue(
None,
trace_data.CreateTestTrace(),
file_path=file_path,
upload_bucket=trace.cloud_storage.PUBLIC_BUCKET,
remote_path='a.html',
cloud_url='http://example.com/a.html') as v:
v.SerializeTraceData()
fh = v.Serialize()
cloud_url = v.UploadToCloud()
d = v.AsDict()
self.assertTrue(os.path.exists(file_path))
self.assertEqual(d['file_id'], fh.id)
self.assertEqual(d['cloud_url'], cloud_url)
insert_mock.assert_called_with(
trace.cloud_storage.PUBLIC_BUCKET, 'a.html', file_path)
def testAsChartDictWithTraceValuesThatHasTirLabel(self):
results = _MakePageTestResults()
try:
results.WillRunPage(self._story_set[0])
v = trace.TraceValue(self._story_set[0],
trace_data.CreateTestTrace())
v.SerializeTraceData()
v.tir_label = 'background'
results.AddValue(v)
results.DidRunPage(self._story_set[0])
d = chart_json_output_formatter.ResultsAsChartDict(
self._benchmark_metadata, results)
self.assertTrue('trace' in d['charts'])
self.assertTrue('http://www.foo.com/' in d['charts']['trace'],
msg=d['charts']['trace'])
self.assertTrue(d['enabled'])
finally:
results.CleanUp()
