def testAsDictWithTwoPages(self):
results = page_test_results.PageTestResults()
results.telemetry_info.benchmark_start_epoch = 1501773200
results.telemetry_info.benchmark_name = 'benchmark_name'
results.WillRunPage(self._story_set[0])
v0 = scalar.ScalarValue(results.current_page, 'foo', 'seconds', 3,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v0)
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
v1 = scalar.ScalarValue(results.current_page, 'bar', 'seconds', 4,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v1)
results.DidRunPage(self._story_set[1])
d = json_3_output_formatter.ResultsAsDict(results)
self.assertTrue(_HasBenchmark(d['tests'], 'benchmark_name'))
self.assertTrue(_HasStory(d['tests']['benchmark_name'], 'Foo'))
story_result = d['tests']['benchmark_name']['Foo']
self.assertEquals(story_result['actual'], 'PASS')
self.assertEquals(story_result['expected'], 'PASS')
self.assertTrue(_HasBenchmark(d['tests'], 'benchmark_name'))
self.assertTrue(_HasStory(d['tests']['benchmark_name'], 'Bar'))
story_result = d['tests']['benchmark_name']['Bar']
self.assertEquals(story_result['actual'], 'PASS')
self.assertEquals(story_result['expected'], 'PASS')
self.assertEquals(d['num_failures_by_type'], {'PASS': 2})
def testAsDictWithRepeatedTests(self):
with self._MakeResults() as results:
results.WillRunPage(self._story_set[0])
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
results.Skip('fake_skip')
results.DidRunPage(self._story_set[1])
results.WillRunPage(self._story_set[0])
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
results.Skip('fake_skip')
results.DidRunPage(self._story_set[1])
d = json_3_output_formatter.ResultsAsDict(results)
foo_story_result = d['tests']['benchmark_name']['Foo']
self.assertEquals(foo_story_result['actual'], 'PASS')
self.assertEquals(foo_story_result['expected'], 'PASS')
bar_story_result = d['tests']['benchmark_name']['Bar']
self.assertEquals(bar_story_result['actual'], 'SKIP')
self.assertEquals(bar_story_result['expected'], 'SKIP')
self.assertEquals(d['num_failures_by_type'], {'SKIP': 2, 'PASS': 2})
def testAsDictWithRepeatedTests(self):
results = page_test_results.PageTestResults()
results.telemetry_info.benchmark_start_epoch = 1501773200
results.telemetry_info.benchmark_name = 'benchmark_name'
results.WillRunPage(self._story_set[0])
v0 = scalar.ScalarValue(results.current_page, 'foo', 'seconds', 3,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v0)
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
results.Skip('fake_skip')
results.DidRunPage(self._story_set[1])
results.WillRunPage(self._story_set[0])
v0 = scalar.ScalarValue(results.current_page, 'foo', 'seconds', 3,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v0)
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
results.Skip('fake_skip')
results.DidRunPage(self._story_set[1])
d = json_3_output_formatter.ResultsAsDict(results)
foo_story_result = d['tests']['benchmark_name']['Foo']
self.assertEquals(foo_story_result['actual'], 'PASS')
self.assertEquals(foo_story_result['expected'], 'PASS')
bar_story_result = d['tests']['benchmark_name']['Bar']
self.assertEquals(bar_story_result['actual'], 'SKIP')
self.assertEquals(bar_story_result['expected'], 'SKIP')
self.assertEquals(d['num_failures_by_type'], {'SKIP': 2, 'PASS': 2})
def testAsDictWithSkippedAndFailedTests_AlsoShardIndex(self):
# Set up shard index. If already running on a shard or fake it
# if not running on a shard.
delete_env_var_after = False
expected_shard_index = 0
if 'GTEST_SHARD_INDEX' in os.environ:
expected_shard_index = int(os.environ['GTEST_SHARD_INDEX'])
else:
os.environ['GTEST_SHARD_INDEX'] = str(expected_shard_index)
delete_env_var_after = True
try:
results = page_test_results.PageTestResults()
results.telemetry_info.benchmark_start_epoch = 1501773200
results.telemetry_info.benchmark_name = 'benchmark_name'
results.WillRunPage(self._story_set[0])
v0 = scalar.ScalarValue(results.current_page, 'foo', 'seconds', 3,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v0)
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
v1 = scalar.ScalarValue(results.current_page, 'bar', 'seconds', 4,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v1)
results.DidRunPage(self._story_set[1])
results.WillRunPage(self._story_set[0])
results.Skip('fake_skip')
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[0])
results.Skip('unexpected_skip', False)
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
results.Fail('fake_failure')
results.DidRunPage(self._story_set[1])
d = json_3_output_formatter.ResultsAsDict(results)
foo_story_result = d['tests']['benchmark_name']['Foo']
self.assertEquals(foo_story_result['actual'], 'PASS SKIP SKIP')
self.assertEquals(foo_story_result['expected'], 'PASS SKIP')
self.assertTrue(foo_story_result['is_unexpected'])
bar_story_result = d['tests']['benchmark_name']['Bar']
self.assertEquals(bar_story_result['actual'], 'PASS FAIL')
self.assertEquals(bar_story_result['expected'], 'PASS')
self.assertEquals(bar_story_result['shard'], expected_shard_index)
self.assertTrue(bar_story_result['is_unexpected'])
self.assertEquals(
d['num_failures_by_type'], {'PASS': 2, 'FAIL': 1, 'SKIP': 2})
finally:
if delete_env_var_after:
del os.environ['GTEST_SHARD_INDEX']
def testAsDictBaseKeys(self):
results = _MakePageTestResults()
d = json_3_output_formatter.ResultsAsDict(results)
self.assertEquals(d['interrupted'], False)
self.assertEquals(d['num_failures_by_type'], {})
self.assertEquals(d['path_delimiter'], '/')
self.assertEquals(d['seconds_since_epoch'], 1501773200)
self.assertEquals(d['tests'], {})
self.assertEquals(d['version'], 3)
def testAsDictWithSkippedAndFailedTests_AlsoShardIndex(self):
shard_index = 42
with mock.patch.dict(os.environ,
{'GTEST_SHARD_INDEX': str(shard_index)}):
with self._MakeResults() as results:
results.WillRunPage(self._story_set[0])
v0 = scalar.ScalarValue(
results.current_story,
'foo',
'seconds',
3,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v0)
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
v1 = scalar.ScalarValue(
results.current_story,
'bar',
'seconds',
4,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v1)
results.DidRunPage(self._story_set[1])
results.WillRunPage(self._story_set[0])
results.Skip('fake_skip')
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[0])
results.Skip('unexpected_skip', False)
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
results.Fail('fake_failure')
results.DidRunPage(self._story_set[1])
d = json_3_output_formatter.ResultsAsDict(results)
foo_story_result = d['tests']['benchmark_name']['Foo']
self.assertEquals(foo_story_result['actual'], 'PASS SKIP SKIP')
self.assertEquals(foo_story_result['expected'], 'PASS SKIP')
self.assertTrue(foo_story_result['is_unexpected'])
bar_story_result = d['tests']['benchmark_name']['Bar']
self.assertEquals(bar_story_result['actual'], 'PASS FAIL')
self.assertEquals(bar_story_result['expected'], 'PASS')
self.assertEquals(bar_story_result['shard'], shard_index)
self.assertTrue(bar_story_result['is_unexpected'])
self.assertEquals(d['num_failures_by_type'], {
'PASS': 2,
'FAIL': 1,
'SKIP': 2
})
def testAsDictBaseKeys(self):
results = page_test_results.PageTestResults()
results.telemetry_info.benchmark_start_epoch = 1501773200
d = json_3_output_formatter.ResultsAsDict(results)
self.assertEquals(d['interrupted'], False)
self.assertEquals(d['num_failures_by_type'], {})
self.assertEquals(d['path_delimiter'], '/')
self.assertEquals(d['seconds_since_epoch'], 1501773200)
self.assertEquals(d['tests'], {})
self.assertEquals(d['version'], 3)
def testAsDictWithSkippedAndFailedTests(self):
results = page_test_results.PageTestResults()
results.telemetry_info.benchmark_start_epoch = 1501773200
results.telemetry_info.benchmark_name = 'benchmark_name'
results.WillRunPage(self._story_set[0])
v0 = scalar.ScalarValue(
results.current_page,
'foo',
'seconds',
3,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v0)
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
v1 = scalar.ScalarValue(
results.current_page,
'bar',
'seconds',
4,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v1)
results.DidRunPage(self._story_set[1])
results.WillRunPage(self._story_set[0])
v0 = skip.SkipValue(results.current_page, 'fake_skip')
results.AddValue(v0)
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
v1 = failure.FailureValue.FromMessage(results.current_page,
'fake_failure')
results.AddValue(v1)
results.DidRunPage(self._story_set[1])
d = json_3_output_formatter.ResultsAsDict(results)
foo_story_result = d['tests']['benchmark_name']['Foo']
self.assertEquals(foo_story_result['actual'], 'PASS SKIP')
self.assertEquals(foo_story_result['expected'], 'PASS SKIP')
self.assertFalse(foo_story_result['is_unexpected'])
bar_story_result = d['tests']['benchmark_name']['Bar']
self.assertEquals(bar_story_result['actual'], 'PASS FAIL')
self.assertEquals(bar_story_result['expected'], 'PASS')
self.assertTrue(bar_story_result['is_unexpected'])
self.assertEquals(d['num_failures_by_type'], {
'PASS': 2,
'FAIL': 1,
'SKIP': 1
})
def testAsDictWithOnePage(self):
with self._MakeResults() as results:
results.WillRunPage(self._story_set[0])
results.DidRunPage(self._story_set[0])
d = json_3_output_formatter.ResultsAsDict(results)
self.assertTrue(_HasBenchmark(d['tests'], 'benchmark_name'))
self.assertTrue(_HasStory(d['tests']['benchmark_name'], 'Foo'))
story_result = d['tests']['benchmark_name']['Foo']
self.assertEquals(story_result['actual'], 'PASS')
self.assertEquals(story_result['expected'], 'PASS')
self.assertEquals(d['num_failures_by_type'], {'PASS': 1})
def testAsDictWithOnePage(self):
results = _MakePageTestResults()
results.WillRunPage(self._story_set[0])
v0 = scalar.ScalarValue(results.current_page, 'foo', 'seconds', 3,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v0)
results.DidRunPage(self._story_set[0])
d = json_3_output_formatter.ResultsAsDict(results)
self.assertTrue(_HasBenchmark(d['tests'], 'benchmark_name'))
self.assertTrue(_HasStory(d['tests']['benchmark_name'], 'Foo'))
story_result = d['tests']['benchmark_name']['Foo']
self.assertEquals(story_result['actual'], 'PASS')
self.assertEquals(story_result['expected'], 'PASS')
self.assertEquals(d['num_failures_by_type'], {'PASS': 1})
def testArtifactsWithRepeatedRuns(self):
with self._MakeResults() as results:
results.WillRunPage(self._story_set[0])
with results.CreateArtifact('log.txt'):
pass
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[0])
with results.CreateArtifact('log.txt'):
pass
with results.CreateArtifact('trace.json'):
pass
results.DidRunPage(self._story_set[0])
d = json_3_output_formatter.ResultsAsDict(results)
foo_story_artifacts = d['tests']['benchmark_name']['Foo']['artifacts']
self.assertEquals(len(foo_story_artifacts['log.txt']), 2)
self.assertEquals(len(foo_story_artifacts['trace.json']), 1)
def testArtifactsWithRepeatedRuns(self):
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
results = _MakePageTestResults(output_dir=tempdir)
results.WillRunPage(self._story_set[0])
with results.CreateArtifact('log'):
pass
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[0])
with results.CreateArtifact('log'):
pass
with results.CreateArtifact('trace'):
pass
results.DidRunPage(self._story_set[0])
d = json_3_output_formatter.ResultsAsDict(results)
foo_story_artifacts = d['tests']['benchmark_name']['Foo']['artifacts']
self.assertEquals(len(foo_story_artifacts['log']), 2)
self.assertEquals(len(foo_story_artifacts['trace']), 1)
