def testPagesetRepeat(self):
us = user_story_set.UserStorySet()
us.AddUserStory(
user_story.UserStory(TestSharedUserStoryState, name='blank'))
us.AddUserStory(
user_story.UserStory(TestSharedUserStoryState, name='green'))
class Measurement(page_test.PageTest):
i = 0
def RunPage(self, page, _, results):
self.i += 1
results.AddValue(
scalar.ScalarValue(page, 'metric', 'unit', self.i))
self.options.page_repeat = 1
self.options.pageset_repeat = 2
self.options.output_formats = ['buildbot']
output = StringIO.StringIO()
real_stdout = sys.stdout
sys.stdout = output
try:
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
results.PrintSummary()
contents = output.getvalue()
self.assertEquals(4, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertIn('RESULT metric: blank= [1,3] unit', contents)
self.assertIn('RESULT metric: green= [2,4] unit', contents)
self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', contents)
finally:
sys.stdout = real_stdout
def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class ExpectedException(Exception):
pass
class Test(page_test.PageTest):
def __init__(self, *args):
super(Test, self).__init__(*args)
self.run_count = 0
def RunPage(self, *_):
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 0:
raise ExpectedException()
us.AddUserStory(user_story.UserStory(TestSharedUserStoryState))
us.AddUserStory(user_story.UserStory(TestSharedUserStoryState))
test = Test()
user_story_runner.Run(test, us, self.expectations, self.options,
self.results)
self.assertEquals(2, test.run_count)
self.assertEquals(1, len(self.results.failures))
self.assertEquals(1, GetNumberOfSuccessfulPageRuns(self.results))
def testSuccefulUserStoryTest(self):
us = user_story_set.UserStorySet()
us.AddUserStory(user_story.UserStory(FooUserStoryState))
us.AddUserStory(user_story.UserStory(FooUserStoryState))
us.AddUserStory(user_story.UserStory(BarUserStoryState))
user_story_runner.Run(DummyTest(), us, self.expectations, self.options,
self.results)
self.assertEquals(0, len(self.results.failures))
self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results))
def testMakeJavaScriptDeterministic(self):
u = user_story.UserStory(SharedStateBar)
self.assertTrue(u.make_javascript_deterministic)
u = user_story.UserStory(SharedStateBar,
make_javascript_deterministic=False)
self.assertFalse(u.make_javascript_deterministic)
u = user_story.UserStory(SharedStateBar,
make_javascript_deterministic=True)
self.assertTrue(u.make_javascript_deterministic)
def testDiscardFirstResult(self):
us = user_story_set.UserStorySet()
us.AddUserStory(user_story.UserStory(TestSharedUserStoryState))
us.AddUserStory(user_story.UserStory(TestSharedUserStoryState))
class Measurement(page_test.PageTest):
@property
def discard_first_result(self):
return True
def RunPage(self, page, _, results):
results.AddValue(
string.StringValue(page, 'test', 't', page.name))
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(0, len(results.all_page_specific_values))
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
self.options.page_repeat = 1
self.options.pageset_repeat = 2
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
self.assertEquals(2, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(2, len(results.all_page_specific_values))
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
self.options.page_repeat = 2
self.options.pageset_repeat = 1
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
self.assertEquals(2, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(2, len(results.all_page_specific_values))
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
self.options.page_repeat = 1
self.options.pageset_repeat = 1
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(0, len(results.all_page_specific_values))
def testGetUserStoryGroupsWithSameSharedUserStoryClass(self):
us = user_story_set.UserStorySet()
us.AddUserStory(user_story.UserStory(FooUserStoryState))
us.AddUserStory(user_story.UserStory(FooUserStoryState))
us.AddUserStory(user_story.UserStory(BarUserStoryState))
us.AddUserStory(user_story.UserStory(FooUserStoryState))
story_groups = (user_story_runner.
GetUserStoryGroupsWithSameSharedUserStoryClass(us))
self.assertEqual(len(story_groups), 3)
self.assertEqual(story_groups[0].shared_user_story_state_class,
FooUserStoryState)
self.assertEqual(story_groups[1].shared_user_story_state_class,
BarUserStoryState)
self.assertEqual(story_groups[2].shared_user_story_state_class,
FooUserStoryState)
def testPageTestWithIncompatibleUserStory(self):
b = TestBenchmark(user_story.UserStory(
shared_state_class=shared_page_state.SharedPageState))
with self.assertRaisesRegexp(
Exception, 'containing only telemetry.page.Page user stories'):
b.Run(browser_options.BrowserFinderOptions())
state_class = shared_state.SharedState
b = TestBenchmark(user_story.UserStory(
shared_state_class=state_class))
with self.assertRaisesRegexp(
Exception, 'containing only telemetry.page.Page user stories'):
b.Run(browser_options.BrowserFinderOptions())
b = TestBenchmark(android.AppStory(start_intent=None))
with self.assertRaisesRegexp(
Exception, 'containing only telemetry.page.Page user stories'):
b.Run(browser_options.BrowserFinderOptions())
def testTearDownIsCalledOnceForEachUserStoryGroupWithPageSetRepeat(self):
self.options.pageset_repeat = 3
us = user_story_set.UserStorySet()
fooz_init_call_counter = [0]
fooz_tear_down_call_counter = [0]
barz_init_call_counter = [0]
barz_tear_down_call_counter = [0]
class FoozUserStoryState(FooUserStoryState):
def __init__(self, test, options, user_story_setz):
super(FoozUserStoryState,
self).__init__(test, options, user_story_setz)
fooz_init_call_counter[0] += 1
def TearDownState(self, _results):
fooz_tear_down_call_counter[0] += 1
class BarzUserStoryState(BarUserStoryState):
def __init__(self, test, options, user_story_setz):
super(BarzUserStoryState,
self).__init__(test, options, user_story_setz)
barz_init_call_counter[0] += 1
def TearDownState(self, _results):
barz_tear_down_call_counter[0] += 1
us.AddUserStory(user_story.UserStory(FoozUserStoryState))
us.AddUserStory(user_story.UserStory(FoozUserStoryState))
us.AddUserStory(user_story.UserStory(BarzUserStoryState))
us.AddUserStory(user_story.UserStory(BarzUserStoryState))
user_story_runner.Run(DummyTest(), us, self.expectations, self.options,
self.results)
self.assertEquals(0, len(self.results.failures))
self.assertEquals(12, GetNumberOfSuccessfulPageRuns(self.results))
self.assertEquals(1, fooz_init_call_counter[0])
self.assertEquals(1, fooz_tear_down_call_counter[0])
self.assertEquals(1, barz_init_call_counter[0])
self.assertEquals(1, barz_tear_down_call_counter[0])
def testHandlingOfCrashedApp(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class SharedUserStoryThatCausesAppCrash(TestSharedUserStoryState):
def WillRunUserStory(self, user_storyz):
raise exceptions.AppCrashException()
us.AddUserStory(
user_story.UserStory(SharedUserStoryThatCausesAppCrash))
user_story_runner.Run(DummyTest(), us, self.expectations, self.options,
self.results)
self.assertEquals(1, len(self.results.failures))
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(self.results))
def testNamedUserStoryAsDict(self):
u = user_story.UserStory(SharedStateBar, 'Foo')
u_dict = u.AsDict()
self.assertEquals(u_dict['id'], u.id)
self.assertEquals('Foo', u_dict['name'])
def testNamelessUserStoryAsDict(self):
u = user_story.UserStory(SharedStateBar)
u_dict = u.AsDict()
self.assertEquals(u_dict['id'], u.id)
self.assertNotIn('name', u_dict)
def testUserStoriesHaveDifferentIds(self):
u0 = user_story.UserStory(SharedStateBar, 'foo')
u1 = user_story.UserStory(SharedStateBar, 'bar')
self.assertNotEqual(u0.id, u1.id)