def testUpdateAndCheckArchives(self): usr_stub = system_stub.Override(user_story_runner, ['cloud_storage']) wpr_stub = system_stub.Override(archive_info, ['cloud_storage']) try: uss = user_story_set.UserStorySet() uss.AddUserStory( page_module.Page('http://www.testurl.com', uss, uss.base_dir)) # Page set missing archive_data_file. self.assertRaises(user_story_runner.ArchiveError, user_story_runner._UpdateAndCheckArchives, uss.archive_data_file, uss.wpr_archive_info, uss.user_stories) uss = user_story_set.UserStorySet( archive_data_file='missing_archive_data_file.json') uss.AddUserStory( page_module.Page('http://www.testurl.com', uss, uss.base_dir)) # Page set missing json file specified in archive_data_file. self.assertRaises(user_story_runner.ArchiveError, user_story_runner._UpdateAndCheckArchives, uss.archive_data_file, uss.wpr_archive_info, uss.user_stories) uss = user_story_set.UserStorySet( archive_data_file='../../unittest_data/archive_files/test.json', cloud_storage_bucket=cloud_storage.PUBLIC_BUCKET) uss.AddUserStory( page_module.Page('http://www.testurl.com', uss, uss.base_dir)) # Page set with valid archive_data_file. self.assertTrue( user_story_runner._UpdateAndCheckArchives( uss.archive_data_file, uss.wpr_archive_info, uss.user_stories)) uss.AddUserStory( page_module.Page('http://www.google.com', uss, uss.base_dir)) # Page set with an archive_data_file which exists but is missing a page. self.assertRaises(user_story_runner.ArchiveError, user_story_runner._UpdateAndCheckArchives, uss.archive_data_file, uss.wpr_archive_info, uss.user_stories) uss = user_story_set.UserStorySet( archive_data_file= '../../unittest_data/test_missing_wpr_file.json', cloud_storage_bucket=cloud_storage.PUBLIC_BUCKET) uss.AddUserStory( page_module.Page('http://www.testurl.com', uss, uss.base_dir)) uss.AddUserStory( page_module.Page('http://www.google.com', uss, uss.base_dir)) # Page set with an archive_data_file which exists and contains all pages # but fails to find a wpr file. self.assertRaises(user_story_runner.ArchiveError, user_story_runner._UpdateAndCheckArchives, uss.archive_data_file, uss.wpr_archive_info, uss.user_stories) finally: usr_stub.Restore() wpr_stub.Restore()
def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self): self.SuppressExceptionFormatting() us = user_story_set.UserStorySet() class ExpectedException(Exception): pass class Test(page_test.PageTest): def __init__(self, *args): super(Test, self).__init__(*args) self.run_count = 0 def RunPage(self, *_): old_run_count = self.run_count self.run_count += 1 if old_run_count == 0: raise ExpectedException() us.AddUserStory(user_story.UserStory(TestSharedUserStoryState)) us.AddUserStory(user_story.UserStory(TestSharedUserStoryState)) test = Test() user_story_runner.Run(test, us, self.expectations, self.options, self.results) self.assertEquals(2, test.run_count) self.assertEquals(1, len(self.results.failures)) self.assertEquals(1, GetNumberOfSuccessfulPageRuns(self.results))
def testUnknownExceptionIsFatal(self): self.SuppressExceptionFormatting() uss = user_story_set.UserStorySet() class UnknownException(Exception): pass # This erroneous test is set up to raise exception for the 2nd user story # run. class Test(page_test.PageTest): def __init__(self, *args): super(Test, self).__init__(*args) self.run_count = 0 def RunPage(self, *_): old_run_count = self.run_count self.run_count += 1 if old_run_count == 1: raise UnknownException('FooBarzException') def ValidateAndMeasurePage(self, page, tab, results): pass us1 = DummyLocalUserStory(TestSharedPageState) us2 = DummyLocalUserStory(TestSharedPageState) uss.AddUserStory(us1) uss.AddUserStory(us2) test = Test() with self.assertRaises(UnknownException): story_runner.Run( test, uss, self.expectations, self.options, self.results) self.assertEqual(set([us2]), self.results.pages_that_failed) self.assertEqual(set([us1]), self.results.pages_that_succeeded) self.assertIn('FooBarzException', self.fake_stdout.getvalue())
def testRaiseBrowserGoneExceptionFromRunPage(self): self.SuppressExceptionFormatting() us = user_story_set.UserStorySet() class Test(page_test.PageTest): def __init__(self, *args): super(Test, self).__init__(*args) self.run_count = 0 def RunPage(self, *_): old_run_count = self.run_count self.run_count += 1 if old_run_count == 0: raise exceptions.BrowserGoneException('i am a browser instance') def ValidateAndMeasurePage(self, page, tab, results): pass us.AddUserStory(DummyLocalUserStory(TestSharedPageState)) us.AddUserStory(DummyLocalUserStory(TestSharedPageState)) test = Test() story_runner.Run( test, us, self.expectations, self.options, self.results) self.assertEquals(2, test.run_count) self.assertEquals(1, len(self.results.failures)) self.assertEquals(1, GetNumberOfSuccessfulPageRuns(self.results))
def testPagesetRepeat(self): us = user_story_set.UserStorySet() us.AddUserStory(DummyLocalUserStory(TestSharedPageState, name='blank')) us.AddUserStory(DummyLocalUserStory(TestSharedPageState, name='green')) class Measurement(page_test.PageTest): i = 0 def RunPage(self, page, _, results): self.i += 1 results.AddValue(scalar.ScalarValue( page, 'metric', 'unit', self.i)) def ValidateAndMeasurePage(self, page, tab, results): pass self.options.page_repeat = 1 self.options.pageset_repeat = 2 self.options.output_formats = ['buildbot'] results = results_options.CreateResults( EmptyMetadataForTest(), self.options) story_runner.Run( Measurement(), us, self.expectations, self.options, results) results.PrintSummary() contents = self.fake_stdout.getvalue() self.assertEquals(4, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(0, len(results.failures)) self.assertIn('RESULT metric: blank= [1,3] unit', contents) self.assertIn('RESULT metric: green= [2,4] unit', contents) self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', contents)
def SetupUserStorySet(allow_multiple_user_story_states, user_story_state_list): if allow_multiple_user_story_states: us = MixedStateStorySet() else: us = user_story_set.UserStorySet() for user_story_state in user_story_state_list: us.AddUserStory(DummyLocalUserStory(user_story_state)) return us
def testBasicAddRemove(self): uss = user_story_set.UserStorySet() foo_story = UserStoryFoo() uss.AddUserStory(foo_story) self.assertEqual([foo_story], uss.user_stories) uss.RemoveUserStory(foo_story) self.assertEqual([], uss.user_stories)
def _testMaxFailuresOptionIsRespectedAndOverridable( self, num_failing_user_stories, runner_max_failures, options_max_failures, expected_num_failures): class SimpleSharedState( shared_state.SharedState): _fake_platform = FakePlatform() _current_user_story = None @property def platform(self): return self._fake_platform def WillRunUserStory(self, story): self._current_user_story = story def RunUserStory(self, results): self._current_user_story.Run() def DidRunUserStory(self, results): pass def GetTestExpectationAndSkipValue(self, expectations): return 'pass', None def TearDownState(self): pass class FailingUserStory(user_story.UserStory): def __init__(self): super(FailingUserStory, self).__init__( shared_state_class=SimpleSharedState, is_local=True) self.was_run = False def Run(self): self.was_run = True raise page_test.Failure self.SuppressExceptionFormatting() uss = user_story_set.UserStorySet() for _ in range(num_failing_user_stories): uss.AddUserStory(FailingUserStory()) options = _GetOptionForUnittest() options.output_formats = ['none'] options.suppress_gtest_report = True if options_max_failures: options.max_failures = options_max_failures results = results_options.CreateResults(EmptyMetadataForTest(), options) story_runner.Run( DummyTest(), uss, test_expectations.TestExpectations(), options, results, max_failures=runner_max_failures) self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(expected_num_failures, len(results.failures)) for ii, story in enumerate(uss.user_stories): self.assertEqual(story.was_run, ii < expected_num_failures)
def testSuccefulUserStoryTest(self): us = user_story_set.UserStorySet() us.AddUserStory(user_story.UserStory(FooUserStoryState)) us.AddUserStory(user_story.UserStory(FooUserStoryState)) us.AddUserStory(user_story.UserStory(BarUserStoryState)) user_story_runner.Run(DummyTest(), us, self.expectations, self.options, self.results) self.assertEquals(0, len(self.results.failures)) self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results))
def testCloudBucket(self): blank_uss = user_story_set.UserStorySet() self.assertEqual(blank_uss.bucket, None) public_uss = user_story_set.UserStorySet( cloud_storage_bucket=cloud_storage.PUBLIC_BUCKET) self.assertEqual(public_uss.bucket, cloud_storage.PUBLIC_BUCKET) partner_uss = user_story_set.UserStorySet( cloud_storage_bucket=cloud_storage.PARTNER_BUCKET) self.assertEqual(partner_uss.bucket, cloud_storage.PARTNER_BUCKET) internal_uss = user_story_set.UserStorySet( cloud_storage_bucket=cloud_storage.INTERNAL_BUCKET) self.assertEqual(internal_uss.bucket, cloud_storage.INTERNAL_BUCKET) with self.assertRaises(ValueError): user_story_set.UserStorySet(cloud_storage_bucket='garbage_bucket')
def testDiscardFirstResult(self): us = user_story_set.UserStorySet() us.AddUserStory(DummyLocalUserStory(TestSharedPageState)) us.AddUserStory(DummyLocalUserStory(TestSharedPageState)) class Measurement(page_test.PageTest): @property def discard_first_result(self): return True def RunPage(self, page, _, results): results.AddValue( string.StringValue(page, 'test', 't', page.name)) def ValidateAndMeasurePage(self, page, tab, results): pass results = results_options.CreateResults(EmptyMetadataForTest(), self.options) user_story_runner.Run(Measurement(), us, self.expectations, self.options, results) self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(0, len(results.failures)) self.assertEquals(0, len(results.all_page_specific_values)) results = results_options.CreateResults(EmptyMetadataForTest(), self.options) self.options.page_repeat = 1 self.options.pageset_repeat = 2 user_story_runner.Run(Measurement(), us, self.expectations, self.options, results) self.assertEquals(2, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(0, len(results.failures)) self.assertEquals(2, len(results.all_page_specific_values)) results = results_options.CreateResults(EmptyMetadataForTest(), self.options) self.options.page_repeat = 2 self.options.pageset_repeat = 1 user_story_runner.Run(Measurement(), us, self.expectations, self.options, results) self.assertEquals(2, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(0, len(results.failures)) self.assertEquals(2, len(results.all_page_specific_values)) results = results_options.CreateResults(EmptyMetadataForTest(), self.options) self.options.page_repeat = 1 self.options.pageset_repeat = 1 user_story_runner.Run(Measurement(), us, self.expectations, self.options, results) self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(0, len(results.failures)) self.assertEquals(0, len(results.all_page_specific_values))
def assertUserAgentSetCorrectly( self, shared_page_state_class, expected_user_agent): us = page.Page( 'http://www.google.com', shared_page_state_class=shared_page_state_class) test = DummyTest() uss = user_story_set.UserStorySet() uss.AddUserStory(us) us.shared_state_class(test, self.options, uss) browser_options = self.options.browser_options actual_user_agent = browser_options.browser_user_agent_type self.assertEqual(expected_user_agent, actual_user_agent)
def testAppCrashExceptionCausesFailureValue(self): self.SuppressExceptionFormatting() us = user_story_set.UserStorySet() class SharedUserStoryThatCausesAppCrash(TestSharedPageState): def WillRunUserStory(self, user_storyz): raise exceptions.AppCrashException('App Foo crashes') us.AddUserStory(DummyLocalUserStory(SharedUserStoryThatCausesAppCrash)) story_runner.Run( DummyTest(), us, self.expectations, self.options, self.results) self.assertEquals(1, len(self.results.failures)) self.assertEquals(0, GetNumberOfSuccessfulPageRuns(self.results)) self.assertIn('App Foo crashes', self.fake_stdout.getvalue())
def testHandlingOfCrashedApp(self): self.SuppressExceptionFormatting() us = user_story_set.UserStorySet() class SharedUserStoryThatCausesAppCrash(TestSharedUserStoryState): def WillRunUserStory(self, user_storyz): raise exceptions.AppCrashException() us.AddUserStory( user_story.UserStory(SharedUserStoryThatCausesAppCrash)) user_story_runner.Run(DummyTest(), us, self.expectations, self.options, self.results) self.assertEquals(1, len(self.results.failures)) self.assertEquals(0, GetNumberOfSuccessfulPageRuns(self.results))
def testGetUserStoryGroupsWithSameSharedUserStoryClass(self): us = user_story_set.UserStorySet() us.AddUserStory(user_story.UserStory(FooUserStoryState)) us.AddUserStory(user_story.UserStory(FooUserStoryState)) us.AddUserStory(user_story.UserStory(BarUserStoryState)) us.AddUserStory(user_story.UserStory(FooUserStoryState)) story_groups = (user_story_runner. GetUserStoryGroupsWithSameSharedUserStoryClass(us)) self.assertEqual(len(story_groups), 3) self.assertEqual(story_groups[0].shared_user_story_state_class, FooUserStoryState) self.assertEqual(story_groups[1].shared_user_story_state_class, BarUserStoryState) self.assertEqual(story_groups[2].shared_user_story_state_class, FooUserStoryState)
def testSuccessfulTimelineBasedMeasurementTest(self): """Check that PageTest is not required for story_runner.Run. Any PageTest related calls or attributes need to only be called for PageTest tests. """ class TestSharedTbmState(TestSharedState): def RunUserStory(self, results): pass test = timeline_based_measurement.TimelineBasedMeasurement( timeline_based_measurement.Options()) us = user_story_set.UserStorySet() us.AddUserStory(DummyLocalUserStory(TestSharedTbmState)) us.AddUserStory(DummyLocalUserStory(TestSharedTbmState)) us.AddUserStory(DummyLocalUserStory(TestSharedTbmState)) story_runner.Run( test, us, self.expectations, self.options, self.results) self.assertEquals(0, len(self.results.failures)) self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results))
def testAppCrashThenRaiseInTearDownFatal(self): self.SuppressExceptionFormatting() us = user_story_set.UserStorySet() class DidRunTestError(Exception): pass class TestTearDownSharedUserStoryState(TestSharedPageState): def TearDownState(self, results): self._test.DidRunTest('app', results) class Test(page_test.PageTest): def __init__(self, *args): super(Test, self).__init__(*args) self.run_count = 0 self._unit_test_events = [] # track what was called when def RunPage(self, *_): old_run_count = self.run_count self.run_count += 1 if old_run_count == 0: self._unit_test_events.append('app-crash') raise exceptions.AppCrashException def ValidateAndMeasurePage(self, page, tab, results): pass def DidRunTest(self, _, __): self._unit_test_events.append('did-run-test') raise DidRunTestError us.AddUserStory(DummyLocalUserStory(TestTearDownSharedUserStoryState)) us.AddUserStory(DummyLocalUserStory(TestTearDownSharedUserStoryState)) test = Test() with self.assertRaises(DidRunTestError): user_story_runner.Run(test, us, self.expectations, self.options, self.results) self.assertEqual(['app-crash', 'did-run-test'], test._unit_test_events) # The AppCrashException gets added as a failure. self.assertEquals(1, len(self.results.failures))
def testTearDownIsCalledOnceForEachUserStoryGroupWithPageSetRepeat(self): self.options.pageset_repeat = 3 us = user_story_set.UserStorySet() fooz_init_call_counter = [0] fooz_tear_down_call_counter = [0] barz_init_call_counter = [0] barz_tear_down_call_counter = [0] class FoozUserStoryState(FooUserStoryState): def __init__(self, test, options, user_story_setz): super(FoozUserStoryState, self).__init__(test, options, user_story_setz) fooz_init_call_counter[0] += 1 def TearDownState(self, _results): fooz_tear_down_call_counter[0] += 1 class BarzUserStoryState(BarUserStoryState): def __init__(self, test, options, user_story_setz): super(BarzUserStoryState, self).__init__(test, options, user_story_setz) barz_init_call_counter[0] += 1 def TearDownState(self, _results): barz_tear_down_call_counter[0] += 1 us.AddUserStory(user_story.UserStory(FoozUserStoryState)) us.AddUserStory(user_story.UserStory(FoozUserStoryState)) us.AddUserStory(user_story.UserStory(BarzUserStoryState)) us.AddUserStory(user_story.UserStory(BarzUserStoryState)) user_story_runner.Run(DummyTest(), us, self.expectations, self.options, self.results) self.assertEquals(0, len(self.results.failures)) self.assertEquals(12, GetNumberOfSuccessfulPageRuns(self.results)) self.assertEquals(1, fooz_init_call_counter[0]) self.assertEquals(1, fooz_tear_down_call_counter[0]) self.assertEquals(1, barz_init_call_counter[0]) self.assertEquals(1, barz_tear_down_call_counter[0])
def setUp(self): self.user_story_set = user_story_set.UserStorySet() self.user_story_set.AddUserStory(UserStoryFoo())
def __init__(self, story): super(TestBenchmark, self).__init__() self._uss = user_story_set.UserStorySet() self._uss.AddUserStory(story)
def testRemoveWithEmptySetRaises(self): uss = user_story_set.UserStorySet() foo_story = UserStoryFoo() with self.assertRaises(ValueError): uss.RemoveUserStory(foo_story)