def testUpdateAndCheckArchives(self):
usr_stub = system_stub.Override(user_story_runner, ['cloud_storage'])
wpr_stub = system_stub.Override(archive_info, ['cloud_storage'])
try:
uss = user_story_set.UserStorySet()
uss.AddUserStory(
page_module.Page('http://www.testurl.com', uss, uss.base_dir))
# Page set missing archive_data_file.
self.assertRaises(user_story_runner.ArchiveError,
user_story_runner._UpdateAndCheckArchives,
uss.archive_data_file, uss.wpr_archive_info,
uss.user_stories)
uss = user_story_set.UserStorySet(
archive_data_file='missing_archive_data_file.json')
uss.AddUserStory(
page_module.Page('http://www.testurl.com', uss, uss.base_dir))
# Page set missing json file specified in archive_data_file.
self.assertRaises(user_story_runner.ArchiveError,
user_story_runner._UpdateAndCheckArchives,
uss.archive_data_file, uss.wpr_archive_info,
uss.user_stories)
uss = user_story_set.UserStorySet(
archive_data_file='../../unittest_data/archive_files/test.json',
cloud_storage_bucket=cloud_storage.PUBLIC_BUCKET)
uss.AddUserStory(
page_module.Page('http://www.testurl.com', uss, uss.base_dir))
# Page set with valid archive_data_file.
self.assertTrue(
user_story_runner._UpdateAndCheckArchives(
uss.archive_data_file, uss.wpr_archive_info,
uss.user_stories))
uss.AddUserStory(
page_module.Page('http://www.google.com', uss, uss.base_dir))
# Page set with an archive_data_file which exists but is missing a page.
self.assertRaises(user_story_runner.ArchiveError,
user_story_runner._UpdateAndCheckArchives,
uss.archive_data_file, uss.wpr_archive_info,
uss.user_stories)
uss = user_story_set.UserStorySet(
archive_data_file=
'../../unittest_data/test_missing_wpr_file.json',
cloud_storage_bucket=cloud_storage.PUBLIC_BUCKET)
uss.AddUserStory(
page_module.Page('http://www.testurl.com', uss, uss.base_dir))
uss.AddUserStory(
page_module.Page('http://www.google.com', uss, uss.base_dir))
# Page set with an archive_data_file which exists and contains all pages
# but fails to find a wpr file.
self.assertRaises(user_story_runner.ArchiveError,
user_story_runner._UpdateAndCheckArchives,
uss.archive_data_file, uss.wpr_archive_info,
uss.user_stories)
finally:
usr_stub.Restore()
wpr_stub.Restore()
def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class ExpectedException(Exception):
pass
class Test(page_test.PageTest):
def __init__(self, *args):
super(Test, self).__init__(*args)
self.run_count = 0
def RunPage(self, *_):
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 0:
raise ExpectedException()
us.AddUserStory(user_story.UserStory(TestSharedUserStoryState))
us.AddUserStory(user_story.UserStory(TestSharedUserStoryState))
test = Test()
user_story_runner.Run(test, us, self.expectations, self.options,
self.results)
self.assertEquals(2, test.run_count)
self.assertEquals(1, len(self.results.failures))
self.assertEquals(1, GetNumberOfSuccessfulPageRuns(self.results))
def testUnknownExceptionIsFatal(self):
self.SuppressExceptionFormatting()
uss = user_story_set.UserStorySet()
class UnknownException(Exception):
pass
# This erroneous test is set up to raise exception for the 2nd user story
# run.
class Test(page_test.PageTest):
def __init__(self, *args):
super(Test, self).__init__(*args)
self.run_count = 0
def RunPage(self, *_):
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 1:
raise UnknownException('FooBarzException')
def ValidateAndMeasurePage(self, page, tab, results):
pass
us1 = DummyLocalUserStory(TestSharedPageState)
us2 = DummyLocalUserStory(TestSharedPageState)
uss.AddUserStory(us1)
uss.AddUserStory(us2)
test = Test()
with self.assertRaises(UnknownException):
story_runner.Run(
test, uss, self.expectations, self.options, self.results)
self.assertEqual(set([us2]), self.results.pages_that_failed)
self.assertEqual(set([us1]), self.results.pages_that_succeeded)
self.assertIn('FooBarzException', self.fake_stdout.getvalue())
def testRaiseBrowserGoneExceptionFromRunPage(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class Test(page_test.PageTest):
def __init__(self, *args):
super(Test, self).__init__(*args)
self.run_count = 0
def RunPage(self, *_):
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 0:
raise exceptions.BrowserGoneException('i am a browser instance')
def ValidateAndMeasurePage(self, page, tab, results):
pass
us.AddUserStory(DummyLocalUserStory(TestSharedPageState))
us.AddUserStory(DummyLocalUserStory(TestSharedPageState))
test = Test()
story_runner.Run(
test, us, self.expectations, self.options, self.results)
self.assertEquals(2, test.run_count)
self.assertEquals(1, len(self.results.failures))
self.assertEquals(1, GetNumberOfSuccessfulPageRuns(self.results))
def testPagesetRepeat(self):
us = user_story_set.UserStorySet()
us.AddUserStory(DummyLocalUserStory(TestSharedPageState, name='blank'))
us.AddUserStory(DummyLocalUserStory(TestSharedPageState, name='green'))
class Measurement(page_test.PageTest):
i = 0
def RunPage(self, page, _, results):
self.i += 1
results.AddValue(scalar.ScalarValue(
page, 'metric', 'unit', self.i))
def ValidateAndMeasurePage(self, page, tab, results):
pass
self.options.page_repeat = 1
self.options.pageset_repeat = 2
self.options.output_formats = ['buildbot']
results = results_options.CreateResults(
EmptyMetadataForTest(), self.options)
story_runner.Run(
Measurement(), us, self.expectations, self.options, results)
results.PrintSummary()
contents = self.fake_stdout.getvalue()
self.assertEquals(4, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertIn('RESULT metric: blank= [1,3] unit', contents)
self.assertIn('RESULT metric: green= [2,4] unit', contents)
self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', contents)
def SetupUserStorySet(allow_multiple_user_story_states, user_story_state_list):
if allow_multiple_user_story_states:
us = MixedStateStorySet()
else:
us = user_story_set.UserStorySet()
for user_story_state in user_story_state_list:
us.AddUserStory(DummyLocalUserStory(user_story_state))
return us
def testBasicAddRemove(self):
uss = user_story_set.UserStorySet()
foo_story = UserStoryFoo()
uss.AddUserStory(foo_story)
self.assertEqual([foo_story], uss.user_stories)
uss.RemoveUserStory(foo_story)
self.assertEqual([], uss.user_stories)
def _testMaxFailuresOptionIsRespectedAndOverridable(
self, num_failing_user_stories, runner_max_failures, options_max_failures,
expected_num_failures):
class SimpleSharedState(
shared_state.SharedState):
_fake_platform = FakePlatform()
_current_user_story = None
@property
def platform(self):
return self._fake_platform
def WillRunUserStory(self, story):
self._current_user_story = story
def RunUserStory(self, results):
self._current_user_story.Run()
def DidRunUserStory(self, results):
pass
def GetTestExpectationAndSkipValue(self, expectations):
return 'pass', None
def TearDownState(self):
pass
class FailingUserStory(user_story.UserStory):
def __init__(self):
super(FailingUserStory, self).__init__(
shared_state_class=SimpleSharedState,
is_local=True)
self.was_run = False
def Run(self):
self.was_run = True
raise page_test.Failure
self.SuppressExceptionFormatting()
uss = user_story_set.UserStorySet()
for _ in range(num_failing_user_stories):
uss.AddUserStory(FailingUserStory())
options = _GetOptionForUnittest()
options.output_formats = ['none']
options.suppress_gtest_report = True
if options_max_failures:
options.max_failures = options_max_failures
results = results_options.CreateResults(EmptyMetadataForTest(), options)
story_runner.Run(
DummyTest(), uss, test_expectations.TestExpectations(), options,
results, max_failures=runner_max_failures)
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(expected_num_failures, len(results.failures))
for ii, story in enumerate(uss.user_stories):
self.assertEqual(story.was_run, ii < expected_num_failures)
def testSuccefulUserStoryTest(self):
us = user_story_set.UserStorySet()
us.AddUserStory(user_story.UserStory(FooUserStoryState))
us.AddUserStory(user_story.UserStory(FooUserStoryState))
us.AddUserStory(user_story.UserStory(BarUserStoryState))
user_story_runner.Run(DummyTest(), us, self.expectations, self.options,
self.results)
self.assertEquals(0, len(self.results.failures))
self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results))
def testCloudBucket(self):
blank_uss = user_story_set.UserStorySet()
self.assertEqual(blank_uss.bucket, None)
public_uss = user_story_set.UserStorySet(
cloud_storage_bucket=cloud_storage.PUBLIC_BUCKET)
self.assertEqual(public_uss.bucket, cloud_storage.PUBLIC_BUCKET)
partner_uss = user_story_set.UserStorySet(
cloud_storage_bucket=cloud_storage.PARTNER_BUCKET)
self.assertEqual(partner_uss.bucket, cloud_storage.PARTNER_BUCKET)
internal_uss = user_story_set.UserStorySet(
cloud_storage_bucket=cloud_storage.INTERNAL_BUCKET)
self.assertEqual(internal_uss.bucket, cloud_storage.INTERNAL_BUCKET)
with self.assertRaises(ValueError):
user_story_set.UserStorySet(cloud_storage_bucket='garbage_bucket')
def testDiscardFirstResult(self):
us = user_story_set.UserStorySet()
us.AddUserStory(DummyLocalUserStory(TestSharedPageState))
us.AddUserStory(DummyLocalUserStory(TestSharedPageState))
class Measurement(page_test.PageTest):
@property
def discard_first_result(self):
return True
def RunPage(self, page, _, results):
results.AddValue(
string.StringValue(page, 'test', 't', page.name))
def ValidateAndMeasurePage(self, page, tab, results):
pass
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(0, len(results.all_page_specific_values))
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
self.options.page_repeat = 1
self.options.pageset_repeat = 2
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
self.assertEquals(2, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(2, len(results.all_page_specific_values))
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
self.options.page_repeat = 2
self.options.pageset_repeat = 1
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
self.assertEquals(2, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(2, len(results.all_page_specific_values))
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
self.options.page_repeat = 1
self.options.pageset_repeat = 1
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(0, len(results.all_page_specific_values))
def assertUserAgentSetCorrectly(
self, shared_page_state_class, expected_user_agent):
us = page.Page(
'http://www.google.com',
shared_page_state_class=shared_page_state_class)
test = DummyTest()
uss = user_story_set.UserStorySet()
uss.AddUserStory(us)
us.shared_state_class(test, self.options, uss)
browser_options = self.options.browser_options
actual_user_agent = browser_options.browser_user_agent_type
self.assertEqual(expected_user_agent, actual_user_agent)
def testAppCrashExceptionCausesFailureValue(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class SharedUserStoryThatCausesAppCrash(TestSharedPageState):
def WillRunUserStory(self, user_storyz):
raise exceptions.AppCrashException('App Foo crashes')
us.AddUserStory(DummyLocalUserStory(SharedUserStoryThatCausesAppCrash))
story_runner.Run(
DummyTest(), us, self.expectations, self.options, self.results)
self.assertEquals(1, len(self.results.failures))
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(self.results))
self.assertIn('App Foo crashes', self.fake_stdout.getvalue())
def testHandlingOfCrashedApp(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class SharedUserStoryThatCausesAppCrash(TestSharedUserStoryState):
def WillRunUserStory(self, user_storyz):
raise exceptions.AppCrashException()
us.AddUserStory(
user_story.UserStory(SharedUserStoryThatCausesAppCrash))
user_story_runner.Run(DummyTest(), us, self.expectations, self.options,
self.results)
self.assertEquals(1, len(self.results.failures))
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(self.results))
def testGetUserStoryGroupsWithSameSharedUserStoryClass(self):
us = user_story_set.UserStorySet()
us.AddUserStory(user_story.UserStory(FooUserStoryState))
us.AddUserStory(user_story.UserStory(FooUserStoryState))
us.AddUserStory(user_story.UserStory(BarUserStoryState))
us.AddUserStory(user_story.UserStory(FooUserStoryState))
story_groups = (user_story_runner.
GetUserStoryGroupsWithSameSharedUserStoryClass(us))
self.assertEqual(len(story_groups), 3)
self.assertEqual(story_groups[0].shared_user_story_state_class,
FooUserStoryState)
self.assertEqual(story_groups[1].shared_user_story_state_class,
BarUserStoryState)
self.assertEqual(story_groups[2].shared_user_story_state_class,
FooUserStoryState)
def testSuccessfulTimelineBasedMeasurementTest(self):
"""Check that PageTest is not required for story_runner.Run.
Any PageTest related calls or attributes need to only be called
for PageTest tests.
"""
class TestSharedTbmState(TestSharedState):
def RunUserStory(self, results):
pass
test = timeline_based_measurement.TimelineBasedMeasurement(
timeline_based_measurement.Options())
us = user_story_set.UserStorySet()
us.AddUserStory(DummyLocalUserStory(TestSharedTbmState))
us.AddUserStory(DummyLocalUserStory(TestSharedTbmState))
us.AddUserStory(DummyLocalUserStory(TestSharedTbmState))
story_runner.Run(
test, us, self.expectations, self.options, self.results)
self.assertEquals(0, len(self.results.failures))
self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results))
def testAppCrashThenRaiseInTearDownFatal(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class DidRunTestError(Exception):
pass
class TestTearDownSharedUserStoryState(TestSharedPageState):
def TearDownState(self, results):
self._test.DidRunTest('app', results)
class Test(page_test.PageTest):
def __init__(self, *args):
super(Test, self).__init__(*args)
self.run_count = 0
self._unit_test_events = [] # track what was called when
def RunPage(self, *_):
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 0:
self._unit_test_events.append('app-crash')
raise exceptions.AppCrashException
def ValidateAndMeasurePage(self, page, tab, results):
pass
def DidRunTest(self, _, __):
self._unit_test_events.append('did-run-test')
raise DidRunTestError
us.AddUserStory(DummyLocalUserStory(TestTearDownSharedUserStoryState))
us.AddUserStory(DummyLocalUserStory(TestTearDownSharedUserStoryState))
test = Test()
with self.assertRaises(DidRunTestError):
user_story_runner.Run(test, us, self.expectations, self.options,
self.results)
self.assertEqual(['app-crash', 'did-run-test'], test._unit_test_events)
# The AppCrashException gets added as a failure.
self.assertEquals(1, len(self.results.failures))
def testTearDownIsCalledOnceForEachUserStoryGroupWithPageSetRepeat(self):
self.options.pageset_repeat = 3
us = user_story_set.UserStorySet()
fooz_init_call_counter = [0]
fooz_tear_down_call_counter = [0]
barz_init_call_counter = [0]
barz_tear_down_call_counter = [0]
class FoozUserStoryState(FooUserStoryState):
def __init__(self, test, options, user_story_setz):
super(FoozUserStoryState,
self).__init__(test, options, user_story_setz)
fooz_init_call_counter[0] += 1
def TearDownState(self, _results):
fooz_tear_down_call_counter[0] += 1
class BarzUserStoryState(BarUserStoryState):
def __init__(self, test, options, user_story_setz):
super(BarzUserStoryState,
self).__init__(test, options, user_story_setz)
barz_init_call_counter[0] += 1
def TearDownState(self, _results):
barz_tear_down_call_counter[0] += 1
us.AddUserStory(user_story.UserStory(FoozUserStoryState))
us.AddUserStory(user_story.UserStory(FoozUserStoryState))
us.AddUserStory(user_story.UserStory(BarzUserStoryState))
us.AddUserStory(user_story.UserStory(BarzUserStoryState))
user_story_runner.Run(DummyTest(), us, self.expectations, self.options,
self.results)
self.assertEquals(0, len(self.results.failures))
self.assertEquals(12, GetNumberOfSuccessfulPageRuns(self.results))
self.assertEquals(1, fooz_init_call_counter[0])
self.assertEquals(1, fooz_tear_down_call_counter[0])
self.assertEquals(1, barz_init_call_counter[0])
self.assertEquals(1, barz_tear_down_call_counter[0])
def setUp(self):
self.user_story_set = user_story_set.UserStorySet()
self.user_story_set.AddUserStory(UserStoryFoo())
def __init__(self, story):
super(TestBenchmark, self).__init__()
self._uss = user_story_set.UserStorySet()
self._uss.AddUserStory(story)
def testRemoveWithEmptySetRaises(self):
uss = user_story_set.UserStorySet()
foo_story = UserStoryFoo()
with self.assertRaises(ValueError):
uss.RemoveUserStory(foo_story)
