def _RunPage(test, page, state, expectation, results):
if expectation == 'skip':
logging.debug('Skipping test: Skip expectation for %s', page.url)
results.AddValue(skip.SkipValue(page, 'Skipped by test expectations'))
return
logging.info('Running %s', page.url)
page_state = PageState(page, test.TabForPage(page, state.browser))
def ProcessError():
if expectation == 'fail':
msg = 'Expected exception while running %s' % page.url
else:
msg = 'Exception while running %s' % page.url
results.AddValue(failure.FailureValue(page, sys.exc_info()))
exception_formatter.PrintFormattedException(msg=msg)
try:
page_state.PreparePage(test)
page_state.ImplicitPageNavigation(test)
test.RunPage(page, page_state.tab, results)
util.CloseConnections(page_state.tab)
except page_test.TestNotSupportedOnPlatformFailure:
raise
except page_test.Failure:
if expectation == 'fail':
exception_formatter.PrintFormattedException(
msg='Expected failure while running %s' % page.url)
else:
exception_formatter.PrintFormattedException(
msg='Failure while running %s' % page.url)
results.AddValue(failure.FailureValue(page, sys.exc_info()))
except (util.TimeoutException, exceptions.LoginException,
exceptions.ProfilingException):
ProcessError()
except (exceptions.TabCrashException, exceptions.BrowserGoneException):
ProcessError()
# Run() catches these exceptions to relaunch the tab/browser, so re-raise.
raise
except page_action.PageActionNotSupported as e:
results.AddValue(
skip.SkipValue(page, 'Unsupported page action: %s' % e))
except Exception:
exception_formatter.PrintFormattedException(
msg='Unhandled exception while running %s' % page.url)
results.AddValue(failure.FailureValue(page, sys.exc_info()))
else:
if expectation == 'fail':
logging.warning('%s was expected to fail, but passed.\n', page.url)
finally:
page_state.CleanUpPage(test)
def ProcessError():
if expectation == 'fail':
msg = 'Expected exception while running %s' % page.url
else:
msg = 'Exception while running %s' % page.url
results.AddValue(failure.FailureValue(page, sys.exc_info()))
exception_formatter.PrintFormattedException(msg=msg)
def testStreamingResults(self):
test_story_set = _MakeStorySet()
results = page_test_results.PageTestResults(
progress_reporter=self._reporter)
exc_info = self.CreateException()
results.WillRunPage(test_story_set.stories[0])
self._fake_timer.SetTime(0.007)
results.DidRunPage(test_story_set.stories[0])
expected = ('[ RUN ] http://www.foo.com/\n'
'[ OK ] http://www.foo.com/ (7 ms)\n')
self.assertEquals(expected, ''.join(self._output_stream.output_data))
results.WillRunPage(test_story_set.stories[1])
self._fake_timer.SetTime(0.009)
exception_trace = ''.join(traceback.format_exception(*exc_info))
results.AddValue(
failure.FailureValue(test_story_set.stories[1], exc_info))
results.DidRunPage(test_story_set.stories[1])
expected = ('[ RUN ] http://www.foo.com/\n'
'[ OK ] http://www.foo.com/ (7 ms)\n'
'[ RUN ] http://www.bar.com/\n'
'%s\n'
'[ FAILED ] http://www.bar.com/ (2 ms)\n' %
exception_trace)
def ProcessError():
if expectation == 'fail':
msg = 'Expected exception while running %s' % user_story.display_name
exception_formatter.PrintFormattedException(msg=msg)
else:
msg = 'Exception while running %s' % user_story.display_name
results.AddValue(failure.FailureValue(user_story, sys.exc_info()))
def testName(self):
v0 = failure.FailureValue.FromMessage(self.pages[0], 'Failure')
self.assertEquals('Exception', v0.name)
try:
raise NotImplementedError()
except Exception:
v1 = failure.FailureValue(self.pages[0], sys.exc_info())
self.assertEquals('NotImplementedError', v1.name)
def Fail(self, exc_info_or_message):
if isinstance(exc_info_or_message, basestring):
self.AddValue(
failure.FailureValue.FromMessage(self.story,
exc_info_or_message))
else:
self.AddValue(failure.FailureValue(self.story,
exc_info_or_message))
def testPassAndFailedPages(self):
test_page_set = _MakePageSet()
results = page_test_results.PageTestResults(
progress_reporter=self._reporter)
exc_info = self.CreateException()
results.WillRunPage(test_page_set.pages[0])
self._mock_timer.SetTime(0.007)
results.DidRunPage(test_page_set.pages[0])
results.WillRunPage(test_page_set.pages[1])
self._mock_timer.SetTime(0.009)
results.AddValue(failure.FailureValue(test_page_set.pages[1],
exc_info))
results.DidRunPage(test_page_set.pages[1])
results.WillRunPage(test_page_set.pages[2])
self._mock_timer.SetTime(0.015)
results.AddValue(failure.FailureValue(test_page_set.pages[2],
exc_info))
results.DidRunPage(test_page_set.pages[2])
results.WillRunPage(test_page_set.pages[3])
self._mock_timer.SetTime(0.020)
results.DidRunPage(test_page_set.pages[3])
results.PrintSummary()
exception_trace = ''.join(traceback.format_exception(*exc_info))
expected = ('[ RUN ] http://www.foo.com/\n'
'[ OK ] http://www.foo.com/ (7 ms)\n'
'[ RUN ] http://www.bar.com/\n'
'%s\n'
'[ FAILED ] http://www.bar.com/ (2 ms)\n'
'[ RUN ] http://www.baz.com/\n'
'%s\n'
'[ FAILED ] http://www.baz.com/ (6 ms)\n'
'[ RUN ] http://www.roz.com/\n'
'[ OK ] http://www.roz.com/ (5 ms)\n'
'[ PASSED ] 2 tests.\n'
'[ FAILED ] 2 tests, listed below:\n'
'[ FAILED ] http://www.bar.com/\n'
'[ FAILED ] http://www.baz.com/\n\n'
'2 FAILED TESTS\n\n' % (exception_trace, exception_trace))
self.assertEquals(expected, ''.join(self._output_stream.output_data))
def Fail(self, exc_info_or_message, handleable=True):
assert self._current_page_run, 'Not currently running test.'
if not handleable:
description = 'Unhandleable exception raised.'
else:
description = None
if isinstance(exc_info_or_message, basestring):
self.AddValue(failure.FailureValue.FromMessage(
self.current_page, exc_info_or_message))
else:
self.AddValue(failure.FailureValue(
self.current_page, exc_info_or_message, description))
def ProcessError(exc=None, description=None):
state.DumpStateUponFailure(story, results)
# Dump app crash, if present
if exc:
if isinstance(exc, exceptions.AppCrashException):
minidump_path = exc.minidump_path
if minidump_path:
results.AddArtifact(story.name, 'minidump', minidump_path)
# Note: adding the FailureValue to the results object also normally
# cause the progress_reporter to log it in the output.
results.AddValue(failure.FailureValue(story, sys.exc_info(), description))
def _RunUserStoryAndProcessErrorIfNeeded(expectations, user_story, results,
state):
def ProcessError():
if expectation == 'fail':
msg = 'Expected exception while running %s' % user_story.display_name
exception_formatter.PrintFormattedException(msg=msg)
else:
msg = 'Exception while running %s' % user_story.display_name
results.AddValue(failure.FailureValue(user_story, sys.exc_info()))
try:
expectation = None
state.WillRunUserStory(user_story)
expectation, skip_value = state.GetTestExpectationAndSkipValue(
expectations)
if expectation == 'skip':
assert skip_value
results.AddValue(skip_value)
return
state.RunUserStory(results)
except (page_test.Failure, exceptions.TimeoutException,
exceptions.LoginException, exceptions.ProfilingException):
ProcessError()
except exceptions.Error:
ProcessError()
raise
except page_action.PageActionNotSupported as e:
results.AddValue(
skip.SkipValue(user_story, 'Unsupported page action: %s' % e))
except Exception:
results.AddValue(
failure.FailureValue(user_story, sys.exc_info(),
'Unhandlable exception raised.'))
raise
else:
if expectation == 'fail':
logging.warning('%s was expected to fail, but passed.\n',
user_story.display_name)
finally:
has_existing_exception = sys.exc_info() is not None
try:
state.DidRunUserStory(results)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from DidRunUserStory: ')
def testFailures(self):
results = page_test_results.PageTestResults()
results.WillRunPage(self.pages[0])
results.AddValue(
failure.FailureValue(self.pages[0], self.CreateException()))
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.DidRunPage(self.pages[1])
self.assertEqual(set([self.pages[0]]), results.pages_that_failed)
self.assertEqual(set([self.pages[1]]), results.pages_that_succeeded)
self.assertEqual(2, len(results.all_page_runs))
self.assertTrue(results.all_page_runs[0].failed)
self.assertTrue(results.all_page_runs[1].ok)
def _RunStoryAndProcessErrorIfNeeded(story, results, state, test):
def ProcessError():
results.AddValue(failure.FailureValue(story, sys.exc_info()))
try:
if isinstance(test, story_test.StoryTest):
test.WillRunStory(state.platform)
state.WillRunStory(story)
if not state.CanRunStory(story):
results.AddValue(
skip.SkipValue(
story, 'Skipped because story is not supported '
'(SharedState.CanRunStory() returns False).'))
return
state.RunStory(results)
if isinstance(test, story_test.StoryTest):
test.Measure(state.platform, results)
except (page_test.Failure, exceptions.TimeoutException,
exceptions.LoginException, exceptions.ProfilingException):
ProcessError()
except exceptions.Error:
ProcessError()
raise
except page_action.PageActionNotSupported as e:
results.AddValue(
skip.SkipValue(story, 'Unsupported page action: %s' % e))
except Exception:
results.AddValue(
failure.FailureValue(story, sys.exc_info(),
'Unhandlable exception raised.'))
raise
finally:
has_existing_exception = (sys.exc_info() != (None, None, None))
try:
state.DidRunStory(results)
# if state.DidRunStory raises exception, things are messed up badly and we
# do not need to run test.DidRunStory at that point.
if isinstance(test, story_test.StoryTest):
test.DidRunStory(state.platform)
else:
test.DidRunPage(state.platform)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception raised when cleaning story run: ')
def testSingleFailedPage(self):
test_story_set = _MakeStorySet()
results = page_test_results.PageTestResults(
progress_reporter=self._reporter)
results.WillRunPage(test_story_set.stories[0])
exc_info = self.CreateException()
results.AddValue(failure.FailureValue(test_story_set.stories[0], exc_info))
results.DidRunPage(test_story_set.stories[0])
results.PrintSummary()
exception_trace = ''.join(traceback.format_exception(*exc_info))
expected = ('[ RUN ] http://www.foo.com/\n'
'%s\n'
'[ FAILED ] http://www.foo.com/ (0 ms)\n'
'[ PASSED ] 0 tests.\n'
'[ FAILED ] 1 test, listed below:\n'
'[ FAILED ] http://www.foo.com/\n\n'
'1 FAILED TEST\n\n' % exception_trace)
self.assertEquals(expected, ''.join(self._output_stream.output_data))
def testPassesNoSkips(self):
results = page_test_results.PageTestResults()
results.WillRunPage(self.pages[0])
results.AddValue(
failure.FailureValue(self.pages[0], self.CreateException()))
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.DidRunPage(self.pages[1])
results.WillRunPage(self.pages[2])
results.AddValue(skip.SkipValue(self.pages[2], 'testing reason'))
results.DidRunPage(self.pages[2])
self.assertEqual(set([self.pages[0]]), results.pages_that_failed)
self.assertEqual(set([self.pages[1], self.pages[2]]),
results.pages_that_succeeded)
self.assertEqual(set([self.pages[1]]),
results.pages_that_succeeded_and_not_skipped)
self.assertEqual(3, len(results.all_page_runs))
self.assertTrue(results.all_page_runs[0].failed)
self.assertTrue(results.all_page_runs[1].ok)
self.assertTrue(results.all_page_runs[2].skipped)
def Run(test, user_story_set, expectations, finder_options, results):
"""Runs a given test against a given page_set with the given options."""
test.ValidatePageSet(user_story_set)
# Reorder page set based on options.
user_stories = _ShuffleAndFilterUserStorySet(user_story_set,
finder_options)
if (not finder_options.use_live_sites
and finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD
and
# TODO(nednguyen): also handle these logic for user_story_set in next
# patch.
isinstance(user_story_set, page_set_module.PageSet)):
_UpdateUserStoryArchivesIfChanged(user_story_set)
if not _CheckArchives(user_story_set.archive_data_file,
user_story_set.wpr_archive_info,
user_story_set.pages):
return
for user_story in list(user_stories):
if not test.CanRunForPage(user_story):
results.WillRunPage(user_story)
logging.debug('Skipping test: it cannot run for %s',
user_story.display_name)
results.AddValue(skip.SkipValue(user_story, 'Test cannot run'))
results.DidRunPage(user_story)
user_stories.remove(user_story)
if not user_stories:
return
user_story_with_discarded_first_results = set()
max_failures = finder_options.max_failures # command-line gets priority
if max_failures is None:
max_failures = test.max_failures # may be None
user_story_groups = GetUserStoryGroupsWithSameSharedUserStoryClass(
user_stories)
test.WillRunTest(finder_options)
for group in user_story_groups:
state = None
try:
state = group.shared_user_story_state_class(
test, finder_options, user_story_set)
for _ in xrange(finder_options.pageset_repeat):
for user_story in group.user_stories:
if test.IsExiting():
break
for _ in xrange(finder_options.page_repeat):
results.WillRunPage(user_story)
try:
_WaitForThermalThrottlingIfNeeded(state.platform)
_RunUserStoryAndProcessErrorIfNeeded(
test, expectations, user_story, results, state)
except Exception:
# Tear down & restart the state for unhandled exceptions thrown by
# _RunUserStoryAndProcessErrorIfNeeded.
results.AddValue(
failure.FailureValue(user_story,
sys.exc_info()))
state.TearDownState(results)
state = group.shared_user_story_state_class(
test, finder_options, user_story_set)
finally:
_CheckThermalThrottling(state.platform)
discard_run = (
test.discard_first_result and user_story
not in user_story_with_discarded_first_results)
if discard_run:
user_story_with_discarded_first_results.add(
user_story)
results.DidRunPage(user_story,
discard_run=discard_run)
if max_failures is not None and len(
results.failures) > max_failures:
logging.error('Too many failures. Aborting.')
test.RequestExit()
finally:
if state:
state.TearDownState(results)
def ProcessError(description=None):
state.DumpStateUponFailure(story, results)
results.AddValue(
failure.FailureValue(story, sys.exc_info(), description))
def ProcessError(): results.AddValue(failure.FailureValue(story, sys.exc_info()))
def ProcessError(description=None):
state.DumpStateUponFailure(story, results)
# Note: adding the FailureValue to the results object also normally
# cause the progress_reporter to log it in the output.
results.AddValue(
failure.FailureValue(story, sys.exc_info(), description))
def testPassAndFailedPages(self):
test_story_set = _MakeStorySet()
results = page_test_results.PageTestResults(
progress_reporter=self._reporter)
exc_info = self.CreateException()
results.WillRunPage(test_story_set.stories[0])
self._fake_timer.SetTime(0.007)
results.DidRunPage(test_story_set.stories[0])
results.WillRunPage(test_story_set.stories[1])
self._fake_timer.SetTime(0.009)
results.AddValue(
failure.FailureValue(test_story_set.stories[1], exc_info))
results.DidRunPage(test_story_set.stories[1])
results.WillRunPage(test_story_set.stories[2])
self._fake_timer.SetTime(0.015)
results.AddValue(
failure.FailureValue(test_story_set.stories[2], exc_info))
results.DidRunPage(test_story_set.stories[2])
results.WillRunPage(test_story_set.stories[3])
self._fake_timer.SetTime(0.020)
results.DidRunPage(test_story_set.stories[3])
results.WillRunPage(test_story_set.stories[4])
self._fake_timer.SetTime(0.025)
results.DidRunPage(test_story_set.stories[4])
results.WillRunPage(test_story_set.stories[5])
self._fake_timer.SetTime(0.030)
results.AddValue(
failure.FailureValue(test_story_set.stories[5], exc_info))
results.DidRunPage(test_story_set.stories[5])
results.PrintSummary()
exception_trace = ''.join(traceback.format_exception(*exc_info))
expected = ("[ RUN ] http://www.foo.com/\n"
"[ OK ] http://www.foo.com/ (7 ms)\n"
"[ RUN ] http://www.bar.com/\n"
"%s\n"
"[ FAILED ] http://www.bar.com/ (2 ms)\n"
"[ RUN ] http://www.baz.com/\n"
"%s\n"
"[ FAILED ] http://www.baz.com/ (6 ms)\n"
"[ RUN ] http://www.roz.com/\n"
"[ OK ] http://www.roz.com/ (5 ms)\n"
"[ RUN ] http://www.fus.com/@{'1': '2'}\n"
"[ OK ] http://www.fus.com/@{'1': '2'} (5 ms)\n"
"[ RUN ] http://www.ro.com/@{'1': '2'}\n"
"%s\n"
"[ FAILED ] http://www.ro.com/@{'1': '2'} (5 ms)\n"
"[ PASSED ] 3 tests.\n"
"[ FAILED ] 3 tests, listed below:\n"
"[ FAILED ] http://www.bar.com/\n"
"[ FAILED ] http://www.baz.com/\n"
"[ FAILED ] http://www.ro.com/@{'1': '2'}\n\n"
"3 FAILED TESTS\n\n" %
(exception_trace, exception_trace, exception_trace))
self.assertEquals(expected, ''.join(self._output_stream.output_data))