def _RunPage(test, page, state, expectation, results): if expectation == 'skip': logging.debug('Skipping test: Skip expectation for %s', page.url) results.AddValue(skip.SkipValue(page, 'Skipped by test expectations')) return logging.info('Running %s', page.url) page_state = PageState(page, test.TabForPage(page, state.browser)) def ProcessError(): if expectation == 'fail': msg = 'Expected exception while running %s' % page.url else: msg = 'Exception while running %s' % page.url results.AddValue(failure.FailureValue(page, sys.exc_info())) exception_formatter.PrintFormattedException(msg=msg) try: page_state.PreparePage(test) page_state.ImplicitPageNavigation(test) test.RunPage(page, page_state.tab, results) util.CloseConnections(page_state.tab) except page_test.TestNotSupportedOnPlatformFailure: raise except page_test.Failure: if expectation == 'fail': exception_formatter.PrintFormattedException( msg='Expected failure while running %s' % page.url) else: exception_formatter.PrintFormattedException( msg='Failure while running %s' % page.url) results.AddValue(failure.FailureValue(page, sys.exc_info())) except (util.TimeoutException, exceptions.LoginException, exceptions.ProfilingException): ProcessError() except (exceptions.TabCrashException, exceptions.BrowserGoneException): ProcessError() # Run() catches these exceptions to relaunch the tab/browser, so re-raise. raise except page_action.PageActionNotSupported as e: results.AddValue( skip.SkipValue(page, 'Unsupported page action: %s' % e)) except Exception: exception_formatter.PrintFormattedException( msg='Unhandled exception while running %s' % page.url) results.AddValue(failure.FailureValue(page, sys.exc_info())) else: if expectation == 'fail': logging.warning('%s was expected to fail, but passed.\n', page.url) finally: page_state.CleanUpPage(test)
def ProcessError(): if expectation == 'fail': msg = 'Expected exception while running %s' % page.url else: msg = 'Exception while running %s' % page.url results.AddValue(failure.FailureValue(page, sys.exc_info())) exception_formatter.PrintFormattedException(msg=msg)
def testStreamingResults(self): test_story_set = _MakeStorySet() results = page_test_results.PageTestResults( progress_reporter=self._reporter) exc_info = self.CreateException() results.WillRunPage(test_story_set.stories[0]) self._fake_timer.SetTime(0.007) results.DidRunPage(test_story_set.stories[0]) expected = ('[ RUN ] http://www.foo.com/\n' '[ OK ] http://www.foo.com/ (7 ms)\n') self.assertEquals(expected, ''.join(self._output_stream.output_data)) results.WillRunPage(test_story_set.stories[1]) self._fake_timer.SetTime(0.009) exception_trace = ''.join(traceback.format_exception(*exc_info)) results.AddValue( failure.FailureValue(test_story_set.stories[1], exc_info)) results.DidRunPage(test_story_set.stories[1]) expected = ('[ RUN ] http://www.foo.com/\n' '[ OK ] http://www.foo.com/ (7 ms)\n' '[ RUN ] http://www.bar.com/\n' '%s\n' '[ FAILED ] http://www.bar.com/ (2 ms)\n' % exception_trace)
def ProcessError(): if expectation == 'fail': msg = 'Expected exception while running %s' % user_story.display_name exception_formatter.PrintFormattedException(msg=msg) else: msg = 'Exception while running %s' % user_story.display_name results.AddValue(failure.FailureValue(user_story, sys.exc_info()))
def testName(self): v0 = failure.FailureValue.FromMessage(self.pages[0], 'Failure') self.assertEquals('Exception', v0.name) try: raise NotImplementedError() except Exception: v1 = failure.FailureValue(self.pages[0], sys.exc_info()) self.assertEquals('NotImplementedError', v1.name)
def Fail(self, exc_info_or_message): if isinstance(exc_info_or_message, basestring): self.AddValue( failure.FailureValue.FromMessage(self.story, exc_info_or_message)) else: self.AddValue(failure.FailureValue(self.story, exc_info_or_message))
def testPassAndFailedPages(self): test_page_set = _MakePageSet() results = page_test_results.PageTestResults( progress_reporter=self._reporter) exc_info = self.CreateException() results.WillRunPage(test_page_set.pages[0]) self._mock_timer.SetTime(0.007) results.DidRunPage(test_page_set.pages[0]) results.WillRunPage(test_page_set.pages[1]) self._mock_timer.SetTime(0.009) results.AddValue(failure.FailureValue(test_page_set.pages[1], exc_info)) results.DidRunPage(test_page_set.pages[1]) results.WillRunPage(test_page_set.pages[2]) self._mock_timer.SetTime(0.015) results.AddValue(failure.FailureValue(test_page_set.pages[2], exc_info)) results.DidRunPage(test_page_set.pages[2]) results.WillRunPage(test_page_set.pages[3]) self._mock_timer.SetTime(0.020) results.DidRunPage(test_page_set.pages[3]) results.PrintSummary() exception_trace = ''.join(traceback.format_exception(*exc_info)) expected = ('[ RUN ] http://www.foo.com/\n' '[ OK ] http://www.foo.com/ (7 ms)\n' '[ RUN ] http://www.bar.com/\n' '%s\n' '[ FAILED ] http://www.bar.com/ (2 ms)\n' '[ RUN ] http://www.baz.com/\n' '%s\n' '[ FAILED ] http://www.baz.com/ (6 ms)\n' '[ RUN ] http://www.roz.com/\n' '[ OK ] http://www.roz.com/ (5 ms)\n' '[ PASSED ] 2 tests.\n' '[ FAILED ] 2 tests, listed below:\n' '[ FAILED ] http://www.bar.com/\n' '[ FAILED ] http://www.baz.com/\n\n' '2 FAILED TESTS\n\n' % (exception_trace, exception_trace)) self.assertEquals(expected, ''.join(self._output_stream.output_data))
def Fail(self, exc_info_or_message, handleable=True): assert self._current_page_run, 'Not currently running test.' if not handleable: description = 'Unhandleable exception raised.' else: description = None if isinstance(exc_info_or_message, basestring): self.AddValue(failure.FailureValue.FromMessage( self.current_page, exc_info_or_message)) else: self.AddValue(failure.FailureValue( self.current_page, exc_info_or_message, description))
def ProcessError(exc=None, description=None): state.DumpStateUponFailure(story, results) # Dump app crash, if present if exc: if isinstance(exc, exceptions.AppCrashException): minidump_path = exc.minidump_path if minidump_path: results.AddArtifact(story.name, 'minidump', minidump_path) # Note: adding the FailureValue to the results object also normally # cause the progress_reporter to log it in the output. results.AddValue(failure.FailureValue(story, sys.exc_info(), description))
def _RunUserStoryAndProcessErrorIfNeeded(expectations, user_story, results, state): def ProcessError(): if expectation == 'fail': msg = 'Expected exception while running %s' % user_story.display_name exception_formatter.PrintFormattedException(msg=msg) else: msg = 'Exception while running %s' % user_story.display_name results.AddValue(failure.FailureValue(user_story, sys.exc_info())) try: expectation = None state.WillRunUserStory(user_story) expectation, skip_value = state.GetTestExpectationAndSkipValue( expectations) if expectation == 'skip': assert skip_value results.AddValue(skip_value) return state.RunUserStory(results) except (page_test.Failure, exceptions.TimeoutException, exceptions.LoginException, exceptions.ProfilingException): ProcessError() except exceptions.Error: ProcessError() raise except page_action.PageActionNotSupported as e: results.AddValue( skip.SkipValue(user_story, 'Unsupported page action: %s' % e)) except Exception: results.AddValue( failure.FailureValue(user_story, sys.exc_info(), 'Unhandlable exception raised.')) raise else: if expectation == 'fail': logging.warning('%s was expected to fail, but passed.\n', user_story.display_name) finally: has_existing_exception = sys.exc_info() is not None try: state.DidRunUserStory(results) except Exception: if not has_existing_exception: raise # Print current exception and propagate existing exception. exception_formatter.PrintFormattedException( msg='Exception from DidRunUserStory: ')
def testFailures(self): results = page_test_results.PageTestResults() results.WillRunPage(self.pages[0]) results.AddValue( failure.FailureValue(self.pages[0], self.CreateException())) results.DidRunPage(self.pages[0]) results.WillRunPage(self.pages[1]) results.DidRunPage(self.pages[1]) self.assertEqual(set([self.pages[0]]), results.pages_that_failed) self.assertEqual(set([self.pages[1]]), results.pages_that_succeeded) self.assertEqual(2, len(results.all_page_runs)) self.assertTrue(results.all_page_runs[0].failed) self.assertTrue(results.all_page_runs[1].ok)
def _RunStoryAndProcessErrorIfNeeded(story, results, state, test): def ProcessError(): results.AddValue(failure.FailureValue(story, sys.exc_info())) try: if isinstance(test, story_test.StoryTest): test.WillRunStory(state.platform) state.WillRunStory(story) if not state.CanRunStory(story): results.AddValue( skip.SkipValue( story, 'Skipped because story is not supported ' '(SharedState.CanRunStory() returns False).')) return state.RunStory(results) if isinstance(test, story_test.StoryTest): test.Measure(state.platform, results) except (page_test.Failure, exceptions.TimeoutException, exceptions.LoginException, exceptions.ProfilingException): ProcessError() except exceptions.Error: ProcessError() raise except page_action.PageActionNotSupported as e: results.AddValue( skip.SkipValue(story, 'Unsupported page action: %s' % e)) except Exception: results.AddValue( failure.FailureValue(story, sys.exc_info(), 'Unhandlable exception raised.')) raise finally: has_existing_exception = (sys.exc_info() != (None, None, None)) try: state.DidRunStory(results) # if state.DidRunStory raises exception, things are messed up badly and we # do not need to run test.DidRunStory at that point. if isinstance(test, story_test.StoryTest): test.DidRunStory(state.platform) else: test.DidRunPage(state.platform) except Exception: if not has_existing_exception: raise # Print current exception and propagate existing exception. exception_formatter.PrintFormattedException( msg='Exception raised when cleaning story run: ')
def testSingleFailedPage(self): test_story_set = _MakeStorySet() results = page_test_results.PageTestResults( progress_reporter=self._reporter) results.WillRunPage(test_story_set.stories[0]) exc_info = self.CreateException() results.AddValue(failure.FailureValue(test_story_set.stories[0], exc_info)) results.DidRunPage(test_story_set.stories[0]) results.PrintSummary() exception_trace = ''.join(traceback.format_exception(*exc_info)) expected = ('[ RUN ] http://www.foo.com/\n' '%s\n' '[ FAILED ] http://www.foo.com/ (0 ms)\n' '[ PASSED ] 0 tests.\n' '[ FAILED ] 1 test, listed below:\n' '[ FAILED ] http://www.foo.com/\n\n' '1 FAILED TEST\n\n' % exception_trace) self.assertEquals(expected, ''.join(self._output_stream.output_data))
def testPassesNoSkips(self): results = page_test_results.PageTestResults() results.WillRunPage(self.pages[0]) results.AddValue( failure.FailureValue(self.pages[0], self.CreateException())) results.DidRunPage(self.pages[0]) results.WillRunPage(self.pages[1]) results.DidRunPage(self.pages[1]) results.WillRunPage(self.pages[2]) results.AddValue(skip.SkipValue(self.pages[2], 'testing reason')) results.DidRunPage(self.pages[2]) self.assertEqual(set([self.pages[0]]), results.pages_that_failed) self.assertEqual(set([self.pages[1], self.pages[2]]), results.pages_that_succeeded) self.assertEqual(set([self.pages[1]]), results.pages_that_succeeded_and_not_skipped) self.assertEqual(3, len(results.all_page_runs)) self.assertTrue(results.all_page_runs[0].failed) self.assertTrue(results.all_page_runs[1].ok) self.assertTrue(results.all_page_runs[2].skipped)
def Run(test, user_story_set, expectations, finder_options, results): """Runs a given test against a given page_set with the given options.""" test.ValidatePageSet(user_story_set) # Reorder page set based on options. user_stories = _ShuffleAndFilterUserStorySet(user_story_set, finder_options) if (not finder_options.use_live_sites and finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD and # TODO(nednguyen): also handle these logic for user_story_set in next # patch. isinstance(user_story_set, page_set_module.PageSet)): _UpdateUserStoryArchivesIfChanged(user_story_set) if not _CheckArchives(user_story_set.archive_data_file, user_story_set.wpr_archive_info, user_story_set.pages): return for user_story in list(user_stories): if not test.CanRunForPage(user_story): results.WillRunPage(user_story) logging.debug('Skipping test: it cannot run for %s', user_story.display_name) results.AddValue(skip.SkipValue(user_story, 'Test cannot run')) results.DidRunPage(user_story) user_stories.remove(user_story) if not user_stories: return user_story_with_discarded_first_results = set() max_failures = finder_options.max_failures # command-line gets priority if max_failures is None: max_failures = test.max_failures # may be None user_story_groups = GetUserStoryGroupsWithSameSharedUserStoryClass( user_stories) test.WillRunTest(finder_options) for group in user_story_groups: state = None try: state = group.shared_user_story_state_class( test, finder_options, user_story_set) for _ in xrange(finder_options.pageset_repeat): for user_story in group.user_stories: if test.IsExiting(): break for _ in xrange(finder_options.page_repeat): results.WillRunPage(user_story) try: _WaitForThermalThrottlingIfNeeded(state.platform) _RunUserStoryAndProcessErrorIfNeeded( test, expectations, user_story, results, state) except Exception: # Tear down & restart the state for unhandled exceptions thrown by # _RunUserStoryAndProcessErrorIfNeeded. results.AddValue( failure.FailureValue(user_story, sys.exc_info())) state.TearDownState(results) state = group.shared_user_story_state_class( test, finder_options, user_story_set) finally: _CheckThermalThrottling(state.platform) discard_run = ( test.discard_first_result and user_story not in user_story_with_discarded_first_results) if discard_run: user_story_with_discarded_first_results.add( user_story) results.DidRunPage(user_story, discard_run=discard_run) if max_failures is not None and len( results.failures) > max_failures: logging.error('Too many failures. Aborting.') test.RequestExit() finally: if state: state.TearDownState(results)
def ProcessError(description=None): state.DumpStateUponFailure(story, results) results.AddValue( failure.FailureValue(story, sys.exc_info(), description))
def ProcessError(): results.AddValue(failure.FailureValue(story, sys.exc_info()))
def ProcessError(description=None): state.DumpStateUponFailure(story, results) # Note: adding the FailureValue to the results object also normally # cause the progress_reporter to log it in the output. results.AddValue( failure.FailureValue(story, sys.exc_info(), description))
def testPassAndFailedPages(self): test_story_set = _MakeStorySet() results = page_test_results.PageTestResults( progress_reporter=self._reporter) exc_info = self.CreateException() results.WillRunPage(test_story_set.stories[0]) self._fake_timer.SetTime(0.007) results.DidRunPage(test_story_set.stories[0]) results.WillRunPage(test_story_set.stories[1]) self._fake_timer.SetTime(0.009) results.AddValue( failure.FailureValue(test_story_set.stories[1], exc_info)) results.DidRunPage(test_story_set.stories[1]) results.WillRunPage(test_story_set.stories[2]) self._fake_timer.SetTime(0.015) results.AddValue( failure.FailureValue(test_story_set.stories[2], exc_info)) results.DidRunPage(test_story_set.stories[2]) results.WillRunPage(test_story_set.stories[3]) self._fake_timer.SetTime(0.020) results.DidRunPage(test_story_set.stories[3]) results.WillRunPage(test_story_set.stories[4]) self._fake_timer.SetTime(0.025) results.DidRunPage(test_story_set.stories[4]) results.WillRunPage(test_story_set.stories[5]) self._fake_timer.SetTime(0.030) results.AddValue( failure.FailureValue(test_story_set.stories[5], exc_info)) results.DidRunPage(test_story_set.stories[5]) results.PrintSummary() exception_trace = ''.join(traceback.format_exception(*exc_info)) expected = ("[ RUN ] http://www.foo.com/\n" "[ OK ] http://www.foo.com/ (7 ms)\n" "[ RUN ] http://www.bar.com/\n" "%s\n" "[ FAILED ] http://www.bar.com/ (2 ms)\n" "[ RUN ] http://www.baz.com/\n" "%s\n" "[ FAILED ] http://www.baz.com/ (6 ms)\n" "[ RUN ] http://www.roz.com/\n" "[ OK ] http://www.roz.com/ (5 ms)\n" "[ RUN ] http://www.fus.com/@{'1': '2'}\n" "[ OK ] http://www.fus.com/@{'1': '2'} (5 ms)\n" "[ RUN ] http://www.ro.com/@{'1': '2'}\n" "%s\n" "[ FAILED ] http://www.ro.com/@{'1': '2'} (5 ms)\n" "[ PASSED ] 3 tests.\n" "[ FAILED ] 3 tests, listed below:\n" "[ FAILED ] http://www.bar.com/\n" "[ FAILED ] http://www.baz.com/\n" "[ FAILED ] http://www.ro.com/@{'1': '2'}\n\n" "3 FAILED TESTS\n\n" % (exception_trace, exception_trace, exception_trace)) self.assertEquals(expected, ''.join(self._output_stream.output_data))