def _RunPage(test, page, state, expectation, results, finder_options): if expectation == 'skip': logging.debug('Skipping test: Skip expectation for %s', page.url) results.AddSkip(page, 'Skipped by test expectations') return logging.info('Running %s' % page.url) page_state = PageState(page, test.TabForPage(page, state.browser)) def ProcessError(): logging.error('%s:', page.url) exception_formatter.PrintFormattedException() if expectation == 'fail': logging.info('Error was expected\n') results.AddSuccess(page) else: results.AddError(page, sys.exc_info()) try: page_state.PreparePage(test) if state.repeat_state.ShouldNavigate( finder_options.skip_navigate_on_repeat): page_state.ImplicitPageNavigation(test) test.RunPage(page, page_state.tab, results) util.CloseConnections(page_state.tab) except page_test.TestNotSupportedOnPlatformFailure: raise except page_test.Failure: if expectation == 'fail': logging.info('%s:', page.url) exception_formatter.PrintFormattedException() logging.info('Failure was expected\n') results.AddSuccess(page) else: logging.warning('%s:', page.url) exception_formatter.PrintFormattedException() results.AddFailure(page, sys.exc_info()) except (util.TimeoutException, exceptions.LoginException, exceptions.ProfilingException): ProcessError() except (exceptions.TabCrashException, exceptions.BrowserGoneException): ProcessError() # Run() catches these exceptions to relaunch the tab/browser, so re-raise. raise except page_action.PageActionNotSupported as e: results.AddSkip(page, 'Unsupported page action: %s' % e) except Exception: logging.warning('While running %s', page.url) exception_formatter.PrintFormattedException() results.AddFailure(page, sys.exc_info()) else: if expectation == 'fail': logging.warning('%s was expected to fail, but passed.\n', page.url) results.AddSuccess(page) finally: page_state.CleanUpPage(test)
def _RunPage(test, page, state, expectation, results): if expectation == 'skip': logging.debug('Skipping test: Skip expectation for %s', page.url) results.AddValue(skip.SkipValue(page, 'Skipped by test expectations')) return logging.info('Running %s', page.url) page_state = PageState(page, test.TabForPage(page, state.browser)) def ProcessError(): if expectation == 'fail': msg = 'Expected exception while running %s' % page.url else: msg = 'Exception while running %s' % page.url results.AddValue(failure.FailureValue(page, sys.exc_info())) exception_formatter.PrintFormattedException(msg=msg) try: page_state.PreparePage(test) page_state.ImplicitPageNavigation(test) test.RunPage(page, page_state.tab, results) util.CloseConnections(page_state.tab) except page_test.TestNotSupportedOnPlatformFailure: raise except page_test.Failure: if expectation == 'fail': exception_formatter.PrintFormattedException( msg='Expected failure while running %s' % page.url) else: exception_formatter.PrintFormattedException( msg='Failure while running %s' % page.url) results.AddValue(failure.FailureValue(page, sys.exc_info())) except (util.TimeoutException, exceptions.LoginException, exceptions.ProfilingException): ProcessError() except (exceptions.TabCrashException, exceptions.BrowserGoneException): ProcessError() # Run() catches these exceptions to relaunch the tab/browser, so re-raise. raise except page_action.PageActionNotSupported as e: results.AddValue( skip.SkipValue(page, 'Unsupported page action: %s' % e)) except Exception: exception_formatter.PrintFormattedException( msg='Unhandled exception while running %s' % page.url) results.AddValue(failure.FailureValue(page, sys.exc_info())) else: if expectation == 'fail': logging.warning('%s was expected to fail, but passed.\n', page.url) finally: page_state.CleanUpPage(test)
def ProcessError(): if expectation == 'fail': msg = 'Expected exception while running %s' % user_story.display_name exception_formatter.PrintFormattedException(msg=msg) else: msg = 'Exception while running %s' % user_story.display_name results.AddValue(failure.FailureValue(user_story, sys.exc_info()))
def ProcessError(): if expectation == 'fail': msg = 'Expected exception while running %s' % page.url else: msg = 'Exception while running %s' % page.url results.AddValue(failure.FailureValue(page, sys.exc_info())) exception_formatter.PrintFormattedException(msg=msg)
def ProcessError(): logging.error('%s:', page.url) exception_formatter.PrintFormattedException() if expectation == 'fail': logging.info('Error was expected\n') results.AddSuccess(page) else: results.AddError(page, sys.exc_info())
def RunUserStory(self, results): try: self._PreparePage() self._ImplicitPageNavigation() self._test.RunPage(self._current_page, self._current_tab, results) except exceptions.Error: if self._test.is_multi_tab_test: # Avoid trying to recover from an unknown multi-tab state. exception_formatter.PrintFormattedException( msg='Telemetry Error during multi tab test:') raise page_test.MultiTabTestAppCrashError raise
def Run(self, finder_options): """Run this test with the given options. Returns: The number of failure values (up to 254) or 255 if there is an uncaught exception. """ self.CustomizeBrowserOptions(finder_options.browser_options) pt = self.CreatePageTest(finder_options) pt.__name__ = self.__class__.__name__ if hasattr(self, '_disabled_strings'): # pylint: disable=protected-access pt._disabled_strings = self._disabled_strings if hasattr(self, '_enabled_strings'): # pylint: disable=protected-access pt._enabled_strings = self._enabled_strings expectations = self.CreateExpectations() us = self.CreateUserStorySet(finder_options) if isinstance(pt, page_test.PageTest): if any(not isinstance(p, page.Page) for p in us.user_stories): raise Exception( 'PageTest must be used with UserStorySet containing only ' 'telemetry.page.Page user stories.') self._DownloadGeneratedProfileArchive(finder_options) benchmark_metadata = self.GetMetadata() with results_options.CreateResults( benchmark_metadata, finder_options, self.ValueCanBeAddedPredicate) as results: try: user_story_runner.Run(pt, us, expectations, finder_options, results, max_failures=self._max_failures) return_code = min(254, len(results.failures)) except Exception: exception_formatter.PrintFormattedException() return_code = 255 bucket = cloud_storage.BUCKET_ALIASES[finder_options.upload_bucket] if finder_options.upload_results: results.UploadTraceFilesToCloud(bucket) results.UploadProfilingFilesToCloud(bucket) results.PrintSummary() return return_code
def _RunUserStoryAndProcessErrorIfNeeded(expectations, user_story, results, state): def ProcessError(): if expectation == 'fail': msg = 'Expected exception while running %s' % user_story.display_name exception_formatter.PrintFormattedException(msg=msg) else: msg = 'Exception while running %s' % user_story.display_name results.AddValue(failure.FailureValue(user_story, sys.exc_info())) try: expectation = None state.WillRunUserStory(user_story) expectation, skip_value = state.GetTestExpectationAndSkipValue( expectations) if expectation == 'skip': assert skip_value results.AddValue(skip_value) return state.RunUserStory(results) except (page_test.Failure, exceptions.TimeoutException, exceptions.LoginException, exceptions.ProfilingException): ProcessError() except exceptions.Error: ProcessError() raise except page_action.PageActionNotSupported as e: results.AddValue( skip.SkipValue(user_story, 'Unsupported page action: %s' % e)) except Exception: results.AddValue( failure.FailureValue(user_story, sys.exc_info(), 'Unhandlable exception raised.')) raise else: if expectation == 'fail': logging.warning('%s was expected to fail, but passed.\n', user_story.display_name) finally: has_existing_exception = sys.exc_info() is not None try: state.DidRunUserStory(results) except Exception: if not has_existing_exception: raise # Print current exception and propagate existing exception. exception_formatter.PrintFormattedException( msg='Exception from DidRunUserStory: ')
def RunUserStory(self, results): try: self._PreparePage() self._ImplicitPageNavigation() action_runner = action_runner_module.ActionRunner( self._current_tab, skip_waits=self._current_page.skip_waits) self._current_page.RunPageInteractions(action_runner) self._test.ValidateAndMeasurePage( self._current_page, self._current_tab, results) except exceptions.Error: if self._test.is_multi_tab_test: # Avoid trying to recover from an unknown multi-tab state. exception_formatter.PrintFormattedException( msg='Telemetry Error during multi tab test:') raise page_test.MultiTabTestAppCrashError raise
def RemoveTestCa(self): """Remove root CA generated by previous call to InstallTestCa(). Removes the test root certificate from both the device and host machine. """ if not self._wpr_ca_cert_path: return if self._is_test_ca_installed: try: self._device_cert_util.remove_cert() except Exception: # Best effort cleanup - show the error and continue. exception_formatter.PrintFormattedException( msg=('Error while trying to remove certificate authority: %s. ' % self._adb.device_serial())) self._is_test_ca_installed = False shutil.rmtree(os.path.dirname(self._wpr_ca_cert_path), ignore_errors=True) self._wpr_ca_cert_path = None self._device_cert_util = None
def Run(test, user_story_set, expectations, finder_options, results, max_failures=None): """Runs a given test against a given page_set with the given options. Stop execution for unexpected exceptions such as KeyboardInterrupt. We "white list" certain exceptions for which the user story runner can continue running the remaining user stories. """ # Filter page set based on options. user_stories = filter(user_story_filter.UserStoryFilter.IsSelected, user_story_set) if (not finder_options.use_live_sites and user_story_set.bucket and finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD): serving_dirs = user_story_set.serving_dirs for directory in serving_dirs: cloud_storage.GetFilesInDirectoryIfChanged(directory, user_story_set.bucket) if not _UpdateAndCheckArchives(user_story_set.archive_data_file, user_story_set.wpr_archive_info, user_stories): return if not user_stories: return # Effective max failures gives priority to command-line flag value. effective_max_failures = finder_options.max_failures if effective_max_failures is None: effective_max_failures = max_failures user_story_groups = StoriesGroupedByStateClass( user_stories, user_story_set.allow_mixed_story_states) for group in user_story_groups: state = None try: for _ in xrange(finder_options.pageset_repeat): for user_story in group.user_stories: for _ in xrange(finder_options.page_repeat): if not state: state = group.shared_user_story_state_class( test, finder_options, user_story_set) results.WillRunPage(user_story) try: _WaitForThermalThrottlingIfNeeded(state.platform) _RunUserStoryAndProcessErrorIfNeeded( expectations, user_story, results, state) except exceptions.Error: # Catch all Telemetry errors to give the story a chance to retry. # The retry is enabled by tearing down the state and creating # a new state instance in the next iteration. try: # If TearDownState raises, do not catch the exception. # (The Error was saved as a failure value.) state.TearDownState(results) finally: # Later finally-blocks use state, so ensure it is cleared. state = None finally: has_existing_exception = sys.exc_info() is not None try: if state: _CheckThermalThrottling(state.platform) results.DidRunPage(user_story) except Exception: if not has_existing_exception: raise # Print current exception and propagate existing exception. exception_formatter.PrintFormattedException( msg='Exception from result processing:') if (effective_max_failures is not None and len(results.failures) > effective_max_failures): logging.error('Too many failures. Aborting.') return finally: if state: has_existing_exception = sys.exc_info() is not None try: state.TearDownState(results) except Exception: if not has_existing_exception: raise # Print current exception and propagate existing exception. exception_formatter.PrintFormattedException( msg='Exception from TearDownState:')
def Failure(self, test, err): exception_formatter.PrintFormattedException(*err) test_name = _FormatTestName(test) self._Print('[ FAILED ]', test_name, '(%0.f ms)' % self._TestTimeMs())
def Run(test, user_story_set, expectations, finder_options, results, max_failures=None): """Runs a given test against a given page_set with the given options. Stop execution for unexpected exceptions such as KeyboardInterrupt. We "white list" certain exceptions for which the user story runner can continue running the remaining user stories. """ # TODO(slamm): Remove special-case for PageTest. https://crbug.com/440101 if isinstance(test, page_test.PageTest): test.ValidatePageSet(user_story_set) # Reorder page set based on options. user_stories = _ShuffleAndFilterUserStorySet(user_story_set, finder_options) if (not finder_options.use_live_sites and finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD): _UpdateUserStoryArchivesIfChanged(user_story_set) if not _UpdateAndCheckArchives( user_story_set.archive_data_file, user_story_set.wpr_archive_info, user_stories): return # TODO(slamm): Remove special-case for PageTest. https://crbug.com/440101 if isinstance(test, page_test.PageTest): for user_story in list(user_stories): if not test.CanRunForPage(user_story): results.WillRunPage(user_story) logging.debug('Skipping test: it cannot run for %s', user_story.display_name) results.AddValue(skip.SkipValue(user_story, 'Test cannot run')) results.DidRunPage(user_story) user_stories.remove(user_story) if not user_stories: return # Effective max failures gives priority to command-line flag value. effective_max_failures = finder_options.max_failures if effective_max_failures is None: effective_max_failures = max_failures user_story_groups = GetUserStoryGroupsWithSameSharedUserStoryClass( user_stories) user_story_with_discarded_first_results = set() for group in user_story_groups: state = None try: for _ in xrange(finder_options.pageset_repeat): for user_story in group.user_stories: for _ in xrange(finder_options.page_repeat): if not state: state = group.shared_user_story_state_class( test, finder_options, user_story_set) results.WillRunPage(user_story) try: _WaitForThermalThrottlingIfNeeded(state.platform) _RunUserStoryAndProcessErrorIfNeeded( expectations, user_story, results, state) except exceptions.AppCrashException: # Catch AppCrashException to give the story a chance to retry. # The retry is enabled by tearing down the state and creating # a new state instance in the next iteration. try: # If TearDownState raises, do not catch the exception. # (The AppCrashException was saved as a failure value.) state.TearDownState(results) finally: # Later finally-blocks use state, so ensure it is cleared. state = None finally: has_existing_exception = sys.exc_info() is not None try: if state: _CheckThermalThrottling(state.platform) # TODO(slamm): Make discard_first_result part of user_story API. # https://crbug.com/440101 discard_current_run = ( getattr(test, 'discard_first_result', False) and user_story not in user_story_with_discarded_first_results) if discard_current_run: user_story_with_discarded_first_results.add(user_story) results.DidRunPage(user_story, discard_run=discard_current_run) except Exception: if not has_existing_exception: raise # Print current exception and propagate existing exception. exception_formatter.PrintFormattedException( msg='Exception from result processing:') if (effective_max_failures is not None and len(results.failures) > effective_max_failures): logging.error('Too many failures. Aborting.') return finally: if state: has_existing_exception = sys.exc_info() is not None try: state.TearDownState(results) except Exception: if not has_existing_exception: raise # Print current exception and propagate existing exception. exception_formatter.PrintFormattedException( msg='Exception from TearDownState:')