def _RunPage(test, page, state, expectation, results, finder_options):
if expectation == 'skip':
logging.debug('Skipping test: Skip expectation for %s', page.url)
results.AddSkip(page, 'Skipped by test expectations')
return
logging.info('Running %s' % page.url)
page_state = PageState(page, test.TabForPage(page, state.browser))
def ProcessError():
logging.error('%s:', page.url)
exception_formatter.PrintFormattedException()
if expectation == 'fail':
logging.info('Error was expected\n')
results.AddSuccess(page)
else:
results.AddError(page, sys.exc_info())
try:
page_state.PreparePage(test)
if state.repeat_state.ShouldNavigate(
finder_options.skip_navigate_on_repeat):
page_state.ImplicitPageNavigation(test)
test.RunPage(page, page_state.tab, results)
util.CloseConnections(page_state.tab)
except page_test.TestNotSupportedOnPlatformFailure:
raise
except page_test.Failure:
if expectation == 'fail':
logging.info('%s:', page.url)
exception_formatter.PrintFormattedException()
logging.info('Failure was expected\n')
results.AddSuccess(page)
else:
logging.warning('%s:', page.url)
exception_formatter.PrintFormattedException()
results.AddFailure(page, sys.exc_info())
except (util.TimeoutException, exceptions.LoginException,
exceptions.ProfilingException):
ProcessError()
except (exceptions.TabCrashException, exceptions.BrowserGoneException):
ProcessError()
# Run() catches these exceptions to relaunch the tab/browser, so re-raise.
raise
except page_action.PageActionNotSupported as e:
results.AddSkip(page, 'Unsupported page action: %s' % e)
except Exception:
logging.warning('While running %s', page.url)
exception_formatter.PrintFormattedException()
results.AddFailure(page, sys.exc_info())
else:
if expectation == 'fail':
logging.warning('%s was expected to fail, but passed.\n', page.url)
results.AddSuccess(page)
finally:
page_state.CleanUpPage(test)
def _RunPage(test, page, state, expectation, results):
if expectation == 'skip':
logging.debug('Skipping test: Skip expectation for %s', page.url)
results.AddValue(skip.SkipValue(page, 'Skipped by test expectations'))
return
logging.info('Running %s', page.url)
page_state = PageState(page, test.TabForPage(page, state.browser))
def ProcessError():
if expectation == 'fail':
msg = 'Expected exception while running %s' % page.url
else:
msg = 'Exception while running %s' % page.url
results.AddValue(failure.FailureValue(page, sys.exc_info()))
exception_formatter.PrintFormattedException(msg=msg)
try:
page_state.PreparePage(test)
page_state.ImplicitPageNavigation(test)
test.RunPage(page, page_state.tab, results)
util.CloseConnections(page_state.tab)
except page_test.TestNotSupportedOnPlatformFailure:
raise
except page_test.Failure:
if expectation == 'fail':
exception_formatter.PrintFormattedException(
msg='Expected failure while running %s' % page.url)
else:
exception_formatter.PrintFormattedException(
msg='Failure while running %s' % page.url)
results.AddValue(failure.FailureValue(page, sys.exc_info()))
except (util.TimeoutException, exceptions.LoginException,
exceptions.ProfilingException):
ProcessError()
except (exceptions.TabCrashException, exceptions.BrowserGoneException):
ProcessError()
# Run() catches these exceptions to relaunch the tab/browser, so re-raise.
raise
except page_action.PageActionNotSupported as e:
results.AddValue(
skip.SkipValue(page, 'Unsupported page action: %s' % e))
except Exception:
exception_formatter.PrintFormattedException(
msg='Unhandled exception while running %s' % page.url)
results.AddValue(failure.FailureValue(page, sys.exc_info()))
else:
if expectation == 'fail':
logging.warning('%s was expected to fail, but passed.\n', page.url)
finally:
page_state.CleanUpPage(test)
def ProcessError():
if expectation == 'fail':
msg = 'Expected exception while running %s' % user_story.display_name
exception_formatter.PrintFormattedException(msg=msg)
else:
msg = 'Exception while running %s' % user_story.display_name
results.AddValue(failure.FailureValue(user_story, sys.exc_info()))
def ProcessError():
if expectation == 'fail':
msg = 'Expected exception while running %s' % page.url
else:
msg = 'Exception while running %s' % page.url
results.AddValue(failure.FailureValue(page, sys.exc_info()))
exception_formatter.PrintFormattedException(msg=msg)
def ProcessError():
logging.error('%s:', page.url)
exception_formatter.PrintFormattedException()
if expectation == 'fail':
logging.info('Error was expected\n')
results.AddSuccess(page)
else:
results.AddError(page, sys.exc_info())
def RunUserStory(self, results):
try:
self._PreparePage()
self._ImplicitPageNavigation()
self._test.RunPage(self._current_page, self._current_tab, results)
except exceptions.Error:
if self._test.is_multi_tab_test:
# Avoid trying to recover from an unknown multi-tab state.
exception_formatter.PrintFormattedException(
msg='Telemetry Error during multi tab test:')
raise page_test.MultiTabTestAppCrashError
raise
def Run(self, finder_options):
"""Run this test with the given options.
Returns:
The number of failure values (up to 254) or 255 if there is an uncaught
exception.
"""
self.CustomizeBrowserOptions(finder_options.browser_options)
pt = self.CreatePageTest(finder_options)
pt.__name__ = self.__class__.__name__
if hasattr(self, '_disabled_strings'):
# pylint: disable=protected-access
pt._disabled_strings = self._disabled_strings
if hasattr(self, '_enabled_strings'):
# pylint: disable=protected-access
pt._enabled_strings = self._enabled_strings
expectations = self.CreateExpectations()
us = self.CreateUserStorySet(finder_options)
if isinstance(pt, page_test.PageTest):
if any(not isinstance(p, page.Page) for p in us.user_stories):
raise Exception(
'PageTest must be used with UserStorySet containing only '
'telemetry.page.Page user stories.')
self._DownloadGeneratedProfileArchive(finder_options)
benchmark_metadata = self.GetMetadata()
with results_options.CreateResults(
benchmark_metadata, finder_options,
self.ValueCanBeAddedPredicate) as results:
try:
user_story_runner.Run(pt,
us,
expectations,
finder_options,
results,
max_failures=self._max_failures)
return_code = min(254, len(results.failures))
except Exception:
exception_formatter.PrintFormattedException()
return_code = 255
bucket = cloud_storage.BUCKET_ALIASES[finder_options.upload_bucket]
if finder_options.upload_results:
results.UploadTraceFilesToCloud(bucket)
results.UploadProfilingFilesToCloud(bucket)
results.PrintSummary()
return return_code
def _RunUserStoryAndProcessErrorIfNeeded(expectations, user_story, results,
state):
def ProcessError():
if expectation == 'fail':
msg = 'Expected exception while running %s' % user_story.display_name
exception_formatter.PrintFormattedException(msg=msg)
else:
msg = 'Exception while running %s' % user_story.display_name
results.AddValue(failure.FailureValue(user_story, sys.exc_info()))
try:
expectation = None
state.WillRunUserStory(user_story)
expectation, skip_value = state.GetTestExpectationAndSkipValue(
expectations)
if expectation == 'skip':
assert skip_value
results.AddValue(skip_value)
return
state.RunUserStory(results)
except (page_test.Failure, exceptions.TimeoutException,
exceptions.LoginException, exceptions.ProfilingException):
ProcessError()
except exceptions.Error:
ProcessError()
raise
except page_action.PageActionNotSupported as e:
results.AddValue(
skip.SkipValue(user_story, 'Unsupported page action: %s' % e))
except Exception:
results.AddValue(
failure.FailureValue(user_story, sys.exc_info(),
'Unhandlable exception raised.'))
raise
else:
if expectation == 'fail':
logging.warning('%s was expected to fail, but passed.\n',
user_story.display_name)
finally:
has_existing_exception = sys.exc_info() is not None
try:
state.DidRunUserStory(results)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from DidRunUserStory: ')
def RunUserStory(self, results):
try:
self._PreparePage()
self._ImplicitPageNavigation()
action_runner = action_runner_module.ActionRunner(
self._current_tab, skip_waits=self._current_page.skip_waits)
self._current_page.RunPageInteractions(action_runner)
self._test.ValidateAndMeasurePage(
self._current_page, self._current_tab, results)
except exceptions.Error:
if self._test.is_multi_tab_test:
# Avoid trying to recover from an unknown multi-tab state.
exception_formatter.PrintFormattedException(
msg='Telemetry Error during multi tab test:')
raise page_test.MultiTabTestAppCrashError
raise
def RemoveTestCa(self):
"""Remove root CA generated by previous call to InstallTestCa().
Removes the test root certificate from both the device and host machine.
"""
if not self._wpr_ca_cert_path:
return
if self._is_test_ca_installed:
try:
self._device_cert_util.remove_cert()
except Exception:
# Best effort cleanup - show the error and continue.
exception_formatter.PrintFormattedException(
msg=('Error while trying to remove certificate authority: %s. '
% self._adb.device_serial()))
self._is_test_ca_installed = False
shutil.rmtree(os.path.dirname(self._wpr_ca_cert_path), ignore_errors=True)
self._wpr_ca_cert_path = None
self._device_cert_util = None
def Run(test,
user_story_set,
expectations,
finder_options,
results,
max_failures=None):
"""Runs a given test against a given page_set with the given options.
Stop execution for unexpected exceptions such as KeyboardInterrupt.
We "white list" certain exceptions for which the user story runner
can continue running the remaining user stories.
"""
# Filter page set based on options.
user_stories = filter(user_story_filter.UserStoryFilter.IsSelected,
user_story_set)
if (not finder_options.use_live_sites and user_story_set.bucket and
finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD):
serving_dirs = user_story_set.serving_dirs
for directory in serving_dirs:
cloud_storage.GetFilesInDirectoryIfChanged(directory,
user_story_set.bucket)
if not _UpdateAndCheckArchives(user_story_set.archive_data_file,
user_story_set.wpr_archive_info,
user_stories):
return
if not user_stories:
return
# Effective max failures gives priority to command-line flag value.
effective_max_failures = finder_options.max_failures
if effective_max_failures is None:
effective_max_failures = max_failures
user_story_groups = StoriesGroupedByStateClass(
user_stories, user_story_set.allow_mixed_story_states)
for group in user_story_groups:
state = None
try:
for _ in xrange(finder_options.pageset_repeat):
for user_story in group.user_stories:
for _ in xrange(finder_options.page_repeat):
if not state:
state = group.shared_user_story_state_class(
test, finder_options, user_story_set)
results.WillRunPage(user_story)
try:
_WaitForThermalThrottlingIfNeeded(state.platform)
_RunUserStoryAndProcessErrorIfNeeded(
expectations, user_story, results, state)
except exceptions.Error:
# Catch all Telemetry errors to give the story a chance to retry.
# The retry is enabled by tearing down the state and creating
# a new state instance in the next iteration.
try:
# If TearDownState raises, do not catch the exception.
# (The Error was saved as a failure value.)
state.TearDownState(results)
finally:
# Later finally-blocks use state, so ensure it is cleared.
state = None
finally:
has_existing_exception = sys.exc_info() is not None
try:
if state:
_CheckThermalThrottling(state.platform)
results.DidRunPage(user_story)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from result processing:')
if (effective_max_failures is not None and
len(results.failures) > effective_max_failures):
logging.error('Too many failures. Aborting.')
return
finally:
if state:
has_existing_exception = sys.exc_info() is not None
try:
state.TearDownState(results)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from TearDownState:')
def Failure(self, test, err):
exception_formatter.PrintFormattedException(*err)
test_name = _FormatTestName(test)
self._Print('[ FAILED ]', test_name, '(%0.f ms)' % self._TestTimeMs())
def Run(test, user_story_set, expectations, finder_options, results,
max_failures=None):
"""Runs a given test against a given page_set with the given options.
Stop execution for unexpected exceptions such as KeyboardInterrupt.
We "white list" certain exceptions for which the user story runner
can continue running the remaining user stories.
"""
# TODO(slamm): Remove special-case for PageTest. https://crbug.com/440101
if isinstance(test, page_test.PageTest):
test.ValidatePageSet(user_story_set)
# Reorder page set based on options.
user_stories = _ShuffleAndFilterUserStorySet(user_story_set, finder_options)
if (not finder_options.use_live_sites and
finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD):
_UpdateUserStoryArchivesIfChanged(user_story_set)
if not _UpdateAndCheckArchives(
user_story_set.archive_data_file, user_story_set.wpr_archive_info,
user_stories):
return
# TODO(slamm): Remove special-case for PageTest. https://crbug.com/440101
if isinstance(test, page_test.PageTest):
for user_story in list(user_stories):
if not test.CanRunForPage(user_story):
results.WillRunPage(user_story)
logging.debug('Skipping test: it cannot run for %s',
user_story.display_name)
results.AddValue(skip.SkipValue(user_story, 'Test cannot run'))
results.DidRunPage(user_story)
user_stories.remove(user_story)
if not user_stories:
return
# Effective max failures gives priority to command-line flag value.
effective_max_failures = finder_options.max_failures
if effective_max_failures is None:
effective_max_failures = max_failures
user_story_groups = GetUserStoryGroupsWithSameSharedUserStoryClass(
user_stories)
user_story_with_discarded_first_results = set()
for group in user_story_groups:
state = None
try:
for _ in xrange(finder_options.pageset_repeat):
for user_story in group.user_stories:
for _ in xrange(finder_options.page_repeat):
if not state:
state = group.shared_user_story_state_class(
test, finder_options, user_story_set)
results.WillRunPage(user_story)
try:
_WaitForThermalThrottlingIfNeeded(state.platform)
_RunUserStoryAndProcessErrorIfNeeded(
expectations, user_story, results, state)
except exceptions.AppCrashException:
# Catch AppCrashException to give the story a chance to retry.
# The retry is enabled by tearing down the state and creating
# a new state instance in the next iteration.
try:
# If TearDownState raises, do not catch the exception.
# (The AppCrashException was saved as a failure value.)
state.TearDownState(results)
finally:
# Later finally-blocks use state, so ensure it is cleared.
state = None
finally:
has_existing_exception = sys.exc_info() is not None
try:
if state:
_CheckThermalThrottling(state.platform)
# TODO(slamm): Make discard_first_result part of user_story API.
# https://crbug.com/440101
discard_current_run = (
getattr(test, 'discard_first_result', False) and
user_story not in user_story_with_discarded_first_results)
if discard_current_run:
user_story_with_discarded_first_results.add(user_story)
results.DidRunPage(user_story, discard_run=discard_current_run)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from result processing:')
if (effective_max_failures is not None and
len(results.failures) > effective_max_failures):
logging.error('Too many failures. Aborting.')
return
finally:
if state:
has_existing_exception = sys.exc_info() is not None
try:
state.TearDownState(results)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from TearDownState:')
