def _RunStoryAndProcessErrorIfNeeded(story, results, state, test):
def ProcessError():
results.AddValue(failure.FailureValue(story, sys.exc_info()))
try:
if isinstance(test, story_test.StoryTest):
test.WillRunStory(state.platform)
state.WillRunStory(story)
if not state.CanRunStory(story):
results.AddValue(
skip.SkipValue(
story, 'Skipped because story is not supported '
'(SharedState.CanRunStory() returns False).'))
return
state.RunStory(results)
if isinstance(test, story_test.StoryTest):
test.Measure(state.platform, results)
except (page_test.Failure, exceptions.TimeoutException,
exceptions.LoginException, exceptions.ProfilingException):
ProcessError()
except exceptions.Error:
ProcessError()
raise
except page_action.PageActionNotSupported as e:
results.AddValue(
skip.SkipValue(story, 'Unsupported page action: %s' % e))
except Exception:
results.AddValue(
failure.FailureValue(story, sys.exc_info(),
'Unhandlable exception raised.'))
raise
finally:
has_existing_exception = (sys.exc_info() != (None, None, None))
try:
state.DidRunStory(results)
# if state.DidRunStory raises exception, things are messed up badly and we
# do not need to run test.DidRunStory at that point.
if isinstance(test, story_test.StoryTest):
test.DidRunStory(state.platform)
else:
test.DidRunPage(state.platform)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception raised when cleaning story run: ')
def testBuildbotAndRepresentativeValue(self):
v = skip.SkipValue(self.pages[0], 'page skipped for testing reason')
self.assertIsNone(v.GetBuildbotValue())
self.assertIsNone(v.GetBuildbotDataType(
value.COMPUTED_PER_PAGE_SUMMARY_OUTPUT_CONTEXT))
self.assertIsNone(v.GetChartAndTraceNameForPerPageResult())
self.assertIsNone(v.GetRepresentativeNumber())
self.assertIsNone(v.GetRepresentativeString())
def testAsDictWithRepeatedTests(self):
results = page_test_results.PageTestResults()
results.telemetry_info.benchmark_start_epoch = 1501773200
results.telemetry_info.benchmark_name = 'benchmark_name'
results.WillRunPage(self._story_set[0])
v0 = scalar.ScalarValue(
results.current_page,
'foo',
'seconds',
3,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v0)
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
v1 = skip.SkipValue(results.current_page, 'fake_skip')
results.AddValue(v1)
results.DidRunPage(self._story_set[1])
results.WillRunPage(self._story_set[0])
v0 = scalar.ScalarValue(
results.current_page,
'foo',
'seconds',
3,
improvement_direction=improvement_direction.DOWN)
results.AddValue(v0)
results.DidRunPage(self._story_set[0])
results.WillRunPage(self._story_set[1])
v1 = skip.SkipValue(results.current_page, 'fake_skip')
results.AddValue(v1)
results.DidRunPage(self._story_set[1])
d = json_3_output_formatter.ResultsAsDict(results)
foo_story_result = d['tests']['benchmark_name']['Foo']
self.assertEquals(foo_story_result['actual'], 'PASS')
self.assertEquals(foo_story_result['expected'], 'PASS')
bar_story_result = d['tests']['benchmark_name']['Bar']
self.assertEquals(bar_story_result['actual'], 'SKIP')
self.assertEquals(bar_story_result['expected'], 'SKIP')
self.assertEquals(d['num_failures_by_type'], {'SKIP': 2, 'PASS': 2})
def testRepr(self):
v = skip.SkipValue(self.pages[0], 'page skipped for testing reason',
description='desc')
expected = ('SkipValue(http://www.bar.com/, '
'page skipped for testing reason, '
'description=desc)')
self.assertEquals(expected, str(v))
def testStoryRunSkipped(self):
run = story_run.StoryRun(self.stories[0])
run.AddValue(failure.FailureValue.FromMessage(self.stories[0], 'test'))
run.AddValue(skip.SkipValue(self.stories[0], 'test'))
self.assertFalse(run.ok)
self.assertFalse(run.failed)
self.assertTrue(run.skipped)
run = story_run.StoryRun(self.stories[0])
run.AddValue(
scalar.ScalarValue(self.stories[0],
'a',
's',
1,
improvement_direction=improvement_direction.UP))
run.AddValue(skip.SkipValue(self.stories[0], 'test'))
self.assertFalse(run.ok)
self.assertFalse(run.failed)
self.assertTrue(run.skipped)
def _RunUserStoryAndProcessErrorIfNeeded(expectations, user_story, results,
state):
def ProcessError():
if expectation == 'fail':
msg = 'Expected exception while running %s' % user_story.display_name
exception_formatter.PrintFormattedException(msg=msg)
else:
msg = 'Exception while running %s' % user_story.display_name
results.AddValue(failure.FailureValue(user_story, sys.exc_info()))
try:
expectation = None
state.WillRunUserStory(user_story)
expectation, skip_value = state.GetTestExpectationAndSkipValue(
expectations)
if expectation == 'skip':
assert skip_value
results.AddValue(skip_value)
return
state.RunUserStory(results)
except (page_test.Failure, exceptions.TimeoutException,
exceptions.LoginException, exceptions.ProfilingException):
ProcessError()
except exceptions.Error:
ProcessError()
raise
except page_action.PageActionNotSupported as e:
results.AddValue(
skip.SkipValue(user_story, 'Unsupported page action: %s' % e))
except Exception:
results.AddValue(
failure.FailureValue(user_story, sys.exc_info(),
'Unhandlable exception raised.'))
raise
else:
if expectation == 'fail':
logging.warning('%s was expected to fail, but passed.\n',
user_story.display_name)
finally:
has_existing_exception = sys.exc_info() is not None
try:
state.DidRunUserStory(results)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from DidRunUserStory: ')
def ValidateAndMeasurePage(self, page, tab, results):
del page # unused
# Trigger GC to get histogram data.
# Seven GCs should be enough to collect any detached context.
# If a detached context survives more GCs then there is a leak.
MAX_AGE = 8
for _ in xrange(MAX_AGE):
tab.CollectGarbage()
value = _GetMaxDetachedContextAge(tab, self._data_start)
if value is None:
results.AddValue(skip.SkipValue(
results.current_page, 'No detached contexts'))
else:
results.AddValue(scalar.ScalarValue(
results.current_page, _DISPLAY_NAME, _UNITS, value,
description=_DESCRIPTION))
def testSingleSkippedPage(self):
test_story_set = _MakeStorySet()
results = page_test_results.PageTestResults(
progress_reporter=self._reporter)
results.WillRunPage(test_story_set.stories[0])
self._fake_timer.SetTime(0.007)
results.AddValue(skip.SkipValue(test_story_set.stories[0],
'Page skipped for testing reason'))
results.DidRunPage(test_story_set.stories[0])
results.PrintSummary()
expected = ('[ RUN ] http://www.foo.com/\n'
'===== SKIPPING TEST http://www.foo.com/:'
' Page skipped for testing reason =====\n'
'[ OK ] http://www.foo.com/ (7 ms)\n'
'[ PASSED ] 1 test.\n\n')
self.assertEquals(expected, ''.join(self._output_stream.output_data))
def testSkips(self):
results = page_test_results.PageTestResults()
results.WillRunPage(self.pages[0])
results.AddValue(skip.SkipValue(self.pages[0], 'testing reason'))
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.DidRunPage(self.pages[1])
self.assertTrue(results.all_page_runs[0].skipped)
self.assertEqual(self.pages[0], results.all_page_runs[0].story)
self.assertEqual(set([self.pages[0], self.pages[1]]),
results.pages_that_succeeded)
self.assertEqual(2, len(results.all_page_runs))
self.assertTrue(results.all_page_runs[0].skipped)
self.assertTrue(results.all_page_runs[1].ok)
def testSkipValueCannotBeFiltered(self):
def AcceptValueNamed_a(value, _):
return value.name == 'a'
results = page_test_results.PageTestResults(
value_can_be_added_predicate=AcceptValueNamed_a)
results.WillRunPage(self.pages[0])
skip_value = skip.SkipValue(self.pages[0], 'skip for testing')
results.AddValue(scalar.ScalarValue(self.pages[0], 'b', 'seconds', 8))
results.AddValue(skip_value)
results.DidRunPage(self.pages[0])
results.PrintSummary()
# Although predicate says only accept value with named 'a', skip value is
# added anyway.
self.assertEquals(len(results.all_page_specific_values), 1)
self.assertIn(skip_value, results.all_page_specific_values)
def _RunUserStoryAndProcessErrorIfNeeded(test, expectations, user_story,
results, state):
expectation = None
def ProcessError():
if expectation == 'fail':
msg = 'Expected exception while running %s' % user_story.display_name
exception_formatter.PrintFormattedException(msg=msg)
else:
msg = 'Exception while running %s' % user_story.display_name
results.AddValue(failure.FailureValue(user_story, sys.exc_info()))
try:
state.WillRunUserStory(user_story)
expectation, skip_value = state.GetTestExpectationAndSkipValue(
expectations)
if expectation == 'skip':
assert skip_value
results.AddValue(skip_value)
return
state.RunUserStory(results)
except page_test.TestNotSupportedOnPlatformFailure:
raise
except (page_test.Failure, util.TimeoutException,
exceptions.LoginException, exceptions.ProfilingException):
ProcessError()
except exceptions.AppCrashException:
ProcessError()
state.TearDownState(results)
if test.is_multi_tab_test:
logging.error(
'Aborting multi-tab test after browser or tab crashed at '
'user story %s' % user_story.display_name)
test.RequestExit()
return
except page_action.PageActionNotSupported as e:
results.AddValue(
skip.SkipValue(user_story, 'Unsupported page action: %s' % e))
else:
if expectation == 'fail':
logging.warning('%s was expected to fail, but passed.\n',
user_story.display_name)
finally:
state.DidRunUserStory(results)
def testOutputSkipInformation(self):
test_page_set = _MakePageSet()
self._reporter = gtest_progress_reporter.GTestProgressReporter(
self._output_stream, output_skipped_tests_summary=True)
results = page_test_results.PageTestResults(
progress_reporter=self._reporter)
results.WillRunPage(test_page_set.pages[0])
self._mock_timer.SetTime(0.007)
results.AddValue(
skip.SkipValue(test_page_set.pages[0],
'Page skipped for testing reason'))
results.DidRunPage(test_page_set.pages[0])
results.PrintSummary()
expected = ('[ RUN ] http://www.foo.com/\n'
'===== SKIPPING TEST http://www.foo.com/:'
' Page skipped for testing reason =====\n'
'[ OK ] http://www.foo.com/ (7 ms)\n'
'[ PASSED ] 1 test.\n'
'\n'
'Skipped pages:\n'
'http://www.foo.com/\n'
'\n')
self.assertEquals(expected, ''.join(self._output_stream.output_data))
def testPassesNoSkips(self):
results = page_test_results.PageTestResults()
results.WillRunPage(self.pages[0])
results.AddValue(
failure.FailureValue(self.pages[0], self.CreateException()))
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.DidRunPage(self.pages[1])
results.WillRunPage(self.pages[2])
results.AddValue(skip.SkipValue(self.pages[2], 'testing reason'))
results.DidRunPage(self.pages[2])
self.assertEqual(set([self.pages[0]]), results.pages_that_failed)
self.assertEqual(set([self.pages[1], self.pages[2]]),
results.pages_that_succeeded)
self.assertEqual(set([self.pages[1]]),
results.pages_that_succeeded_and_not_skipped)
self.assertEqual(3, len(results.all_page_runs))
self.assertTrue(results.all_page_runs[0].failed)
self.assertTrue(results.all_page_runs[1].ok)
self.assertTrue(results.all_page_runs[2].skipped)
def Skip(self, reason):
assert self._current_page_run, 'Not currently running test.'
self.AddValue(skip.SkipValue(self.current_page, reason))
def Run(test, user_story_set, expectations, finder_options, results):
"""Runs a given test against a given page_set with the given options."""
test.ValidatePageSet(user_story_set)
# Reorder page set based on options.
user_stories = _ShuffleAndFilterUserStorySet(user_story_set,
finder_options)
if (not finder_options.use_live_sites
and finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD
and
# TODO(nednguyen): also handle these logic for user_story_set in next
# patch.
isinstance(user_story_set, page_set_module.PageSet)):
_UpdateUserStoryArchivesIfChanged(user_story_set)
if not _CheckArchives(user_story_set.archive_data_file,
user_story_set.wpr_archive_info,
user_story_set.pages):
return
for user_story in list(user_stories):
if not test.CanRunForPage(user_story):
results.WillRunPage(user_story)
logging.debug('Skipping test: it cannot run for %s',
user_story.display_name)
results.AddValue(skip.SkipValue(user_story, 'Test cannot run'))
results.DidRunPage(user_story)
user_stories.remove(user_story)
if not user_stories:
return
user_story_with_discarded_first_results = set()
max_failures = finder_options.max_failures # command-line gets priority
if max_failures is None:
max_failures = test.max_failures # may be None
user_story_groups = GetUserStoryGroupsWithSameSharedUserStoryClass(
user_stories)
test.WillRunTest(finder_options)
for group in user_story_groups:
state = None
try:
state = group.shared_user_story_state_class(
test, finder_options, user_story_set)
for _ in xrange(finder_options.pageset_repeat):
for user_story in group.user_stories:
if test.IsExiting():
break
for _ in xrange(finder_options.page_repeat):
results.WillRunPage(user_story)
try:
_WaitForThermalThrottlingIfNeeded(state.platform)
_RunUserStoryAndProcessErrorIfNeeded(
test, expectations, user_story, results, state)
except Exception:
# Tear down & restart the state for unhandled exceptions thrown by
# _RunUserStoryAndProcessErrorIfNeeded.
results.AddValue(
failure.FailureValue(user_story,
sys.exc_info()))
state.TearDownState(results)
state = group.shared_user_story_state_class(
test, finder_options, user_story_set)
finally:
_CheckThermalThrottling(state.platform)
discard_run = (
test.discard_first_result and user_story
not in user_story_with_discarded_first_results)
if discard_run:
user_story_with_discarded_first_results.add(
user_story)
results.DidRunPage(user_story,
discard_run=discard_run)
if max_failures is not None and len(
results.failures) > max_failures:
logging.error('Too many failures. Aborting.')
test.RequestExit()
finally:
if state:
state.TearDownState(results)
if not os.path.exists(credentials_path):
credentials_path = None
# Set up user agent.
browser_options.browser_user_agent_type = page_set.user_agent_type or None
if finder_options.profiler:
profiler_class = profiler_finder.FindProfiler(finder_options.profiler)
profiler_class.CustomizeBrowserOptions(browser_options.browser_type,
finder_options)
for page in list(pages):
if not test.CanRunForPage(page):
results.WillRunPage(page)
logging.debug('Skipping test: it cannot run for %s', page.url)
results.AddValue(skip.SkipValue(page, 'Test cannot run'))
results.DidRunPage(page)
pages.remove(page)
if not pages:
return
state = _RunState()
pages_with_discarded_first_result = set()
max_failures = finder_options.max_failures # command-line gets priority
if max_failures is None:
max_failures = test.max_failures # may be None
try:
test.WillRunTest(finder_options)
for _ in xrange(finder_options.pageset_repeat):
def testAsDict(self): v = skip.SkipValue(self.pages[0], 'page skipped for testing reason') d = v.AsDictWithoutBaseClassEntries() self.assertEquals(d['reason'], 'page skipped for testing reason')
def Run(test, user_story_set, expectations, finder_options, results,
max_failures=None):
"""Runs a given test against a given page_set with the given options.
Stop execution for unexpected exceptions such as KeyboardInterrupt.
We "white list" certain exceptions for which the user story runner
can continue running the remaining user stories.
"""
# TODO(slamm): Remove special-case for PageTest. https://crbug.com/440101
if isinstance(test, page_test.PageTest):
test.ValidatePageSet(user_story_set)
# Reorder page set based on options.
user_stories = _ShuffleAndFilterUserStorySet(user_story_set, finder_options)
if (not finder_options.use_live_sites and
finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD):
_UpdateUserStoryArchivesIfChanged(user_story_set)
if not _UpdateAndCheckArchives(
user_story_set.archive_data_file, user_story_set.wpr_archive_info,
user_stories):
return
# TODO(slamm): Remove special-case for PageTest. https://crbug.com/440101
if isinstance(test, page_test.PageTest):
for user_story in list(user_stories):
if not test.CanRunForPage(user_story):
results.WillRunPage(user_story)
logging.debug('Skipping test: it cannot run for %s',
user_story.display_name)
results.AddValue(skip.SkipValue(user_story, 'Test cannot run'))
results.DidRunPage(user_story)
user_stories.remove(user_story)
if not user_stories:
return
# Effective max failures gives priority to command-line flag value.
effective_max_failures = finder_options.max_failures
if effective_max_failures is None:
effective_max_failures = max_failures
user_story_groups = GetUserStoryGroupsWithSameSharedUserStoryClass(
user_stories)
user_story_with_discarded_first_results = set()
for group in user_story_groups:
state = None
try:
for _ in xrange(finder_options.pageset_repeat):
for user_story in group.user_stories:
for _ in xrange(finder_options.page_repeat):
if not state:
state = group.shared_user_story_state_class(
test, finder_options, user_story_set)
results.WillRunPage(user_story)
try:
_WaitForThermalThrottlingIfNeeded(state.platform)
_RunUserStoryAndProcessErrorIfNeeded(
expectations, user_story, results, state)
except exceptions.AppCrashException:
# Catch AppCrashException to give the story a chance to retry.
# The retry is enabled by tearing down the state and creating
# a new state instance in the next iteration.
try:
# If TearDownState raises, do not catch the exception.
# (The AppCrashException was saved as a failure value.)
state.TearDownState(results)
finally:
# Later finally-blocks use state, so ensure it is cleared.
state = None
finally:
has_existing_exception = sys.exc_info() is not None
try:
if state:
_CheckThermalThrottling(state.platform)
# TODO(slamm): Make discard_first_result part of user_story API.
# https://crbug.com/440101
discard_current_run = (
getattr(test, 'discard_first_result', False) and
user_story not in user_story_with_discarded_first_results)
if discard_current_run:
user_story_with_discarded_first_results.add(user_story)
results.DidRunPage(user_story, discard_run=discard_current_run)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from result processing:')
if (effective_max_failures is not None and
len(results.failures) > effective_max_failures):
logging.error('Too many failures. Aborting.')
return
finally:
if state:
has_existing_exception = sys.exc_info() is not None
try:
state.TearDownState(results)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from TearDownState:')
def Run(test,
story_set,
finder_options,
results,
max_failures=None,
expectations=None,
metadata=None,
max_num_values=sys.maxint):
"""Runs a given test against a given page_set with the given options.
Stop execution for unexpected exceptions such as KeyboardInterrupt.
We "white list" certain exceptions for which the story runner
can continue running the remaining stories.
"""
for s in story_set:
ValidateStory(s)
# Filter page set based on options.
stories = story_module.StoryFilter.FilterStorySet(story_set)
if (not finder_options.use_live_sites and
finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD):
serving_dirs = story_set.serving_dirs
if story_set.bucket:
for directory in serving_dirs:
cloud_storage.GetFilesInDirectoryIfChanged(
directory, story_set.bucket)
if story_set.archive_data_file and not _UpdateAndCheckArchives(
story_set.archive_data_file, story_set.wpr_archive_info,
stories):
return
if not stories:
return
# Effective max failures gives priority to command-line flag value.
effective_max_failures = finder_options.max_failures
if effective_max_failures is None:
effective_max_failures = max_failures
state = None
device_info_diags = {}
try:
for storyset_repeat_counter in xrange(finder_options.pageset_repeat):
for story in stories:
if not state:
# Construct shared state by using a copy of finder_options. Shared
# state may update the finder_options. If we tear down the shared
# state after this story run, we want to construct the shared
# state for the next story from the original finder_options.
state = story_set.shared_state_class(
test, finder_options.Copy(), story_set)
results.WillRunPage(story, storyset_repeat_counter)
if expectations:
disabled = expectations.IsStoryDisabled(
story, state.platform, finder_options)
if disabled and not finder_options.run_disabled_tests:
results.AddValue(skip.SkipValue(story, disabled))
results.DidRunPage(story)
continue
try:
state.platform.WaitForBatteryTemperature(35)
_WaitForThermalThrottlingIfNeeded(state.platform)
_RunStoryAndProcessErrorIfNeeded(story, results, state,
test)
num_values = len(results.all_page_specific_values)
if num_values > max_num_values:
msg = 'Too many values: %d > %d' % (num_values,
max_num_values)
results.AddValue(
failure.FailureValue.FromMessage(None, msg))
device_info_diags = _MakeDeviceInfoDiagnostics(state)
except exceptions.Error:
# Catch all Telemetry errors to give the story a chance to retry.
# The retry is enabled by tearing down the state and creating
# a new state instance in the next iteration.
try:
# If TearDownState raises, do not catch the exception.
# (The Error was saved as a failure value.)
state.TearDownState()
finally:
# Later finally-blocks use state, so ensure it is cleared.
state = None
finally:
has_existing_exception = sys.exc_info() != (None, None,
None)
try:
if state:
_CheckThermalThrottling(state.platform)
results.DidRunPage(story)
except Exception: # pylint: disable=broad-except
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from result processing:')
if (effective_max_failures is not None
and len(results.failures) > effective_max_failures):
logging.error('Too many failures. Aborting.')
return
finally:
results.PopulateHistogramSet(metadata)
for name, diag in device_info_diags.iteritems():
results.histograms.AddSharedDiagnostic(name, diag)
tagmap = _GenerateTagMapFromStorySet(stories)
if tagmap.tags_to_story_names:
results.histograms.AddSharedDiagnostic(reserved_infos.TAG_MAP.name,
tagmap)
if state:
has_existing_exception = sys.exc_info() != (None, None, None)
try:
state.TearDownState()
except Exception: # pylint: disable=broad-except
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from TearDownState:')
def Run(test,
story_set,
finder_options,
results,
max_failures=None,
tear_down_after_story=False,
tear_down_after_story_set=False,
expectations=None):
"""Runs a given test against a given page_set with the given options.
Stop execution for unexpected exceptions such as KeyboardInterrupt.
We "white list" certain exceptions for which the story runner
can continue running the remaining stories.
"""
for s in story_set:
ValidateStory(s)
# Filter page set based on options.
stories = filter(story_module.StoryFilter.IsSelected, story_set)
if (not finder_options.use_live_sites and
finder_options.browser_options.wpr_mode != wpr_modes.WPR_RECORD):
serving_dirs = story_set.serving_dirs
if story_set.bucket:
for directory in serving_dirs:
cloud_storage.GetFilesInDirectoryIfChanged(
directory, story_set.bucket)
if story_set.archive_data_file and not _UpdateAndCheckArchives(
story_set.archive_data_file, story_set.wpr_archive_info,
stories):
return
if not stories:
return
# Effective max failures gives priority to command-line flag value.
effective_max_failures = finder_options.max_failures
if effective_max_failures is None:
effective_max_failures = max_failures
story_groups = StoriesGroupedByStateClass(
stories, story_set.allow_mixed_story_states)
for group in story_groups:
state = None
try:
for storyset_repeat_counter in xrange(
finder_options.pageset_repeat):
for story in group.stories:
if not state:
# Construct shared state by using a copy of finder_options. Shared
# state may update the finder_options. If we tear down the shared
# state after this story run, we want to construct the shared
# state for the next story from the original finder_options.
state = group.shared_state_class(
test, finder_options.Copy(), story_set)
results.WillRunPage(story, storyset_repeat_counter)
if expectations:
disabled = expectations.IsStoryDisabled(
story, state.platform)
if disabled and not finder_options.run_disabled_tests:
results.AddValue(skip.SkipValue(story, disabled))
results.DidRunPage(story)
continue
try:
state.platform.WaitForBatteryTemperature(35)
_WaitForThermalThrottlingIfNeeded(state.platform)
_RunStoryAndProcessErrorIfNeeded(
story, results, state, test)
except exceptions.Error:
# Catch all Telemetry errors to give the story a chance to retry.
# The retry is enabled by tearing down the state and creating
# a new state instance in the next iteration.
try:
# If TearDownState raises, do not catch the exception.
# (The Error was saved as a failure value.)
state.TearDownState()
finally:
# Later finally-blocks use state, so ensure it is cleared.
state = None
finally:
has_existing_exception = sys.exc_info() != (None, None,
None)
try:
if state:
_CheckThermalThrottling(state.platform)
results.DidRunPage(story)
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from result processing:')
if state and tear_down_after_story:
state.TearDownState()
state = None
if (effective_max_failures is not None and
len(results.failures) > effective_max_failures):
logging.error('Too many failures. Aborting.')
return
if state and tear_down_after_story_set:
state.TearDownState()
state = None
finally:
if state:
has_existing_exception = sys.exc_info() != (None, None, None)
try:
state.TearDownState()
except Exception:
if not has_existing_exception:
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception from TearDownState:')
def _RunStoryAndProcessErrorIfNeeded(story, results, state, test):
def ProcessError(description=None):
state.DumpStateUponFailure(story, results)
# Note: adding the FailureValue to the results object also normally
# cause the progress_reporter to log it in the output.
results.AddValue(
failure.FailureValue(story, sys.exc_info(), description))
try:
# TODO(mikecase): Remove this logging once Android perf bots are swarmed.
# crbug.com/678282
if state.platform.GetOSName() == 'android':
state.platform._platform_backend.Log(
'START %s' % (story.name if story.name else str(story)))
if isinstance(test, story_test.StoryTest):
test.WillRunStory(state.platform)
state.WillRunStory(story)
if not state.CanRunStory(story):
results.AddValue(
skip.SkipValue(
story, 'Skipped because story is not supported '
'(SharedState.CanRunStory() returns False).'))
return
state.RunStory(results)
if isinstance(test, story_test.StoryTest):
test.Measure(state.platform, results)
except (legacy_page_test.Failure, exceptions.TimeoutException,
exceptions.LoginException, exceptions.ProfilingException,
py_utils.TimeoutException):
ProcessError()
except exceptions.Error:
ProcessError()
raise
except page_action.PageActionNotSupported as e:
results.AddValue(
skip.SkipValue(story, 'Unsupported page action: %s' % e))
except Exception:
ProcessError(description='Unhandlable exception raised.')
raise
finally:
has_existing_exception = (sys.exc_info() != (None, None, None))
try:
# We attempt to stop tracing and/or metric collecting before possibly
# closing the browser. Closing the browser first and stopping tracing
# later appeared to cause issues where subsequent browser instances would
# not launch correctly on some devices (see: crbug.com/720317).
# The following normally cause tracing and/or metric collecting to stop.
if isinstance(test, story_test.StoryTest):
test.DidRunStory(state.platform, results)
else:
test.DidRunPage(state.platform)
# And the following normally causes the browser to be closed.
state.DidRunStory(results)
# TODO(mikecase): Remove this logging once Android perf bots are swarmed.
# crbug.com/678282
if state.platform.GetOSName() == 'android':
state.platform._platform_backend.Log(
'END %s' % (story.name if story.name else str(story)))
except Exception:
if not has_existing_exception:
state.DumpStateUponFailure(story, results)
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception raised when cleaning story run: ')
def Skip(self, reason, is_expected=True):
self.AddValue(skip.SkipValue(self.story, reason, is_expected))
def _RunStoryAndProcessErrorIfNeeded(story, results, state, test):
def ProcessError(exc=None, description=None):
state.DumpStateUponFailure(story, results)
# Dump app crash, if present
if exc:
if isinstance(exc, exceptions.AppCrashException):
minidump_path = exc.minidump_path
if minidump_path:
results.AddArtifact(story.name, 'minidump', minidump_path)
# Note: adding the FailureValue to the results object also normally
# cause the progress_reporter to log it in the output.
results.AddValue(
failure.FailureValue(story, sys.exc_info(), description))
with CaptureLogsAsArtifacts(results, story.name):
try:
if isinstance(test, story_test.StoryTest):
test.WillRunStory(state.platform)
state.WillRunStory(story)
if not state.CanRunStory(story):
results.AddValue(
skip.SkipValue(
story, 'Skipped because story is not supported '
'(SharedState.CanRunStory() returns False).'))
return
state.RunStory(results)
if isinstance(test, story_test.StoryTest):
test.Measure(state.platform, results)
except (legacy_page_test.Failure, exceptions.TimeoutException,
exceptions.LoginException, exceptions.ProfilingException,
py_utils.TimeoutException) as exc:
ProcessError(exc)
except exceptions.Error as exc:
ProcessError(exc)
raise
except page_action.PageActionNotSupported as exc:
results.AddValue(
skip.SkipValue(story, 'Unsupported page action: %s' % exc))
except Exception:
ProcessError(description='Unhandlable exception raised.')
raise
finally:
has_existing_exception = (sys.exc_info() != (None, None, None))
try:
# We attempt to stop tracing and/or metric collecting before possibly
# closing the browser. Closing the browser first and stopping tracing
# later appeared to cause issues where subsequent browser instances
# would not launch correctly on some devices (see: crbug.com/720317).
# The following normally cause tracing and/or metric collecting to stop.
if isinstance(test, story_test.StoryTest):
test.DidRunStory(state.platform, results)
else:
test.DidRunPage(state.platform)
# And the following normally causes the browser to be closed.
state.DidRunStory(results)
except Exception as exc: # pylint: disable=broad-except
if not has_existing_exception:
state.DumpStateUponFailure(story, results, exc)
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception raised when cleaning story run: ')
def testAsDict(self):
v = skip.SkipValue(self.pages[0], 'page skipped for testing reason',
False)
d = v.AsDict()
self.assertEquals(d['reason'], 'page skipped for testing reason')
self.assertEquals(d['is_expected'], False)
def Skip(self, reason, is_expected=True):
assert self._current_page_run, 'Not currently running test.'
self.AddValue(skip.SkipValue(self.current_page, reason, is_expected))
def Skip(self, reason):
self.AddValue(skip.SkipValue(self.story, reason))
def _RunStoryAndProcessErrorIfNeeded(story, results, state, test):
def ProcessError(description=None):
state.DumpStateUponFailure(story, results)
# Note: adding the FailureValue to the results object also normally
# cause the progress_reporter to log it in the output.
results.AddValue(
failure.FailureValue(story, sys.exc_info(), description))
try:
# TODO(mikecase): Remove this logging once Android perf bots are swarmed.
# crbug.com/678282
if state.platform.GetOSName() == 'android':
state.platform._platform_backend.Log(
'START %s' % (story.name if story.name else str(story)))
if isinstance(test, story_test.StoryTest):
test.WillRunStory(state.platform)
state.WillRunStory(story)
if not state.CanRunStory(story):
results.AddValue(
skip.SkipValue(
story, 'Skipped because story is not supported '
'(SharedState.CanRunStory() returns False).'))
return
state.RunStory(results)
if isinstance(test, story_test.StoryTest):
test.Measure(state.platform, results)
except (legacy_page_test.Failure, exceptions.TimeoutException,
exceptions.LoginException, exceptions.ProfilingException,
py_utils.TimeoutException):
ProcessError()
except exceptions.Error:
ProcessError()
raise
except page_action.PageActionNotSupported as e:
results.AddValue(
skip.SkipValue(story, 'Unsupported page action: %s' % e))
except Exception:
ProcessError(description='Unhandlable exception raised.')
raise
finally:
has_existing_exception = (sys.exc_info() != (None, None, None))
try:
state.DidRunStory(results)
# if state.DidRunStory raises exception, things are messed up badly and we
# do not need to run test.DidRunStory at that point.
if isinstance(test, story_test.StoryTest):
test.DidRunStory(state.platform, results)
else:
test.DidRunPage(state.platform)
# TODO(mikecase): Remove this logging once Android perf bots are swarmed.
# crbug.com/678282
if state.platform.GetOSName() == 'android':
state.platform._platform_backend.Log(
'END %s' % (story.name if story.name else str(story)))
except Exception:
if not has_existing_exception:
state.DumpStateUponFailure(story, results)
raise
# Print current exception and propagate existing exception.
exception_formatter.PrintFormattedException(
msg='Exception raised when cleaning story run: ')
