def _RunGpuTest(self, url, test_name, args):
temp_page = _EmulatedPage(url, test_name)
expectations = self.__class__.GetExpectations()
expectation = expectations.GetExpectationForPage(
self.browser, temp_page)
if expectation == 'skip':
# skipTest in Python's unittest harness raises an exception, so
# aborts the control flow here.
self.skipTest('SKIPPING TEST due to test expectations')
try:
self.RunActualGpuTest(url, *args)
except Exception:
if expectation == 'pass':
# This is not an expected exception or test failure, so print
# the detail to the console.
exception_formatter.PrintFormattedException()
# This failure might have been caused by a browser or renderer
# crash, so restart the browser to make sure any state doesn't
# propagate to the next test iteration.
self._RestartBrowser('unexpected test failure')
raise
elif expectation == 'fail':
msg = 'Expected exception while running %s' % test_name
exception_formatter.PrintFormattedException(msg=msg)
return
if expectation != 'flaky':
logging.warning(
'Unknown expectation %s while handling exception for %s',
expectation, test_name)
raise
# Flaky tests are handled here.
num_retries = expectations.GetFlakyRetriesForPage(
self.browser, temp_page)
if not num_retries:
# Re-raise the exception.
raise
# Re-run the test up to |num_retries| times.
for ii in xrange(0, num_retries):
print 'FLAKY TEST FAILURE, retrying: ' + test_name
try:
# For robustness, shut down the browser and restart it
# between flaky test failures, to make sure any state
# doesn't propagate to the next iteration.
self._RestartBrowser('flaky test failure')
self.RunActualGpuTest(url, *args)
break
except Exception:
# Squelch any exceptions from any but the last retry.
if ii == num_retries - 1:
# Restart the browser after the last failure to make sure
# any state doesn't propagate to the next iteration.
self._RestartBrowser('excessive flaky test failures')
raise
else:
if expectation == 'fail':
logging.warning('%s was expected to fail, but passed.\n',
test_name)
def _RunGpuTest(self, url, test_name, *args):
expected_results, should_retry_on_failure = (
self.GetExpectationsForTest())
try:
# TODO(nednguyen): For some reason the arguments are getting wrapped
# in another tuple sometimes (like in the WebGL extension tests).
# Perhaps only if multiple arguments are yielded in the test
# generator?
if len(args) == 1 and isinstance(args[0], tuple):
args = args[0]
self.RunActualGpuTest(url, *args)
except Exception:
if ResultType.Failure in expected_results or should_retry_on_failure:
if should_retry_on_failure:
# For robustness, shut down the browser and restart it
# between flaky test failures, to make sure any state
# doesn't propagate to the next iteration.
self._RestartBrowser('flaky test failure')
else:
msg = 'Expected exception while running %s' % test_name
exception_formatter.PrintFormattedException(msg=msg)
# Even though this is a known failure, the browser might still
# be in a bad state; for example, certain kinds of timeouts
# will affect the next test. Restart the browser to prevent
# these kinds of failures propagating to the next test.
self._RestartBrowser('expected test failure')
else:
# This is not an expected exception or test failure, so print
# the detail to the console.
exception_formatter.PrintFormattedException()
# Symbolize any crash dump (like from the GPU process) that
# might have happened but wasn't detected above. Note we don't
# do this for either 'fail' or 'flaky' expectations because
# there are still quite a few flaky failures in the WebGL test
# expectations, and since minidump symbolization is slow
# (upwards of one minute on a fast laptop), symbolizing all the
# stacks could slow down the tests' running time unacceptably.
# We also don't do this if the browser failed to startup.
if self.browser is not None:
self.browser.LogSymbolizedUnsymbolizedMinidumps(
logging.ERROR)
# This failure might have been caused by a browser or renderer
# crash, so restart the browser to make sure any state doesn't
# propagate to the next test iteration.
self._RestartBrowser('unexpected test failure')
self.fail()
else:
if ResultType.Failure in expected_results:
logging.warning('%s was expected to fail, but passed.\n',
test_name)
def RunStoryWithRetries(cls, shared_page_state, results):
page = shared_page_state.current_page
expectations = page.GetExpectations()
expectation = 'pass'
if expectations:
expectation = expectations.GetExpectationForPage(
shared_page_state.browser, page)
if expectation == 'skip':
raise Exception(
'Skip expectations should have been handled in CanRunOnBrowser')
try:
super(cls, shared_page_state).RunStory(results)
except Exception:
if expectation == 'pass':
raise
elif expectation == 'fail':
msg = 'Expected exception while running %s' % page.display_name
exception_formatter.PrintFormattedException(msg=msg)
return
if expectation != 'flaky':
logging.warning(
'Unknown expectation %s while handling exception for %s',
expectation, page.display_name)
raise
# Flaky tests are handled here.
num_retries = expectations.GetFlakyRetriesForPage(
shared_page_state.browser, page)
if not num_retries:
# Re-raise the exception.
raise
# Re-run the test up to |num_retries| times.
for ii in xrange(0, num_retries):
print 'FLAKY TEST FAILURE, retrying: ' + page.display_name
try:
super(cls, shared_page_state).RunStory(results)
break
except Exception:
# Squelch any exceptions from any but the last retry.
if ii == num_retries - 1:
raise
else:
if expectation == 'fail':
logging.warning('%s was expected to fail, but passed.\n',
page.display_name)
def _RunGpuTest(self, url, test_name, *args):
expectations = self.__class__.GetExpectations()
expectation = expectations.GetExpectationForTest(
self.browser, url, test_name)
if self.__class__._also_run_disabled_tests:
# Ignore test expectations if the user has requested it.
expectation = 'pass'
if expectation == 'skip':
# skipTest in Python's unittest harness raises an exception, so
# aborts the control flow here.
self.skipTest('SKIPPING TEST due to test expectations')
try:
# TODO(nednguyen): For some reason the arguments are getting wrapped
# in another tuple sometimes (like in the WebGL extension tests).
# Perhaps only if multiple arguments are yielded in the test
# generator?
if len(args) == 1 and isinstance(args[0], tuple):
args = args[0]
self.RunActualGpuTest(url, *args)
except Exception:
if expectation == 'pass':
# This is not an expected exception or test failure, so print
# the detail to the console.
exception_formatter.PrintFormattedException()
# Symbolize any crash dump (like from the GPU process) that
# might have happened but wasn't detected above. Note we don't
# do this for either 'fail' or 'flaky' expectations because
# there are still quite a few flaky failures in the WebGL test
# expectations, and since minidump symbolization is slow
# (upwards of one minute on a fast laptop), symbolizing all the
# stacks could slow down the tests' running time unacceptably.
self.browser.LogSymbolizedUnsymbolizedMinidumps(logging.ERROR)
# This failure might have been caused by a browser or renderer
# crash, so restart the browser to make sure any state doesn't
# propagate to the next test iteration.
self._RestartBrowser('unexpected test failure')
raise
elif expectation == 'fail':
msg = 'Expected exception while running %s' % test_name
exception_formatter.PrintFormattedException(msg=msg)
# Even though this is a known failure, the browser might still
# be in a bad state; for example, certain kinds of timeouts
# will affect the next test. Restart the browser to prevent
# these kinds of failures propagating to the next test.
self._RestartBrowser('expected test failure')
return
if expectation != 'flaky':
logging.warning(
'Unknown expectation %s while handling exception for %s',
expectation, test_name)
raise
# Flaky tests are handled here.
num_retries = expectations.GetFlakyRetriesForTest(
self.browser, url, test_name)
if not num_retries:
# Re-raise the exception.
raise
# Re-run the test up to |num_retries| times.
for ii in xrange(0, num_retries):
print 'FLAKY TEST FAILURE, retrying: ' + test_name
try:
# For robustness, shut down the browser and restart it
# between flaky test failures, to make sure any state
# doesn't propagate to the next iteration.
self._RestartBrowser('flaky test failure')
self.RunActualGpuTest(url, *args)
break
except Exception:
# Squelch any exceptions from any but the last retry.
if ii == num_retries - 1:
# Restart the browser after the last failure to make sure
# any state doesn't propagate to the next iteration.
self._RestartBrowser('excessive flaky test failures')
raise
else:
if expectation == 'fail':
logging.warning('%s was expected to fail, but passed.\n',
test_name)
def _RunGpuTest(self, url, test_name, *args):
temp_page = _EmulatedPage(url, test_name)
expectations = self.__class__.GetExpectations()
expectation = expectations.GetExpectationForPage(
self.browser, temp_page)
if expectation == 'skip':
# skipTest in Python's unittest harness raises an exception, so
# aborts the control flow here.
self.skipTest('SKIPPING TEST due to test expectations')
try:
# TODO(nednguyen): For some reason the arguments are getting wrapped
# in another tuple sometimes (like in the WebGL extension tests).
# Perhaps only if multiple arguments are yielded in the test
# generator?
if len(args) == 1 and isinstance(args[0], tuple):
args = args[0]
self.RunActualGpuTest(url, *args)
except Exception:
if expectation == 'pass':
# This is not an expected exception or test failure, so print
# the detail to the console.
exception_formatter.PrintFormattedException()
# This failure might have been caused by a browser or renderer
# crash, so restart the browser to make sure any state doesn't
# propagate to the next test iteration.
self._RestartBrowser('unexpected test failure')
raise
elif expectation == 'fail':
msg = 'Expected exception while running %s' % test_name
exception_formatter.PrintFormattedException(msg=msg)
# Even though this is a known failure, the browser might still
# be in a bad state; for example, certain kinds of timeouts
# will affect the next test. Restart the browser to prevent
# these kinds of failures propagating to the next test.
self._RestartBrowser('expected test failure')
return
if expectation != 'flaky':
logging.warning(
'Unknown expectation %s while handling exception for %s',
expectation, test_name)
raise
# Flaky tests are handled here.
num_retries = expectations.GetFlakyRetriesForPage(
self.browser, temp_page)
if not num_retries:
# Re-raise the exception.
raise
# Re-run the test up to |num_retries| times.
for ii in xrange(0, num_retries):
print 'FLAKY TEST FAILURE, retrying: ' + test_name
try:
# For robustness, shut down the browser and restart it
# between flaky test failures, to make sure any state
# doesn't propagate to the next iteration.
self._RestartBrowser('flaky test failure')
self.RunActualGpuTest(url, *args)
break
except Exception:
# Squelch any exceptions from any but the last retry.
if ii == num_retries - 1:
# Restart the browser after the last failure to make sure
# any state doesn't propagate to the next iteration.
self._RestartBrowser('excessive flaky test failures')
raise
else:
if expectation == 'fail':
logging.warning('%s was expected to fail, but passed.\n',
test_name)
