def testDiscardFirstResult(self):
us = user_story_set.UserStorySet()
us.AddUserStory(DummyLocalUserStory(TestSharedPageState))
us.AddUserStory(DummyLocalUserStory(TestSharedPageState))
class Measurement(page_test.PageTest):
@property
def discard_first_result(self):
return True
def RunPage(self, page, _, results):
results.AddValue(
string.StringValue(page, 'test', 't', page.name))
def ValidateAndMeasurePage(self, page, tab, results):
pass
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(0, len(results.all_page_specific_values))
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
self.options.page_repeat = 1
self.options.pageset_repeat = 2
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
self.assertEquals(2, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(2, len(results.all_page_specific_values))
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
self.options.page_repeat = 2
self.options.pageset_repeat = 1
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
self.assertEquals(2, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(2, len(results.all_page_specific_values))
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
self.options.page_repeat = 1
self.options.pageset_repeat = 1
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(0, len(results.all_page_specific_values))
def RunMeasurement(self,
measurement,
ps,
expectations=test_expectations.TestExpectations(),
options=None):
"""Runs a measurement against a pageset, returning the rows its outputs."""
if options is None:
options = options_for_unittests.GetCopy()
assert options
temp_parser = options.CreateParser()
user_story_runner.AddCommandLineArgs(temp_parser)
defaults = temp_parser.get_default_values()
for k, v in defaults.__dict__.items():
if hasattr(options, k):
continue
setattr(options, k, v)
measurement.CustomizeBrowserOptions(options.browser_options)
options.output_file = None
options.output_formats = ['none']
options.suppress_gtest_report = True
options.output_trace_tag = None
user_story_runner.ProcessCommandLineArgs(temp_parser, options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(measurement, ps, expectations, options, results)
return results
def testCleanUpPage(self):
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
page = page_module.Page('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir())
ps.pages.append(page)
class Test(page_test.PageTest):
def __init__(self):
super(Test, self).__init__()
self.did_call_clean_up = False
def ValidatePage(self, *_):
raise exceptions.IntentionalException
def CleanUpAfterPage(self, page, tab):
self.did_call_clean_up = True
test = Test()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpUserStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(test, ps, expectations, options, results)
assert test.did_call_clean_up
def testBrowserBeforeLaunch(self):
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
page = page_module.Page('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir())
ps.pages.append(page)
class TestBeforeLaunch(page_test.PageTest):
def __init__(self):
super(TestBeforeLaunch, self).__init__()
self._did_call_will_start = False
self._did_call_did_start = False
def WillStartBrowser(self, platform):
self._did_call_will_start = True
# TODO(simonjam): Test that the profile is available.
def DidStartBrowser(self, browser):
assert self._did_call_will_start
self._did_call_did_start = True
def ValidatePage(self, *_):
assert self._did_call_did_start
test = TestBeforeLaunch()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpUserStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(test, ps, expectations, options, results)
def testOneTab(self):
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
page = page_module.Page('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir())
ps.pages.append(page)
class TestOneTab(page_test.PageTest):
def __init__(self):
super(TestOneTab, self).__init__()
self._browser = None
def DidStartBrowser(self, browser):
self._browser = browser
self._browser.tabs.New()
def ValidatePage(self, *_):
assert len(self._browser.tabs) == 1
test = TestOneTab()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpUserStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(test, ps, expectations, options, results)
def testUserAgent(self):
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
page = page_module.Page('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir())
ps.pages.append(page)
ps.user_agent_type = 'tablet'
class TestUserAgent(page_test.PageTest):
def ValidatePage(self, _1, tab, _2):
actual_user_agent = tab.EvaluateJavaScript(
'window.navigator.userAgent')
expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
assert actual_user_agent.strip() == expected_user_agent
# This is so we can check later that the test actually made it into this
# function. Previously it was timing out before even getting here, which
# should fail, but since it skipped all the asserts, it slipped by.
self.hasRun = True # pylint: disable=W0201
test = TestUserAgent()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpUserStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(test, ps, expectations, options, results)
self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
def testRaiseBrowserGoneExceptionFromRunPage(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class Test(page_test.PageTest):
def __init__(self, *args):
super(Test, self).__init__(*args)
self.run_count = 0
def RunPage(self, *_):
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 0:
raise exceptions.BrowserGoneException(
'i am a browser instance')
def ValidateAndMeasurePage(self, page, tab, results):
pass
us.AddUserStory(DummyLocalUserStory(TestSharedPageState))
us.AddUserStory(DummyLocalUserStory(TestSharedPageState))
test = Test()
user_story_runner.Run(test, us, self.expectations, self.options,
self.results)
self.assertEquals(2, test.run_count)
self.assertEquals(1, len(self.results.failures))
self.assertEquals(1, GetNumberOfSuccessfulPageRuns(self.results))
def testUnknownExceptionIsFatal(self):
self.SuppressExceptionFormatting()
uss = user_story_set.UserStorySet()
class UnknownException(Exception):
pass
# This erroneous test is set up to raise exception for the 2nd user story
# run.
class Test(page_test.PageTest):
def __init__(self, *args):
super(Test, self).__init__(*args)
self.run_count = 0
def RunPage(self, *_):
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 1:
raise UnknownException('FooBarzException')
def ValidateAndMeasurePage(self, page, tab, results):
pass
us1 = DummyLocalUserStory(TestSharedPageState)
us2 = DummyLocalUserStory(TestSharedPageState)
uss.AddUserStory(us1)
uss.AddUserStory(us2)
test = Test()
with self.assertRaises(UnknownException):
user_story_runner.Run(test, uss, self.expectations, self.options,
self.results)
self.assertEqual(set([us2]), self.results.pages_that_failed)
self.assertEqual(set([us1]), self.results.pages_that_succeeded)
self.assertIn('FooBarzException', self.fake_stdout.getvalue())
def _RunPageTestThatRaisesAppCrashException(self, test, max_failures):
class TestPage(page_module.Page):
def RunNavigateSteps(self, _):
raise exceptions.AppCrashException
ps = page_set.PageSet()
for _ in range(5):
ps.AddUserStory(
TestPage('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir()))
expectations = test_expectations.TestExpectations()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpUserStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(test,
ps,
expectations,
options,
results,
max_failures=max_failures)
return results
def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class ExpectedException(Exception):
pass
class Test(page_test.PageTest):
def __init__(self, *args):
super(Test, self).__init__(*args)
self.run_count = 0
def RunPage(self, *_):
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 0:
raise ExpectedException()
us.AddUserStory(user_story.UserStory(TestSharedUserStoryState))
us.AddUserStory(user_story.UserStory(TestSharedUserStoryState))
test = Test()
user_story_runner.Run(test, us, self.expectations, self.options,
self.results)
self.assertEquals(2, test.run_count)
self.assertEquals(1, len(self.results.failures))
self.assertEquals(1, GetNumberOfSuccessfulPageRuns(self.results))
def Run(self, finder_options):
"""Run this test with the given options."""
self.CustomizeBrowserOptions(finder_options.browser_options)
pt = self.CreatePageTest(finder_options)
pt.__name__ = self.__class__.__name__
if hasattr(self, '_disabled_strings'):
# pylint: disable=protected-access
pt._disabled_strings = self._disabled_strings
if hasattr(self, '_enabled_strings'):
# pylint: disable=protected-access
pt._enabled_strings = self._enabled_strings
expectations = self.CreateExpectations()
us = self.CreateUserStorySet(finder_options)
self._DownloadGeneratedProfileArchive(finder_options)
benchmark_metadata = self.GetMetadata()
results = results_options.CreateResults(benchmark_metadata, finder_options)
try:
user_story_runner.Run(pt, us, expectations, finder_options, results)
except page_test.TestNotSupportedOnPlatformFailure as failure:
logging.warning(str(failure))
bucket = cloud_storage.INTERNAL_BUCKET
if finder_options.upload_results:
results.UploadTraceFilesToCloud(bucket)
results.UploadProfilingFilesToCloud(bucket)
results.PrintSummary()
return len(results.failures)
def RunUserStoryTest(self, us, expected_successes):
test = DummyTest()
user_story_runner.Run(test, us, self.expectations, self.options,
self.results)
self.assertEquals(0, len(self.results.failures))
self.assertEquals(expected_successes,
GetNumberOfSuccessfulPageRuns(self.results))
def testPagesetRepeat(self):
us = user_story_set.UserStorySet()
us.AddUserStory(
user_story.UserStory(TestSharedUserStoryState, name='blank'))
us.AddUserStory(
user_story.UserStory(TestSharedUserStoryState, name='green'))
class Measurement(page_test.PageTest):
i = 0
def RunPage(self, page, _, results):
self.i += 1
results.AddValue(
scalar.ScalarValue(page, 'metric', 'unit', self.i))
self.options.page_repeat = 1
self.options.pageset_repeat = 2
self.options.output_formats = ['buildbot']
output = StringIO.StringIO()
real_stdout = sys.stdout
sys.stdout = output
try:
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
user_story_runner.Run(Measurement(), us, self.expectations,
self.options, results)
results.PrintSummary()
contents = output.getvalue()
self.assertEquals(4, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertIn('RESULT metric: blank= [1,3] unit', contents)
self.assertIn('RESULT metric: green= [2,4] unit', contents)
self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', contents)
finally:
sys.stdout = real_stdout
def testNeedsBrowserRestartAfterEachPage(self):
self.SuppressExceptionFormatting()
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
ps.pages.append(
page_module.Page('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir()))
ps.pages.append(
page_module.Page('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir()))
class Test(page_test.PageTest):
def __init__(self, *args, **kwargs):
super(Test, self).__init__(*args, **kwargs)
self.browser_starts = 0
def DidStartBrowser(self, *args):
super(Test, self).DidStartBrowser(*args)
self.browser_starts += 1
def ValidateAndMeasurePage(self, page, tab, results):
pass
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
test = Test(needs_browser_restart_after_each_page=True)
SetUpUserStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(test, ps, expectations, options, results)
self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
self.assertEquals(2, test.browser_starts)
def testRunPageWithProfilingFlag(self):
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
ps.pages.append(
page_module.Page('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir()))
class Measurement(page_test.PageTest):
pass
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
options.reset_results = None
options.upload_results = None
options.results_label = None
options.output_dir = tempfile.mkdtemp()
options.profiler = 'trace'
try:
SetUpUserStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(Measurement(), ps, expectations, options,
results)
self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
self.assertEquals(0, len(results.failures))
self.assertEquals(0, len(results.all_page_specific_values))
self.assertTrue(
os.path.isfile(
os.path.join(options.output_dir, 'blank_html.json')))
finally:
shutil.rmtree(options.output_dir)
def testWebPageReplay(self):
ps = example_domain.ExampleDomainPageSet()
expectations = test_expectations.TestExpectations()
body = []
class TestWpr(page_test.PageTest):
def ValidateAndMeasurePage(self, _, tab, __):
body.append(tab.EvaluateJavaScript('document.body.innerText'))
test = TestWpr()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpUserStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(test, ps, expectations, options, results)
self.longMessage = True
self.assertIn('Example Domain',
body[0],
msg='URL: %s' % ps.pages[0].url)
self.assertIn('Example Domain',
body[1],
msg='URL: %s' % ps.pages[1].url)
self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
self.assertEquals(0, len(results.failures))
def GenerateProfiles(profile_creator_class, profile_creator_name, options):
"""Generate a profile"""
expectations = test_expectations.TestExpectations()
test = profile_creator_class()
temp_output_directory = tempfile.mkdtemp()
options.output_profile_path = temp_output_directory
results = results_options.CreateResults(
benchmark.BenchmarkMetadata(test.__class__.__name__), options)
user_story_runner.Run(test, test.page_set, expectations, options, results)
if results.failures:
logging.warning('Some pages failed.')
logging.warning('Failed pages:\n%s',
'\n'.join(map(str, results.pages_that_failed)))
return 1
# Everything is a-ok, move results to final destination.
generated_profiles_dir = os.path.abspath(options.output_dir)
if not os.path.exists(generated_profiles_dir):
os.makedirs(generated_profiles_dir)
out_path = os.path.join(generated_profiles_dir, profile_creator_name)
if os.path.exists(out_path):
shutil.rmtree(out_path)
shutil.copytree(temp_output_directory, out_path, ignore=_IsPseudoFile)
shutil.rmtree(temp_output_directory)
sys.stderr.write("SUCCESS: Generated profile copied to: '%s'.\n" %
out_path)
return 0
def Run(self, options):
# Installing extensions requires that the profile directory exist before
# the browser is launched.
if not options.browser_options.profile_dir:
options.browser_options.profile_dir = tempfile.mkdtemp()
options.browser_options.disable_default_apps = False
self._PrepareExtensionInstallFiles(options.browser_options.profile_dir)
expectations = test_expectations.TestExpectations()
results = results_options.CreateResults(
benchmark.BenchmarkMetadata(profile_creator.__class__.__name__),
options)
extension_page_test = _ExtensionPageTest()
extension_page_test._expected_extension_count = len(
self._extensions_to_install)
user_story_runner.Run(extension_page_test,
extension_page_test._page_set, expectations,
options, results)
self._CleanupExtensionInstallFiles()
# Check that files on this list exist and have content.
expected_files = [os.path.join('Default', 'Network Action Predictor')]
for filename in expected_files:
filename = os.path.join(options.output_profile_path, filename)
if not os.path.getsize(filename) > 0:
raise Exception("Profile not complete: %s is zero length." %
filename)
if results.failures:
logging.warning('Some pages failed.')
logging.warning('Failed pages:\n%s',
'\n'.join(map(str, results.pages_that_failed)))
raise Exception('ExtensionsProfileCreator failed.')
def Record(self, results):
assert self._page_set.wpr_archive_info, (
'Pageset archive_data_file path must be specified.')
self._page_set.wpr_archive_info.AddNewTemporaryRecording()
self._record_page_test.CustomizeBrowserOptions(self._options)
user_story_runner.Run(self._record_page_test, self._page_set,
test_expectations.TestExpectations(),
self._options, results)
def testSuccefulUserStoryTest(self):
us = user_story_set.UserStorySet()
us.AddUserStory(user_story.UserStory(FooUserStoryState))
us.AddUserStory(user_story.UserStory(FooUserStoryState))
us.AddUserStory(user_story.UserStory(BarUserStoryState))
user_story_runner.Run(DummyTest(), us, self.expectations, self.options,
self.results)
self.assertEquals(0, len(self.results.failures))
self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results))
def Run(self, finder_options):
"""Run this test with the given options.
Returns:
The number of failure values (up to 254) or 255 if there is an uncaught
exception.
"""
self.CustomizeBrowserOptions(finder_options.browser_options)
pt = self.CreatePageTest(finder_options)
pt.__name__ = self.__class__.__name__
if hasattr(self, '_disabled_strings'):
# pylint: disable=protected-access
pt._disabled_strings = self._disabled_strings
if hasattr(self, '_enabled_strings'):
# pylint: disable=protected-access
pt._enabled_strings = self._enabled_strings
expectations = self.CreateExpectations()
us = self.CreateUserStorySet(finder_options)
if isinstance(pt, page_test.PageTest):
if any(not isinstance(p, page.Page) for p in us.user_stories):
raise Exception(
'PageTest must be used with UserStorySet containing only '
'telemetry.page.Page user stories.')
self._DownloadGeneratedProfileArchive(finder_options)
benchmark_metadata = self.GetMetadata()
with results_options.CreateResults(
benchmark_metadata, finder_options,
self.ValueCanBeAddedPredicate) as results:
try:
user_story_runner.Run(pt,
us,
expectations,
finder_options,
results,
max_failures=self._max_failures)
return_code = min(254, len(results.failures))
except Exception:
exception_formatter.PrintFormattedException()
return_code = 255
bucket = cloud_storage.BUCKET_ALIASES[finder_options.upload_bucket]
if finder_options.upload_results:
results.UploadTraceFilesToCloud(bucket)
results.UploadProfilingFilesToCloud(bucket)
results.PrintSummary()
return return_code
def Run(self, options):
expectations = test_expectations.TestExpectations()
results = results_options.CreateResults(
benchmark.BenchmarkMetadata(profile_creator.__class__.__name__),
options)
user_story_runner.Run(self._page_test, self._page_test._page_set,
expectations, options, results)
if results.failures:
logging.warning('Some pages failed to load.')
logging.warning('Failed pages:\n%s',
'\n'.join(map(str, results.pages_that_failed)))
raise Exception('SmallProfileCreator failed.')
def testHandlingOfCrashedApp(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class SharedUserStoryThatCausesAppCrash(TestSharedUserStoryState):
def WillRunUserStory(self, user_storyz):
raise exceptions.AppCrashException()
us.AddUserStory(
user_story.UserStory(SharedUserStoryThatCausesAppCrash))
user_story_runner.Run(DummyTest(), us, self.expectations, self.options,
self.results)
self.assertEquals(1, len(self.results.failures))
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(self.results))
def testAppCrashExceptionCausesFailureValue(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class SharedUserStoryThatCausesAppCrash(TestSharedPageState):
def WillRunUserStory(self, user_storyz):
raise exceptions.AppCrashException('App Foo crashes')
us.AddUserStory(DummyLocalUserStory(SharedUserStoryThatCausesAppCrash))
user_story_runner.Run(DummyTest(), us, self.expectations, self.options,
self.results)
self.assertEquals(1, len(self.results.failures))
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(self.results))
self.assertIn('App Foo crashes', self.fake_stdout.getvalue())
def _testMaxFailuresOptionIsRespectedAndOverridable(
self, max_failures=None):
self.SuppressExceptionFormatting()
class TestPage(page_module.Page):
def __init__(self, *args, **kwargs):
super(TestPage, self).__init__(*args, **kwargs)
self.was_run = False
def RunNavigateSteps(self, action_runner): # pylint: disable=W0613
self.was_run = True
raise Exception('Test exception')
class Test(page_test.PageTest):
def ValidatePage(self, *args):
pass
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
for ii in range(5):
ps.pages.append(
TestPage('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir()))
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
expected_max_failures = 2
if not max_failures is None:
options.max_failures = max_failures
expected_max_failures = max_failures
SetUpUserStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(Test(max_failures=2), ps, expectations, options,
results)
self.assertEquals(0, len(GetSuccessfulPageRuns(results)))
# Runs up to max_failures+1 failing tests before stopping, since
# every tests after max_failures failures have been encountered
# may all be passing.
self.assertEquals(expected_max_failures + 1, len(results.failures))
for ii in range(len(ps.pages)):
if ii <= expected_max_failures:
self.assertTrue(ps.pages[ii].was_run)
else:
self.assertFalse(ps.pages[ii].was_run)
def testHandlingOfCrashedTabWithExpectedFailure(self):
self.SuppressExceptionFormatting()
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
expectations.Fail('chrome://crash')
page1 = page_module.Page('chrome://crash', ps)
ps.pages.append(page1)
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpUserStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(DummyTest(), ps, expectations, options, results)
self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
self.assertEquals(0, len(results.failures))
def testSuccessfulTimelineBasedMeasurementTest(self):
"""Check that PageTest is not required for user_story_runner.Run.
Any PageTest related calls or attributes need to only be called
for PageTest tests.
"""
class TestSharedTbmState(TestSharedUserStoryState):
def RunUserStory(self, results):
pass
test = timeline_based_measurement.TimelineBasedMeasurement(
timeline_based_measurement.Options())
us = user_story_set.UserStorySet()
us.AddUserStory(DummyLocalUserStory(TestSharedTbmState))
us.AddUserStory(DummyLocalUserStory(TestSharedTbmState))
us.AddUserStory(DummyLocalUserStory(TestSharedTbmState))
user_story_runner.Run(test, us, self.expectations, self.options,
self.results)
self.assertEquals(0, len(self.results.failures))
self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results))
def testRaiseBrowserGoneExceptionFromRestartBrowserBeforeEachPage(self):
self.CaptureFormattedException()
ps = page_set.PageSet()
expectations = test_expectations.TestExpectations()
ps.pages.append(
page_module.Page('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir()))
ps.pages.append(
page_module.Page('file://blank.html',
ps,
base_dir=util.GetUnittestDataDir()))
class Test(page_test.PageTest):
def __init__(self, *args):
super(Test, self).__init__(
*args, needs_browser_restart_after_each_page=True)
self.run_count = 0
def RestartBrowserBeforeEachPage(self):
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 0:
raise exceptions.BrowserGoneException(None)
return self._needs_browser_restart_after_each_page
def ValidateAndMeasurePage(self, page, tab, results):
pass
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
test = Test()
SetUpUserStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(test, ps, expectations, options, results)
self.assertEquals(2, test.run_count)
self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
self.assertEquals(1, len(results.failures))
self.assertFormattedExceptionIsEmpty()
def testAppCrashThenRaiseInTearDownFatal(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class DidRunTestError(Exception):
pass
class TestTearDownSharedUserStoryState(TestSharedPageState):
def TearDownState(self, results):
self._test.DidRunTest('app', results)
class Test(page_test.PageTest):
def __init__(self, *args):
super(Test, self).__init__(*args)
self.run_count = 0
self._unit_test_events = [] # track what was called when
def RunPage(self, *_):
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 0:
self._unit_test_events.append('app-crash')
raise exceptions.AppCrashException
def ValidateAndMeasurePage(self, page, tab, results):
pass
def DidRunTest(self, _, __):
self._unit_test_events.append('did-run-test')
raise DidRunTestError
us.AddUserStory(DummyLocalUserStory(TestTearDownSharedUserStoryState))
us.AddUserStory(DummyLocalUserStory(TestTearDownSharedUserStoryState))
test = Test()
with self.assertRaises(DidRunTestError):
user_story_runner.Run(test, us, self.expectations, self.options,
self.results)
self.assertEqual(['app-crash', 'did-run-test'], test._unit_test_events)
# The AppCrashException gets added as a failure.
self.assertEquals(1, len(self.results.failures))
def testSharedPageStateCannotRunOnBrowser(self):
ps = page_set.PageSet()
class UnrunnableSharedState(shared_page_state.SharedPageState):
def CanRunOnBrowser(self, _):
return False
def ValidateAndMeasurePage(self, _):
pass
ps.AddUserStory(
page_module.Page(url='file://blank.html',
page_set=ps,
base_dir=util.GetUnittestDataDir(),
shared_page_state_class=UnrunnableSharedState))
expectations = test_expectations.TestExpectations()
class Test(page_test.PageTest):
def __init__(self, *args, **kwargs):
super(Test, self).__init__(*args, **kwargs)
self.will_navigate_to_page_called = False
def ValidateAndMeasurePage(self, *_args):
raise Exception('Exception should not be thrown')
def WillNavigateToPage(self, _1, _2):
self.will_navigate_to_page_called = True
test = Test()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpUserStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
user_story_runner.Run(test, ps, expectations, options, results)
self.assertFalse(test.will_navigate_to_page_called)
self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
self.assertEquals(1, len(results.skipped_values))
self.assertEquals(0, len(results.failures))