def testDiscardFirstResult(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() ps.pages.append(page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) ps.pages.append(page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) class Measurement(page_test.PageTest): @property def discard_first_result(self): return True def ValidateAndMeasurePage(self, page, _, results): results.AddValue(string.StringValue(page, 'test', 't', page.url)) options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True options.reset_results = None options.upload_results = None options.results_label = None options.page_repeat = 1 options.pageset_repeat = 1 SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(Measurement(), ps, expectations, options, results) self.assertEquals(0, len(GetSuccessfulPageRuns(results))) self.assertEquals(0, len(results.failures)) self.assertEquals(0, len(results.all_page_specific_values)) options.page_repeat = 1 options.pageset_repeat = 2 SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(Measurement(), ps, expectations, options, results) self.assertEquals(2, len(GetSuccessfulPageRuns(results))) self.assertEquals(0, len(results.failures)) self.assertEquals(2, len(results.all_page_specific_values)) options.page_repeat = 2 options.pageset_repeat = 1 SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(Measurement(), ps, expectations, options, results) self.assertEquals(2, len(GetSuccessfulPageRuns(results))) self.assertEquals(0, len(results.failures)) self.assertEquals(2, len(results.all_page_specific_values)) options.output_formats = ['html'] options.suppress_gtest_report = True options.page_repeat = 1 options.pageset_repeat = 1 SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(Measurement(), ps, expectations, options, results) self.assertEquals(0, len(GetSuccessfulPageRuns(results))) self.assertEquals(0, len(results.failures)) self.assertEquals(0, len(results.all_page_specific_values))
def testDiscardFirstResult(self): us = user_story_set.UserStorySet() us.AddUserStory(DummyLocalUserStory(TestSharedPageState)) us.AddUserStory(DummyLocalUserStory(TestSharedPageState)) class Measurement(page_test.PageTest): @property def discard_first_result(self): return True def RunPage(self, page, _, results): results.AddValue( string.StringValue(page, 'test', 't', page.name)) def ValidateAndMeasurePage(self, page, tab, results): pass results = results_options.CreateResults(EmptyMetadataForTest(), self.options) user_story_runner.Run(Measurement(), us, self.expectations, self.options, results) self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(0, len(results.failures)) self.assertEquals(0, len(results.all_page_specific_values)) results = results_options.CreateResults(EmptyMetadataForTest(), self.options) self.options.page_repeat = 1 self.options.pageset_repeat = 2 user_story_runner.Run(Measurement(), us, self.expectations, self.options, results) self.assertEquals(2, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(0, len(results.failures)) self.assertEquals(2, len(results.all_page_specific_values)) results = results_options.CreateResults(EmptyMetadataForTest(), self.options) self.options.page_repeat = 2 self.options.pageset_repeat = 1 user_story_runner.Run(Measurement(), us, self.expectations, self.options, results) self.assertEquals(2, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(0, len(results.failures)) self.assertEquals(2, len(results.all_page_specific_values)) results = results_options.CreateResults(EmptyMetadataForTest(), self.options) self.options.page_repeat = 1 self.options.pageset_repeat = 1 user_story_runner.Run(Measurement(), us, self.expectations, self.options, results) self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(0, len(results.failures)) self.assertEquals(0, len(results.all_page_specific_values))
def Run(self, finder_options): """Run this test with the given options.""" self.CustomizeBrowserOptions(finder_options.browser_options) pt = self.PageTestClass()() pt.__name__ = self.__class__.__name__ if hasattr(self, '_disabled_strings'): pt._disabled_strings = self._disabled_strings if hasattr(self, '_enabled_strings'): pt._enabled_strings = self._enabled_strings ps = self.CreatePageSet(finder_options) expectations = self.CreateExpectations(ps) self._DownloadGeneratedProfileArchive(finder_options) results = results_options.CreateResults(self.GetMetadata(), finder_options) try: page_runner.Run(pt, ps, expectations, finder_options, results) except page_test.TestNotSupportedOnPlatformFailure as failure: logging.warning(str(failure)) results.PrintSummary() return len(results.failures)
def testHandlingOfTestThatRaisesWithNonFatalUnknownExceptions(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() ps.pages.append(page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) ps.pages.append(page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) class ExpectedException(Exception): pass class Test(page_test.PageTest): def __init__(self, *args): super(Test, self).__init__(*args) self.run_count = 0 def ValidatePage(self, *_): old_run_count = self.run_count self.run_count += 1 if old_run_count == 0: raise ExpectedException() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True test = Test() SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(test, ps, expectations, options, results) self.assertEquals(2, test.run_count) self.assertEquals(1, len(GetSuccessfulPageRuns(results))) self.assertEquals(1, len(results.failures))
def testWebPageReplay(self): ps = example_domain.ExampleDomainPageSet() expectations = test_expectations.TestExpectations() body = [] class TestWpr(page_test.PageTest): def ValidateAndMeasurePage(self, _, tab, __): body.append(tab.EvaluateJavaScript('document.body.innerText')) test = TestWpr() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(test, ps, expectations, options, results) self.longMessage = True self.assertIn('Example Domain', body[0], msg='URL: %s' % ps.pages[0].url) self.assertIn('Example Domain', body[1], msg='URL: %s' % ps.pages[1].url) self.assertEquals(2, len(GetSuccessfulPageRuns(results))) self.assertEquals(0, len(results.failures))
def testNeedsBrowserRestartAfterEachPage(self): self.CaptureFormattedException() ps = page_set.PageSet() expectations = test_expectations.TestExpectations() ps.pages.append(page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) ps.pages.append(page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) class Test(page_test.PageTest): def __init__(self, *args, **kwargs): super(Test, self).__init__(*args, **kwargs) self.browser_starts = 0 def DidStartBrowser(self, *args): super(Test, self).DidStartBrowser(*args) self.browser_starts += 1 def ValidateAndMeasurePage(self, page, tab, results): pass options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True test = Test(needs_browser_restart_after_each_page=True) SetUpStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) story_runner.Run(test, ps, expectations, options, results) self.assertEquals(2, len(GetSuccessfulPageRuns(results))) self.assertEquals(2, test.browser_starts) self.assertFormattedExceptionIsEmpty()
def _RunPageTestThatRaisesAppCrashException(self, test, max_failures): class TestPage(page_module.Page): def RunNavigateSteps(self, _): raise exceptions.AppCrashException ps = page_set.PageSet() for _ in range(5): ps.AddUserStory( TestPage('file://blank.html', ps, base_dir=util.GetUnittestDataDir())) expectations = test_expectations.TestExpectations() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(test, ps, expectations, options, results, max_failures=max_failures) return results
def CreateResults(self): if self._benchmark is not None: benchmark_metadata = self._benchmark.GetMetadata() else: benchmark_metadata = benchmark.BenchmarkMetadata('record_wpr') return results_options.CreateResults(benchmark_metadata, self._options)
def Run(self, options): # Installing extensions requires that the profile directory exist before # the browser is launched. if not options.browser_options.profile_dir: options.browser_options.profile_dir = tempfile.mkdtemp() options.browser_options.disable_default_apps = False self._PrepareExtensionInstallFiles(options.browser_options.profile_dir) expectations = test_expectations.TestExpectations() results = results_options.CreateResults( benchmark.BenchmarkMetadata(profile_creator.__class__.__name__), options) extension_page_test = _ExtensionPageTest() extension_page_test._expected_extension_count = len( self._extensions_to_install) user_story_runner.Run(extension_page_test, extension_page_test._page_set, expectations, options, results) self._CleanupExtensionInstallFiles() # Check that files on this list exist and have content. expected_files = [os.path.join('Default', 'Network Action Predictor')] for filename in expected_files: filename = os.path.join(options.output_profile_path, filename) if not os.path.getsize(filename) > 0: raise Exception("Profile not complete: %s is zero length." % filename) if results.failures: logging.warning('Some pages failed.') logging.warning('Failed pages:\n%s', '\n'.join(map(str, results.pages_that_failed))) raise Exception('ExtensionsProfileCreator failed.')
def testBrowserBeforeLaunch(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() page = page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page) class TestBeforeLaunch(page_test.PageTest): def __init__(self): super(TestBeforeLaunch, self).__init__() self._did_call_will_start = False self._did_call_did_start = False def WillStartBrowser(self, platform): self._did_call_will_start = True # TODO(simonjam): Test that the profile is available. def DidStartBrowser(self, browser): assert self._did_call_will_start self._did_call_did_start = True def ValidatePage(self, *_): assert self._did_call_did_start test = TestBeforeLaunch() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(test, ps, expectations, options, results)
def testUserAgent(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() page = page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page) ps.user_agent_type = 'tablet' class TestUserAgent(page_test.PageTest): def ValidatePage(self, _1, tab, _2): actual_user_agent = tab.EvaluateJavaScript('window.navigator.userAgent') expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet'] assert actual_user_agent.strip() == expected_user_agent # This is so we can check later that the test actually made it into this # function. Previously it was timing out before even getting here, which # should fail, but since it skipped all the asserts, it slipped by. self.hasRun = True # pylint: disable=W0201 test = TestUserAgent() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(test, ps, expectations, options, results) self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
def GenerateProfiles(profile_creator_class, profile_creator_name, options): """Generate a profile""" expectations = test_expectations.TestExpectations() test = profile_creator_class() temp_output_directory = tempfile.mkdtemp() options.output_profile_path = temp_output_directory results = results_options.CreateResults( benchmark.BenchmarkMetadata(test.__class__.__name__), options) page_runner.Run(test, test.page_set, expectations, options, results) if results.failures: logging.warning('Some pages failed.') logging.warning('Failed pages:\n%s', '\n'.join(results.pages_that_failed)) return 1 # Everything is a-ok, move results to final destination. generated_profiles_dir = os.path.abspath(options.output_dir) if not os.path.exists(generated_profiles_dir): os.makedirs(generated_profiles_dir) out_path = os.path.join(generated_profiles_dir, profile_creator_name) if os.path.exists(out_path): shutil.rmtree(out_path) shutil.copytree(temp_output_directory, out_path, ignore=_IsPseudoFile) shutil.rmtree(temp_output_directory) sys.stderr.write("SUCCESS: Generated profile copied to: '%s'.\n" % out_path) return 0
def testRetryOnBrowserCrash(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() ps.pages.append(page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir())) class CrashyMeasurement(page_test.PageTest): has_crashed = False def ValidateAndMeasurePage(self, page, tab, results): # This value should be discarded on the first run when the # browser crashed. results.AddValue( string.StringValue(page, 'test', 't', str(self.has_crashed))) if not self.has_crashed: self.has_crashed = True raise exceptions.BrowserGoneException(tab.browser) options = options_for_unittests.GetCopy() options.output_formats = ['csv'] options.suppress_gtest_report = True SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(CrashyMeasurement(), ps, expectations, options, results) self.assertEquals(1, len(GetSuccessfulPageRuns(results))) self.assertEquals(0, len(results.failures)) self.assertEquals(1, len(results.all_page_specific_values)) self.assertEquals( 'True', results.all_page_specific_values[0].GetRepresentativeString())
def testRunPageWithStartupUrl(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() expectations = test_expectations.TestExpectations() page = page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) page.startup_url = 'about:blank' ps.pages.append(page) class Measurement(page_test.PageTest): def __init__(self): super(Measurement, self).__init__() self.browser_restarted = False def CustomizeBrowserOptionsForSinglePage(self, ps, options): self.browser_restarted = True super(Measurement, self).CustomizeBrowserOptionsForSinglePage(ps, options) def ValidateAndMeasurePage(self, page, tab, results): pass options = options_for_unittests.GetCopy() options.page_repeat = 2 options.output_formats = ['none'] options.suppress_gtest_report = True if not browser_finder.FindBrowser(options): return test = Measurement() SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(test, ps, expectations, options, results) self.assertEquals('about:blank', options.browser_options.startup_url) self.assertTrue(test.browser_restarted)
def testCleanUpPage(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() page = page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page) class Test(page_test.PageTest): def __init__(self): super(Test, self).__init__() self.did_call_clean_up = False def ValidatePage(self, *_): raise exceptions.IntentionalException def CleanUpAfterPage(self, page, tab): self.did_call_clean_up = True test = Test() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(test, ps, expectations, options, results) assert test.did_call_clean_up
def Run(self, finder_options): """Run this test with the given options.""" self.CustomizeBrowserOptions(finder_options.browser_options) pt = self.CreatePageTest(finder_options) pt.__name__ = self.__class__.__name__ if hasattr(self, '_disabled_strings'): # pylint: disable=protected-access pt._disabled_strings = self._disabled_strings if hasattr(self, '_enabled_strings'): # pylint: disable=protected-access pt._enabled_strings = self._enabled_strings expectations = self.CreateExpectations() us = self.CreateUserStorySet(finder_options) self._DownloadGeneratedProfileArchive(finder_options) benchmark_metadata = self.GetMetadata() results = results_options.CreateResults(benchmark_metadata, finder_options) try: user_story_runner.Run(pt, us, expectations, finder_options, results) except page_test.TestNotSupportedOnPlatformFailure as failure: logging.warning(str(failure)) bucket = cloud_storage.INTERNAL_BUCKET if finder_options.upload_results: results.UploadTraceFilesToCloud(bucket) results.UploadProfilingFilesToCloud(bucket) results.PrintSummary() return len(results.failures)
def RunMeasurement(self, measurement, ps, expectations=test_expectations.TestExpectations(), options=None): """Runs a measurement against a pageset, returning the rows its outputs.""" if options is None: options = options_for_unittests.GetCopy() assert options temp_parser = options.CreateParser() story_runner.AddCommandLineArgs(temp_parser) defaults = temp_parser.get_default_values() for k, v in defaults.__dict__.items(): if hasattr(options, k): continue setattr(options, k, v) measurement.CustomizeBrowserOptions(options.browser_options) options.output_file = None options.output_formats = ['none'] options.suppress_gtest_report = True options.output_trace_tag = None story_runner.ProcessCommandLineArgs(temp_parser, options) results = results_options.CreateResults(EmptyMetadataForTest(), options) story_runner.Run(measurement, ps, expectations, options, results) return results
def testRunPageWithProfilingFlag(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() ps.pages.append( page_module.Page('file://blank.html', ps, base_dir=util.GetUnittestDataDir())) class Measurement(page_test.PageTest): pass options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True options.reset_results = None options.upload_results = None options.results_label = None options.output_dir = tempfile.mkdtemp() options.profiler = 'trace' try: SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(Measurement(), ps, expectations, options, results) self.assertEquals(1, len(GetSuccessfulPageRuns(results))) self.assertEquals(0, len(results.failures)) self.assertEquals(0, len(results.all_page_specific_values)) self.assertTrue( os.path.isfile( os.path.join(options.output_dir, 'blank_html.json'))) finally: shutil.rmtree(options.output_dir)
def testOneTab(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() page = page_module.Page( 'file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page) class TestOneTab(page_test.PageTest): def __init__(self): super(TestOneTab, self).__init__() self._browser = None def DidStartBrowser(self, browser): self._browser = browser self._browser.tabs.New() def ValidatePage(self, *_): assert len(self._browser.tabs) == 1 test = TestOneTab() options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(test, ps, expectations, options, results)
def testPagesetRepeat(self): us = user_story_set.UserStorySet() us.AddUserStory(DummyLocalUserStory(TestSharedPageState, name='blank')) us.AddUserStory(DummyLocalUserStory(TestSharedPageState, name='green')) class Measurement(page_test.PageTest): i = 0 def RunPage(self, page, _, results): self.i += 1 results.AddValue(scalar.ScalarValue( page, 'metric', 'unit', self.i)) def ValidateAndMeasurePage(self, page, tab, results): pass self.options.page_repeat = 1 self.options.pageset_repeat = 2 self.options.output_formats = ['buildbot'] results = results_options.CreateResults( EmptyMetadataForTest(), self.options) story_runner.Run( Measurement(), us, self.expectations, self.options, results) results.PrintSummary() contents = self.fake_stdout.getvalue() self.assertEquals(4, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(0, len(results.failures)) self.assertIn('RESULT metric: blank= [1,3] unit', contents) self.assertIn('RESULT metric: green= [2,4] unit', contents) self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', contents)
def _testMaxFailuresOptionIsRespectedAndOverridable( self, num_failing_user_stories, runner_max_failures, options_max_failures, expected_num_failures): class SimpleSharedState( shared_state.SharedState): _fake_platform = FakePlatform() _current_user_story = None @property def platform(self): return self._fake_platform def WillRunUserStory(self, story): self._current_user_story = story def RunUserStory(self, results): self._current_user_story.Run() def DidRunUserStory(self, results): pass def GetTestExpectationAndSkipValue(self, expectations): return 'pass', None def TearDownState(self): pass class FailingUserStory(user_story.UserStory): def __init__(self): super(FailingUserStory, self).__init__( shared_state_class=SimpleSharedState, is_local=True) self.was_run = False def Run(self): self.was_run = True raise page_test.Failure self.SuppressExceptionFormatting() uss = user_story_set.UserStorySet() for _ in range(num_failing_user_stories): uss.AddUserStory(FailingUserStory()) options = _GetOptionForUnittest() options.output_formats = ['none'] options.suppress_gtest_report = True if options_max_failures: options.max_failures = options_max_failures results = results_options.CreateResults(EmptyMetadataForTest(), options) story_runner.Run( DummyTest(), uss, test_expectations.TestExpectations(), options, results, max_failures=runner_max_failures) self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results)) self.assertEquals(expected_num_failures, len(results.failures)) for ii, story in enumerate(uss.user_stories): self.assertEqual(story.was_run, ii < expected_num_failures)
def setUp(self): self.fake_stdout = StringIO.StringIO() self.actual_stdout = sys.stdout sys.stdout = self.fake_stdout self.options = _GetOptionForUnittest() self.expectations = test_expectations.TestExpectations() self.results = results_options.CreateResults( EmptyMetadataForTest(), self.options) self._story_runner_logging_stub = None
def testMaxFailuresOptionIsRespected(self): class TestPage(page_module.Page): def __init__(self, *args, **kwargs): super(TestPage, self).__init__(*args, **kwargs) self.was_run = False def RunNavigateSteps(self, action_runner): self.was_run = True raise Exception('Test exception') class Test(page_test.PageTest): def ValidatePage(self, *args): pass ps = page_set.PageSet() expectations = test_expectations.TestExpectations() page1 = TestPage('file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page1) page2 = TestPage('file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page2) page3 = TestPage('file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page3) page4 = TestPage('file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page4) page5 = TestPage('file://blank.html', ps, base_dir=util.GetUnittestDataDir()) ps.pages.append(page5) options = options_for_unittests.GetCopy() options.output_format = 'none' options.suppress_gtest_report = True SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(Test(max_failures=2), ps, expectations, options, results) self.assertEquals(0, len(GetSuccessfulPageRuns(results))) # Runs up to max_failures+1 failing tests before stopping, since # every tests after max_failures failures have been encountered # may all be passing. self.assertEquals(3, len(results.failures)) self.assertTrue(page1.was_run) self.assertTrue(page2.was_run) self.assertTrue(page3.was_run) self.assertFalse(page4.was_run) self.assertFalse(page5.was_run)
def Run(self, finder_options): """Run this test with the given options. Returns: The number of failure values (up to 254) or 255 if there is an uncaught exception. """ self.CustomizeBrowserOptions(finder_options.browser_options) pt = self.CreatePageTest(finder_options) pt.__name__ = self.__class__.__name__ if hasattr(self, '_disabled_strings'): # pylint: disable=protected-access pt._disabled_strings = self._disabled_strings if hasattr(self, '_enabled_strings'): # pylint: disable=protected-access pt._enabled_strings = self._enabled_strings expectations = self.CreateExpectations() us = self.CreateUserStorySet(finder_options) if isinstance(pt, page_test.PageTest): if any(not isinstance(p, page.Page) for p in us.user_stories): raise Exception( 'PageTest must be used with UserStorySet containing only ' 'telemetry.page.Page user stories.') self._DownloadGeneratedProfileArchive(finder_options) benchmark_metadata = self.GetMetadata() with results_options.CreateResults( benchmark_metadata, finder_options, self.ValueCanBeAddedPredicate) as results: try: user_story_runner.Run(pt, us, expectations, finder_options, results, max_failures=self._max_failures) return_code = min(254, len(results.failures)) except Exception: exception_formatter.PrintFormattedException() return_code = 255 bucket = cloud_storage.BUCKET_ALIASES[finder_options.upload_bucket] if finder_options.upload_results: results.UploadTraceFilesToCloud(bucket) results.UploadProfilingFilesToCloud(bucket) results.PrintSummary() return return_code
def Run(self, options): expectations = test_expectations.TestExpectations() results = results_options.CreateResults( benchmark.BenchmarkMetadata(profile_creator.__class__.__name__), options) user_story_runner.Run(self._page_test, self._page_test._page_set, expectations, options, results) if results.failures: logging.warning('Some pages failed to load.') logging.warning('Failed pages:\n%s', '\n'.join(map(str, results.pages_that_failed))) raise Exception('SmallProfileCreator failed.')
def testPagesetRepeat(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() ps.pages.append( page_module.Page('file://blank.html', ps, base_dir=util.GetUnittestDataDir())) ps.pages.append( page_module.Page('file://green_rect.html', ps, base_dir=util.GetUnittestDataDir())) class Measurement(page_test.PageTest): i = 0 def ValidateAndMeasurePage(self, page, _, results): self.i += 1 results.AddValue( scalar.ScalarValue(page, 'metric', 'unit', self.i)) output_file = tempfile.NamedTemporaryFile(delete=False).name try: options = options_for_unittests.GetCopy() options.output_format = 'buildbot' options.output_file = output_file options.suppress_gtest_report = True options.reset_results = None options.upload_results = None options.results_label = None options.page_repeat = 1 options.pageset_repeat = 2 SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(Measurement(), ps, expectations, options, results) results.PrintSummary() self.assertEquals(4, len(GetSuccessfulPageRuns(results))) self.assertEquals(0, len(results.failures)) with open(output_file) as f: stdout = f.read() self.assertIn('RESULT metric: blank.html= [1,3] unit', stdout) self.assertIn('RESULT metric: green_rect.html= [2,4] unit', stdout) self.assertIn('*RESULT metric: metric= [1,2,3,4] unit', stdout) finally: # TODO(chrishenry): This is a HACK!!1 Really, the right way to # do this is for page_runner (or output formatter) to close any # files it has opened. for formatter in results._output_formatters: # pylint: disable=W0212 formatter.output_stream.close() os.remove(output_file)
def _testMaxFailuresOptionIsRespectedAndOverridable( self, max_failures=None): self.SuppressExceptionFormatting() class TestPage(page_module.Page): def __init__(self, *args, **kwargs): super(TestPage, self).__init__(*args, **kwargs) self.was_run = False def RunNavigateSteps(self, action_runner): # pylint: disable=W0613 self.was_run = True raise Exception('Test exception') class Test(page_test.PageTest): def ValidatePage(self, *args): pass ps = page_set.PageSet() expectations = test_expectations.TestExpectations() for ii in range(5): ps.pages.append( TestPage('file://blank.html', ps, base_dir=util.GetUnittestDataDir())) options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True expected_max_failures = 2 if not max_failures is None: options.max_failures = max_failures expected_max_failures = max_failures SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(Test(max_failures=2), ps, expectations, options, results) self.assertEquals(0, len(GetSuccessfulPageRuns(results))) # Runs up to max_failures+1 failing tests before stopping, since # every tests after max_failures failures have been encountered # may all be passing. self.assertEquals(expected_max_failures + 1, len(results.failures)) for ii in range(len(ps.pages)): if ii <= expected_max_failures: self.assertTrue(ps.pages[ii].was_run) else: self.assertFalse(ps.pages[ii].was_run)
def testHandlingOfCrashedTabWithExpectedFailure(self): self.SuppressExceptionFormatting() ps = page_set.PageSet() expectations = test_expectations.TestExpectations() expectations.Fail('chrome://crash') page1 = page_module.Page('chrome://crash', ps) ps.pages.append(page1) options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(DummyTest(), ps, expectations, options, results) self.assertEquals(1, len(GetSuccessfulPageRuns(results))) self.assertEquals(0, len(results.failures))
def testHandlingOfCrashedTab(self): ps = page_set.PageSet() expectations = test_expectations.TestExpectations() page1 = page_module.Page('chrome://crash', ps) ps.pages.append(page1) class Test(page_test.PageTest): def ValidatePage(self, *args): pass options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True SetUpPageRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) page_runner.Run(Test(), ps, expectations, options, results) self.assertEquals(0, len(GetSuccessfulPageRuns(results))) self.assertEquals(1, len(results.failures))
def testRaiseBrowserGoneExceptionFromRestartBrowserBeforeEachPage(self): self.CaptureFormattedException() ps = page_set.PageSet() expectations = test_expectations.TestExpectations() ps.pages.append( page_module.Page('file://blank.html', ps, base_dir=util.GetUnittestDataDir())) ps.pages.append( page_module.Page('file://blank.html', ps, base_dir=util.GetUnittestDataDir())) class Test(page_test.PageTest): def __init__(self, *args): super(Test, self).__init__( *args, needs_browser_restart_after_each_page=True) self.run_count = 0 def RestartBrowserBeforeEachPage(self): old_run_count = self.run_count self.run_count += 1 if old_run_count == 0: raise exceptions.BrowserGoneException(None) return self._needs_browser_restart_after_each_page def ValidateAndMeasurePage(self, page, tab, results): pass options = options_for_unittests.GetCopy() options.output_formats = ['none'] options.suppress_gtest_report = True test = Test() SetUpUserStoryRunnerArguments(options) results = results_options.CreateResults(EmptyMetadataForTest(), options) user_story_runner.Run(test, ps, expectations, options, results) self.assertEquals(2, test.run_count) self.assertEquals(1, len(GetSuccessfulPageRuns(results))) self.assertEquals(1, len(results.failures)) self.assertFormattedExceptionIsEmpty()