def __init__(self, max_failures=None): """Creates a new Benchmark. Args: max_failures: The number of story run's failures before bailing from executing subsequent page runs. If None, we never bail. """ self._expectations = typ_expectations.StoryExpectations(self.Name()) self._max_failures = max_failures
def testDisableBenchmark(self): expectations = typ_expectations.StoryExpectations( 'fake_benchmark_name') raw_expectations = ( '# tags: [ all ]\n' '# results: [ Skip ]\n' 'crbug.com/123 [ all ] fake_benchmark_name/* [ Skip ]\n') expectations.GetBenchmarkExpectationsFromParser(raw_expectations) expectations.SetTags(['All']) reason = expectations.IsBenchmarkDisabled() self.assertTrue(reason) self.assertEqual(reason, 'crbug.com/123')
def BuildStoryFilter(cls, benchmark_name, platform_tags, abridged_story_set_tag): expectations = typ_expectations.StoryExpectations(benchmark_name) expectations.SetTags(platform_tags or []) if cls._expectations_file and os.path.exists(cls._expectations_file): with open(cls._expectations_file) as fh: expectations.GetBenchmarkExpectationsFromParser(fh.read()) if not cls._run_abridged_story_set: abridged_story_set_tag = None return StoryFilter( expectations, abridged_story_set_tag, cls._story_filter, cls._story_filter_exclude, cls._story_tag_filter, cls._story_tag_filter_exclude, cls._shard_begin_index, cls._shard_end_index, cls._run_disabled_stories, stories=cls._stories)
def testDisableStory_NoReasonGiven(self): expectations = typ_expectations.StoryExpectations( 'fake_benchmark_name') raw_expectations = ( '# tags: [ linux win ]\n' '# results: [ Skip ]\n' '[ linux ] fake_benchmark_name/one [ Skip ]\n' 'crbug.com/123 [ win ] fake_benchmark_name/on* [ Skip ]\n') expectations.GetBenchmarkExpectationsFromParser(raw_expectations) expectations.SetTags(['linux']) story = mock.MagicMock() story.name = 'one' reason = expectations.IsStoryDisabled(story) self.assertTrue(reason) self.assertEqual(reason, 'No reason given')
def AugmentExpectationsWithFile(self, raw_data): typ_parser = typ_expectations_parser.TestExpectations() error, _ = typ_parser.parse_tagged_list(raw_data) if not error: self._expectations = typ_expectations.StoryExpectations(self.Name()) self._expectations.GetBenchmarkExpectationsFromParser(raw_data) else: # If we can't parse the file using typ's expectation parser # then we fall back to using py_util's expectation parser # TODO(crbug.com/973936): When all expectations files have # been migrated, we will remove this if else statement and # only use typ's expectations parser. self._expectations.GetBenchmarkExpectationsFromParser( expectations_parser.TestExpectationParser( raw_data).expectations, self.Name())
def testValidateTags(self): story_expectations = typ_expectations.StoryExpectations('fake') story_expectations.GetBenchmarkExpectationsFromParser('# tags: [ all ]') with self.assertRaises(ValueError) as context: story_expectations.SetTags(['abc'], True) self.assertIn('not declared in the expectations', str(context.exception))