def testExpectationRemoval(self): """Tests that expectations are properly removed from a file.""" contents = validate_tag_consistency.TAG_HEADER + """ # This is a test comment crbug.com/1234 [ win ] foo/test [ Failure ] crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] # Another comment [ linux ] bar/test [ RetryOnFailure ] [ win ] bar/test [ RetryOnFailure ] """ stale_expectations = [ data_types.Expectation('foo/test', ['win'], ['Failure']), data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) ] expected_contents = validate_tag_consistency.TAG_HEADER + """ # This is a test comment crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] # Another comment [ win ] bar/test [ RetryOnFailure ] """ with open(self.filename, 'w') as f: f.write(contents) expectations.RemoveExpectationsFromFile(stale_expectations, self.filename) with open(self.filename) as f: self.assertEqual(f.read(), expected_contents)
def testInlineComments(self): """Tests that expectations with inline disable comments are not removed.""" contents = validate_tag_consistency.TAG_HEADER + """ crbug.com/1234 [ win ] foo/test [ Failure ] crbug.com/2345 [ win ] foo/test [ Failure ] # finder:disable crbug.com/3456 [ win ] foo/test [ Failure ] """ stale_expectations = [ data_types.Expectation('foo/test', ['win'], ['Failure'], 'crbug.com/1234'), data_types.Expectation('foo/test', ['win'], ['Failure'], 'crbug.com/2345'), data_types.Expectation('foo/test', ['win'], ['Failure'], 'crbug.com/3456'), ] expected_contents = validate_tag_consistency.TAG_HEADER + """ crbug.com/2345 [ win ] foo/test [ Failure ] # finder:disable """ with open(self.filename, 'w') as f: f.write(contents) removed_urls = expectations.RemoveExpectationsFromFile( stale_expectations, self.filename) self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456'])) with open(self.filename) as f: self.assertEqual(f.read(), expected_contents)
def testNestedBlockComments(self): """Tests that nested disable block comments throw exceptions.""" contents = validate_tag_consistency.TAG_HEADER + """ # finder:disable # finder:disable crbug.com/1234 [ win ] foo/test [ Failure ] # finder:enable # finder:enable """ with open(self.filename, 'w') as f: f.write(contents) with self.assertRaises(RuntimeError): expectations.RemoveExpectationsFromFile([], self.filename) contents = validate_tag_consistency.TAG_HEADER + """ # finder:enable crbug.com/1234 [ win ] foo/test [ Failure ] """ with open(self.filename, 'w') as f: f.write(contents) with self.assertRaises(RuntimeError): expectations.RemoveExpectationsFromFile([], self.filename)
def main(): args = ParseArgs() test_expectation_map = expectations.CreateTestExpectationMap( args.expectation_file, args.tests) ci_builders = builders.GetCiBuilders( SUITE_TO_TELEMETRY_SUITE_MAP.get(args.suite, args.suite)) querier = queries.BigQueryQuerier(args.suite, args.project, args.num_samples, args.large_query_mode) # Unmatched results are mainly useful for script maintainers, as they don't # provide any additional information for the purposes of finding unexpectedly # passing tests or unused expectations. unmatched = querier.FillExpectationMapForCiBuilders( test_expectation_map, ci_builders) try_builders = builders.GetTryBuilders(ci_builders) unmatched.update( querier.FillExpectationMapForTryBuilders(test_expectation_map, try_builders)) unused_expectations = expectations.FilterOutUnusedExpectations( test_expectation_map) stale, semi_stale, active = expectations.SplitExpectationsByStaleness( test_expectation_map) result_output.OutputResults(stale, semi_stale, active, unmatched, unused_expectations, args.output_format) affected_urls = set() stale_message = '' if args.remove_stale_expectations: stale_expectations = [] for _, expectation_map in stale.iteritems(): stale_expectations.extend(expectation_map.keys()) stale_expectations.extend(unused_expectations) affected_urls |= expectations.RemoveExpectationsFromFile( stale_expectations, args.expectation_file) stale_message += ( 'Stale expectations removed from %s. Stale comments, ' 'etc. may still need to be removed.\n' % args.expectation_file) if args.modify_semi_stale_expectations: affected_urls |= expectations.ModifySemiStaleExpectations( semi_stale, args.expectation_file) stale_message += ('Semi-stale expectations modified in %s. Stale ' 'comments, etc. may still need to be removed.\n' % args.expectation_file) if stale_message: print(stale_message) if affected_urls: result_output.OutputAffectedUrls(affected_urls)
def main(): args = ParseArgs() # TODO(crbug.com/1108016): Remove this warning once ResultDB is enabled on all # builders and there is enough data for the results to be trusted. WarnUserOfIncompleteRollout() test_expectation_map = expectations.CreateTestExpectationMap( args.expectation_file, args.tests) ci_builders = builders.GetCiBuilders( SUITE_TO_TELEMETRY_SUITE_MAP.get(args.suite, args.suite)) # Unmatched results are mainly useful for script maintainers, as they don't # provide any additional information for the purposes of finding unexpectedly # passing tests or unused expectations. unmatched = queries.FillExpectationMapForCiBuilders( test_expectation_map, ci_builders, args.suite, args.project, args.num_samples) try_builders = builders.GetTryBuilders(ci_builders) unmatched.update( queries.FillExpectationMapForTryBuilders(test_expectation_map, try_builders, args.suite, args.project, args.num_samples)) unused_expectations = expectations.FilterOutUnusedExpectations( test_expectation_map) stale, semi_stale, active = expectations.SplitExpectationsByStaleness( test_expectation_map) result_output.OutputResults(stale, semi_stale, active, unmatched, unused_expectations, args.output_format) if args.remove_stale_expectations: stale_expectations = [] for _, expectation_map in stale.iteritems(): stale_expectations.extend(expectation_map.keys()) stale_expectations.extend(unused_expectations) removed_urls = expectations.RemoveExpectationsFromFile( stale_expectations, args.expectation_file) print( 'Stale expectations removed from %s. Stale comments, etc. may still ' 'need to be removed.' % args.expectation_file) result_output.OutputRemovedUrls(removed_urls)