def testEmptyInput(self): """Tests that nothing blows up with empty input.""" stale_dict, semi_stale_dict, active_dict =\ expectations.SplitExpectationsByStaleness({}) self.assertEqual(stale_dict, {}) self.assertEqual(semi_stale_dict, {}) self.assertEqual(active_dict, {})
def testActiveExpectations(self): """Tests output when only active expectations are provided.""" expectation_map = { 'foo': { data_types.Expectation('foo', ['win'], ['Failure']): { 'foo_builder': { 'step1': uu.CreateStatsWithPassFails(0, 1), 'step2': uu.CreateStatsWithPassFails(0, 2), }, 'bar_builder': { 'step1': uu.CreateStatsWithPassFails(0, 3), 'step2': uu.CreateStatsWithPassFails(0, 4) }, }, data_types.Expectation('foo', ['linux'], ['RetryOnFailure']): { 'foo_builder': { 'step1': uu.CreateStatsWithPassFails(0, 5), 'step2': uu.CreateStatsWithPassFails(0, 6), }, }, }, 'bar': { data_types.Expectation('bar', ['win'], ['Failure']): { 'foo_builder': { 'step1': uu.CreateStatsWithPassFails(0, 7), }, }, }, } expected_active_dict = copy.deepcopy(expectation_map) stale_dict, semi_stale_dict, active_dict =\ expectations.SplitExpectationsByStaleness(expectation_map) self.assertEqual(stale_dict, {}) self.assertEqual(semi_stale_dict, {}) self.assertEqual(active_dict, expected_active_dict)
def testOutputResultsSmoketest(self): """Test that nothing blows up when outputting.""" expectation_map = { 'foo': { data_types.Expectation('foo', ['win', 'intel'], 'RetryOnFailure'): { 'stale': { 'all_pass': uu.CreateStatsWithPassFails(2, 0), }, }, data_types.Expectation('foo', ['linux'], 'Failure'): { 'semi_stale': { 'all_pass': uu.CreateStatsWithPassFails(2, 0), 'some_pass': uu.CreateStatsWithPassFails(1, 1), 'none_pass': uu.CreateStatsWithPassFails(0, 2), }, }, data_types.Expectation('foo', ['mac'], 'Failure'): { 'active': { 'none_pass': uu.CreateStatsWithPassFails(0, 2), }, }, }, } unmatched_results = { 'builder': [ data_types.Result('foo', ['win', 'intel'], 'Failure', 'step_name', 'build_id'), ], } unmatched_expectations = [ data_types.Expectation('foo', ['linux'], 'RetryOnFailure') ] stale, semi_stale, active = expectations.SplitExpectationsByStaleness( expectation_map) result_output.OutputResults(stale, semi_stale, active, {}, [], 'print', self._file_handle) result_output.OutputResults(stale, semi_stale, active, unmatched_results, [], 'print', self._file_handle) result_output.OutputResults(stale, semi_stale, active, {}, unmatched_expectations, 'print', self._file_handle) result_output.OutputResults(stale, semi_stale, active, unmatched_results, unmatched_expectations, 'print', self._file_handle) result_output.OutputResults(stale, semi_stale, active, {}, [], 'html', self._file_handle) result_output.OutputResults(stale, semi_stale, active, unmatched_results, [], 'html', self._file_handle) result_output.OutputResults(stale, semi_stale, active, {}, unmatched_expectations, 'html', self._file_handle) result_output.OutputResults(stale, semi_stale, active, unmatched_results, unmatched_expectations, 'html', self._file_handle)
def testSemiStaleExpectations(self): """Tests output when only semi-stale expectations are provided.""" expectation_map = data_types.TestExpectationMap({ 'foo': data_types.ExpectationBuilderMap({ data_types.Expectation('foo', ['win'], ['Failure']): data_types.BuilderStepMap({ 'foo_builder': data_types.StepBuildStatsMap({ 'step1': uu.CreateStatsWithPassFails(1, 0), 'step2': uu.CreateStatsWithPassFails(2, 2), }), 'bar_builder': data_types.StepBuildStatsMap({ 'step1': uu.CreateStatsWithPassFails(3, 0), 'step2': uu.CreateStatsWithPassFails(0, 4) }), }), data_types.Expectation('foo', ['linux'], ['RetryOnFailure']): data_types.BuilderStepMap({ 'foo_builder': data_types.StepBuildStatsMap({ 'step1': uu.CreateStatsWithPassFails(5, 0), 'step2': uu.CreateStatsWithPassFails(6, 6), }), }), }), 'bar': data_types.ExpectationBuilderMap({ data_types.Expectation('bar', ['win'], ['Failure']): data_types.BuilderStepMap({ 'foo_builder': data_types.StepBuildStatsMap({ 'step1': uu.CreateStatsWithPassFails(7, 0), }), 'bar_builder': data_types.StepBuildStatsMap({ 'step1': uu.CreateStatsWithPassFails(0, 8), }), }), }), }) expected_semi_stale_dict = copy.deepcopy(expectation_map) stale_dict, semi_stale_dict, active_dict =\ expectations.SplitExpectationsByStaleness(expectation_map) self.assertEqual(stale_dict, {}) self.assertEqual(semi_stale_dict, expected_semi_stale_dict) self.assertEqual(active_dict, {})
def testEmptyInput(self): """Tests that nothing blows up with empty input.""" stale_dict, semi_stale_dict, active_dict =\ expectations.SplitExpectationsByStaleness( data_types.TestExpectationMap()) self.assertEqual(stale_dict, {}) self.assertEqual(semi_stale_dict, {}) self.assertEqual(active_dict, {}) self.assertIsInstance(stale_dict, data_types.TestExpectationMap) self.assertIsInstance(semi_stale_dict, data_types.TestExpectationMap) self.assertIsInstance(active_dict, data_types.TestExpectationMap)
def main(): args = ParseArgs() test_expectation_map = expectations.CreateTestExpectationMap( args.expectation_file, args.tests) ci_builders = builders.GetCiBuilders( SUITE_TO_TELEMETRY_SUITE_MAP.get(args.suite, args.suite)) querier = queries.BigQueryQuerier(args.suite, args.project, args.num_samples, args.large_query_mode) # Unmatched results are mainly useful for script maintainers, as they don't # provide any additional information for the purposes of finding unexpectedly # passing tests or unused expectations. unmatched = querier.FillExpectationMapForCiBuilders( test_expectation_map, ci_builders) try_builders = builders.GetTryBuilders(ci_builders) unmatched.update( querier.FillExpectationMapForTryBuilders(test_expectation_map, try_builders)) unused_expectations = expectations.FilterOutUnusedExpectations( test_expectation_map) stale, semi_stale, active = expectations.SplitExpectationsByStaleness( test_expectation_map) result_output.OutputResults(stale, semi_stale, active, unmatched, unused_expectations, args.output_format) affected_urls = set() stale_message = '' if args.remove_stale_expectations: stale_expectations = [] for _, expectation_map in stale.iteritems(): stale_expectations.extend(expectation_map.keys()) stale_expectations.extend(unused_expectations) affected_urls |= expectations.RemoveExpectationsFromFile( stale_expectations, args.expectation_file) stale_message += ( 'Stale expectations removed from %s. Stale comments, ' 'etc. may still need to be removed.\n' % args.expectation_file) if args.modify_semi_stale_expectations: affected_urls |= expectations.ModifySemiStaleExpectations( semi_stale, args.expectation_file) stale_message += ('Semi-stale expectations modified in %s. Stale ' 'comments, etc. may still need to be removed.\n' % args.expectation_file) if stale_message: print(stale_message) if affected_urls: result_output.OutputAffectedUrls(affected_urls)
def main(): args = ParseArgs() # TODO(crbug.com/1108016): Remove this warning once ResultDB is enabled on all # builders and there is enough data for the results to be trusted. WarnUserOfIncompleteRollout() test_expectation_map = expectations.CreateTestExpectationMap( args.expectation_file, args.tests) ci_builders = builders.GetCiBuilders( SUITE_TO_TELEMETRY_SUITE_MAP.get(args.suite, args.suite)) # Unmatched results are mainly useful for script maintainers, as they don't # provide any additional information for the purposes of finding unexpectedly # passing tests or unused expectations. unmatched = queries.FillExpectationMapForCiBuilders( test_expectation_map, ci_builders, args.suite, args.project, args.num_samples) try_builders = builders.GetTryBuilders(ci_builders) unmatched.update( queries.FillExpectationMapForTryBuilders(test_expectation_map, try_builders, args.suite, args.project, args.num_samples)) unused_expectations = expectations.FilterOutUnusedExpectations( test_expectation_map) stale, semi_stale, active = expectations.SplitExpectationsByStaleness( test_expectation_map) result_output.OutputResults(stale, semi_stale, active, unmatched, unused_expectations, args.output_format) if args.remove_stale_expectations: stale_expectations = [] for _, expectation_map in stale.iteritems(): stale_expectations.extend(expectation_map.keys()) stale_expectations.extend(unused_expectations) removed_urls = expectations.RemoveExpectationsFromFile( stale_expectations, args.expectation_file) print( 'Stale expectations removed from %s. Stale comments, etc. may still ' 'need to be removed.' % args.expectation_file) result_output.OutputRemovedUrls(removed_urls)
def testAllExpectations(self): """Tests output when all three types of expectations are provided.""" expectation_map = { 'foo': { data_types.Expectation('foo', ['stale'], 'Failure'): { 'foo_builder': { 'step1': uu.CreateStatsWithPassFails(1, 0), 'step2': uu.CreateStatsWithPassFails(2, 0), }, 'bar_builder': { 'step1': uu.CreateStatsWithPassFails(3, 0), 'step2': uu.CreateStatsWithPassFails(4, 0) }, }, data_types.Expectation('foo', ['semistale'], 'Failure'): { 'foo_builder': { 'step1': uu.CreateStatsWithPassFails(1, 0), 'step2': uu.CreateStatsWithPassFails(2, 2), }, 'bar_builder': { 'step1': uu.CreateStatsWithPassFails(3, 0), 'step2': uu.CreateStatsWithPassFails(0, 4) }, }, data_types.Expectation('foo', ['active'], 'Failure'): { 'foo_builder': { 'step1': uu.CreateStatsWithPassFails(1, 1), 'step2': uu.CreateStatsWithPassFails(2, 2), }, 'bar_builder': { 'step1': uu.CreateStatsWithPassFails(3, 3), 'step2': uu.CreateStatsWithPassFails(0, 4) }, }, }, } expected_stale = { 'foo': { data_types.Expectation('foo', ['stale'], 'Failure'): { 'foo_builder': { 'step1': uu.CreateStatsWithPassFails(1, 0), 'step2': uu.CreateStatsWithPassFails(2, 0), }, 'bar_builder': { 'step1': uu.CreateStatsWithPassFails(3, 0), 'step2': uu.CreateStatsWithPassFails(4, 0) }, }, }, } expected_semi_stale = { 'foo': { data_types.Expectation('foo', ['semistale'], 'Failure'): { 'foo_builder': { 'step1': uu.CreateStatsWithPassFails(1, 0), 'step2': uu.CreateStatsWithPassFails(2, 2), }, 'bar_builder': { 'step1': uu.CreateStatsWithPassFails(3, 0), 'step2': uu.CreateStatsWithPassFails(0, 4) }, }, }, } expected_active = { 'foo': { data_types.Expectation('foo', ['active'], 'Failure'): { 'foo_builder': { 'step1': uu.CreateStatsWithPassFails(1, 1), 'step2': uu.CreateStatsWithPassFails(2, 2), }, 'bar_builder': { 'step1': uu.CreateStatsWithPassFails(3, 3), 'step2': uu.CreateStatsWithPassFails(0, 4) }, }, }, } stale_dict, semi_stale_dict, active_dict =\ expectations.SplitExpectationsByStaleness(expectation_map) self.assertEqual(stale_dict, expected_stale) self.assertEqual(semi_stale_dict, expected_semi_stale) self.assertEqual(active_dict, expected_active)