def main(args=None): """Run tests, rerunning at most MAX_RETRY_COUNT times if they flake.""" parsed_args = _PARSER.parse_args(args=args) policy = RERUN_POLICIES[parsed_args.suite.lower()] with servers.managed_portserver(): for attempt_num in python_utils.RANGE(1, MAX_RETRY_COUNT + 1): python_utils.PRINT('***Attempt %d.***' % attempt_num) output, return_code = run_tests(parsed_args) if not flake_checker.check_if_on_ci(): # Don't rerun off of CI. python_utils.PRINT('No reruns because not running on CI.') break if return_code == 0: # Don't rerun passing tests. flake_checker.report_pass(parsed_args.suite) break # Check whether we should rerun based on this suite's policy. test_is_flaky = flake_checker.is_test_output_flaky( output, parsed_args.suite) if policy == RERUN_POLICY_NEVER: break if policy == RERUN_POLICY_KNOWN_FLAKES and not test_is_flaky: break sys.exit(return_code)
def main(args=None): """Run tests, rerunning at most MAX_RETRY_COUNT times if they flake.""" parsed_args = _PARSER.parse_args(args=args) portserver_process = start_portserver() atexit.register(cleanup_portserver, portserver_process) for attempt_num in python_utils.RANGE(MAX_RETRY_COUNT): python_utils.PRINT('***Attempt %s.***' % (attempt_num + 1)) output, return_code = run_tests(parsed_args) # Don't rerun passing tests. if return_code == 0: flake_checker.report_pass(parsed_args.suite) break # Don't rerun off of CI. if not flake_checker.check_if_on_ci(): python_utils.PRINT('No reruns because not running on CI.') break flaky = flake_checker.is_test_output_flaky(output, parsed_args.suite) # Don't rerun if the test was non-flaky and we are not # rerunning non-flaky tests. if not flaky and not RERUN_NON_FLAKY: break # Prepare for rerun. cleanup() sys.exit(return_code)
def main(args=None): """Run tests, rerunning at most MAX_RETRY_COUNT times if they flake.""" parsed_args = _PARSER.parse_args(args=args) with servers.managed_portserver(): for attempt_num in python_utils.RANGE(1, MAX_RETRY_COUNT + 1): python_utils.PRINT('***Attempt %d.***' % attempt_num) output, return_code = run_tests(parsed_args) if not flake_checker.check_if_on_ci(): # Don't rerun off of CI. python_utils.PRINT('No reruns because not running on CI.') break if return_code == 0: # Don't rerun passing tests. flake_checker.report_pass(parsed_args.suite) break test_is_flaky = flake_checker.is_test_output_flaky( output, parsed_args.suite) if not test_is_flaky and not RERUN_NON_FLAKY: # Don't rerun if the test was non-flaky and we are not rerunning # non-flaky tests. break sys.exit(return_code)
def test_returns_false_when_off_ci(self): def mock_getenv(unused_variable): return False getenv_swap = self.swap_with_checks( os, 'getenv', mock_getenv, expected_args=[ ('GITHUB_ACTIONS',), ('CIRCLECI',), ]) with getenv_swap: on_ci = flake_checker.check_if_on_ci() self.assertFalse(on_ci)
def test_returns_true_when_on_ci(self): def mock_getenv(variable): return variable == 'CIRCLECI' getenv_swap = self.swap_with_checks( os, 'getenv', mock_getenv, expected_args=[ ('GITHUB_ACTIONS',), ('CIRCLECI',), ]) with getenv_swap: on_ci = flake_checker.check_if_on_ci() self.assertTrue(on_ci)
def main(args=None): """Run tests, rerunning at most MAX_RETRY_COUNT times if they flake.""" parsed_args = _PARSER.parse_args(args=args) policy = RERUN_POLICIES[parsed_args.suite.lower()] with servers.managed_portserver(): for attempt_num in range(1, MAX_RETRY_COUNT + 1): print('***Attempt %d.***' % attempt_num) output, return_code = run_tests(parsed_args) if not flake_checker.check_if_on_ci(): # Don't rerun off of CI. print('No reruns because not running on CI.') break if return_code == 0: # Don't rerun passing tests. flake_checker.report_pass(parsed_args.suite) break # Check whether we should rerun based on this suite's policy # and override instructions from the flake checker server. test_is_flaky, rerun_override = flake_checker.is_test_output_flaky( output, parsed_args.suite) if rerun_override == flake_checker.RERUN_YES: print('Rerunning as per instructions from logging ' 'server.') elif rerun_override == flake_checker.RERUN_NO: print('Not rerunning as per instructions from ' 'logging server.') break # No rerun override, so follow rerun policies. elif policy == RERUN_POLICY_NEVER: print('Not rerunning because the policy is to never ' 'rerun the {} suite'.format(parsed_args.suite)) break elif policy == RERUN_POLICY_KNOWN_FLAKES and not test_is_flaky: print(('Not rerunning because the policy is to only ' 'rerun the %s suite on known flakes, and this ' 'failure did not match any known flakes') % parsed_args.suite) break # Else rerun policy is either always or we had a known flake # with a known flakes rerun policy, so rerun. sys.exit(return_code)