def main(args=None): """Run tests, rerunning at most MAX_RETRY_COUNT times if they flake.""" parsed_args = _PARSER.parse_args(args=args) policy = RERUN_POLICIES[parsed_args.suite.lower()] with servers.managed_portserver(): for attempt_num in python_utils.RANGE(1, MAX_RETRY_COUNT + 1): python_utils.PRINT('***Attempt %d.***' % attempt_num) output, return_code = run_tests(parsed_args) if not flake_checker.check_if_on_ci(): # Don't rerun off of CI. python_utils.PRINT('No reruns because not running on CI.') break if return_code == 0: # Don't rerun passing tests. flake_checker.report_pass(parsed_args.suite) break # Check whether we should rerun based on this suite's policy. test_is_flaky = flake_checker.is_test_output_flaky( output, parsed_args.suite) if policy == RERUN_POLICY_NEVER: break if policy == RERUN_POLICY_KNOWN_FLAKES and not test_is_flaky: break sys.exit(return_code)
def main(args=None): """Run tests, rerunning at most MAX_RETRY_COUNT times if they flake.""" parsed_args = _PARSER.parse_args(args=args) portserver_process = start_portserver() atexit.register(cleanup_portserver, portserver_process) for attempt_num in python_utils.RANGE(MAX_RETRY_COUNT): python_utils.PRINT('***Attempt %s.***' % (attempt_num + 1)) output, return_code = run_tests(parsed_args) # Don't rerun passing tests. if return_code == 0: flake_checker.report_pass(parsed_args.suite) break # Don't rerun off of CI. if not flake_checker.check_if_on_ci(): python_utils.PRINT('No reruns because not running on CI.') break flaky = flake_checker.is_test_output_flaky(output, parsed_args.suite) # Don't rerun if the test was non-flaky and we are not # rerunning non-flaky tests. if not flaky and not RERUN_NON_FLAKY: break # Prepare for rerun. cleanup() sys.exit(return_code)
def main(args=None): """Run tests, rerunning at most MAX_RETRY_COUNT times if they flake.""" parsed_args = _PARSER.parse_args(args=args) with servers.managed_portserver(): for attempt_num in python_utils.RANGE(1, MAX_RETRY_COUNT + 1): python_utils.PRINT('***Attempt %d.***' % attempt_num) output, return_code = run_tests(parsed_args) if not flake_checker.check_if_on_ci(): # Don't rerun off of CI. python_utils.PRINT('No reruns because not running on CI.') break if return_code == 0: # Don't rerun passing tests. flake_checker.report_pass(parsed_args.suite) break test_is_flaky = flake_checker.is_test_output_flaky( output, parsed_args.suite) if not test_is_flaky and not RERUN_NON_FLAKY: # Don't rerun if the test was non-flaky and we are not rerunning # non-flaky tests. break sys.exit(return_code)
def test_unknown_build_environment(self): def mock_getenv(variable): environment_vars = {} return environment_vars.get(variable) def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument raise AssertionError('requests.post called.') getenv_swap = self.swap(os, 'getenv', mock_getenv) post_swap = self.swap(requests, 'post', mock_post) with getenv_swap, post_swap: with self.assertRaisesRegex(Exception, 'Unknown build environment.'): flake_checker.report_pass('suiteName')
def main(args=None): """Run tests, rerunning at most MAX_RETRY_COUNT times if they flake.""" parsed_args = _PARSER.parse_args(args=args) policy = RERUN_POLICIES[parsed_args.suite.lower()] with servers.managed_portserver(): for attempt_num in range(1, MAX_RETRY_COUNT + 1): print('***Attempt %d.***' % attempt_num) output, return_code = run_tests(parsed_args) if not flake_checker.check_if_on_ci(): # Don't rerun off of CI. print('No reruns because not running on CI.') break if return_code == 0: # Don't rerun passing tests. flake_checker.report_pass(parsed_args.suite) break # Check whether we should rerun based on this suite's policy # and override instructions from the flake checker server. test_is_flaky, rerun_override = flake_checker.is_test_output_flaky( output, parsed_args.suite) if rerun_override == flake_checker.RERUN_YES: print('Rerunning as per instructions from logging ' 'server.') elif rerun_override == flake_checker.RERUN_NO: print('Not rerunning as per instructions from ' 'logging server.') break # No rerun override, so follow rerun policies. elif policy == RERUN_POLICY_NEVER: print('Not rerunning because the policy is to never ' 'rerun the {} suite'.format(parsed_args.suite)) break elif policy == RERUN_POLICY_KNOWN_FLAKES and not test_is_flaky: print(('Not rerunning because the policy is to only ' 'rerun the %s suite on known flakes, and this ' 'failure did not match any known flakes') % parsed_args.suite) break # Else rerun policy is either always or we had a known flake # with a known flakes rerun policy, so rerun. sys.exit(return_code)
def test_successful_report_construct_url(self): def mock_getenv(variable): environment_vars = { 'GITHUB_ACTIONS': 1, 'GITHUB_ACTOR': 'user', 'GITHUB_RUN_ID': 1234, 'GITHUB_REF': 'develop', 'GITHUB_REPOSITORY': 'foo/oppia', } return environment_vars.get(variable) def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument pass expected_payload = { 'suite': 'suiteName', 'metadata': { 'username': '******', 'build_url': 'https://github.com/foo/oppia/actions/runs/1234', 'timestamp': '2020-01-01T00:00:00.000001+00:00', 'branch': 'develop', } } getenv_swap = self.swap(os, 'getenv', mock_getenv) datetime_swap = self.swap(datetime, 'datetime', MockDatetime(self.example_date)) post_swap = self.swap_with_checks(requests, 'post', mock_post, expected_args=[ (flake_checker.PASS_REPORT_URL, ) ], expected_kwargs=[{ 'json': expected_payload, 'allow_redirects': False, 'headers': { 'report_key': flake_checker.REPORT_API_KEY }, }]) with getenv_swap, datetime_swap, post_swap: flake_checker.report_pass('suiteName')
def test_successful_report(self): def mock_getenv(variable): environment_vars = { 'CIRCLECI': 1, 'CIRCLE_USERNAME': '******', 'CIRCLE_BUILD_URL': 'https://example.com', 'CIRCLE_BRANCH': 'develop', } return environment_vars.get(variable) def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument pass expected_payload = { 'suite': 'suiteName', 'metadata': { 'username': '******', 'build_url': 'https://example.com', 'timestamp': '2020-01-01T00:00:00.000001+00:00', 'branch': 'develop', } } getenv_swap = self.swap(os, 'getenv', mock_getenv) datetime_swap = self.swap(datetime, 'datetime', MockDatetime(self.example_date)) post_swap = self.swap_with_checks(requests, 'post', mock_post, expected_args=[ (flake_checker.PASS_REPORT_URL, ) ], expected_kwargs=[{ 'json': expected_payload, 'allow_redirects': False, 'headers': { 'report_key': flake_checker.REPORT_API_KEY }, }]) with getenv_swap, datetime_swap, post_swap: flake_checker.report_pass('suiteName')
def test_missing_environment_variable(self): def mock_getenv(variable): environment_vars = { 'CIRCLECI': 1, 'CIRCLE_USERNAME': '******', 'CIRCLE_BRANCH': 'develop', } return environment_vars.get(variable) def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument raise AssertionError('requests.post called.') getenv_swap = self.swap(os, 'getenv', mock_getenv) post_swap = self.swap(requests, 'post', mock_post) with getenv_swap, post_swap: with self.assertRaisesRegex( RuntimeError, 'Expected environment variable CIRCLE_BUILD_URL missing'): flake_checker.report_pass('suiteName')