def main(args=None): """Run tests, rerunning at most MAX_RETRY_COUNT times if they flake.""" parsed_args = _PARSER.parse_args(args=args) policy = RERUN_POLICIES[parsed_args.suite.lower()] with servers.managed_portserver(): for attempt_num in python_utils.RANGE(1, MAX_RETRY_COUNT + 1): python_utils.PRINT('***Attempt %d.***' % attempt_num) output, return_code = run_tests(parsed_args) if not flake_checker.check_if_on_ci(): # Don't rerun off of CI. python_utils.PRINT('No reruns because not running on CI.') break if return_code == 0: # Don't rerun passing tests. flake_checker.report_pass(parsed_args.suite) break # Check whether we should rerun based on this suite's policy. test_is_flaky = flake_checker.is_test_output_flaky( output, parsed_args.suite) if policy == RERUN_POLICY_NEVER: break if policy == RERUN_POLICY_KNOWN_FLAKES and not test_is_flaky: break sys.exit(return_code)
def main(args=None): """Run tests, rerunning at most MAX_RETRY_COUNT times if they flake.""" parsed_args = _PARSER.parse_args(args=args) portserver_process = start_portserver() atexit.register(cleanup_portserver, portserver_process) for attempt_num in python_utils.RANGE(MAX_RETRY_COUNT): python_utils.PRINT('***Attempt %s.***' % (attempt_num + 1)) output, return_code = run_tests(parsed_args) # Don't rerun passing tests. if return_code == 0: flake_checker.report_pass(parsed_args.suite) break # Don't rerun off of CI. if not flake_checker.check_if_on_ci(): python_utils.PRINT('No reruns because not running on CI.') break flaky = flake_checker.is_test_output_flaky(output, parsed_args.suite) # Don't rerun if the test was non-flaky and we are not # rerunning non-flaky tests. if not flaky and not RERUN_NON_FLAKY: break # Prepare for rerun. cleanup() sys.exit(return_code)
def main(args=None): """Run tests, rerunning at most MAX_RETRY_COUNT times if they flake.""" parsed_args = _PARSER.parse_args(args=args) with servers.managed_portserver(): for attempt_num in python_utils.RANGE(1, MAX_RETRY_COUNT + 1): python_utils.PRINT('***Attempt %d.***' % attempt_num) output, return_code = run_tests(parsed_args) if not flake_checker.check_if_on_ci(): # Don't rerun off of CI. python_utils.PRINT('No reruns because not running on CI.') break if return_code == 0: # Don't rerun passing tests. flake_checker.report_pass(parsed_args.suite) break test_is_flaky = flake_checker.is_test_output_flaky( output, parsed_args.suite) if not test_is_flaky and not RERUN_NON_FLAKY: # Don't rerun if the test was non-flaky and we are not rerunning # non-flaky tests. break sys.exit(return_code)
def test_successful_report_construct_url(self): def mock_getenv(variable): environment_vars = { 'GITHUB_ACTIONS': 1, 'GITHUB_ACTOR': 'user', 'GITHUB_RUN_ID': 1234, 'GITHUB_REF': 'develop', 'GITHUB_REPOSITORY': 'foo/oppia' } return environment_vars.get(variable) def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument response = { 'log': ['log1', 'log2'], 'result': True, 'flake': { 'suite': 'suiteName', 'test': 'testName', 'flake_id': 'flake', }, } return MockResponse(True, response) expected_payload = { 'suite': 'suiteName', 'output_lines': ['line1', 'line2'], 'metadata': { 'username': '******', 'build_url': 'https://github.com/foo/oppia/actions/runs/1234', 'timestamp': '2020-01-01T00:00:00.000001+00:00', 'branch': 'develop', } } getenv_swap = self.swap(os, 'getenv', mock_getenv) datetime_swap = self.swap(datetime, 'datetime', MockDatetime(self.example_date)) post_swap = self.swap_with_checks( requests, 'post', mock_post, expected_args=[(flake_checker.FLAKE_CHECK_AND_REPORT_URL, )], expected_kwargs=[{ 'json': expected_payload, 'allow_redirects': False, 'headers': { 'report_key': flake_checker.REPORT_API_KEY }, }]) with getenv_swap, datetime_swap, post_swap: flaky, rerun = flake_checker.is_test_output_flaky( ['line1', 'line2'], 'suiteName') self.assertTrue(flaky) self.assertEqual(rerun, flake_checker.RERUN_UNKNOWN)
def test_successful_report(self): def mock_getenv(variable): environment_vars = { 'CIRCLECI': 1, 'CIRCLE_USERNAME': '******', 'CIRCLE_BUILD_URL': 'https://example.com', 'CIRCLE_BRANCH': 'develop', } return environment_vars.get(variable) def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument response = { 'log': ['log1', 'log2'], 'result': True, 'flake': { 'suite': 'suiteName', 'test': 'testName', 'flake_id': 'flake', }, } return MockResponse(True, response) expected_payload = { 'suite': 'suiteName', 'output_lines': ['line1', 'line2'], 'metadata': { 'username': '******', 'build_url': 'https://example.com', 'timestamp': '2020-01-01T00:00:00.000001+00:00', 'branch': 'develop', } } getenv_swap = self.swap(os, 'getenv', mock_getenv) datetime_swap = self.swap(datetime, 'datetime', MockDatetime(self.example_date)) post_swap = self.swap_with_checks( requests, 'post', mock_post, expected_args=[(flake_checker.FLAKE_CHECK_AND_REPORT_URL, )], expected_kwargs=[{ 'json': expected_payload, 'allow_redirects': False, 'headers': { 'report_key': flake_checker.REPORT_API_KEY }, }]) with getenv_swap, datetime_swap, post_swap: flaky = flake_checker.is_test_output_flaky(['line1', 'line2'], 'suiteName') self.assertTrue(flaky)
def main(args=None): """Run tests, rerunning at most MAX_RETRY_COUNT times if they flake.""" parsed_args = _PARSER.parse_args(args=args) policy = RERUN_POLICIES[parsed_args.suite.lower()] with servers.managed_portserver(): for attempt_num in range(1, MAX_RETRY_COUNT + 1): print('***Attempt %d.***' % attempt_num) output, return_code = run_tests(parsed_args) if not flake_checker.check_if_on_ci(): # Don't rerun off of CI. print('No reruns because not running on CI.') break if return_code == 0: # Don't rerun passing tests. flake_checker.report_pass(parsed_args.suite) break # Check whether we should rerun based on this suite's policy # and override instructions from the flake checker server. test_is_flaky, rerun_override = flake_checker.is_test_output_flaky( output, parsed_args.suite) if rerun_override == flake_checker.RERUN_YES: print('Rerunning as per instructions from logging ' 'server.') elif rerun_override == flake_checker.RERUN_NO: print('Not rerunning as per instructions from ' 'logging server.') break # No rerun override, so follow rerun policies. elif policy == RERUN_POLICY_NEVER: print('Not rerunning because the policy is to never ' 'rerun the {} suite'.format(parsed_args.suite)) break elif policy == RERUN_POLICY_KNOWN_FLAKES and not test_is_flaky: print(('Not rerunning because the policy is to only ' 'rerun the %s suite on known flakes, and this ' 'failure did not match any known flakes') % parsed_args.suite) break # Else rerun policy is either always or we had a known flake # with a known flakes rerun policy, so rerun. sys.exit(return_code)