def main(args=None):
"""Run tests, rerunning at most MAX_RETRY_COUNT times if they flake."""
parsed_args = _PARSER.parse_args(args=args)
policy = RERUN_POLICIES[parsed_args.suite.lower()]
with servers.managed_portserver():
for attempt_num in python_utils.RANGE(1, MAX_RETRY_COUNT + 1):
python_utils.PRINT('***Attempt %d.***' % attempt_num)
output, return_code = run_tests(parsed_args)
if not flake_checker.check_if_on_ci():
# Don't rerun off of CI.
python_utils.PRINT('No reruns because not running on CI.')
break
if return_code == 0:
# Don't rerun passing tests.
flake_checker.report_pass(parsed_args.suite)
break
# Check whether we should rerun based on this suite's policy.
test_is_flaky = flake_checker.is_test_output_flaky(
output, parsed_args.suite)
if policy == RERUN_POLICY_NEVER:
break
if policy == RERUN_POLICY_KNOWN_FLAKES and not test_is_flaky:
break
sys.exit(return_code)
def main(args=None):
"""Run tests, rerunning at most MAX_RETRY_COUNT times if they flake."""
parsed_args = _PARSER.parse_args(args=args)
portserver_process = start_portserver()
atexit.register(cleanup_portserver, portserver_process)
for attempt_num in python_utils.RANGE(MAX_RETRY_COUNT):
python_utils.PRINT('***Attempt %s.***' % (attempt_num + 1))
output, return_code = run_tests(parsed_args)
# Don't rerun passing tests.
if return_code == 0:
flake_checker.report_pass(parsed_args.suite)
break
# Don't rerun off of CI.
if not flake_checker.check_if_on_ci():
python_utils.PRINT('No reruns because not running on CI.')
break
flaky = flake_checker.is_test_output_flaky(output, parsed_args.suite)
# Don't rerun if the test was non-flaky and we are not
# rerunning non-flaky tests.
if not flaky and not RERUN_NON_FLAKY:
break
# Prepare for rerun.
cleanup()
sys.exit(return_code)
def main(args=None):
"""Run tests, rerunning at most MAX_RETRY_COUNT times if they flake."""
parsed_args = _PARSER.parse_args(args=args)
with servers.managed_portserver():
for attempt_num in python_utils.RANGE(1, MAX_RETRY_COUNT + 1):
python_utils.PRINT('***Attempt %d.***' % attempt_num)
output, return_code = run_tests(parsed_args)
if not flake_checker.check_if_on_ci():
# Don't rerun off of CI.
python_utils.PRINT('No reruns because not running on CI.')
break
if return_code == 0:
# Don't rerun passing tests.
flake_checker.report_pass(parsed_args.suite)
break
test_is_flaky = flake_checker.is_test_output_flaky(
output, parsed_args.suite)
if not test_is_flaky and not RERUN_NON_FLAKY:
# Don't rerun if the test was non-flaky and we are not rerunning
# non-flaky tests.
break
sys.exit(return_code)
def test_unknown_build_environment(self):
def mock_getenv(variable):
environment_vars = {}
return environment_vars.get(variable)
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
raise AssertionError('requests.post called.')
getenv_swap = self.swap(os, 'getenv', mock_getenv)
post_swap = self.swap(requests, 'post', mock_post)
with getenv_swap, post_swap:
with self.assertRaisesRegex(Exception,
'Unknown build environment.'):
flake_checker.report_pass('suiteName')
def main(args=None):
"""Run tests, rerunning at most MAX_RETRY_COUNT times if they flake."""
parsed_args = _PARSER.parse_args(args=args)
policy = RERUN_POLICIES[parsed_args.suite.lower()]
with servers.managed_portserver():
for attempt_num in range(1, MAX_RETRY_COUNT + 1):
print('***Attempt %d.***' % attempt_num)
output, return_code = run_tests(parsed_args)
if not flake_checker.check_if_on_ci():
# Don't rerun off of CI.
print('No reruns because not running on CI.')
break
if return_code == 0:
# Don't rerun passing tests.
flake_checker.report_pass(parsed_args.suite)
break
# Check whether we should rerun based on this suite's policy
# and override instructions from the flake checker server.
test_is_flaky, rerun_override = flake_checker.is_test_output_flaky(
output, parsed_args.suite)
if rerun_override == flake_checker.RERUN_YES:
print('Rerunning as per instructions from logging ' 'server.')
elif rerun_override == flake_checker.RERUN_NO:
print('Not rerunning as per instructions from '
'logging server.')
break
# No rerun override, so follow rerun policies.
elif policy == RERUN_POLICY_NEVER:
print('Not rerunning because the policy is to never '
'rerun the {} suite'.format(parsed_args.suite))
break
elif policy == RERUN_POLICY_KNOWN_FLAKES and not test_is_flaky:
print(('Not rerunning because the policy is to only '
'rerun the %s suite on known flakes, and this '
'failure did not match any known flakes') %
parsed_args.suite)
break
# Else rerun policy is either always or we had a known flake
# with a known flakes rerun policy, so rerun.
sys.exit(return_code)
def test_successful_report_construct_url(self):
def mock_getenv(variable):
environment_vars = {
'GITHUB_ACTIONS': 1,
'GITHUB_ACTOR': 'user',
'GITHUB_RUN_ID': 1234,
'GITHUB_REF': 'develop',
'GITHUB_REPOSITORY': 'foo/oppia',
}
return environment_vars.get(variable)
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
pass
expected_payload = {
'suite': 'suiteName',
'metadata': {
'username': '******',
'build_url': 'https://github.com/foo/oppia/actions/runs/1234',
'timestamp': '2020-01-01T00:00:00.000001+00:00',
'branch': 'develop',
}
}
getenv_swap = self.swap(os, 'getenv', mock_getenv)
datetime_swap = self.swap(datetime, 'datetime',
MockDatetime(self.example_date))
post_swap = self.swap_with_checks(requests,
'post',
mock_post,
expected_args=[
(flake_checker.PASS_REPORT_URL, )
],
expected_kwargs=[{
'json': expected_payload,
'allow_redirects': False,
'headers': {
'report_key':
flake_checker.REPORT_API_KEY
},
}])
with getenv_swap, datetime_swap, post_swap:
flake_checker.report_pass('suiteName')
def test_successful_report(self):
def mock_getenv(variable):
environment_vars = {
'CIRCLECI': 1,
'CIRCLE_USERNAME': '******',
'CIRCLE_BUILD_URL': 'https://example.com',
'CIRCLE_BRANCH': 'develop',
}
return environment_vars.get(variable)
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
pass
expected_payload = {
'suite': 'suiteName',
'metadata': {
'username': '******',
'build_url': 'https://example.com',
'timestamp': '2020-01-01T00:00:00.000001+00:00',
'branch': 'develop',
}
}
getenv_swap = self.swap(os, 'getenv', mock_getenv)
datetime_swap = self.swap(datetime, 'datetime',
MockDatetime(self.example_date))
post_swap = self.swap_with_checks(requests,
'post',
mock_post,
expected_args=[
(flake_checker.PASS_REPORT_URL, )
],
expected_kwargs=[{
'json': expected_payload,
'allow_redirects': False,
'headers': {
'report_key':
flake_checker.REPORT_API_KEY
},
}])
with getenv_swap, datetime_swap, post_swap:
flake_checker.report_pass('suiteName')
def test_missing_environment_variable(self):
def mock_getenv(variable):
environment_vars = {
'CIRCLECI': 1,
'CIRCLE_USERNAME': '******',
'CIRCLE_BRANCH': 'develop',
}
return environment_vars.get(variable)
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
raise AssertionError('requests.post called.')
getenv_swap = self.swap(os, 'getenv', mock_getenv)
post_swap = self.swap(requests, 'post', mock_post)
with getenv_swap, post_swap:
with self.assertRaisesRegex(
RuntimeError,
'Expected environment variable CIRCLE_BUILD_URL missing'):
flake_checker.report_pass('suiteName')
