def main(args=None):
"""Run tests, rerunning at most MAX_RETRY_COUNT times if they flake."""
parsed_args = _PARSER.parse_args(args=args)
policy = RERUN_POLICIES[parsed_args.suite.lower()]
with servers.managed_portserver():
for attempt_num in python_utils.RANGE(1, MAX_RETRY_COUNT + 1):
python_utils.PRINT('***Attempt %d.***' % attempt_num)
output, return_code = run_tests(parsed_args)
if not flake_checker.check_if_on_ci():
# Don't rerun off of CI.
python_utils.PRINT('No reruns because not running on CI.')
break
if return_code == 0:
# Don't rerun passing tests.
flake_checker.report_pass(parsed_args.suite)
break
# Check whether we should rerun based on this suite's policy.
test_is_flaky = flake_checker.is_test_output_flaky(
output, parsed_args.suite)
if policy == RERUN_POLICY_NEVER:
break
if policy == RERUN_POLICY_KNOWN_FLAKES and not test_is_flaky:
break
sys.exit(return_code)
def main(args=None):
"""Run tests, rerunning at most MAX_RETRY_COUNT times if they flake."""
parsed_args = _PARSER.parse_args(args=args)
portserver_process = start_portserver()
atexit.register(cleanup_portserver, portserver_process)
for attempt_num in python_utils.RANGE(MAX_RETRY_COUNT):
python_utils.PRINT('***Attempt %s.***' % (attempt_num + 1))
output, return_code = run_tests(parsed_args)
# Don't rerun passing tests.
if return_code == 0:
flake_checker.report_pass(parsed_args.suite)
break
# Don't rerun off of CI.
if not flake_checker.check_if_on_ci():
python_utils.PRINT('No reruns because not running on CI.')
break
flaky = flake_checker.is_test_output_flaky(output, parsed_args.suite)
# Don't rerun if the test was non-flaky and we are not
# rerunning non-flaky tests.
if not flaky and not RERUN_NON_FLAKY:
break
# Prepare for rerun.
cleanup()
sys.exit(return_code)
def main(args=None):
"""Run tests, rerunning at most MAX_RETRY_COUNT times if they flake."""
parsed_args = _PARSER.parse_args(args=args)
with servers.managed_portserver():
for attempt_num in python_utils.RANGE(1, MAX_RETRY_COUNT + 1):
python_utils.PRINT('***Attempt %d.***' % attempt_num)
output, return_code = run_tests(parsed_args)
if not flake_checker.check_if_on_ci():
# Don't rerun off of CI.
python_utils.PRINT('No reruns because not running on CI.')
break
if return_code == 0:
# Don't rerun passing tests.
flake_checker.report_pass(parsed_args.suite)
break
test_is_flaky = flake_checker.is_test_output_flaky(
output, parsed_args.suite)
if not test_is_flaky and not RERUN_NON_FLAKY:
# Don't rerun if the test was non-flaky and we are not rerunning
# non-flaky tests.
break
sys.exit(return_code)
def test_successful_report_construct_url(self):
def mock_getenv(variable):
environment_vars = {
'GITHUB_ACTIONS': 1,
'GITHUB_ACTOR': 'user',
'GITHUB_RUN_ID': 1234,
'GITHUB_REF': 'develop',
'GITHUB_REPOSITORY': 'foo/oppia'
}
return environment_vars.get(variable)
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
response = {
'log': ['log1', 'log2'],
'result': True,
'flake': {
'suite': 'suiteName',
'test': 'testName',
'flake_id': 'flake',
},
}
return MockResponse(True, response)
expected_payload = {
'suite': 'suiteName',
'output_lines': ['line1', 'line2'],
'metadata': {
'username': '******',
'build_url': 'https://github.com/foo/oppia/actions/runs/1234',
'timestamp': '2020-01-01T00:00:00.000001+00:00',
'branch': 'develop',
}
}
getenv_swap = self.swap(os, 'getenv', mock_getenv)
datetime_swap = self.swap(datetime, 'datetime',
MockDatetime(self.example_date))
post_swap = self.swap_with_checks(
requests,
'post',
mock_post,
expected_args=[(flake_checker.FLAKE_CHECK_AND_REPORT_URL, )],
expected_kwargs=[{
'json': expected_payload,
'allow_redirects': False,
'headers': {
'report_key': flake_checker.REPORT_API_KEY
},
}])
with getenv_swap, datetime_swap, post_swap:
flaky, rerun = flake_checker.is_test_output_flaky(
['line1', 'line2'], 'suiteName')
self.assertTrue(flaky)
self.assertEqual(rerun, flake_checker.RERUN_UNKNOWN)
def test_successful_report(self):
def mock_getenv(variable):
environment_vars = {
'CIRCLECI': 1,
'CIRCLE_USERNAME': '******',
'CIRCLE_BUILD_URL': 'https://example.com',
'CIRCLE_BRANCH': 'develop',
}
return environment_vars.get(variable)
def mock_post(url, json, allow_redirects, headers): # pylint: disable=unused-argument
response = {
'log': ['log1', 'log2'],
'result': True,
'flake': {
'suite': 'suiteName',
'test': 'testName',
'flake_id': 'flake',
},
}
return MockResponse(True, response)
expected_payload = {
'suite': 'suiteName',
'output_lines': ['line1', 'line2'],
'metadata': {
'username': '******',
'build_url': 'https://example.com',
'timestamp': '2020-01-01T00:00:00.000001+00:00',
'branch': 'develop',
}
}
getenv_swap = self.swap(os, 'getenv', mock_getenv)
datetime_swap = self.swap(datetime, 'datetime',
MockDatetime(self.example_date))
post_swap = self.swap_with_checks(
requests,
'post',
mock_post,
expected_args=[(flake_checker.FLAKE_CHECK_AND_REPORT_URL, )],
expected_kwargs=[{
'json': expected_payload,
'allow_redirects': False,
'headers': {
'report_key': flake_checker.REPORT_API_KEY
},
}])
with getenv_swap, datetime_swap, post_swap:
flaky = flake_checker.is_test_output_flaky(['line1', 'line2'],
'suiteName')
self.assertTrue(flaky)
def main(args=None):
"""Run tests, rerunning at most MAX_RETRY_COUNT times if they flake."""
parsed_args = _PARSER.parse_args(args=args)
policy = RERUN_POLICIES[parsed_args.suite.lower()]
with servers.managed_portserver():
for attempt_num in range(1, MAX_RETRY_COUNT + 1):
print('***Attempt %d.***' % attempt_num)
output, return_code = run_tests(parsed_args)
if not flake_checker.check_if_on_ci():
# Don't rerun off of CI.
print('No reruns because not running on CI.')
break
if return_code == 0:
# Don't rerun passing tests.
flake_checker.report_pass(parsed_args.suite)
break
# Check whether we should rerun based on this suite's policy
# and override instructions from the flake checker server.
test_is_flaky, rerun_override = flake_checker.is_test_output_flaky(
output, parsed_args.suite)
if rerun_override == flake_checker.RERUN_YES:
print('Rerunning as per instructions from logging ' 'server.')
elif rerun_override == flake_checker.RERUN_NO:
print('Not rerunning as per instructions from '
'logging server.')
break
# No rerun override, so follow rerun policies.
elif policy == RERUN_POLICY_NEVER:
print('Not rerunning because the policy is to never '
'rerun the {} suite'.format(parsed_args.suite))
break
elif policy == RERUN_POLICY_KNOWN_FLAKES and not test_is_flaky:
print(('Not rerunning because the policy is to only '
'rerun the %s suite on known flakes, and this '
'failure did not match any known flakes') %
parsed_args.suite)
break
# Else rerun policy is either always or we had a known flake
# with a known flakes rerun policy, so rerun.
sys.exit(return_code)
