def test_start_tests_in_debug_mode(self):
self.exit_stack.enter_context(
self.swap_with_checks(run_e2e_tests,
'is_oppia_server_already_running',
lambda *_: False))
self.exit_stack.enter_context(
self.swap_with_checks(run_e2e_tests,
'install_third_party_libraries',
lambda _: None,
expected_args=[(False, )]))
self.exit_stack.enter_context(
self.swap_with_checks(run_e2e_tests,
'build_js_files',
lambda *_, **__: None,
expected_args=[(True, )]))
self.exit_stack.enter_context(
self.swap_with_checks(servers, 'managed_elasticsearch_dev_server',
mock_managed_process))
self.exit_stack.enter_context(
self.swap_with_checks(servers, 'managed_firebase_auth_emulator',
mock_managed_process))
self.exit_stack.enter_context(
self.swap_with_checks(servers, 'managed_dev_appserver',
mock_managed_process))
self.exit_stack.enter_context(
self.swap_with_checks(servers, 'managed_redis_server',
mock_managed_process))
self.exit_stack.enter_context(
self.swap_with_checks(servers, 'managed_portserver',
mock_managed_process))
self.exit_stack.enter_context(
self.swap_with_checks(servers, 'managed_webdriver_server',
mock_managed_process))
self.exit_stack.enter_context(
self.swap_with_checks(servers,
'managed_protractor_server',
mock_managed_process,
expected_kwargs=[
{
'dev_mode': True,
'suite_name': 'full',
'sharding_instances': 3,
'debug_mode': True,
'stdout': subprocess.PIPE,
},
]))
self.exit_stack.enter_context(
self.swap(flake_checker, 'check_if_on_ci', lambda: True))
self.exit_stack.enter_context(
self.swap_with_checks(flake_checker,
'report_pass',
lambda _: None,
expected_args=[('full', )]))
self.exit_stack.enter_context(
self.swap_with_checks(sys,
'exit',
lambda _: None,
expected_args=[(0, )]))
run_e2e_tests.main(args=['--debug_mode'])
def test_no_reruns_off_ci_pass(self):
def mock_run_tests(unused_args):
return 'sample\noutput', 0
def mock_report_pass(unused_suite_name):
raise AssertionError('Tried to Report Pass')
self.exit_stack.enter_context(
self.swap_with_checks(servers, 'managed_portserver',
mock_managed_process))
self.exit_stack.enter_context(
self.swap(run_e2e_tests, 'run_tests', mock_run_tests))
self.exit_stack.enter_context(
self.swap(flake_checker, 'report_pass', mock_report_pass))
self.exit_stack.enter_context(
self.swap(flake_checker, 'check_if_on_ci', lambda: False))
self.exit_stack.enter_context(
self.swap_with_checks(sys,
'exit',
lambda _: None,
expected_args=[(0, )]))
self.exit_stack.enter_context(
self.swap(run_e2e_tests, 'RERUN_POLICIES', MOCK_RERUN_POLICIES))
run_e2e_tests.main(args=['--suite', 'always'])
def test_rerun_when_tests_flake(self):
def mock_run_tests(unused_args):
return 'sample\noutput', 1
self.exit_stack.enter_context(
self.swap_with_checks(servers, 'managed_portserver',
mock_managed_process))
self.exit_stack.enter_context(
self.swap(run_e2e_tests, 'run_tests', mock_run_tests))
self.exit_stack.enter_context(
self.swap_with_checks(flake_checker,
'is_test_output_flaky',
lambda *_: True,
expected_args=[
('sample\noutput', 'mySuite'),
('sample\noutput', 'mySuite'),
('sample\noutput', 'mySuite'),
]))
self.exit_stack.enter_context(
self.swap(flake_checker, 'check_if_on_ci', lambda: True))
self.exit_stack.enter_context(
self.swap_with_checks(sys,
'exit',
lambda _: None,
expected_args=[(1, )]))
run_e2e_tests.main(args=['--suite', 'mySuite'])
def test_do_not_rerun_when_tests_fail_with_never_policy(self):
def mock_run_tests(unused_args):
return 'sample\noutput', 1
self.exit_stack.enter_context(
self.swap_with_checks(servers, 'managed_portserver',
mock_managed_process))
self.exit_stack.enter_context(
self.swap(run_e2e_tests, 'run_tests', mock_run_tests))
self.exit_stack.enter_context(
self.swap_with_checks(flake_checker,
'is_test_output_flaky',
lambda *_: False,
expected_args=[
('sample\noutput', 'never'),
]))
self.exit_stack.enter_context(
self.swap(flake_checker, 'check_if_on_ci', lambda: True))
self.exit_stack.enter_context(
self.swap_with_checks(sys,
'exit',
lambda _: None,
expected_args=[(1, )]))
self.exit_stack.enter_context(
self.swap(run_e2e_tests, 'RERUN_POLICIES', MOCK_RERUN_POLICIES))
run_e2e_tests.main(args=['--suite', 'never'])
def test_rerun_when_tests_flake_with_known_flakes_policy(self):
def mock_run_tests(unused_args):
return 'sample\noutput', 1
def mock_is_test_output_flaky(*_):
return True, flake_checker.RERUN_UNKNOWN
self.exit_stack.enter_context(
self.swap(run_e2e_tests, 'run_tests', mock_run_tests))
self.exit_stack.enter_context(
self.swap_with_checks(flake_checker,
'is_test_output_flaky',
mock_is_test_output_flaky,
expected_args=[
('sample\noutput', 'known_flakes'),
('sample\noutput', 'known_flakes'),
('sample\noutput', 'known_flakes'),
]))
self.exit_stack.enter_context(
self.swap(flake_checker, 'check_if_on_ci', lambda: True))
self.exit_stack.enter_context(
self.swap_with_checks(sys,
'exit',
lambda _: None,
expected_args=[(1, )]))
self.exit_stack.enter_context(
self.swap_with_checks(servers, 'managed_portserver',
mock_managed_process))
self.exit_stack.enter_context(
self.swap(run_e2e_tests, 'RERUN_POLICIES', MOCK_RERUN_POLICIES))
run_e2e_tests.main(args=['--suite', 'known_flakes'])
def test_start_tests_when_other_instances_not_stopped(self):
self.exit_stack.enter_context(self.swap_with_checks(
run_e2e_tests, 'is_oppia_server_already_running', lambda *_: True))
self.exit_stack.enter_context(self.swap_with_checks(
servers, 'managed_portserver', mock_managed_process))
with self.assertRaisesRegexp(SystemExit, '1'):
run_e2e_tests.main(args=[])
def test_start_tests_when_other_instances_not_stopped(self):
def mock_exit(unused_exit_code):
raise Exception('sys.exit(1)')
def mock_is_oppia_server_already_running(*unused_args):
return True
check_swap = self.swap_with_checks(
run_e2e_tests, 'is_oppia_server_already_running',
mock_is_oppia_server_already_running)
exit_swap = self.swap(sys, 'exit', mock_exit)
with check_swap, exit_swap:
with self.assertRaisesRegexp(Exception, r'sys\.exit\(1\)'):
run_e2e_tests.main(args=[])
def test_no_reruns_off_ci_fail(self):
def mock_run_tests(unused_args):
return 'sample\noutput', 1
def mock_is_test_output_flaky(unused_output, unused_suite_name):
raise AssertionError('Tried to Check Flakiness.')
self.exit_stack.enter_context(self.swap_with_checks(
servers, 'managed_portserver', mock_managed_process))
self.exit_stack.enter_context(self.swap(
run_e2e_tests, 'run_tests', mock_run_tests))
self.exit_stack.enter_context(self.swap(
flake_checker, 'is_test_output_flaky', mock_is_test_output_flaky))
self.exit_stack.enter_context(self.swap(
flake_checker, 'check_if_on_ci', lambda: False))
self.exit_stack.enter_context(self.swap_with_checks(
sys, 'exit', lambda _: None, expected_args=[(1,)]))
run_e2e_tests.main(args=['--suite', 'mySuite'])
def test_start_tests_when_no_other_instance_running(self):
def mock_is_oppia_server_already_running(*unused_args):
return False
def mock_setup_and_install_dependencies(unused_arg):
return
def mock_register(unused_func):
return
def mock_cleanup():
return
def mock_build_js_files(unused_arg, deparallelize_terser=False): # pylint: disable=unused-argument
return
def mock_start_webdriver_manager():
return
def mock_start_google_app_engine_server(unused_arg):
return
def mock_wait_for_port_to_be_open(unused_port):
return
def mock_ensure_screenshots_dir_is_removed():
return
def mock_get_e2e_test_parameters(unused_sharding_instances,
unused_suite, unused_dev_mode):
return ['commands']
def mock_popen(unused_commands):
def mock_communicate():
return
result = MockProcessClass()
result.communicate = mock_communicate # pylint: disable=attribute-defined-outside-init
result.returncode = 0 # pylint: disable=attribute-defined-outside-init
return result
def mock_exit(unused_code):
return
check_swap = self.swap_with_checks(
run_e2e_tests, 'is_oppia_server_already_running',
mock_is_oppia_server_already_running)
setup_and_install_swap = self.swap_with_checks(
run_e2e_tests,
'setup_and_install_dependencies',
mock_setup_and_install_dependencies,
expected_args=[(False, )])
register_swap = self.swap_with_checks(atexit,
'register',
mock_register,
expected_args=[(mock_cleanup, )])
cleanup_swap = self.swap(run_e2e_tests, 'cleanup', mock_cleanup)
build_swap = self.swap_with_checks(run_e2e_tests,
'build_js_files',
mock_build_js_files,
expected_args=[(True, )])
start_webdriver_swap = self.swap_with_checks(
run_e2e_tests, 'start_webdriver_manager',
mock_start_webdriver_manager)
start_google_app_engine_server_swap = self.swap_with_checks(
run_e2e_tests,
'start_google_app_engine_server',
mock_start_google_app_engine_server,
expected_args=[(True, )])
wait_swap = self.swap_with_checks(
run_e2e_tests,
'wait_for_port_to_be_open',
mock_wait_for_port_to_be_open,
expected_args=[(run_e2e_tests.WEB_DRIVER_PORT, ),
(run_e2e_tests.GOOGLE_APP_ENGINE_PORT, )])
ensure_screenshots_dir_is_removed_swap = self.swap_with_checks(
run_e2e_tests, 'ensure_screenshots_dir_is_removed',
mock_ensure_screenshots_dir_is_removed)
get_parameters_swap = self.swap_with_checks(
run_e2e_tests,
'get_e2e_test_parameters',
mock_get_e2e_test_parameters,
expected_args=[(3, 'full', True)])
popen_swap = self.swap_with_checks(
subprocess,
'Popen',
mock_popen,
expected_args=[([
common.NODE_BIN_PATH, '--unhandled-rejections=strict',
run_e2e_tests.PROTRACTOR_BIN_PATH, 'commands'
], )])
exit_swap = self.swap_with_checks(sys,
'exit',
mock_exit,
expected_args=[(0, )])
with check_swap, setup_and_install_swap, register_swap, cleanup_swap:
with build_swap, start_webdriver_swap:
with start_google_app_engine_server_swap:
with wait_swap, ensure_screenshots_dir_is_removed_swap:
with get_parameters_swap, popen_swap, exit_swap:
run_e2e_tests.main(args=[])
def test_start_tests_skip_build(self):
def mock_is_oppia_server_already_running(*unused_args):
return False
def mock_setup_and_install_dependencies(unused_arg):
return
def mock_register(unused_func):
return
def mock_cleanup():
return
def mock_modify_constants(prod_env, maintenance_mode=False): # pylint: disable=unused-argument
return
def mock_start_webdriver_manager(unused_arg):
return
def mock_start_google_app_engine_server(unused_arg, unused_log_level):
return
def mock_wait_for_port_to_be_open(unused_port):
return
def mock_ensure_screenshots_dir_is_removed():
return
def mock_get_e2e_test_parameters(
unused_sharding_instances, unused_suite, unused_dev_mode):
return ['commands']
def mock_popen(unused_commands):
def mock_communicate():
return
result = MockProcessClass()
result.communicate = mock_communicate # pylint: disable=attribute-defined-outside-init
result.returncode = 0 # pylint: disable=attribute-defined-outside-init
return result
def mock_exit(unused_code):
return
def mock_get_chrome_driver_version():
return CHROME_DRIVER_VERSION
get_chrome_driver_version_swap = self.swap(
run_e2e_tests, 'get_chrome_driver_version',
mock_get_chrome_driver_version)
check_swap = self.swap_with_checks(
run_e2e_tests, 'is_oppia_server_already_running',
mock_is_oppia_server_already_running)
setup_and_install_swap = self.swap_with_checks(
run_e2e_tests, 'setup_and_install_dependencies',
mock_setup_and_install_dependencies, expected_args=[(True,)])
register_swap = self.swap_with_checks(
atexit, 'register', mock_register, expected_args=[(mock_cleanup,)])
cleanup_swap = self.swap(run_e2e_tests, 'cleanup', mock_cleanup)
modify_constants_swap = self.swap_with_checks(
build, 'modify_constants', mock_modify_constants,
expected_kwargs=[{'prod_env': False}])
start_webdriver_swap = self.swap_with_checks(
run_e2e_tests, 'start_webdriver_manager',
mock_start_webdriver_manager,
expected_args=[(CHROME_DRIVER_VERSION,)])
start_google_app_engine_server_swap = self.swap_with_checks(
run_e2e_tests, 'start_google_app_engine_server',
mock_start_google_app_engine_server,
expected_args=[(True, 'critical')])
wait_swap = self.swap_with_checks(
run_e2e_tests, 'wait_for_port_to_be_open',
mock_wait_for_port_to_be_open,
expected_args=[
(run_e2e_tests.WEB_DRIVER_PORT,),
(run_e2e_tests.GOOGLE_APP_ENGINE_PORT,)])
ensure_screenshots_dir_is_removed_swap = self.swap_with_checks(
run_e2e_tests, 'ensure_screenshots_dir_is_removed',
mock_ensure_screenshots_dir_is_removed)
get_parameters_swap = self.swap_with_checks(
run_e2e_tests, 'get_e2e_test_parameters',
mock_get_e2e_test_parameters, expected_args=[(3, 'full', True)])
popen_swap = self.swap_with_checks(
subprocess, 'Popen', mock_popen, expected_args=[([
common.NODE_BIN_PATH, '--unhandled-rejections=strict',
run_e2e_tests.PROTRACTOR_BIN_PATH,
'commands'],)])
exit_swap = self.swap_with_checks(
sys, 'exit', mock_exit, expected_args=[(0,)])
with check_swap, setup_and_install_swap, register_swap, cleanup_swap:
with modify_constants_swap, start_webdriver_swap:
with start_google_app_engine_server_swap:
with wait_swap, ensure_screenshots_dir_is_removed_swap:
with get_parameters_swap, popen_swap, exit_swap:
with get_chrome_driver_version_swap:
run_e2e_tests.main(
args=['--skip-install', '--skip-build'])