def test_exception_raised_while_waiting_causes_termination_and_adds_error_message_to_output(
self):
exception_message = 'Something terribly horrible just happened!'
value_err_exc = ValueError(exception_message)
timeout_exc = TimeoutExpired(cmd=None, timeout=1)
fake_failing_return_code = -15
# Simulate Popen.wait() timing out twice before raising a ValueError exception.
self.mock_popen.wait.side_effect = [
timeout_exc, timeout_exc, value_err_exc, fake_failing_return_code
]
self.mock_popen.returncode = fake_failing_return_code
self.mock_popen.pid = 55555
self._mock_stdout_and_stderr(b'', b'')
project_type = ProjectType()
actual_output, actual_return_code = project_type.execute_command_in_project(
'echo The power is yours!')
self.assertEqual(
self.mock_killpg.call_count, 1,
'os.killpg should be called when wait() raises exception.')
self.assertIn(
exception_message, actual_output,
'ClusterRunner exception message should be included in output.')
self.assertEqual(actual_return_code, fake_failing_return_code,
'Actual return code should match expected.')
def test_calling_kill_subprocesses_will_break_out_of_command_execution_wait_loop(
self):
self._mock_stdout_and_stderr(b'fake_output', b'fake_error')
self.mock_popen.pid = 55555
self._simulate_hanging_popen_process()
project_type = ProjectType()
command_thread = SafeThread(
target=project_type.execute_command_in_project,
args=('echo The power is yours!', ))
# This calls execute_command_in_project() on one thread, and calls kill_subprocesses() on another. The
# kill_subprocesses() call should cause the first thread to exit.
command_thread.start()
project_type.kill_subprocesses()
# This *should* join immediately, but we specify a timeout just in case something goes wrong so that the test
# doesn't hang. A successful join implies success. We also use the UnhandledExceptionHandler so that exceptions
# propagate from the child thread to the test thread and fail the test.
with UnhandledExceptionHandler.singleton():
command_thread.join(timeout=10)
if command_thread.is_alive():
self.mock_killpg(
) # Calling killpg() causes the command thread to end.
self.fail(
'project_type.kill_subprocesses should cause the command execution wait loop to exit.'
)
self.mock_killpg.assert_called_once_with(
55555, ANY) # Note: os.killpg does not accept keyword args.
def test_timing_out_will_break_out_of_command_execution_wait_loop_and_kill_subprocesses(
self):
mock_time = self.patch('time.time')
mock_time.side_effect = [
0.0, 100.0, 200.0, 300.0
] # time increases by 100 seconds with each loop
expected_return_code = 1
self._simulate_hanging_popen_process(
fake_returncode=expected_return_code)
self.mock_popen.pid = 55555
self._mock_stdout_and_stderr(b'fake output', b'fake error')
project_type = ProjectType()
actual_output, actual_return_code = project_type.execute_command_in_project(
command='sleep 99', timeout=250)
self.assertEqual(
self.mock_killpg.call_count, 1,
'os.killpg should be called when execution times out.')
self.assertEqual(actual_output, 'fake output\nfake error',
'Output should contain stdout and stderr.')
self.assertEqual(actual_return_code, expected_return_code,
'Actual return code should match expected.')
self.assertTrue(
all([file.close.called for file in self.mock_temporary_files]),
'All created TemporaryFiles should be closed so that they are removed from the filesystem.'
)
def test_calling_kill_subprocesses_will_break_out_of_command_execution_wait_loop(self):
def fake_communicate(timeout=None):
# The fake implementation is that communicate() times out forever until os.killpg is called.
if mock_killpg.call_count == 0 and timeout is not None:
raise TimeoutExpired(None, timeout)
elif mock_killpg.call_count > 0:
return b'fake output', b'fake error'
self.fail('Popen.communicate() should not be called without a timeout before os.killpg has been called.')
mock_killpg = self.patch('os.killpg')
self.mock_popen.communicate.side_effect = fake_communicate
self.mock_popen.returncode = 1
self.mock_popen.pid = 55555
project_type = ProjectType()
command_thread = SafeThread(target=project_type.execute_command_in_project, args=('echo The power is yours!',))
# This calls execute_command_in_project() on one thread, and calls kill_subprocesses() on another. The
# kill_subprocesses() call should cause the first thread to exit.
command_thread.start()
project_type.kill_subprocesses()
# This *should* join immediately, but we specify a timeout just in case something goes wrong so that the test
# doesn't hang. A successful join implies success. We also use the UnhandledExceptionHandler so that exceptions
# propagate from the child thread to the test thread and fail the test.
with UnhandledExceptionHandler.singleton():
command_thread.join(timeout=10)
if command_thread.is_alive():
mock_killpg() # Calling killpg() causes the command thread to end.
self.fail('project_type.kill_subprocesses should cause the command execution wait loop to exit.')
mock_killpg.assert_called_once_with(pgid=55555, sig=ANY)
def test_command_exiting_normally_will_break_out_of_command_execution_wait_loop(
self):
timeout_exc = TimeoutExpired(cmd=None, timeout=1)
expected_return_code = 0
# Simulate Popen.wait() timing out twice before command completes and returns output.
self.mock_popen.wait.side_effect = [
timeout_exc, timeout_exc, expected_return_code
]
self.mock_popen.returncode = expected_return_code
self._mock_stdout_and_stderr(b'fake_output', b'fake_error')
project_type = ProjectType()
actual_output, actual_return_code = project_type.execute_command_in_project(
'echo The power is yours!')
self.assertEqual(
self.mock_killpg.call_count, 0,
'os.killpg should not be called when command exits normally.')
self.assertEqual(actual_output, 'fake_output\nfake_error',
'Output should contain stdout and stderr.')
self.assertEqual(actual_return_code, expected_return_code,
'Actual return code should match expected.')
self.assertTrue(
all([file.close.called for file in self.mock_temporary_files]),
'All created TemporaryFiles should be closed so that they are removed from the filesystem.'
)
def test_execute_command_in_project_does_not_choke_on_weird_command_output(self):
some_weird_output = b'\xbf\xe2\x98\x82' # the byte \xbf is invalid unicode
self.mock_popen.communicate.return_value = (some_weird_output, None)
self.mock_popen.returncode = (some_weird_output, None)
project_type = ProjectType()
project_type.execute_command_in_project('fake command')
def test_teardown_build_runs_teardown(self):
job_config = JobConfig('name', 'setup', 'teardown', 'command', 'atomizer', 10)
project_type = ProjectType()
project_type.job_config = MagicMock(return_value=job_config)
project_type.execute_command_in_project = MagicMock(return_value=('', 0))
project_type.teardown_build()
project_type.execute_command_in_project.assert_called_with('teardown')
def test_teardown_build_runs_teardown(self):
job_config = JobConfig("name", "setup", "teardown", "command", "atomizer", 10, 10)
project_type = ProjectType()
project_type.job_config = MagicMock(return_value=job_config)
project_type.execute_command_in_project = MagicMock(return_value=("", 0))
project_type.teardown_build()
project_type.execute_command_in_project.assert_called_with("teardown", timeout=None)
def test_failing_exit_code_is_injected_when_no_return_code_available(self):
self.mock_popen.returncode = None # This will happen if we were not able to terminate the process.
self._mock_stdout_and_stderr(b'', b'')
project_type = ProjectType()
actual_output, actual_return_code = project_type.execute_command_in_project('echo The power is yours!')
self.assertIsInstance(actual_return_code, int, 'Returned exit code should always be an int.')
self.assertNotEqual(actual_return_code, 0, 'Returned exit code should be failing (non-zero) when subprocess '
'returncode is not available.')
def test_execute_command_in_project_does_not_choke_on_weird_command_output(self):
some_weird_output = b'\xbf\xe2\x98\x82' # the byte \xbf is invalid unicode
self._mock_stdout_and_stderr(some_weird_output, b'')
self.mock_popen.returncode = 0
project_type = ProjectType()
actual_output, _ = project_type.execute_command_in_project('fake command')
# Part of this test is just proving that no exception is raised on invalid output.
self.assertIsInstance(actual_output, str, 'Invalid output from a process should still be converted to string.')
def test_execute_command_in_project_does_not_choke_on_weird_command_output(self):
some_weird_output = b"\xbf\xe2\x98\x82" # the byte \xbf is invalid unicode
self._mock_stdout_and_stderr(some_weird_output, b"")
self.mock_popen.returncode = 0
project_type = ProjectType()
actual_output, _ = project_type.execute_command_in_project("fake command")
# Part of this test is just proving that no exception is raised on invalid output.
self.assertIsInstance(actual_output, str, "Invalid output from a process should still be converted to string.")
def test_failing_exit_code_is_injected_when_no_return_code_available(self):
self.mock_popen.returncode = None # This will happen if we were not able to terminate the process.
self._mock_stdout_and_stderr(b'', b'')
project_type = ProjectType()
actual_output, actual_return_code = project_type.execute_command_in_project('echo The power is yours!')
self.assertIsInstance(actual_return_code, int, 'Returned exit code should always be an int.')
self.assertNotEqual(actual_return_code, 0, 'Returned exit code should be failing (non-zero) when subprocess '
'returncode is not available.')
def test_command_exiting_normally_will_break_out_of_command_execution_wait_loop(self):
# Simulate Popen.communicate() timing out twice before command completes and returns output.
timeout_exc = TimeoutExpired(None, 1)
self.mock_popen.communicate.side_effect = [timeout_exc, timeout_exc, (b'fake_output', b'fake_error')]
self.mock_popen.returncode = 0
self.mock_popen.pid = 55555
project_type = ProjectType()
actual_output, actual_return_code = project_type.execute_command_in_project('echo The power is yours!')
self.assertEqual(self.mock_killpg.call_count, 0, 'os.killpg should not be called when command exits normally.')
self.assertEqual(actual_output, 'fake_output\nfake_error', 'Output should contain stdout and stderr.')
def test_timing_out_will_break_out_of_command_execution_wait_loop_and_kill_subprocesses(self):
mock_time = self.patch('time.time')
mock_time.side_effect = [0.0, 100.0, 200.0, 300.0] # time increases by 100 seconds with each loop
self._mock_out_popen_communicate()
project_type = ProjectType()
actual_output, actual_return_code = project_type.execute_command_in_project(
command='sleep 99',
timeout=250,
)
self.assertEqual(self.mock_killpg.call_count, 1, 'os.killpg should be called when execution times out.')
self.assertEqual(actual_output, 'fake output\nfake error', 'Output should contain stdout and stderr.')
def test_job_config_uses_passed_in_config_instead_of_clusterrunner_yaml(self):
config_dict = {
'commands': ['shell command 1', 'shell command 2;'],
'atomizers': [{'TESTPATH': 'atomizer command'}],
'max_executors': 100,
'max_executors_per_slave': 2,
}
project_type = ProjectType(config=config_dict, job_name='some_job_name')
job_config = project_type.job_config()
self.assertEquals(job_config.name, 'some_job_name')
self.assertEquals(job_config.command, 'shell command 1 && shell command 2')
self.assertEquals(job_config.max_executors, 100)
self.assertEquals(job_config.max_executors_per_slave, 2)
def test_exception_raised_while_waiting_for_termination_adds_error_message_to_output(self):
mock_time = self.patch('time.time')
mock_time.side_effect = [0.0, 100.0, 200.0, 300.0] # time increases by 100 seconds with each loop
fake_failing_return_code = -15
self.mock_popen.pid = 55555
self._mock_stdout_and_stderr(b'', b'')
exception_message = 'Something terribly horrible just happened!'
self._simulate_hanging_popen_process(
fake_returncode=fake_failing_return_code, wait_exception=ValueError(exception_message))
project_type = ProjectType()
actual_output, actual_return_code = project_type.execute_command_in_project(
'echo The power is yours!', timeout=250)
self.assertIn(exception_message, actual_output, 'ClusterRunner exception message should be included in output.')
self.assertEqual(actual_return_code, fake_failing_return_code, 'Actual return code should match expected.')
def test_command_exiting_normally_will_break_out_of_command_execution_wait_loop(self):
timeout_exc = TimeoutExpired(cmd=None, timeout=1)
expected_return_code = 0
# Simulate Popen.wait() timing out twice before command completes and returns output.
self.mock_popen.wait.side_effect = [timeout_exc, timeout_exc, expected_return_code]
self.mock_popen.returncode = expected_return_code
self._mock_stdout_and_stderr(b'fake_output', b'fake_error')
project_type = ProjectType()
actual_output, actual_return_code = project_type.execute_command_in_project('echo The power is yours!')
self.assertEqual(self.mock_killpg.call_count, 0, 'os.killpg should not be called when command exits normally.')
self.assertEqual(actual_output, 'fake_output\nfake_error', 'Output should contain stdout and stderr.')
self.assertEqual(actual_return_code, expected_return_code, 'Actual return code should match expected.')
self.assertTrue(all([file.close.called for file in self.mock_temporary_files]),
'All created TemporaryFiles should be closed so that they are removed from the filesystem.')
def test_compute_subjobs_for_build_only_atomizes_if_override_not_specified(
self, atoms_override, atomizer_output, atomizer_called):
"""
:type atoms_override: list[str] | None
:type atomizer_output: list[Atom] | None
:type atomizer_called: bool
"""
self.patch('os.path.isfile').return_value = False
mock_project = Mock(spec_set=ProjectType())
mock_project.atoms_override = atoms_override
mock_project.timing_file_path.return_value = '/some/path/doesnt/matter'
mock_project.project_directory = '/some/project/directory'
mock_atomizer = Mock(spec_set=Atomizer)
mock_atomizer.atomize_in_project.return_value = atomizer_output
mock_job_config = Mock(spec=JobConfig)
mock_job_config.name = 'some_config'
mock_job_config.max_executors = 1
mock_job_config.atomizer = mock_atomizer
subjob_calculator = SubjobCalculator()
subjob_calculator.compute_subjobs_for_build(build_id=1,
job_config=mock_job_config,
project_type=mock_project)
self.assertEquals(mock_atomizer.atomize_in_project.called,
atomizer_called)
def test_exception_raised_while_waiting_for_termination_adds_error_message_to_output(self):
mock_time = self.patch('time.time')
mock_time.side_effect = [0.0, 100.0, 200.0, 300.0] # time increases by 100 seconds with each loop
fake_failing_return_code = -15
self.mock_popen.pid = 55555
self._mock_stdout_and_stderr(b'', b'')
exception_message = 'Something terribly horrible just happened!'
self._simulate_hanging_popen_process(
fake_returncode=fake_failing_return_code, wait_exception=ValueError(exception_message))
project_type = ProjectType()
actual_output, actual_return_code = project_type.execute_command_in_project(
'echo The power is yours!', timeout=250)
self.assertIn(exception_message, actual_output, 'ClusterRunner exception message should be included in output.')
self.assertEqual(actual_return_code, fake_failing_return_code, 'Actual return code should match expected.')
def test_exception_raised_while_waiting_causes_termination_and_adds_error_message_to_output(self):
exception_message = "Something terribly horrible just happened!"
value_err_exc = ValueError(exception_message)
timeout_exc = TimeoutExpired(cmd=None, timeout=1)
fake_failing_return_code = -15
# Simulate Popen.wait() timing out twice before raising a ValueError exception.
self.mock_popen.wait.side_effect = [timeout_exc, timeout_exc, value_err_exc, fake_failing_return_code]
self.mock_popen.returncode = fake_failing_return_code
self.mock_popen.pid = 55555
self._mock_stdout_and_stderr(b"", b"")
project_type = ProjectType()
actual_output, actual_return_code = project_type.execute_command_in_project("echo The power is yours!")
self.assertEqual(self.mock_kill.call_count, 1, "os.killpg should be called when wait() raises exception.")
self.assertIn(exception_message, actual_output, "ClusterRunner exception message should be included in output.")
self.assertEqual(actual_return_code, fake_failing_return_code, "Actual return code should match expected.")
def test_timing_out_will_break_out_of_command_execution_wait_loop_and_kill_subprocesses(self):
mock_time = self.patch('time.time')
mock_time.side_effect = [0.0, 100.0, 200.0, 300.0] # time increases by 100 seconds with each loop
expected_return_code = 1
self._simulate_hanging_popen_process(fake_returncode=expected_return_code)
self.mock_popen.pid = 55555
self._mock_stdout_and_stderr(b'fake output', b'fake error')
project_type = ProjectType()
actual_output, actual_return_code = project_type.execute_command_in_project(
command='sleep 99', timeout=250)
self.assertEqual(self.mock_killpg.call_count, 1, 'os.killpg should be called when execution times out.')
self.assertEqual(actual_output, 'fake output\nfake error', 'Output should contain stdout and stderr.')
self.assertEqual(actual_return_code, expected_return_code, 'Actual return code should match expected.')
self.assertTrue(all([file.close.called for file in self.mock_temporary_files]),
'All created TemporaryFiles should be closed so that they are removed from the filesystem.')
def test_teardown_build_executes_teardown_command(self, expected_timeout):
project_type = ProjectType()
mock_execute = MagicMock(return_value=('fake output', 0))
project_type.execute_command_in_project = mock_execute
project_type.job_config = MagicMock()
if expected_timeout:
project_type.teardown_build(timeout=expected_timeout)
else:
project_type.teardown_build()
mock_execute.assert_called_once_with(ANY, timeout=expected_timeout)
def test_calling_kill_subprocesses_will_break_out_of_command_execution_wait_loop(self):
self._mock_out_popen_communicate()
project_type = ProjectType()
command_thread = SafeThread(target=project_type.execute_command_in_project, args=('echo The power is yours!',))
# This calls execute_command_in_project() on one thread, and calls kill_subprocesses() on another. The
# kill_subprocesses() call should cause the first thread to exit.
command_thread.start()
project_type.kill_subprocesses()
# This *should* join immediately, but we specify a timeout just in case something goes wrong so that the test
# doesn't hang. A successful join implies success. We also use the UnhandledExceptionHandler so that exceptions
# propagate from the child thread to the test thread and fail the test.
with UnhandledExceptionHandler.singleton():
command_thread.join(timeout=10)
if command_thread.is_alive():
self.mock_killpg() # Calling killpg() causes the command thread to end.
self.fail('project_type.kill_subprocesses should cause the command execution wait loop to exit.')
self.mock_killpg.assert_called_once_with(55555, ANY) # Note: os.killpg does not accept keyword args.
def test_teardown_build_executes_teardown_command(self, expected_timeout):
project_type = ProjectType()
mock_execute = MagicMock(return_value=("fake output", 0))
project_type.execute_command_in_project = mock_execute
project_type.job_config = MagicMock()
if expected_timeout:
project_type.teardown_build(timeout=expected_timeout)
else:
project_type.teardown_build()
mock_execute.assert_called_once_with(ANY, timeout=expected_timeout)
def test_teardown_build_runs_teardown(self):
job_config = JobConfig('name', 'setup', 'teardown', 'command', 'atomizer', 10, 10)
project_type = ProjectType()
project_type.job_config = MagicMock(return_value=job_config)
project_type.execute_command_in_project = MagicMock(return_value=('', 0))
project_type.teardown_build()
project_type.execute_command_in_project.assert_called_with('teardown', timeout=None)
def _create_mock_project_type(self):
return MagicMock(spec_set=ProjectType())