def test_012_check_stdin_fail_close(
self,
popen, # type: mock.MagicMock
_, # type: mock.MagicMock
select, # type: mock.MagicMock
logger # type: mock.MagicMock
): # type: (...) -> None
stdin = b'this is a line'
popen_obj, exp_result = self.prepare_close(popen,
cmd=print_stdin,
stdout_override=[stdin])
pipe_error = OSError()
stdin_mock = mock.Mock()
stdin_mock.attach_mock(mock.Mock(side_effect=pipe_error), 'close')
popen_obj.attach_mock(stdin_mock, 'stdin')
select.return_value = [popen_obj.stdout, popen_obj.stderr], [], []
runner = exec_helpers.Subprocess()
with self.assertRaises(OSError):
# noinspection PyTypeChecker
runner.execute_async(print_stdin, stdin=stdin)
popen_obj.kill.assert_called_once()
def test_001_check_call(self, execute, logger):
exit_code = 0
return_value = exec_helpers.ExecResult(
cmd=command,
stdout=stdout_list,
stderr=stdout_list,
exit_code=exit_code,
)
execute.return_value = return_value
verbose = False
runner = exec_helpers.Subprocess()
# noinspection PyTypeChecker
result = runner.check_call(
command=command, verbose=verbose, timeout=None)
execute.assert_called_once_with(command, verbose, None)
self.assertEqual(result, return_value)
exit_code = 1
return_value = exec_helpers.ExecResult(
cmd=command,
stdout=stdout_list,
stderr=stdout_list,
exit_code=exit_code,
)
execute.reset_mock()
execute.return_value = return_value
with self.assertRaises(exec_helpers.CalledProcessError):
# noinspection PyTypeChecker
runner.check_call(command=command, verbose=verbose, timeout=None)
execute.assert_called_once_with(command, verbose, None)
def test_004_execute_timeout_fail(
self,
sleep,
popen, _, select, logger
):
popen_obj, exp_result = self.prepare_close(popen)
popen_obj.configure_mock(returncode=None)
select.return_value = [popen_obj.stdout, popen_obj.stderr], [], []
runner = exec_helpers.Subprocess()
# noinspection PyTypeChecker
with self.assertRaises(exec_helpers.ExecHelperTimeoutError) as cm:
# noinspection PyTypeChecker
runner.execute(command, timeout=0.2)
self.assertEqual(cm.exception.timeout, 0.2)
self.assertEqual(cm.exception.cmd, command)
self.assertEqual(cm.exception.stdout, exp_result.stdout_str)
self.assertEqual(cm.exception.stderr, exp_result.stderr_str)
popen.assert_has_calls((
mock.call(
args=[command],
cwd=None,
env=None,
shell=True,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
universal_newlines=False,
),
))
def test_003_context_manager(
self,
popen, # type: mock.MagicMock
_, # type: mock.MagicMock
select, # type: mock.MagicMock
logger # type: mock.MagicMock
): # type: (...) -> None
popen_obj, exp_result = self.prepare_close(popen)
select.return_value = [popen_obj.stdout, popen_obj.stderr], [], []
subprocess_runner.SingletonMeta._instances.clear()
with mock.patch('threading.RLock', autospec=True):
with exec_helpers.Subprocess() as runner:
runner.lock.acquire.assert_called()
self.assertEqual(
mock.call.acquire(), runner.lock.mock_calls[0]
)
result = runner.execute(command)
self.assertEqual(
result, exp_result
)
runner.lock.release.assert_called()
subprocess_runner.SingletonMeta._instances.clear()
def test_007_execute_no_stdout_stderr(self, popen, _, select, logger):
popen_obj, exp_result = self.prepare_close(popen,
open_stdout=False,
open_stderr=False)
select.return_value = [popen_obj.stdout, popen_obj.stderr], [], []
runner = exec_helpers.Subprocess()
# noinspection PyTypeChecker
result = runner.execute(command, open_stdout=False, open_stderr=False)
self.assertEqual(result, exp_result)
popen.assert_has_calls((mock.call(
args=[command],
cwd=None,
env=None,
shell=True,
stderr=subprocess_runner.devnull,
stdin=subprocess.PIPE,
stdout=subprocess_runner.devnull,
universal_newlines=False,
), ))
logger.assert_has_calls([
mock.call.log(level=logging.DEBUG, msg=command_log),
] + [
mock.call.log(level=logging.DEBUG,
msg=self.gen_cmd_result_log_message(result)),
])
self.assertIn(mock.call.poll(), popen_obj.mock_calls)
def test_006_check_stdin_bytearray(
self,
popen, # type: mock.MagicMock
_, # type: mock.MagicMock
select, # type: mock.MagicMock
logger # type: mock.MagicMock
): # type: (...) -> None
stdin = bytearray(b'this is a line')
popen_obj, exp_result = self.prepare_close(
popen, cmd=print_stdin, stdout_override=[bytes(l) for l in stdin])
stdin_mock = mock.Mock()
popen_obj.attach_mock(stdin_mock, 'stdin')
select.return_value = [popen_obj.stdout, popen_obj.stderr], [], []
runner = exec_helpers.Subprocess()
# noinspection PyTypeChecker
result = runner.execute(print_stdin, stdin=stdin)
self.assertEqual(result, exp_result)
popen.assert_has_calls((mock.call(
args=[print_stdin],
cwd=None,
env=None,
shell=True,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
universal_newlines=False,
), ))
stdin_mock.assert_has_calls(
[mock.call.write(stdin), mock.call.close()])
def test_007_check_stderr(execute, exec_result, subprocess_logger) -> None:
"""Test STDERR content validator."""
runner = exec_helpers.Subprocess()
if not exec_result.stderr:
assert runner.check_stderr(command,
stdin=exec_result.stdin,
expected=[exec_result.exit_code
]) == exec_result
else:
with pytest.raises(exec_helpers.CalledProcessError) as e:
runner.check_stderr(command,
stdin=exec_result.stdin,
expected=[exec_result.exit_code])
exc: exec_helpers.CalledProcessError = e.value
assert exc.result == exec_result
assert exc.cmd == exec_result.cmd
assert exc.returncode == exec_result.exit_code
assert exc.stdout == exec_result.stdout_str
assert exc.stderr == exec_result.stderr_str
assert exc.result == exec_result
assert subprocess_logger.mock_calls[-1] == mock.call.error(
msg=
f"Command {exc.result.cmd!r} output contains STDERR while not expected\n"
f"\texit code: {exc.result.exit_code!s}")
def test_006_check_call_expect(execute, exec_result,
subprocess_logger) -> None:
"""Test exit code validator with custom return codes."""
runner = exec_helpers.Subprocess()
assert runner.check_call(command,
stdin=exec_result.stdin,
expected=[exec_result.exit_code]) == exec_result
def test_013_execute_timeout_done(
self,
sleep,
popen, _, select, logger
):
popen_obj, exp_result = self.prepare_close(popen, ec=exec_helpers.ExitCodes.EX_INVALID)
popen_obj.configure_mock(returncode=None)
popen_obj.attach_mock(mock.Mock(side_effect=OSError), 'kill')
select.return_value = [popen_obj.stdout, popen_obj.stderr], [], []
runner = exec_helpers.Subprocess()
# noinspection PyTypeChecker
res = runner.execute(command, timeout=0.2)
self.assertEqual(res, exp_result)
popen.assert_has_calls((
mock.call(
args=[command],
cwd=None,
env=None,
shell=True,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
universal_newlines=False,
),
))
def test_002_call_verbose(
self,
popen, # type: mock.MagicMock
_, # type: mock.MagicMock
select, # type: mock.MagicMock
logger # type: mock.MagicMock
): # type: (...) -> None
popen_obj, _ = self.prepare_close(popen)
select.return_value = [popen_obj.stdout, popen_obj.stderr], [], []
runner = exec_helpers.Subprocess()
# noinspection PyTypeChecker
result = runner.execute(command, verbose=True)
logger.assert_has_calls(
[
mock.call.log(level=logging.INFO, msg=command_log),
] + [
mock.call.log(
level=logging.INFO,
msg=str(x.rstrip().decode('utf-8')))
for x in stdout_list
] + [
mock.call.log(
level=logging.INFO,
msg=str(x.rstrip().decode('utf-8')))
for x in stderr_list
] + [
mock.call.log(
level=logging.INFO,
msg=self.gen_cmd_result_log_message(result)),
])
def test_008_check_stderr_no_raise(execute, exec_result,
subprocess_logger) -> None:
"""Test STDERR content validator in permissive mode."""
runner = exec_helpers.Subprocess()
assert (runner.check_stderr(command,
stdin=exec_result.stdin,
expected=[exec_result.exit_code],
raise_on_err=False) == exec_result)
def test_008_execute_mask_global(self, popen, _, select, logger):
cmd = "USE='secret=secret_pass' do task"
log_mask_re = r"secret\s*=\s*([A-Z-a-z0-9_\-]+)"
masked_cmd = "USE='secret=<*masked*>' do task"
cmd_log = u"Executing command:\n{!r}\n".format(masked_cmd)
popen_obj, exp_result = self.prepare_close(
popen,
cmd=cmd,
cmd_in_result=masked_cmd
)
select.return_value = [popen_obj.stdout, popen_obj.stderr], [], []
runner = exec_helpers.Subprocess(
log_mask_re=log_mask_re
)
# noinspection PyTypeChecker
result = runner.execute(cmd)
self.assertEqual(result, exp_result)
popen.assert_has_calls((
mock.call(
args=[cmd],
cwd=None,
env=None,
shell=True,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
universal_newlines=False,
),
))
logger.assert_has_calls(
[
mock.call.log(level=logging.DEBUG, msg=cmd_log),
] + [
mock.call.log(
level=logging.DEBUG,
msg=str(x.rstrip().decode('utf-8'))
)
for x in stdout_list
] + [
mock.call.log(
level=logging.DEBUG,
msg=str(x.rstrip().decode('utf-8')))
for x in stderr_list
] + [
mock.call.log(
level=logging.DEBUG,
msg=self.gen_cmd_result_log_message(result)),
])
self.assertIn(
mock.call.poll(), popen_obj.mock_calls
)
def test_001_execute_async(popen, subprocess_logger, run_parameters) -> None:
"""Test low level API."""
runner = exec_helpers.Subprocess()
res = runner._execute_async(
command,
stdin=run_parameters["stdin"],
open_stdout=run_parameters["open_stdout"],
open_stderr=run_parameters["open_stderr"],
)
assert isinstance(res, SubprocessExecuteAsyncResult)
assert res.interface.wait() == run_parameters["ec"]
assert res.interface.returncode == run_parameters["ec"]
stdout = run_parameters["stdout"]
stderr = run_parameters["stderr"]
if stdout is not None:
assert read_stream(res.stdout) == stdout
else:
assert res.stdout is stdout
if stderr is not None:
assert read_stream(res.stderr) == stderr
else:
assert res.stderr is stderr
if run_parameters["stdin"] is None:
stdin = None
elif isinstance(run_parameters["stdin"], bytes):
stdin = run_parameters["stdin"]
elif isinstance(run_parameters["stdin"], str):
stdin = run_parameters["stdin"].encode(encoding="utf-8")
else:
stdin = bytes(run_parameters["stdin"])
if stdin:
assert res.stdin is None
popen.assert_called_once_with(
args=[command],
stdout=subprocess.PIPE
if run_parameters["open_stdout"] else subprocess.DEVNULL,
stderr=subprocess.PIPE
if run_parameters["open_stderr"] else subprocess.DEVNULL,
stdin=subprocess.PIPE,
shell=True,
cwd=run_parameters.get("cwd", None),
env=run_parameters.get("env", None),
universal_newlines=False,
**_subprocess_helpers.subprocess_kw,
)
if stdin is not None:
res.interface.stdin.write.assert_called_once_with(stdin)
res.interface.stdin.close.assert_called_once()
def test_003_context_manager(mocker, popen, subprocess_logger, exec_result,
run_parameters) -> None:
"""Test context manager for threads synchronization."""
lock_mock = mocker.patch("threading.RLock")
with exec_helpers.Subprocess() as runner:
res = runner.execute(command, stdin=run_parameters["stdin"])
lock_mock.acquire_assert_called_once()
lock_mock.release_assert_called_once()
assert isinstance(res, exec_helpers.ExecResult)
assert res == exec_result
def test_009_call(popen, subprocess_logger, exec_result,
run_parameters) -> None:
"""Test callable."""
runner = exec_helpers.Subprocess()
res = runner(
command,
stdin=run_parameters["stdin"],
open_stdout=run_parameters["open_stdout"],
open_stderr=run_parameters["open_stderr"],
)
assert isinstance(res, exec_helpers.ExecResult)
assert res == exec_result
popen().wait.assert_called_once_with(timeout=default_timeout)
def test_001_call(
self,
popen, # type: mock.MagicMock
_, # type: mock.MagicMock
select, # type: mock.MagicMock
logger # type: mock.MagicMock
): # type: (...) -> None
popen_obj, exp_result = self.prepare_close(popen)
select.return_value = [popen_obj.stdout, popen_obj.stderr], [], []
runner = exec_helpers.Subprocess()
# noinspection PyTypeChecker
result = runner.execute(command)
self.assertEqual(result, exp_result)
popen.assert_has_calls((
mock.call(
args=[command],
cwd=None,
env=None,
shell=True,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
universal_newlines=False,
),
))
logger.assert_has_calls(
[
mock.call.log(level=logging.DEBUG, msg=command_log),
] + [
mock.call.log(
level=logging.DEBUG,
msg=str(x.rstrip().decode('utf-8'))
)
for x in stdout_list
] + [
mock.call.log(
level=logging.DEBUG,
msg=str(x.rstrip().decode('utf-8')))
for x in stderr_list
] + [
mock.call.log(
level=logging.DEBUG,
msg=self.gen_cmd_result_log_message(result)),
])
self.assertIn(
mock.call.poll(), popen_obj.mock_calls
)
def test_005_check_call_no_raise(execute, exec_result,
subprocess_logger) -> None:
"""Test exit code validator in permissive mode."""
runner = exec_helpers.Subprocess()
res = runner.check_call(command,
stdin=exec_result.stdin,
raise_on_err=False)
assert res == exec_result
if exec_result.exit_code != exec_helpers.ExitCodes.EX_OK:
expected = (proc_enums.EXPECTED, )
assert subprocess_logger.mock_calls[-1] == mock.call.error(
msg=
f"Command {res.cmd!r} returned exit code {res.exit_code!s} while expected {expected!r}"
)
def test_002_execute(popen, subprocess_logger, exec_result,
run_parameters) -> None:
"""Test API without checkers."""
runner = exec_helpers.Subprocess()
res = runner.execute(
command,
stdin=run_parameters["stdin"],
open_stdout=run_parameters["open_stdout"],
open_stderr=run_parameters["open_stderr"],
)
assert isinstance(res, exec_helpers.ExecResult)
assert res == exec_result
popen().wait.assert_called_once_with(timeout=default_timeout)
assert subprocess_logger.mock_calls[0] == mock.call.log(
level=logging.DEBUG, msg=command_log)
def test_003_check_stderr(self, check_call, _, logger):
return_value = exec_helpers.ExecResult(
cmd=command,
stdout=stdout_list,
exit_code=0,
)
check_call.return_value = return_value
verbose = False
raise_on_err = True
runner = exec_helpers.Subprocess()
# noinspection PyTypeChecker
result = runner.check_stderr(command=command,
verbose=verbose,
timeout=None,
raise_on_err=raise_on_err)
check_call.assert_called_once_with(command,
verbose,
timeout=None,
error_info=None,
raise_on_err=raise_on_err)
self.assertEqual(result, return_value)
return_value = exec_helpers.ExecResult(
cmd=command,
stdout=stdout_list,
stderr=stdout_list,
exit_code=0,
)
check_call.reset_mock()
check_call.return_value = return_value
with self.assertRaises(exec_helpers.CalledProcessError):
# noinspection PyTypeChecker
runner.check_stderr(command=command,
verbose=verbose,
timeout=None,
raise_on_err=raise_on_err)
check_call.assert_called_once_with(command,
verbose,
timeout=None,
error_info=None,
raise_on_err=raise_on_err)
def test_010_check_stdin_fail_close_pipe(
self,
popen, # type: mock.MagicMock
_, # type: mock.MagicMock
select, # type: mock.MagicMock
logger # type: mock.MagicMock
): # type: (...) -> None
stdin = b'this is a line'
popen_obj, exp_result = self.prepare_close(popen, cmd=print_stdin, stdout_override=[stdin])
pipe_err = BrokenPipeError()
pipe_err.errno = errno.EPIPE
stdin_mock = mock.Mock()
stdin_mock.attach_mock(mock.Mock(side_effect=pipe_err), 'close')
popen_obj.attach_mock(stdin_mock, 'stdin')
select.return_value = [popen_obj.stdout, popen_obj.stderr], [], []
runner = exec_helpers.Subprocess()
# noinspection PyTypeChecker
result = runner.execute(print_stdin, stdin=stdin)
self.assertEqual(result, exp_result)
popen.assert_has_calls((
mock.call(
args=[print_stdin],
cwd=None,
env=None,
shell=True,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
universal_newlines=False,
),
))
stdin_mock.assert_has_calls([
mock.call.write(stdin),
mock.call.close()
])
logger.warning.assert_not_called()
def test_004_check_call(execute, exec_result, subprocess_logger) -> None:
"""Test exit code validator."""
runner = exec_helpers.Subprocess()
if exec_result.exit_code == exec_helpers.ExitCodes.EX_OK:
assert runner.check_call(command,
stdin=exec_result.stdin) == exec_result
else:
with pytest.raises(exec_helpers.CalledProcessError) as e:
runner.check_call(command, stdin=exec_result.stdin)
exc: exec_helpers.CalledProcessError = e.value
assert exc.cmd == exec_result.cmd
assert exc.returncode == exec_result.exit_code
assert exc.stdout == exec_result.stdout_str
assert exc.stderr == exec_result.stderr_str
assert exc.result == exec_result
assert exc.expected == (proc_enums.EXPECTED, )
assert subprocess_logger.mock_calls[-1] == mock.call.error(
msg=
f"Command {exc.result.cmd!r} returned exit code {exc.result.exit_code!s} "
f"while expected {exc.expected!r}")
def test_special_cases(create_subprocess_shell, exec_result, subprocess_logger,
run_parameters) -> None:
"""Parametrized validation of special cases."""
command_parameters: CommandParameters = run_parameters[
"command_parameters"]
runner = exec_helpers.Subprocess(
log_mask_re=run_parameters.get("init_log_mask_re", None))
if "expect_exc" not in run_parameters:
res = runner.execute(**command_parameters.as_dict())
level = logging.INFO if command_parameters.verbose else logging.DEBUG
command_for_log = run_parameters.get("masked_cmd", command)
command_log = f"Executing command:\n{command_for_log.rstrip()!r}\n"
result_log = "Command {command!r} exit code: {result.exit_code!s}".format(
command=command_for_log.rstrip(), result=res)
assert subprocess_logger.mock_calls[0] == mock.call.log(
level=level, msg=command_log)
assert subprocess_logger.mock_calls[-1] == mock.call.log(
level=level, msg=result_log)
assert res == exec_result
else:
with pytest.raises(run_parameters["expect_exc"]):
runner.execute(**command_parameters.as_dict())
