def test_execute_with_bad_param_key():
"`param_key` values must be strings"
def fn():
return
cases = [None, [], {}, (), 1, lambda x: x]
for bad_param_key in cases:
with pytest.raises(ValueError):
execute.execute(fn, param_key=bad_param_key, param_values=["foo"])
def test_execute_with_missing():
"both `param_key` and `param_values` should be provided or neither"
def fn():
return
with pytest.raises(ValueError):
execute.execute(fn, param_key="good_key", param_values=None)
with pytest.raises(ValueError):
execute.execute(fn, param_values=["good", "values"])
def test_parallel_with_prompts__raise_errors():
"prompts issued while executing a worker function in parallel return the PromptedException"
@execute.parallel
def workerfn():
return operations.prompt("gimmie")
with pytest.raises(PromptedException) as e:
execute.execute(workerfn)
expected = "prompted with: gimmie"
assert expected == str(e)
def test_parallel_with_prompts_custom__raise_errors():
"prompts issued while executing a worker function in parallel with `abort_exception` return a custom exception"
@execute.parallel
def workerfn():
return operations.prompt("gimmie")
with settings(abort_exception=ValueError):
with pytest.raises(ValueError) as e:
execute.execute(workerfn)
expected = "prompted with: gimmie"
assert expected == str(e)
def test_execute_with_prompts_override__raise_errors():
"""prompts issued while executing a worker function in parallel with `abort_on_prompts` set to `False` will get the unsupported `EOFError` raised.
When `raise_unhandled_errors` is set to `True` (default) the `EOFError` will be re-thrown."""
@execute.parallel
def workerfn():
with settings(abort_on_prompts=False):
return operations.prompt("gimmie")
with pytest.raises(EOFError) as e:
execute.execute(workerfn)
expected = "EOF when reading a line"
assert expected == str(e)
def test_execute_with_bad_param_values():
"`param_values` must be a list, tuple or set of values"
def fn():
return
cases = [None, 1, "", {}, lambda x: x]
for bad_param_values in cases:
with pytest.raises(ValueError):
execute.execute(fn,
param_key="mykey",
param_values=bad_param_values)
def test_parallel_worker_exceptions__raise_errors():
"exceptions in worker functions are raised when encountered in the results"
exc_msg = "omg. dead"
@execute.parallel
def workerfn():
raise EnvironmentError(exc_msg)
with pytest.raises(EnvironmentError) as e:
execute.execute(workerfn)
expected = exc_msg
assert expected == str(e)
def test_execute_workerfn_exception():
"exceptions thrown by worker functions while being executed serially are left uncaught"
exc_msg = "omg. dead"
def workerfn():
raise EnvironmentError(exc_msg)
# what should the behaviour here be? consistent with `parallel`?
# in that case, the exception should be returned as a result.
# I think builder expects exceptions to be thrown rather than returned however.
with pytest.raises(EnvironmentError) as exc:
execute.execute(workerfn)
assert isinstance(exc, EnvironmentError)
assert str(exc) == exc_msg
def test_remote_exceptions_in_parallel__raise_errors():
"""Remote commands that raise exceptions while executing in parallel are re-raised when encountered in the results."""
def workerfn():
with state.settings():
return remote("exit 1")
workerfn = execute.parallel(workerfn, pool_size=1)
with test_settings():
expected = RuntimeError(
"remote() encountered an error (return code 1) while executing '/bin/bash -l -c \"exit 1\"'"
)
with pytest.raises(RuntimeError) as e:
execute.execute(workerfn)
assert str(expected) == str(e)
def test_execute_many_parallel_with_params():
"`parallel` will wrap a given function and run it `pool_size` times in parallel, but is ignored when `execute` is given a list of values"
def fn():
with settings() as local_env:
return local_env
parallel_fn = execute.parallel(fn, pool_size=1)
param_key = "mykey"
param_values = [1, 2, 3]
expected = [
{
"parallel": True,
"abort_on_prompts": True,
"parent": "environment",
"mykey": 1,
},
{
"parallel": True,
"abort_on_prompts": True,
"parent": "environment",
"mykey": 2,
},
{
"parallel": True,
"abort_on_prompts": True,
"parent": "environment",
"mykey": 3,
},
]
with settings(parent="environment"):
assert expected == execute.execute(parallel_fn, param_key,
param_values)
def test_execute_process_not_terminating(caplog):
"""we've had a case where a parallel process doesn't terminate despite the
worker function having finished and returned a result.
This test simulates that scenario (because I can't replicate it under test conditions)
by patching `process_status` to always return `alive=True`"""
# keep a reference before mock.patch overrides it
orig_fn = execute.process_status
def patch_fn(running_p):
result = orig_fn(running_p)
# why on earth are you *still* alive??
result["alive"] = True
return result
@execute.parallel
def worker_fn():
return "foo"
expected = "foo"
with patch("threadbare.execute.process_status", new=patch_fn):
results = execute.execute(worker_fn)
assert [expected] == results
# ensure a warning was logged
_, log_level, log_msg = caplog.record_tuples[0]
assert log_level == logging.WARNING
expected_warning_text = "process is still alive despite worker having completed. terminating process: process--1"
assert expected_warning_text == log_msg
def test_execute_serial():
"`execute` will call a regular function and return a list of results"
def fn():
return "hello, world"
expected = ["hello, world"]
assert expected == execute.execute(fn)
def test_parallel_with_prompts__swallow_errors():
"prompts issued while executing a worker function in parallel return the PromptedException"
@execute.parallel
def workerfn():
return operations.prompt("gimmie")
results = execute.execute(workerfn, raise_unhandled_errors=False)
expected = str(PromptedException("prompted with: gimmie"))
assert expected == str(results[0])
def test_parallel_with_prompts_custom__swallow_errors():
"prompts issued while executing a worker function in parallel with `abort_exception` set returns the custom exception"
@execute.parallel
def workerfn():
return operations.prompt("gimmie")
with settings(abort_exception=ValueError):
results = execute.execute(workerfn, raise_unhandled_errors=False)
expected = "prompted with: gimmie"
assert expected == str(results[0])
def test_execute_with_prompts_override__swallow_errors():
"""prompts issued while executing a worker function in parallel with `abort_on_prompts` set to `False` will get the unsupported `EOFError` raised.
When `raise_unhandled_errors` is set to `False` the `EOFError` is available in the results."""
@execute.parallel
def workerfn():
with settings(abort_on_prompts=False):
return operations.prompt("gimmie")
results = execute.execute(workerfn, raise_unhandled_errors=False)
expected = "EOF when reading a line"
assert expected == str(results[0])
def test_parallel_worker_exceptions__swallow_errors():
"exceptions in worker functions are returned as results when `raise_unhandled_errors` is `False`"
exc_msg = "omg. dead"
@execute.parallel
def workerfn():
raise EnvironmentError(exc_msg)
results = execute.execute(workerfn, raise_unhandled_errors=False)
unhandled_exception = results[0]
assert isinstance(unhandled_exception, EnvironmentError)
assert str(unhandled_exception) == exc_msg
def test_execute_many_serial_with_params():
"`serial` will wrap a given function and run it `pool_size` times, but is ignored when `execute` is given a list of values"
def fn():
with settings() as local_env:
return "foo" + local_env["mykey"]
wrapped_fn = execute.serial(fn, pool_size=1)
param_key = "mykey"
param_values = ["bar", "baz", "bop"]
expected = ["foobar", "foobaz", "foobop"]
assert expected == execute.execute(wrapped_fn, param_key, param_values)
def test_remote_exceptions_in_parallel__swallow_errors():
"""Remote commands that raise exceptions while executing in parallel return the exception object when `raise_unhandled_errors` is `False`."""
def workerfn():
with state.settings():
return remote("exit 1")
workerfn = execute.parallel(workerfn, pool_size=1)
with test_settings():
expected = RuntimeError(
"remote() encountered an error (return code 1) while executing '/bin/bash -l -c \"exit 1\"'"
)
result_list = execute.execute(workerfn, raise_unhandled_errors=False)
result = result_list[0]
assert str(expected) == str(result)
def test_run_many_local_commands_serially():
"run a list of commands serially. `serial` exists to complement `parallel`"
command_list = [
"echo all",
"echo these commands",
"echo are executed",
"echo in serially",
]
def myfn():
return local(state.ENV["cmd"], capture=True)
results = execute.execute(myfn, param_key="cmd", param_values=command_list)
assert len(results) == len(command_list)
assert results[-2]["stdout"] == ["are executed"]
def test_run_many_local_commands_in_parallel():
"run a set of commands in parallel using Python's multiprocessing"
command_list = [
"echo all",
"echo these commands",
"echo are executed",
"echo in parallel",
]
@execute.parallel
def myfn():
return local(state.ENV["cmd"], capture=True)
results = execute.execute(myfn, param_key="cmd", param_values=command_list)
assert len(results) == len(command_list)
assert results[-2]["stdout"] == ["are executed"]
def test_line_formatting():
# todo: not a great test. how do I capture and test the formatted line while preserving the original output?
num_workers = 2
@execute.parallel
def workerfn():
iterations = 2
cmd = 'for run in {1..%s}; do echo "I am %s, iteration $run"; done' % (
iterations,
state.ENV["worker_num"],
)
return remote(cmd)
expected = [
{
"command":
'/bin/bash -l -c "for run in {1..2}; do echo \\"I am 1, iteration \\$run\\"; done"',
"failed": False,
"return_code": 0,
"stderr": [],
"stdout": [
"I am 1, iteration 1",
"I am 1, iteration 2",
],
"succeeded": True,
},
{
"command":
'/bin/bash -l -c "for run in {1..2}; do echo \\"I am 2, iteration \\$run\\"; done"',
"failed": False,
"return_code": 0,
"stderr": [],
"stdout": ["I am 2, iteration 1", "I am 2, iteration 2"],
"succeeded": True,
},
]
with test_settings(line_template="[{host}] {pipe}: {line}\n"):
results = execute.execute(
workerfn,
param_key="worker_num",
param_values=list(range(1, num_workers + 1)),
)
assert expected == results
def test_run_many_remote_commands_serially():
"""run a list of `remote` commands serially. The `execute` module is aimed at
running commands in parallel.
Serial execution exists only as a sensible default and offers nothing extra."""
command_list = [
"echo all",
"echo these commands",
"echo are executed",
"echo serially and remotely",
]
def myfn():
return remote(state.ENV["cmd"], capture=True)
with test_settings():
results = execute.execute(myfn,
param_key="cmd",
param_values=command_list)
assert len(results) == len(command_list)
assert results[-2]["stdout"] == ["are executed"]
def test_run_many_remote_commands_in_parallel():
"""run a list of `remote` commands in parallel.
`remote` commands run in parallel do not share a ssh connection.
the order of results can be guaranteed but not the order in which output is emitted"""
command_list = [
"echo all",
"echo these commands",
"echo are executed",
"echo remotely and in parallel",
]
@execute.parallel
def myfn():
return remote(state.ENV["cmd"], capture=True)
with test_settings(quiet=True):
results = execute.execute(myfn,
param_key="cmd",
param_values=command_list)
assert len(results) == len(command_list)
assert results[-2]["stdout"] == ["are executed"]
def test_check_many_remote_files():
"checks multiple remote files for existence in parallel"
@execute.parallel
def workerfn():
with state.settings() as env:
return remote_file_exists(env["remote_file"], use_sudo=True)
with remote_fixture() as remote_env:
remote_file_list = [
remote_env["temp-files"]["small-file"], # True, exists
join(remote_env["temp-dir"],
"doesnot.exist"), # False, doesn't exist
]
expected = [True, False]
with test_settings():
result = execute.execute(workerfn,
param_key="remote_file",
param_values=remote_file_list)
assert expected == result
def test_execute_many_parallel():
"`parallel` will wrap a given function and run it `pool_size` times in parallel. complements `serial`"
pool_size = 3
parallel_fn = execute.parallel(lambda: "foo", pool_size)
expected = ["foo", "foo", "foo"]
assert expected == execute.execute(parallel_fn)
def test_execute_many_serial():
"`serial` will wrap a given function and run it `pool_size` times, one after the other. complements `parallel`"
pool_size = 3
wrapped_fn = execute.serial(lambda: "foo", pool_size)
expected = ["foo", "foo", "foo"]
assert expected == execute.execute(wrapped_fn)
