def test_ses_calls_process_manager_as_expected(self,
ses: ScriptExecutionService,
main_hang_script):
"""
Verify ScriptExecutionService interactions with ProcessManager for
standard script execution.
"""
mgr = mock.Mock(wraps=ses._process_manager)
ses._process_manager = mgr
main_running = multiprocessing.Barrier(2)
prepare_cmd = PrepareProcessCommand(
script=main_hang_script, init_args=ProcedureInput(main_running))
summary = ses.prepare(prepare_cmd)
pid = summary.id
mgr.create.assert_called_once_with(prepare_cmd.script,
init_args=prepare_cmd.init_args)
ses._wait_for_state(pid, ProcedureState.READY)
run_cmd = StartProcessCommand(process_uid=pid,
fn_name="main",
run_args=ProcedureInput())
_ = ses.start(run_cmd)
mgr.run.assert_called_once_with(run_cmd.process_uid,
call=run_cmd.fn_name,
run_args=run_cmd.run_args)
ses._wait_for_state(pid, ProcedureState.RUNNING)
main_running.wait(0.1)
stop_cmd = StopProcessCommand(process_uid=pid, run_abort=False)
_ = ses.stop(stop_cmd)
mgr.stop.assert_called_once_with(stop_cmd.process_uid)
# assert no other calls to ProcessManager
assert len(mgr.method_calls) == 3
def test_running_to_none_when_process_completes(self, manager,
barrier_script):
"""
Verify that ProcessManager sets running procedure attribute to None
when process completes.
"""
init_running = multiprocessing.Barrier(2)
main_running = multiprocessing.Barrier(2)
resume = multiprocessing.Barrier(2)
init_args = ProcedureInput(init_running, main_running, resume)
pid = manager.create(barrier_script, init_args=init_args)
init_running.wait(0.1)
resume.wait(0.1)
resume.reset() # reset to pause main method call
wait_for_state(manager, pid, ProcedureState.READY)
# now set main running
manager.run(pid, call="main", run_args=ProcedureInput())
main_running.wait(0.1)
wait_for_state(manager, pid, ProcedureState.RUNNING)
assert manager.running is not None
resume.wait(0.1)
wait_for_state(manager, pid, ProcedureState.COMPLETE)
assert manager.running is None
def test_run_args_are_captured_in_history(self, ses):
"""
Verify that arguments to start() are captured and stored
"""
timestamp = 12345
pid = 456
fn_name = "foo"
run_args = ProcedureInput(5, 6, 7, kw3="c", kw4="d")
# must create process for history entries to be present
script = FileSystemScript("file://test.py")
cmd = PrepareProcessCommand(script=script, init_args=ProcedureInput())
with patch.object(ProcessManager, "create", return_value=pid):
with patch.object(ScriptExecutionService, "_summarise"):
_ = ses.prepare(cmd)
# now we can test when method invocation args are recorded
cmd = StartProcessCommand(pid, fn_name=fn_name, run_args=run_args)
expected = ArgCapture(fn=fn_name, fn_args=run_args, time=timestamp)
with patch("time.time", MagicMock(return_value=timestamp)):
with patch.object(ScriptExecutionService, "_summarise"):
with patch.object(ProcessManager, "run"):
_ = ses.start(cmd)
assert len(ses.script_args[pid]) == 2
assert ses.script_args[pid][1] == expected
def test_stop_during_main_sets_lifecycle_state_to_stopped(
self, manager, main_hang_script):
"""
Verify that procedure state changes to STOPPED
when terminate() is called
"""
helper = PubSubHelper()
main_running = multiprocessing.Barrier(2)
init_args = ProcedureInput(main_running)
pid = manager.create(main_hang_script, init_args=init_args)
helper.wait_for_lifecycle(ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput())
main_running.wait(0.5)
helper.wait_for_lifecycle(ProcedureState.RUNNING)
manager.stop(pid)
helper.wait_for_lifecycle(ProcedureState.STOPPED)
expected = [
ProcedureState.CREATING, # ScriptWorker initialising
ProcedureState.IDLE, # ScriptWorker ready
ProcedureState.LOADING, # load user module
ProcedureState.IDLE, # user module loaded
ProcedureState.RUNNING, # init running
ProcedureState.READY, # init complete
ProcedureState.RUNNING, # main running
ProcedureState.STOPPED, # main stopped
]
self.assert_states(helper, pid, expected)
def test_shared_environment_waits_for_creation_to_complete(
self, mock_subprocess_fn, mock_clone_fn, git_script,
git_script_branch, manager):
"""
Verify calls to subprocess to install environment are only run if the environment does not yet exist.
The first two scripts run in the same environment and the third one creates a new environment.
"""
environment = Environment(
"123",
multiprocessing.Event(),
multiprocessing.Event(),
"/",
"/python/site_packages",
)
environment2 = Environment(
"456",
multiprocessing.Event(),
multiprocessing.Event(),
"/",
"/python/site_packages",
)
manager.em.create_env = MagicMock()
manager.em.create_env.side_effect = [
environment, environment, environment2
]
# Return path to git file from clone call in env creation and module load
mock_clone_fn.return_value = "/"
calls = multiprocessing.Value("i", 0)
def called(*args, **kwargs):
calls.value += 1
mock_subprocess_fn.side_effect = called
pid1 = manager.create(git_script, init_args=ProcedureInput())
env1 = manager.environments[pid1]
pid2 = manager.create(git_script, init_args=ProcedureInput())
env2 = manager.environments[pid2]
assert env1 == env2
wait_for_state(manager, pid1, ProcedureState.READY)
wait_for_state(manager, pid2, ProcedureState.READY)
# Subprocess should only be called twice because second script should
# just wait for first one to create the environment
assert calls.value == 2
pid3 = manager.create(git_script_branch, init_args=ProcedureInput())
env3 = manager.environments[pid3]
wait_for_state(manager, pid3, ProcedureState.READY)
assert env1 != env3
assert env3.created.is_set()
# Call count should go up because the new script should run in a new environment
assert calls.value == 4
def test_procedure_input_equality(self):
"""
Verify ProcedureInput equality
"""
pi1 = ProcedureInput(1, 2, 3, a=1, b=2)
pi2 = ProcedureInput(1, 2, 3, a=1, b=2)
pi3 = ProcedureInput(4, a=1)
assert pi1 == pi2
assert pi1 != pi3
assert pi1 != object()
def test_stop_fails_on_process_that_is_not_running(self, manager, script):
"""
Verify that an exception is raised when requesting stop() for a procedure
that is not running
"""
pid = manager.create(script, init_args=ProcedureInput())
wait_for_state(manager, pid, ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput())
wait_for_state(manager, pid, ProcedureState.COMPLETE)
with pytest.raises(ValueError):
manager.stop(pid)
def test_cleanup_on_failed(self, manager, fail_script):
pid = manager.create(fail_script, init_args=ProcedureInput())
wait_for_state(manager, pid, ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput("foo"))
wait_for_state(manager, pid, ProcedureState.FAILED)
# TODO how can we synchronise with the cleanup function running in another thread?
time.sleep(0.1)
assert pid not in manager.states
assert pid not in manager.script_queues
assert pid not in manager.procedures
def test_sad_path_events(self, ses, fail_script):
"""
Verify that OET events are published at the appropriate times for a
script that raises an exception.
"""
helper = PubSubHelper()
prepare_cmd = PrepareProcessCommand(script=fail_script,
init_args=ProcedureInput())
summary = ses.prepare(prepare_cmd)
helper.wait_for_message_on_topic(topics.procedure.lifecycle.created,
timeout=3.0)
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.created)) == 1
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.started)) == 0
assert len(
helper.messages_on_topic(topics.procedure.lifecycle.complete)) == 0
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.failed)) == 0
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.stopped)) == 0
run_cmd = StartProcessCommand(process_uid=summary.id,
fn_name="main",
run_args=ProcedureInput(msg="foo"))
_ = ses.start(run_cmd)
helper.wait_for_message_on_topic(topics.procedure.lifecycle.stacktrace,
timeout=3.0)
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.created)) == 1
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.started)) == 1
assert len(
helper.messages_on_topic(topics.procedure.lifecycle.complete)) == 0
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.failed)) == 1
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.stopped)) == 0
filtered = [
t for t in helper.topic_list
if t != topics.procedure.lifecycle.statechange
]
assert filtered == [
topics.procedure.lifecycle.created,
topics.procedure.lifecycle.started,
topics.procedure.lifecycle.failed,
topics.procedure.lifecycle.stacktrace,
]
def test_happy_path_script_execution_lifecycle_states(
self, manager: ProcessManager, barrier_script):
"""
Verify that a new ScriptWorker sends the appropriate lifecycle states.
The mptools ProcWorker tests already verify that events are set at the
appropriate times. This test is to confirm that lifecycle EventMessages
are sent at the appropriate times too.
"""
helper = PubSubHelper()
init_running = multiprocessing.Barrier(2)
main_running = multiprocessing.Barrier(2)
resume = multiprocessing.Barrier(2)
init_args = ProcedureInput(init_running, main_running, resume)
pid = manager.create(barrier_script, init_args=init_args)
init_running.wait(0.1)
wait_for_state(manager, pid, ProcedureState.RUNNING)
expected = [
ProcedureState.CREATING, # ScriptWorker initialising
ProcedureState.IDLE, # ScriptWorker ready
ProcedureState.LOADING, # load user module
ProcedureState.IDLE, # user module loaded
ProcedureState.RUNNING, # init present and called
]
self.assert_states(helper, pid, expected)
# let init complete, then check for completion
resume.wait(0.1)
resume.reset() # reset to pause main method call
wait_for_state(manager, pid, ProcedureState.READY)
expected.append(ProcedureState.READY) # init complete
self.assert_states(helper, pid, expected)
# now set main running
manager.run(pid, call="main", run_args=ProcedureInput())
expected.append(ProcedureState.RUNNING) # main running
main_running.wait(0.1)
wait_for_state(manager, pid, ProcedureState.RUNNING)
self.assert_states(helper, pid, expected)
# wait for ScriptWorker process to complete
resume.wait(0.1)
resume.reset() # reset to pause main method call
wait_for_state(manager, pid, ProcedureState.COMPLETE)
expected.extend([
ProcedureState.READY, # main complete
ProcedureState.COMPLETE, # script complete
])
self.assert_states(helper, pid, expected)
def test_scalar_summarise(self, ses, script): # pylint: disable=protected-access
"""
Verify that the private _summarise method compiles the execution
history correctly.
Note that this test exercises the SES and ProcessManager, hence this
also functions as an integration test between the two.
"""
t = 12345
init_args = ProcedureInput(1, 2, a="b", c="d")
run_args = ProcedureInput(3, 4, e="f", g="h")
history = ProcedureHistory(
[
(ProcedureState.CREATING, t),
(ProcedureState.IDLE, t),
(ProcedureState.LOADING, t),
(ProcedureState.IDLE, t),
(ProcedureState.READY, t),
(ProcedureState.RUNNING, t),
(ProcedureState.READY, t),
(ProcedureState.COMPLETE, t),
],
stacktrace=None,
)
prepare_cmd = PrepareProcessCommand(script, init_args=init_args)
with patch("time.time", MagicMock(return_value=t)):
summary = ses.prepare(prepare_cmd)
pid = summary.id
ses._wait_for_state(pid, ProcedureState.READY)
run_cmd = StartProcessCommand(process_uid=pid,
fn_name="main",
run_args=run_args)
ses.start(run_cmd)
ses._wait_for_state(pid, ProcedureState.COMPLETE)
expected = ProcedureSummary(
id=pid,
script=script,
script_args=[
ArgCapture(fn="init", fn_args=init_args, time=t),
ArgCapture(fn="main", fn_args=run_args, time=t),
],
history=history,
state=ProcedureState.COMPLETE,
)
summary = ses._summarise(pid)
assert summary == expected
def test_run_fails_for_loading_process(self, manager, script):
"""
Verify that an exception is raised if requesting run() for a procedure
that has not been loaded or is still loading
"""
with patch(
"ska_oso_oet.procedure.domain.ModuleFactory.get_module") as fn:
fn.side_effect = lambda _: time.sleep(3)
pid = manager.create(script, init_args=ProcedureInput())
wait_for_state(manager, pid, ProcedureState.LOADING)
assert manager.states[pid] == ProcedureState.LOADING
with pytest.raises(ValueError):
manager.run(pid, call="main", run_args=ProcedureInput())
def test_events_emitted_from_scripts_are_republished(
self, manager, pubsub_script):
"""
Verify that OET events are published at the appropriate times for a
sad-path script.
"""
helper = PubSubHelper()
pid = manager.create(pubsub_script, init_args=ProcedureInput())
wait_for_state(manager, pid, ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput(msg="foo"))
helper.wait_for_message_on_topic(topics.user.script.announce)
user_msgs = helper.messages_on_topic(topics.user.script.announce)
assert len(user_msgs) == 1
def test_stop_during_init_sets_lifecycle_state_to_stopped(
self, manager, init_hang_script):
"""
Verify that procedure terminate changes to STOPPED
when terminate() is called
"""
helper = PubSubHelper()
init_running = multiprocessing.Barrier(2)
init_args = ProcedureInput(init_running)
pid = manager.create(init_hang_script, init_args=init_args)
init_running.wait(0.1)
wait_for_empty_message_queue(manager)
manager.stop(pid)
expected = [
ProcedureState.CREATING, # ScriptWorker initialising
ProcedureState.IDLE, # ScriptWorker ready
ProcedureState.LOADING, # load user module
ProcedureState.IDLE, # user module loaded
ProcedureState.RUNNING, # init running
ProcedureState.STOPPED, # init stopped
]
helper.wait_for_lifecycle(ProcedureState.STOPPED)
self.assert_states(helper, pid, expected)
def test_environment_created_condition_is_set(self, mock_subprocess_fn,
mock_clone_fn, git_script,
manager):
"""
Verify event is correctly set on Environment object when env is being created.
"""
environment = Environment(
"123",
multiprocessing.Event(),
multiprocessing.Event(),
"/",
"/python/site_packages",
)
manager.em.create_env = MagicMock()
manager.em.create_env.return_value = environment
# Return path to git file from clone call in env creation and module load
mock_clone_fn.side_effect = ["", ""]
pid = manager.create(git_script, init_args=ProcedureInput())
env = manager.environments[pid]
assert not env.created.is_set()
assert env.env_id == environment.env_id
wait_for_state(manager, pid, ProcedureState.READY)
assert env.created.is_set()
def test_scan_id_persists_between_executions(
manager,
script_that_increments_and_returns_scan_id,
):
"""
The scan ID should be shared and persisted between process executions.
"""
queue = multiprocessing.Queue()
init_args = ProcedureInput(queue)
def run_script():
pid = manager.create(
script=script_that_increments_and_returns_scan_id,
init_args=init_args,
)
wait_for_state(manager, pid, ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput())
wait_for_state(manager, pid, ProcedureState.COMPLETE)
run_script()
scan_id = queue.get(timeout=1)
run_script()
next_scan_id = queue.get(timeout=1)
assert next_scan_id == scan_id + 1
def test_run_fails_on_invalid_pid(self, manager):
"""
Verify that an exception is raised when run() is requested for an invalid
PID
"""
with pytest.raises(ValueError):
manager.run(321, call="foo", run_args=ProcedureInput())
def test_ses_get_subarray_id_for_requested_pid(self, ses):
"""
Verify that the private method _get_subarray_id returns
subarray id correctly
"""
subarray_id = 123
process_pid = 456
init_args = ArgCapture(fn="init",
fn_args=ProcedureInput(subarray_id=subarray_id),
time=1)
process_summary = ProcedureSummary(
id=process_pid,
script=FileSystemScript("file://a"),
script_args=[init_args],
history=mock.MagicMock(),
state=ProcedureState.IDLE,
)
expected = [process_summary]
with patch.object(ScriptExecutionService,
"_summarise",
return_value=process_summary) as method:
returned = ses._get_subarray_id(process_pid)
assert method.called_with(process_pid)
assert returned == expected[0].script_args[0].fn_args.kwargs[
"subarray_id"]
def test_exceeding_history_limit_removes_oldest_deletable_state(
self, ses, script):
"""
Verify that SES removes the oldest deletable state and history when
the record limit is reached.
"""
prepare_cmd = PrepareProcessCommand(script=script,
init_args=ProcedureInput())
# reduce max history to make test quicker
limit = 3
with patch(
"ska_oso_oet.procedure.application.application.HISTORY_MAX_LENGTH",
new=limit,
):
for _ in range(limit):
summary = ses.prepare(prepare_cmd)
pid = summary.id
ses._wait_for_state(pid, ProcedureState.READY)
run_cmd = StartProcessCommand(process_uid=pid,
fn_name="main",
run_args=ProcedureInput())
summary = ses.start(run_cmd)
pid = summary.id
ses._wait_for_state(pid, ProcedureState.COMPLETE)
assert len(ses.history) == limit
assert len(ses.script_args) == limit
assert len(ses.states) == limit
assert len(ses.scripts) == limit
oldest_pid = next(iter(ses.states.keys()))
assert oldest_pid in ses.history
assert oldest_pid in ses.script_args
assert oldest_pid in ses.states
assert oldest_pid in ses.scripts
_ = ses.prepare(prepare_cmd)
# adding procedure should not increase the number of procedures
# and should remove the oldest procedure
assert len(ses.history) == limit
assert oldest_pid not in ses.history
assert oldest_pid not in ses.script_args
assert oldest_pid not in ses.states
assert oldest_pid not in ses.scripts
def test_procedure_input_accepts_expected_constructor_values(self):
"""
Verify that ProcedureInput arguments are slurped into positional and
keyword/value attributes.
"""
procedure_input = ProcedureInput(1, 2, 3, a=1, b=2)
assert procedure_input.args == (1, 2, 3)
assert procedure_input.kwargs == dict(a=1, b=2)
def run_script():
pid = manager.create(
script=script_that_increments_and_returns_scan_id,
init_args=init_args,
)
wait_for_state(manager, pid, ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput())
wait_for_state(manager, pid, ProcedureState.COMPLETE)
def test_two_phase_abort_calls_process_manager_as_expected(
self, sleep_script, script):
"""
Verify ScriptExecutionService interactions with ProcessManager when
terminating a script with a request to run the follow-on abort script
"""
pubsub.pub.unsubAll()
# fixture accepts any positional args and kwargs, which is useful as the
# subarray ID will be passed to the abort script
ses = ScriptExecutionService(abort_script=script)
subarray_id = 3
try:
prepare_cmd = PrepareProcessCommand(
script=sleep_script,
init_args=ProcedureInput(subarray_id=subarray_id))
summary = ses.prepare(prepare_cmd)
pid = summary.id
ses._wait_for_state(pid, ProcedureState.READY)
run_cmd = StartProcessCommand(
process_uid=pid,
fn_name="main",
run_args=ProcedureInput(
2), # max 2 seconds before process completes naturally
)
_ = ses.start(run_cmd)
ses._wait_for_state(pid, ProcedureState.RUNNING)
# wrap the manager here so that we only capture calls related to stop
mgr = mock.Mock(wraps=ses._process_manager)
ses._process_manager = mgr
stop_cmd = StopProcessCommand(process_uid=pid, run_abort=True)
_ = ses.stop(stop_cmd)
assert len(mgr.method_calls) == 3
mgr.stop.assert_called_once_with(1)
mgr.create.assert_called_once_with(
script, init_args=ProcedureInput(subarray_id=subarray_id))
mgr.run.assert_called_once_with(pid + 1,
call="main",
run_args=ProcedureInput())
finally:
ses.shutdown()
def test_create_adds_to_procedures(self, manager, script):
"""
Verify that ProcessManager.procedures references the processes it creates
"""
for _ in range(3):
len_before = len(manager.procedures)
pid = manager.create(script, init_args=ProcedureInput())
assert len(manager.procedures) == len_before + 1
assert pid in manager.procedures
def test_stop_terminates_the_process(self, manager, abort_script):
"""
Verify that ProcessManager stops a script execution
"""
helper = PubSubHelper()
with Manager() as mgr:
q = mgr.Queue()
is_running = multiprocessing.Barrier(2)
pid = manager.create(abort_script,
init_args=ProcedureInput(q, is_running))
wait_for_state(manager, pid, ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput())
is_running.wait(0.1)
helper.wait_for_lifecycle(ProcedureState.RUNNING)
manager.stop(pid)
helper.wait_for_lifecycle(ProcedureState.STOPPED)
assert manager.running is None
assert q.empty()
def test_failure_and_stacktrace_recorded_in_history(
self, ses: ScriptExecutionService, fail_script):
t = 12345
random_exc_string = str(uuid.uuid4())
init_args = ProcedureInput()
run_args = ProcedureInput(random_exc_string)
expected_states = [
(ProcedureState.CREATING, t), # ScriptWorker initialising
(ProcedureState.IDLE, t), # ScriptWorker ready
(ProcedureState.LOADING, t), # load user module
(ProcedureState.IDLE, t), # user module loaded
# fail script has no init so no IDLE->READY expected
(ProcedureState.READY, t), # init complete
(ProcedureState.RUNNING, t), # main running
(ProcedureState.FAILED, t), # exception raised
]
with patch("time.time", MagicMock(return_value=t)):
prepare_cmd = PrepareProcessCommand(script=fail_script,
init_args=init_args)
summary = ses.prepare(prepare_cmd)
pid = summary.id
ses._wait_for_state(pid, ProcedureState.READY)
assert summary.history.stacktrace is None
cmd = StartProcessCommand(process_uid=pid,
fn_name="main",
run_args=run_args)
_ = ses.start(cmd)
ses._wait_for_state(pid, ProcedureState.FAILED)
time.sleep(1.0)
summary = ses._summarise(pid)
assert summary.history.process_states == expected_states
# most recent stacktrace should also have been captured and recorded in history
assert random_exc_string in summary.history.stacktrace
def test_run_fails_for_running_process(self, manager, barrier_script):
"""
Verify that an exception is raised when requesting run() for a procedure
that is already running
"""
init_running = multiprocessing.Barrier(2)
main_running = multiprocessing.Barrier(2)
resume = multiprocessing.Barrier(2)
init_args = ProcedureInput(init_running, main_running, resume)
pid = manager.create(barrier_script, init_args=init_args)
init_running.wait(0.1)
resume.wait()
resume.reset()
wait_for_state(manager, pid, ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput())
main_running.wait(0.1)
wait_for_state(manager, pid, ProcedureState.RUNNING)
with pytest.raises(ValueError):
manager.run(pid, call="main", run_args=ProcedureInput())
resume.wait()
def test_error_in_main_lifecycles_states(self, manager: ProcessManager,
fail_script):
helper = PubSubHelper()
pid = manager.create(fail_script, init_args=ProcedureInput())
wait_for_state(manager, pid, ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput("foo"))
wait_for_state(manager, pid, ProcedureState.FAILED)
expected = [
ProcedureState.CREATING, # ScriptWorker initialising
ProcedureState.IDLE, # ScriptWorker ready
ProcedureState.LOADING, # load user module
ProcedureState.IDLE, # user module loaded
# fail script has no init so no IDLE->READY expected
ProcedureState.READY, # init complete
ProcedureState.RUNNING, # main running
ProcedureState.FAILED, # exception raised
]
helper.wait_for_lifecycle(ProcedureState.FAILED)
# wait_for_state(manager, pid, ProcedureState.FAILED)
# helper.wait_for_message_on_topic(topics.procedure.lifecycle.stacktrace)
self.assert_states(helper, pid, expected)
def test_shared_environment_sys_path_is_set(self, mock_subprocess_fn,
mock_clone_fn,
git_sys_path_script, manager):
"""
Verify site packages are added to sys.path correctly for scripts sharing an environment.
"""
site_pkg = "/python/site_packages"
env = Environment(
"123",
multiprocessing.Event(),
multiprocessing.Event(),
"/",
site_pkg,
)
manager.em.create_env = MagicMock()
manager.em.create_env.side_effect = [env, env]
# Return path to git file from clone call in env creation and module load
mock_clone_fn.return_value = "/"
pid1 = manager.create(git_sys_path_script,
init_args=ProcedureInput(site_pkg))
pid2 = manager.create(git_sys_path_script,
init_args=ProcedureInput(site_pkg))
wait_for_state(manager, pid1, ProcedureState.READY)
wait_for_state(manager, pid2, ProcedureState.READY)
# Running the main function asserts the site_pkg is in sys.path
# If assertion fails, script state goes to FAILED, else it goes
# to COMPLETE
helper = PubSubHelper()
manager.run(pid1, call="main", run_args=ProcedureInput(site_pkg))
assert helper.wait_for_lifecycle(ProcedureState.COMPLETE, msg_src=pid1)
manager.run(pid2, call="main", run_args=ProcedureInput(site_pkg))
assert helper.wait_for_lifecycle(ProcedureState.COMPLETE, msg_src=pid2)
def test_cleanup_on_stopped(self, manager, init_hang_script):
init_running = multiprocessing.Barrier(2)
init_args = ProcedureInput(init_running)
pid = manager.create(init_hang_script, init_args=init_args)
init_running.wait(0.1)
wait_for_state(manager, pid, ProcedureState.RUNNING)
manager.stop(pid)
wait_for_state(manager, pid, ProcedureState.STOPPED)
# TODO how can we synchronise with the cleanup function running in another thread?
time.sleep(0.1)
assert pid not in manager.states
assert pid not in manager.script_queues
assert pid not in manager.procedures
def test_run_sends_run_message(self, manager):
"""
Verify that a call to ProcessManager.run() sends the run message to the
ScriptWorker.
"""
q = manager.ctx.MPQueue()
manager.procedures[1] = MagicMock()
manager.states[1] = ProcedureState.READY
manager.script_queues[1] = q
method = "foo"
run_args = ProcedureInput("a", "b", kw1="c", kw2="d")
manager.run(1, call=method, run_args=run_args)
msg = q.safe_get(timeout=0.1)
assert msg.msg_type == "RUN"
assert msg.msg == (method, run_args)
