def test_stop_during_init_sets_lifecycle_state_to_stopped(
self, manager, init_hang_script):
"""
Verify that procedure terminate changes to STOPPED
when terminate() is called
"""
helper = PubSubHelper()
init_running = multiprocessing.Barrier(2)
init_args = ProcedureInput(init_running)
pid = manager.create(init_hang_script, init_args=init_args)
init_running.wait(0.1)
wait_for_empty_message_queue(manager)
manager.stop(pid)
expected = [
ProcedureState.CREATING, # ScriptWorker initialising
ProcedureState.IDLE, # ScriptWorker ready
ProcedureState.LOADING, # load user module
ProcedureState.IDLE, # user module loaded
ProcedureState.RUNNING, # init running
ProcedureState.STOPPED, # init stopped
]
helper.wait_for_lifecycle(ProcedureState.STOPPED)
self.assert_states(helper, pid, expected)
def test_internal_messages_not_republished(self, mp, caplog):
"""
Verify that message event is not published if the event originates from
an internal source.
"""
helper = PubSubHelper()
work_q = MPQueue(ctx=mp)
# TEST is the default component name assigned in
# _proc_worker_wrapper_helper. This message should not be published to pypubsub
msg = EventMessage(
"TEST",
"PUBSUB",
dict(topic=topics.request.procedure.list,
kwargs={"request_id": "123"}),
)
work_q.put(msg)
_proc_worker_wrapper_helper(mp,
caplog,
ScriptWorker,
args=(work_q, ),
expect_shutdown_evt=True)
msgs_on_topic = helper.messages_on_topic(topics.request.procedure.list)
assert len(msgs_on_topic) == 0
def test_stop_during_main_sets_lifecycle_state_to_stopped(
self, manager, main_hang_script):
"""
Verify that procedure state changes to STOPPED
when terminate() is called
"""
helper = PubSubHelper()
main_running = multiprocessing.Barrier(2)
init_args = ProcedureInput(main_running)
pid = manager.create(main_hang_script, init_args=init_args)
helper.wait_for_lifecycle(ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput())
main_running.wait(0.5)
helper.wait_for_lifecycle(ProcedureState.RUNNING)
manager.stop(pid)
helper.wait_for_lifecycle(ProcedureState.STOPPED)
expected = [
ProcedureState.CREATING, # ScriptWorker initialising
ProcedureState.IDLE, # ScriptWorker ready
ProcedureState.LOADING, # load user module
ProcedureState.IDLE, # user module loaded
ProcedureState.RUNNING, # init running
ProcedureState.READY, # init complete
ProcedureState.RUNNING, # main running
ProcedureState.STOPPED, # main stopped
]
self.assert_states(helper, pid, expected)
def test_events_emitted_from_scripts_are_republished(
self, manager, pubsub_script):
"""
Verify that OET events are published at the appropriate times for a
sad-path script.
"""
helper = PubSubHelper()
pid = manager.create(pubsub_script, init_args=ProcedureInput())
wait_for_state(manager, pid, ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput(msg="foo"))
helper.wait_for_message_on_topic(topics.user.script.announce)
user_msgs = helper.messages_on_topic(topics.user.script.announce)
assert len(user_msgs) == 1
def test_external_messages_are_published_locally(self, mp_fixture, caplog):
"""
Verify that message event is published if the event originates from an
external source.
"""
pubsub.pub.unsubAll()
helper = PubSubHelper()
work_q = MPQueue(ctx=mp_fixture)
msg = EventMessage(
"EXTERNAL COMPONENT",
"PUBSUB",
dict(topic=topics.request.procedure.list,
kwargs={"request_id": "123"}),
)
work_q.put(msg)
with mock.patch.object(pubsub.pub, "unsubAll", return_value=[]):
_proc_worker_wrapper_helper(
mp_fixture,
caplog,
EventBusWorker,
args=(work_q, ),
expect_shutdown_evt=True,
)
assert topics.request.procedure.list in helper.topic_list
work_q.safe_close()
def assert_command_request_and_response(mp_fixture, caplog, mock_method,
request_topic, response_topic, cmd):
pubsub.pub.unsubAll()
helper = PubSubHelper()
work_q = MPQueue(ctx=mp_fixture)
msg = EventMessage(
"UNITTEST",
"PUBSUB",
dict(topic=request_topic, kwargs={
"request_id": "1234",
"cmd": cmd
}),
)
work_q.put(msg)
event = mp_fixture.Event()
mock_method.side_effect = partial(set_event, event)
with mock.patch.object(pubsub.pub, "unsubAll", return_value=[]):
_proc_worker_wrapper_helper(
mp_fixture,
caplog,
ScriptExecutionServiceWorker,
args=(work_q, mp_fixture),
expect_shutdown_evt=True,
)
assert event.is_set()
mock_method.assert_called_once()
assert mock_method.call_args[0][0] == cmd
assert helper.topic_list == [request_topic, response_topic]
work_q.safe_close()
def assert_states(helper: PubSubHelper, pid: int,
expected: List[ProcedureState]):
msgs = helper.messages_on_topic(topics.procedure.lifecycle.statechange)
states = [
msg["new_state"] for msg in msgs if int(msg["msg_src"]) == pid
]
assert states == expected
def test_happy_path_script_execution_lifecycle_states(
self, manager: ProcessManager, barrier_script):
"""
Verify that a new ScriptWorker sends the appropriate lifecycle states.
The mptools ProcWorker tests already verify that events are set at the
appropriate times. This test is to confirm that lifecycle EventMessages
are sent at the appropriate times too.
"""
helper = PubSubHelper()
init_running = multiprocessing.Barrier(2)
main_running = multiprocessing.Barrier(2)
resume = multiprocessing.Barrier(2)
init_args = ProcedureInput(init_running, main_running, resume)
pid = manager.create(barrier_script, init_args=init_args)
init_running.wait(0.1)
wait_for_state(manager, pid, ProcedureState.RUNNING)
expected = [
ProcedureState.CREATING, # ScriptWorker initialising
ProcedureState.IDLE, # ScriptWorker ready
ProcedureState.LOADING, # load user module
ProcedureState.IDLE, # user module loaded
ProcedureState.RUNNING, # init present and called
]
self.assert_states(helper, pid, expected)
# let init complete, then check for completion
resume.wait(0.1)
resume.reset() # reset to pause main method call
wait_for_state(manager, pid, ProcedureState.READY)
expected.append(ProcedureState.READY) # init complete
self.assert_states(helper, pid, expected)
# now set main running
manager.run(pid, call="main", run_args=ProcedureInput())
expected.append(ProcedureState.RUNNING) # main running
main_running.wait(0.1)
wait_for_state(manager, pid, ProcedureState.RUNNING)
self.assert_states(helper, pid, expected)
# wait for ScriptWorker process to complete
resume.wait(0.1)
resume.reset() # reset to pause main method call
wait_for_state(manager, pid, ProcedureState.COMPLETE)
expected.extend([
ProcedureState.READY, # main complete
ProcedureState.COMPLETE, # script complete
])
self.assert_states(helper, pid, expected)
def test_shared_environment_sys_path_is_set(self, mock_subprocess_fn,
mock_clone_fn,
git_sys_path_script, manager):
"""
Verify site packages are added to sys.path correctly for scripts sharing an environment.
"""
site_pkg = "/python/site_packages"
env = Environment(
"123",
multiprocessing.Event(),
multiprocessing.Event(),
"/",
site_pkg,
)
manager.em.create_env = MagicMock()
manager.em.create_env.side_effect = [env, env]
# Return path to git file from clone call in env creation and module load
mock_clone_fn.return_value = "/"
pid1 = manager.create(git_sys_path_script,
init_args=ProcedureInput(site_pkg))
pid2 = manager.create(git_sys_path_script,
init_args=ProcedureInput(site_pkg))
wait_for_state(manager, pid1, ProcedureState.READY)
wait_for_state(manager, pid2, ProcedureState.READY)
# Running the main function asserts the site_pkg is in sys.path
# If assertion fails, script state goes to FAILED, else it goes
# to COMPLETE
helper = PubSubHelper()
manager.run(pid1, call="main", run_args=ProcedureInput(site_pkg))
assert helper.wait_for_lifecycle(ProcedureState.COMPLETE, msg_src=pid1)
manager.run(pid2, call="main", run_args=ProcedureInput(site_pkg))
assert helper.wait_for_lifecycle(ProcedureState.COMPLETE, msg_src=pid2)
def test_error_in_main_lifecycles_states(self, manager: ProcessManager,
fail_script):
helper = PubSubHelper()
pid = manager.create(fail_script, init_args=ProcedureInput())
wait_for_state(manager, pid, ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput("foo"))
wait_for_state(manager, pid, ProcedureState.FAILED)
expected = [
ProcedureState.CREATING, # ScriptWorker initialising
ProcedureState.IDLE, # ScriptWorker ready
ProcedureState.LOADING, # load user module
ProcedureState.IDLE, # user module loaded
# fail script has no init so no IDLE->READY expected
ProcedureState.READY, # init complete
ProcedureState.RUNNING, # main running
ProcedureState.FAILED, # exception raised
]
helper.wait_for_lifecycle(ProcedureState.FAILED)
# wait_for_state(manager, pid, ProcedureState.FAILED)
# helper.wait_for_message_on_topic(topics.procedure.lifecycle.stacktrace)
self.assert_states(helper, pid, expected)
def test_internal_messages_not_republished(self, mp_fixture, caplog):
"""
Verify that message event is not published if the event originates from
an internal source.
"""
pubsub.pub.unsubAll()
helper = PubSubHelper()
work_q = MPQueue(ctx=mp_fixture)
# TEST is the default component name assigned in
# _proc_worker_wrapper_helper. This message should be ignored.
msg = EventMessage(
"TEST",
"PUBSUB",
dict(topic=topics.request.procedure.list,
kwargs={"request_id": "123"}),
)
work_q.put(msg)
# But coming from NONTEST, this message should be republished.
msg = EventMessage(
"NONTEST",
"PUBSUB",
dict(topic=topics.request.procedure.list,
kwargs={"request_id": "456"}),
)
work_q.put(msg)
with mock.patch.object(pubsub.pub, "unsubAll", return_value=[]):
_proc_worker_wrapper_helper(
mp_fixture,
caplog,
EventBusWorker,
args=(work_q, ),
expect_shutdown_evt=True,
)
assert len(helper.messages) == 1
assert helper.messages[0][1] == dict(msg_src="NONTEST",
request_id="456")
work_q.safe_close()
def test_handles_request_to_list_invalid_id(self, mp_fixture, caplog):
"""
The ValueError raised when SES.summarise is given an invalid PID should be handled.
"""
pubsub.pub.unsubAll()
helper = PubSubHelper()
work_q = MPQueue(ctx=mp_fixture)
msg = EventMessage(
"TEST_SUMMARY",
"PUBSUB",
dict(topic=topics.request.procedure.list,
kwargs={"request_id": "123"}),
)
work_q.put(msg)
with mock.patch(
"ska_oso_oet.procedure.application.main.ScriptExecutionService.summarise"
) as mock_cls:
with mock.patch.object(pubsub.pub, "unsubAll", return_value=[]):
mock_cls.side_effect = ValueError
_proc_worker_wrapper_helper(
mp_fixture,
caplog,
ScriptExecutionServiceWorker,
args=(work_q, mp_fixture),
expect_shutdown_evt=True,
)
mock_cls.assert_called_once()
assert helper.topic_list == [
topics.request.procedure.list, # list requested
topics.procedure.pool.list, # response published
]
assert helper.messages[1][1] == dict(msg_src="TEST",
request_id="123",
result=[])
work_q.safe_close()
def test_list_method_called(self, mp_fixture, caplog):
"""
SES.summarise should be called when 'request.procedure.list' message is received
"""
pubsub.pub.unsubAll()
helper = PubSubHelper()
work_q = MPQueue(ctx=mp_fixture)
msg = EventMessage(
"TEST_SUMMARY",
"PUBSUB",
dict(topic=topics.request.procedure.list,
kwargs={"request_id": "123"}),
)
work_q.put(msg)
event = mp_fixture.Event()
with mock.patch(
"ska_oso_oet.procedure.application.main.ScriptExecutionService.summarise"
) as mock_cls:
with mock.patch.object(pubsub.pub, "unsubAll", return_value=[]):
mock_cls.side_effect = partial(set_event, event)
_proc_worker_wrapper_helper(
mp_fixture,
caplog,
ScriptExecutionServiceWorker,
args=(work_q, mp_fixture),
expect_shutdown_evt=True,
)
assert event.is_set() is True
mock_cls.assert_called_once()
assert helper.topic_list == [
topics.request.procedure.list, # list requested
topics.procedure.pool.list, # response published
]
work_q.safe_close()
def test_stop_terminates_the_process(self, manager, abort_script):
"""
Verify that ProcessManager stops a script execution
"""
helper = PubSubHelper()
with Manager() as mgr:
q = mgr.Queue()
is_running = multiprocessing.Barrier(2)
pid = manager.create(abort_script,
init_args=ProcedureInput(q, is_running))
wait_for_state(manager, pid, ProcedureState.READY)
manager.run(pid, call="main", run_args=ProcedureInput())
is_running.wait(0.1)
helper.wait_for_lifecycle(ProcedureState.RUNNING)
manager.stop(pid)
helper.wait_for_lifecycle(ProcedureState.STOPPED)
assert manager.running is None
assert q.empty()
def test_stop_events(self, ses, init_hang_script):
"""
Verify the behaviour of SES.stop(), confirming that OET events are published
at the appropriate times when a script is terminated.
"""
helper = PubSubHelper()
init_running = multiprocessing.Barrier(2)
prepare_cmd = PrepareProcessCommand(
script=init_hang_script, init_args=ProcedureInput(init_running))
summary = ses.prepare(prepare_cmd)
pid = summary.id
ses._wait_for_state(pid, ProcedureState.IDLE)
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.created)) == 1
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.started)) == 1
assert len(
helper.messages_on_topic(topics.procedure.lifecycle.complete)) == 0
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.failed)) == 0
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.stopped)) == 0
init_running.wait(0.1)
ses._wait_for_state(pid, ProcedureState.READY)
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.created)) == 1
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.started)) == 1
assert len(
helper.messages_on_topic(topics.procedure.lifecycle.complete)) == 0
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.failed)) == 0
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.stopped)) == 0
stop_cmd = StopProcessCommand(process_uid=pid, run_abort=False)
_ = ses.stop(stop_cmd)
ses._wait_for_state(pid, ProcedureState.STOPPED)
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.created)) == 1
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.started)) == 1
assert len(
helper.messages_on_topic(topics.procedure.lifecycle.complete)) == 0
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.failed)) == 0
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.stopped)) == 1
filtered = [
t for t in helper.topic_list
if t != topics.procedure.lifecycle.statechange
]
assert filtered == [
topics.procedure.lifecycle.created,
topics.procedure.lifecycle.started,
topics.procedure.lifecycle.stopped,
]
def test_sad_path_events(self, ses, fail_script):
"""
Verify that OET events are published at the appropriate times for a
script that raises an exception.
"""
helper = PubSubHelper()
prepare_cmd = PrepareProcessCommand(script=fail_script,
init_args=ProcedureInput())
summary = ses.prepare(prepare_cmd)
helper.wait_for_message_on_topic(topics.procedure.lifecycle.created,
timeout=3.0)
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.created)) == 1
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.started)) == 0
assert len(
helper.messages_on_topic(topics.procedure.lifecycle.complete)) == 0
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.failed)) == 0
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.stopped)) == 0
run_cmd = StartProcessCommand(process_uid=summary.id,
fn_name="main",
run_args=ProcedureInput(msg="foo"))
_ = ses.start(run_cmd)
helper.wait_for_message_on_topic(topics.procedure.lifecycle.stacktrace,
timeout=3.0)
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.created)) == 1
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.started)) == 1
assert len(
helper.messages_on_topic(topics.procedure.lifecycle.complete)) == 0
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.failed)) == 1
assert len(helper.messages_on_topic(
topics.procedure.lifecycle.stopped)) == 0
filtered = [
t for t in helper.topic_list
if t != topics.procedure.lifecycle.statechange
]
assert filtered == [
topics.procedure.lifecycle.created,
topics.procedure.lifecycle.started,
topics.procedure.lifecycle.failed,
topics.procedure.lifecycle.stacktrace,
]
