def test_uninitialized_system_boot_with_state(self):
self.store.set_system_boot(True)
self.core.node_state("node1", domain_id_from_engine("engine1"),
InstanceState.RUNNING)
resource_id = "eeagent_1"
self.core.ee_heartbeat(resource_id, make_beat("node1"))
p0 = ProcessRecord.new(None, "proc0", {}, ProcessState.RUNNING,
configuration=nosystemrestart_process_config(),
assigned=resource_id,
restart_mode=RestartMode.ALWAYS)
self.store.add_process(p0)
p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.RUNNING,
assigned=resource_id)
self.store.add_process(p1)
p2 = ProcessRecord.new(None, "proc2", {}, ProcessState.PENDING,
assigned=resource_id)
self.store.add_process(p2)
p3 = ProcessRecord.new(None, "proc3", {}, ProcessState.TERMINATING,
assigned=resource_id)
self.store.add_process(p3)
# this one shouldn't restart
p4 = ProcessRecord.new(None, "proc4", {}, ProcessState.RUNNING,
configuration=nosystemrestart_process_config(),
assigned=resource_id,
restart_mode=RestartMode.ABNORMAL)
self.store.add_process(p4)
# non-running proceses should also potentially be restarted on boot
p5 = ProcessRecord.new(None, "proc5", {}, ProcessState.WAITING)
self.store.add_process(p5)
self.store.enqueue_process(*p5.key)
p6 = ProcessRecord.new(None, "proc6", {}, ProcessState.REQUESTED)
self.store.add_process(p6)
# not this one, due to RestartMode
p7 = ProcessRecord.new(None, "proc7", {}, ProcessState.REQUESTED,
configuration=nosystemrestart_process_config(),
restart_mode=RestartMode.ALWAYS)
self.store.add_process(p7)
self.store.enqueue_process(*p7.key)
resource = self.store.get_resource(resource_id)
resource.assigned = [p0.key, p1.key, p2.key, p3.key, p4.key]
self.store.update_resource(resource)
restartable_procs = ["proc1", "proc2", "proc5", "proc6"]
dead_procs = ["proc0", "proc4", "proc7"]
self._run_in_thread()
assert self.store.wait_initialized(timeout=10)
self.assertEqual(len(self.store.get_queued_processes()), 0)
self.assertEqual(len(self.store.get_node_ids()), 0)
self.assertEqual(len(self.store.get_resource_ids()), 0)
for proc in restartable_procs:
self.assertEqual(self.store.get_process(None, proc).state,
ProcessState.UNSCHEDULED_PENDING)
for proc in dead_procs:
self.assertEqual(self.store.get_process(None, proc).state,
ProcessState.TERMINATED)
self.assertEqual(self.store.get_process(None, "proc3").state,
ProcessState.TERMINATED)
self.assertEqual(self.store.get_pd_state(),
ProcessDispatcherState.SYSTEM_BOOTING)
# now end system boot
self.store.set_system_boot(False)
wait(lambda: self.store.get_pd_state() == ProcessDispatcherState.OK)
# check that pending processes were correctly rescheduled
self.assertEqual(len(self.store.get_queued_processes()), len(restartable_procs))
for proc in restartable_procs:
self.assertEqual(self.store.get_process(None, proc).state,
ProcessState.REQUESTED)
def test_process_should_restart(self):
definition = "def1"
self.core.create_definition(definition, None, None)
abnormal_states = (ProcessState.TERMINATED, ProcessState.TERMINATING, ProcessState.FAILED)
all_states = (ProcessState.TERMINATED, ProcessState.TERMINATING, ProcessState.FAILED, ProcessState.EXITED)
# default behavior is to restart processes that exit abnormally
process = self.core.schedule_process(None, uuid.uuid4().hex, definition)
for state in abnormal_states:
self.assertTrue(self.core.process_should_restart(process, state))
# system restart mode doesn't matter
self.assertTrue(self.core.process_should_restart(process, state, is_system_restart=True))
self.assertFalse(self.core.process_should_restart(process, ProcessState.EXITED))
self.assertFalse(self.core.process_should_restart(process, ProcessState.EXITED, is_system_restart=True))
# same with explicit RestartMode.ABNORMAL specified
process = self.core.schedule_process(None, uuid.uuid4().hex, definition, restart_mode=RestartMode.ABNORMAL)
for state in abnormal_states:
self.assertTrue(self.core.process_should_restart(process, state))
self.assertTrue(self.core.process_should_restart(process, state, is_system_restart=True))
self.assertFalse(self.core.process_should_restart(process, ProcessState.EXITED))
self.assertFalse(self.core.process_should_restart(process, ProcessState.EXITED, is_system_restart=True))
# RestartMode.NEVER
process = self.core.schedule_process(None, uuid.uuid4().hex, definition, restart_mode=RestartMode.NEVER)
for state in all_states:
self.assertFalse(self.core.process_should_restart(process, state))
self.assertFalse(self.core.process_should_restart(process, state, is_system_restart=True))
# RestartMode.ALWAYS
process = self.core.schedule_process(None, uuid.uuid4().hex, definition, restart_mode=RestartMode.ALWAYS)
for state in all_states:
self.assertTrue(self.core.process_should_restart(process, state))
self.assertTrue(self.core.process_should_restart(process, state, is_system_restart=True))
# RestartMode.ALWAYS with process.omit_from_system_restart
process = self.core.schedule_process(
None,
uuid.uuid4().hex,
definition,
restart_mode=RestartMode.ALWAYS,
configuration=nosystemrestart_process_config(),
)
for state in all_states:
self.assertTrue(self.core.process_should_restart(process, state))
self.assertFalse(self.core.process_should_restart(process, state, is_system_restart=True))
# RestartMode.ABNORMAL with process.omit_from_system_restart
process = self.core.schedule_process(
None,
uuid.uuid4().hex,
definition,
restart_mode=RestartMode.ABNORMAL,
configuration=nosystemrestart_process_config(),
)
for state in abnormal_states:
self.assertTrue(self.core.process_should_restart(process, state))
self.assertFalse(self.core.process_should_restart(process, state, is_system_restart=True))
self.assertFalse(self.core.process_should_restart(process, ProcessState.EXITED))
self.assertFalse(self.core.process_should_restart(process, ProcessState.EXITED, is_system_restart=True))
# ensure that a process with a busted config doesn't raise an error
process = self.core.schedule_process(
None,
uuid.uuid4().hex,
definition,
restart_mode=RestartMode.ALWAYS,
configuration={"process": ["what is a list doing here??"]},
)
for state in all_states:
self.assertTrue(self.core.process_should_restart(process, state))