def test_heartbeat_node_update_race(self):
# test processing two beats simultaneously, for eeagents in the same node.
# check that they don't collide updating the node record
node_id = uuid.uuid4().hex
self.core.node_state(node_id, domain_id_from_engine("engine1"), InstanceState.RUNNING)
beat = make_beat(node_id)
# this beat gets injected while the other is in the midst of processing
sneaky_beat = make_beat(node_id)
# when the PD attempts to update the process, sneak in an update
# first so the request conflicts
original_update_node = self.store.update_node
def patched_update_node(node):
# unpatch ourself first so we don't recurse forever
self.store.update_node = original_update_node
self.core.ee_heartbeat("eeagent2", sneaky_beat)
original_update_node(node)
self.store.update_node = patched_update_node
self.core.ee_heartbeat("eeagent1", beat)
node = self.store.get_node(node_id)
self.assertEqual(set(["eeagent1", "eeagent2"]), set(node.resources))
def test_add_remove_node_with_resource(self):
self.core.node_state("node1", domain_id_from_engine("engine1"), InstanceState.RUNNING)
resource_id = "eeagent_1"
self.core.ee_heartbeat(resource_id, make_beat("node1"))
resource = self.store.get_resource(resource_id)
self.assertIsNotNone(resource)
self.assertEqual(resource.state, ExecutionResourceState.OK)
# now send a terminated state for the node. resource should be removed.
self.core.node_state("node1", domain_id_from_engine("engine1"), InstanceState.TERMINATED)
self.assertTrue(self.store.get_resource(resource_id) is None)
self.assertTrue(self.store.get_node("node1") is None)
def test_heartbeat_timestamps(self):
# test processing a heartbeat where node is removed partway through
node_id = uuid.uuid4().hex
self.core.node_state(node_id, domain_id_from_engine("engine1"), InstanceState.RUNNING)
d1 = parse_datetime("2013-04-02T19:37:57.617734+00:00")
d2 = parse_datetime("2013-04-02T19:38:57.617734+00:00")
d3 = parse_datetime("2013-04-02T19:39:57.617734+00:00")
self.core.ee_heartbeat("eeagent1", make_beat(node_id, timestamp=d1.isoformat()))
resource = self.store.get_resource("eeagent1")
self.assertEqual(resource.last_heartbeat_datetime, d1)
self.core.ee_heartbeat("eeagent1", make_beat(node_id, timestamp=d3.isoformat()))
resource = self.store.get_resource("eeagent1")
self.assertEqual(resource.last_heartbeat_datetime, d3)
# out of order hbeat. time shouln't be updated
self.core.ee_heartbeat("eeagent1", make_beat(node_id, timestamp=d2.isoformat()))
resource = self.store.get_resource("eeagent1")
self.assertEqual(resource.last_heartbeat_datetime, d3)
def test_add_remove_node_with_resource_and_processes(self):
self.core.node_state("node1", domain_id_from_engine("engine1"),
InstanceState.RUNNING)
resource_id = "eeagent_1"
self.core.ee_heartbeat(resource_id, make_beat("node1"))
# set up a few of processes on the resource
p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.RUNNING,
assigned=resource_id)
self.store.add_process(p1)
p2 = ProcessRecord.new(None, "proc2", {}, ProcessState.PENDING,
assigned=resource_id)
self.store.add_process(p2)
p3 = ProcessRecord.new(None, "proc3", {}, ProcessState.TERMINATING,
assigned=resource_id)
self.store.add_process(p3)
resource = self.store.get_resource(resource_id)
resource.assigned = [p1.key, p2.key, p3.key]
self.store.update_resource(resource)
# now send a terminated state for the node. resource should be removed.
self.core.node_state("node1", domain_id_from_engine("engine1"),
InstanceState.TERMINATED)
self.assertTrue(self.store.get_resource(resource_id) is None)
self.assertTrue(self.store.get_node("node1") is None)
queued_processes = set(self.store.get_queued_processes())
# these two should have been rescheduled
for procname in ("proc1", "proc2"):
proc = self.store.get_process(None, procname)
self.assertEqual(proc.state, ProcessState.DIED_REQUESTED)
self.assertEqual(proc.round, 1)
self.assertIn(proc.key, queued_processes)
self.notifier.assert_process_state(procname, ProcessState.DIED_REQUESTED)
# this one should be terminated
proc3 = self.store.get_process(None, "proc3")
self.assertEqual(proc3.state, ProcessState.TERMINATED)
self.assertEqual(proc3.round, 0)
self.assertNotIn(proc3.key, queued_processes)
self.notifier.assert_process_state("proc3", ProcessState.TERMINATED)
def test_heartbeat_node_removed(self):
# test processing a heartbeat where node is removed partway through
node_id = uuid.uuid4().hex
self.core.node_state(node_id, domain_id_from_engine("engine1"), InstanceState.RUNNING)
beat = make_beat(node_id)
original_update_node = self.store.update_node
def patched_update_node(node):
# unpatch ourself first so we don't recurse forever
self.store.update_node = original_update_node
self.store.remove_node(node.node_id)
original_update_node(node)
self.store.update_node = patched_update_node
# this shouldn't blow up, and no resource should be added
self.core.ee_heartbeat("eeagent1", beat)
self.assertEqual(self.store.get_resource("eeagent1"), None)
def test_uninitialized_system_boot_with_state(self):
self.store.set_system_boot(True)
self.core.node_state("node1", domain_id_from_engine("engine1"),
InstanceState.RUNNING)
resource_id = "eeagent_1"
self.core.ee_heartbeat(resource_id, make_beat("node1"))
p0 = ProcessRecord.new(None, "proc0", {}, ProcessState.RUNNING,
configuration=nosystemrestart_process_config(),
assigned=resource_id,
restart_mode=RestartMode.ALWAYS)
self.store.add_process(p0)
p1 = ProcessRecord.new(None, "proc1", {}, ProcessState.RUNNING,
assigned=resource_id)
self.store.add_process(p1)
p2 = ProcessRecord.new(None, "proc2", {}, ProcessState.PENDING,
assigned=resource_id)
self.store.add_process(p2)
p3 = ProcessRecord.new(None, "proc3", {}, ProcessState.TERMINATING,
assigned=resource_id)
self.store.add_process(p3)
# this one shouldn't restart
p4 = ProcessRecord.new(None, "proc4", {}, ProcessState.RUNNING,
configuration=nosystemrestart_process_config(),
assigned=resource_id,
restart_mode=RestartMode.ABNORMAL)
self.store.add_process(p4)
# non-running proceses should also potentially be restarted on boot
p5 = ProcessRecord.new(None, "proc5", {}, ProcessState.WAITING)
self.store.add_process(p5)
self.store.enqueue_process(*p5.key)
p6 = ProcessRecord.new(None, "proc6", {}, ProcessState.REQUESTED)
self.store.add_process(p6)
# not this one, due to RestartMode
p7 = ProcessRecord.new(None, "proc7", {}, ProcessState.REQUESTED,
configuration=nosystemrestart_process_config(),
restart_mode=RestartMode.ALWAYS)
self.store.add_process(p7)
self.store.enqueue_process(*p7.key)
resource = self.store.get_resource(resource_id)
resource.assigned = [p0.key, p1.key, p2.key, p3.key, p4.key]
self.store.update_resource(resource)
restartable_procs = ["proc1", "proc2", "proc5", "proc6"]
dead_procs = ["proc0", "proc4", "proc7"]
self._run_in_thread()
assert self.store.wait_initialized(timeout=10)
self.assertEqual(len(self.store.get_queued_processes()), 0)
self.assertEqual(len(self.store.get_node_ids()), 0)
self.assertEqual(len(self.store.get_resource_ids()), 0)
for proc in restartable_procs:
self.assertEqual(self.store.get_process(None, proc).state,
ProcessState.UNSCHEDULED_PENDING)
for proc in dead_procs:
self.assertEqual(self.store.get_process(None, proc).state,
ProcessState.TERMINATED)
self.assertEqual(self.store.get_process(None, "proc3").state,
ProcessState.TERMINATED)
self.assertEqual(self.store.get_pd_state(),
ProcessDispatcherState.SYSTEM_BOOTING)
# now end system boot
self.store.set_system_boot(False)
wait(lambda: self.store.get_pd_state() == ProcessDispatcherState.OK)
# check that pending processes were correctly rescheduled
self.assertEqual(len(self.store.get_queued_processes()), len(restartable_procs))
for proc in restartable_procs:
self.assertEqual(self.store.get_process(None, proc).state,
ProcessState.REQUESTED)
def test_resource_monitor(self):
t0 = datetime(2012, 3, 13, 9, 30, 0, tzinfo=UTC)
mock_now = Mock()
mock_now.return_value = t0
def increment_now(seconds):
t = mock_now.return_value + timedelta(seconds=seconds)
mock_now.return_value = t
log.debug("THE TIME IS NOW: %s", t)
return t
monitor = self._setup_resource_monitor()
monitor._now_func = mock_now
# before there are any resources, monitor should work but return a None delay
self.assertIsNone(monitor.monitor_cycle())
self.core.node_state("node1", domain_id_from_engine("engine1"),
InstanceState.RUNNING)
# 3 resources. all report in at t0
r1, r2, r3 = "eeagent_1", "eeagent_2", "eeagent_3"
self._send_heartbeat(r1, "node1", t0)
self._send_heartbeat(r2, "node1", t0)
self._send_heartbeat(r3, "node1", t0)
states = {r1: ExecutionResourceState.OK, r2: ExecutionResourceState.OK,
r3: ExecutionResourceState.OK}
self.assert_monitor_cycle(10, states)
t1 = increment_now(5) # :05
# heartbeat comes in for r1 5 seconds later
self._send_heartbeat(r1, "node1", t1)
self.assert_monitor_cycle(5, states)
increment_now(5) # :10
# no heartbeats for r2 and r3. they should be marked WARNING
states[r2] = ExecutionResourceState.WARNING
states[r3] = ExecutionResourceState.WARNING
self.assert_monitor_cycle(5, states)
increment_now(4) # :14
# r2 gets a heartbeat through, but its timestamp puts it still in the warning threshold
self._send_heartbeat(r2, "node1", t0 + timedelta(seconds=1))
self.assert_monitor_cycle(1, states)
increment_now(6) # :20
# r1 should go warning, r3 should go missing
states[r1] = ExecutionResourceState.WARNING
states[r3] = ExecutionResourceState.MISSING
self.assert_monitor_cycle(4, states)
t2 = increment_now(3) # :23
self._send_heartbeat(r1, "node1", t2)
states[r1] = ExecutionResourceState.OK
self.assert_monitor_cycle(1, states)
t3 = increment_now(2) # :25
self._send_heartbeat(r3, "node1", t3)
states[r2] = ExecutionResourceState.MISSING
states[r3] = ExecutionResourceState.OK
self.assert_monitor_cycle(8, states)
increment_now(5) # :30
# hearbeat r2 enough to go back to WARNING, but still late
self.core.ee_heartbeat(r2, make_beat("node1", timestamp=t0 + timedelta(seconds=15)))
self._send_heartbeat(r2, "node1", t0 + timedelta(seconds=15))
states[r2] = ExecutionResourceState.WARNING
self.assert_monitor_cycle(3, states)
t4 = increment_now(5) # :35
# disable r2 and heartbeat r1 and r3 (heartbeats arrive late, but that's ok)
self._send_heartbeat(r1, "node1", t4)
self._send_heartbeat(r3, "node1", t4)
self.core.resource_change_state(self.store.get_resource(r2),
ExecutionResourceState.DISABLED)
states[r2] = ExecutionResourceState.DISABLED
self.assert_monitor_cycle(10, states)
def _send_heartbeat(self, resource_id, node_id, timestamp):
self.core.ee_heartbeat(resource_id, make_beat(node_id, timestamp=timestamp))