def test_heartbeat_current_op_over_limit(self):
self.patch_cfg('pyon.ion.process.CFG', {'container':{'timeout':{'heartbeat_proc_count_threshold':2}}})
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target
def fake_op(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
self.addCleanup(listeninev.set) # allow graceful termination
self.addCleanup(p.stop)
ar = p._routing_call(fake_op, None, listenoutev, listeninev)
listenoutev.wait(timeout=5) # wait for ctrl thread to run our op
# make sure it's over the threshold
for x in xrange(3):
hb = p.heartbeat()
self.assertEquals((True, True, False), hb)
def test_heartbeat_listener_dead(self):
mocklistener = Mock(spec=ProcessRPCServer)
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc)
readyev = Event()
readyev.set()
mocklistener.get_ready_event.return_value = readyev
def fake_listen(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
p.start()
p.get_ready_event().wait(timeout=5)
p.start_listeners()
listenoutev.wait(timeout=5) # wait for listen loop to start
self.addCleanup(listeninev.set) # makes listen loop fall out on shutdown
self.addCleanup(p.stop)
listeninev.set() # stop the listen loop
p.thread_manager.children[1].join(timeout=5) # wait for listen loop to terminate
hb = p.heartbeat()
self.assertEquals((False, True, True), hb)
self.assertEquals(0, p._heartbeat_count)
self.assertIsNone(p._heartbeat_op)
def test_heartbeat_with_current_op_multiple_times(self):
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target
def fake_op(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
self.addCleanup(listeninev.set) # allow graceful termination
self.addCleanup(p.stop)
ar = p._routing_call(fake_op, None, listenoutev, listeninev)
listenoutev.wait(timeout=5) # wait for ctrl thread to run our op
for x in xrange(5):
hb = p.heartbeat()
self.assertEquals((True, True, True), hb)
self.assertEquals(5, p._heartbeat_count)
self.assertEquals(ar, p._heartbeat_op)
def test_heartbeat_current_op_over_limit(self):
self.patch_cfg(
'pyon.ion.process.CFG',
{'cc': {
'timeout': {
'heartbeat_proc_count_threshold': 2
}
}})
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target
def fake_op(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
self.addCleanup(listeninev.set) # allow graceful termination
self.addCleanup(p.stop)
ar = p._routing_call(fake_op, None, listenoutev, listeninev)
listenoutev.wait(timeout=5) # wait for ctrl thread to run our op
# make sure it's over the threshold
for x in xrange(3):
hb = p.heartbeat()
self.assertEquals((True, True, False), hb)
def test_heartbeat_with_listeners(self):
mocklistener = Mock(spec=ProcessRPCServer)
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc)
readyev = Event()
readyev.set()
mocklistener.get_ready_event.return_value = readyev
def fake_listen(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
mocklistener.listen = lambda *a, **kw: fake_listen(listenoutev, listeninev)
p.start()
p.get_ready_event().wait(timeout=5)
p.start_listeners()
listenoutev.wait(timeout=5) # wait for listen loop to start
self.addCleanup(listeninev.set) # makes listen loop fall out on shutdown
self.addCleanup(p.stop)
# now test heartbeat!
hb = p.heartbeat()
self.assertEquals((True, True, True), hb)
self.assertEquals(0, p._heartbeat_count)
self.assertIsNone(p._heartbeat_op)
def test_heartbeat_with_current_op_multiple_times(self):
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target
def fake_op(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
self.addCleanup(listeninev.set) # allow graceful termination
self.addCleanup(p.stop)
ar = p._routing_call(fake_op, None, listenoutev, listeninev)
listenoutev.wait(timeout=5) # wait for ctrl thread to run our op
for x in xrange(5):
hb = p.heartbeat()
self.assertEquals((True, True, True), hb)
self.assertEquals(5, p._heartbeat_count)
self.assertEquals(ar, p._heartbeat_op)
def test_heartbeat_no_listeners(self):
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
self.addCleanup(p.stop)
hb = p.heartbeat()
self.assertEquals((True, True, True), hb)
self.assertEquals(0, p._heartbeat_count)
self.assertIsNone(p._heartbeat_op)
def test_heartbeat_no_listeners(self):
svc = self._make_service()
p = IonProcessThread(name=sentinel.name, listeners=[], service=svc)
p.start()
p.get_ready_event().wait(timeout=5)
self.addCleanup(p.stop)
hb = p.heartbeat()
self.assertEquals((True, True, True), hb)
self.assertEquals(0, p._heartbeat_count)
self.assertIsNone(p._heartbeat_op)
def test_heartbeat_listener_dead(self):
mocklistener = Mock(spec=ProcessRPCServer)
svc = self._make_service()
p = IonProcessThread(name=sentinel.name,
listeners=[mocklistener],
service=svc)
readyev = Event()
readyev.set()
mocklistener.get_ready_event.return_value = readyev
def fake_listen(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
p.start()
p.get_ready_event().wait(timeout=5)
p.start_listeners()
listenoutev.wait(timeout=5) # wait for listen loop to start
self.addCleanup(
listeninev.set) # makes listen loop fall out on shutdown
self.addCleanup(p.stop)
listeninev.set() # stop the listen loop
p.thread_manager.children[1].join(
timeout=5) # wait for listen loop to terminate
hb = p.heartbeat()
self.assertEquals((False, True, True), hb)
self.assertEquals(0, p._heartbeat_count)
self.assertIsNone(p._heartbeat_op)
def test_heartbeat_with_listeners(self):
mocklistener = Mock(spec=ProcessRPCServer)
svc = self._make_service()
p = IonProcessThread(name=sentinel.name,
listeners=[mocklistener],
service=svc)
readyev = Event()
readyev.set()
mocklistener.get_ready_event.return_value = readyev
def fake_listen(evout, evin):
evout.set(True)
evin.wait()
listenoutev = AsyncResult()
listeninev = Event()
mocklistener.listen = lambda *a, **kw: fake_listen(
listenoutev, listeninev)
p.start()
p.get_ready_event().wait(timeout=5)
p.start_listeners()
listenoutev.wait(timeout=5) # wait for listen loop to start
self.addCleanup(
listeninev.set) # makes listen loop fall out on shutdown
self.addCleanup(p.stop)
# now test heartbeat!
hb = p.heartbeat()
self.assertEquals((True, True, True), hb)
self.assertEquals(0, p._heartbeat_count)
self.assertIsNone(p._heartbeat_op)