def test_heartbeat_current_op_over_limit(self): self.patch_cfg('pyon.ion.process.CFG', {'container':{'timeout':{'heartbeat_proc_count_threshold':2}}}) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target def fake_op(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() self.addCleanup(listeninev.set) # allow graceful termination self.addCleanup(p.stop) ar = p._routing_call(fake_op, None, listenoutev, listeninev) listenoutev.wait(timeout=5) # wait for ctrl thread to run our op # make sure it's over the threshold for x in xrange(3): hb = p.heartbeat() self.assertEquals((True, True, False), hb)
def test_heartbeat_listener_dead(self): mocklistener = Mock(spec=ProcessRPCServer) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc) readyev = Event() readyev.set() mocklistener.get_ready_event.return_value = readyev def fake_listen(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() p.start() p.get_ready_event().wait(timeout=5) p.start_listeners() listenoutev.wait(timeout=5) # wait for listen loop to start self.addCleanup(listeninev.set) # makes listen loop fall out on shutdown self.addCleanup(p.stop) listeninev.set() # stop the listen loop p.thread_manager.children[1].join(timeout=5) # wait for listen loop to terminate hb = p.heartbeat() self.assertEquals((False, True, True), hb) self.assertEquals(0, p._heartbeat_count) self.assertIsNone(p._heartbeat_op)
def test_heartbeat_with_current_op_multiple_times(self): svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target def fake_op(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() self.addCleanup(listeninev.set) # allow graceful termination self.addCleanup(p.stop) ar = p._routing_call(fake_op, None, listenoutev, listeninev) listenoutev.wait(timeout=5) # wait for ctrl thread to run our op for x in xrange(5): hb = p.heartbeat() self.assertEquals((True, True, True), hb) self.assertEquals(5, p._heartbeat_count) self.assertEquals(ar, p._heartbeat_op)
def test_heartbeat_current_op_over_limit(self): self.patch_cfg( 'pyon.ion.process.CFG', {'cc': { 'timeout': { 'heartbeat_proc_count_threshold': 2 } }}) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target def fake_op(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() self.addCleanup(listeninev.set) # allow graceful termination self.addCleanup(p.stop) ar = p._routing_call(fake_op, None, listenoutev, listeninev) listenoutev.wait(timeout=5) # wait for ctrl thread to run our op # make sure it's over the threshold for x in xrange(3): hb = p.heartbeat() self.assertEquals((True, True, False), hb)
def test_heartbeat_with_listeners(self): mocklistener = Mock(spec=ProcessRPCServer) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc) readyev = Event() readyev.set() mocklistener.get_ready_event.return_value = readyev def fake_listen(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() mocklistener.listen = lambda *a, **kw: fake_listen(listenoutev, listeninev) p.start() p.get_ready_event().wait(timeout=5) p.start_listeners() listenoutev.wait(timeout=5) # wait for listen loop to start self.addCleanup(listeninev.set) # makes listen loop fall out on shutdown self.addCleanup(p.stop) # now test heartbeat! hb = p.heartbeat() self.assertEquals((True, True, True), hb) self.assertEquals(0, p._heartbeat_count) self.assertIsNone(p._heartbeat_op)
def test_heartbeat_with_current_op_multiple_times(self): svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) p._ctrl_thread.ev_exit.set() # prevent heartbeat loop in proc's target def fake_op(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() self.addCleanup(listeninev.set) # allow graceful termination self.addCleanup(p.stop) ar = p._routing_call(fake_op, None, listenoutev, listeninev) listenoutev.wait(timeout=5) # wait for ctrl thread to run our op for x in xrange(5): hb = p.heartbeat() self.assertEquals((True, True, True), hb) self.assertEquals(5, p._heartbeat_count) self.assertEquals(ar, p._heartbeat_op)
def test_heartbeat_no_listeners(self): svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) self.addCleanup(p.stop) hb = p.heartbeat() self.assertEquals((True, True, True), hb) self.assertEquals(0, p._heartbeat_count) self.assertIsNone(p._heartbeat_op)
def test_heartbeat_no_listeners(self): svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[], service=svc) p.start() p.get_ready_event().wait(timeout=5) self.addCleanup(p.stop) hb = p.heartbeat() self.assertEquals((True, True, True), hb) self.assertEquals(0, p._heartbeat_count) self.assertIsNone(p._heartbeat_op)
def test_heartbeat_listener_dead(self): mocklistener = Mock(spec=ProcessRPCServer) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc) readyev = Event() readyev.set() mocklistener.get_ready_event.return_value = readyev def fake_listen(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() p.start() p.get_ready_event().wait(timeout=5) p.start_listeners() listenoutev.wait(timeout=5) # wait for listen loop to start self.addCleanup( listeninev.set) # makes listen loop fall out on shutdown self.addCleanup(p.stop) listeninev.set() # stop the listen loop p.thread_manager.children[1].join( timeout=5) # wait for listen loop to terminate hb = p.heartbeat() self.assertEquals((False, True, True), hb) self.assertEquals(0, p._heartbeat_count) self.assertIsNone(p._heartbeat_op)
def test_heartbeat_with_listeners(self): mocklistener = Mock(spec=ProcessRPCServer) svc = self._make_service() p = IonProcessThread(name=sentinel.name, listeners=[mocklistener], service=svc) readyev = Event() readyev.set() mocklistener.get_ready_event.return_value = readyev def fake_listen(evout, evin): evout.set(True) evin.wait() listenoutev = AsyncResult() listeninev = Event() mocklistener.listen = lambda *a, **kw: fake_listen( listenoutev, listeninev) p.start() p.get_ready_event().wait(timeout=5) p.start_listeners() listenoutev.wait(timeout=5) # wait for listen loop to start self.addCleanup( listeninev.set) # makes listen loop fall out on shutdown self.addCleanup(p.stop) # now test heartbeat! hb = p.heartbeat() self.assertEquals((True, True, True), hb) self.assertEquals(0, p._heartbeat_count) self.assertIsNone(p._heartbeat_op)