def start(self): """ Populate pool with connections. """ self.queue = queue_six.Queue() for i in xrange_six(self.pool_size): ctx = ClusterRpcProxyPool.RpcContext(self, self.config) self.queue.put(ctx)
def start(self): """ Populate pool with connections. """ self.queue = queue_six.Queue() for i in xrange_six(self.pool_size): ctx = ClusterRpcProxyPool.RpcContext(self, self.config) self.queue.put(ctx) self.state = 'STARTED' if self.heartbeat: self._heartbeat_check_thread = Thread(target=self.heartbeat_check) self._heartbeat_check_thread.start() _logger.debug("Heart beat check thread started")
def _reload(self, num_of_worker=0): """ Reload into pool's queue with number of new worker :param num_of_worker: :return: """ if num_of_worker <= 0: num_of_worker = self.pool_size count = 0 for i in xrange_six(num_of_worker): if self.queue.full() is False: ctx = ClusterRpcProxyPool.RpcContext(self, self.config) self.queue.put_nowait(ctx) count += 1 _logger.debug("Reload %d worker", count)
def heartbeat_check(self): RATE = 2 + math.log(self.heartbeat, 30) if self.heartbeat > 30 else 2. MIN_SLEEP = 3 # better sleep between 3 seconds, if this loop is running too frequent it may affect performance while self.heartbeat and self.state == 'STARTED': time.sleep(max(self.heartbeat / abs(RATE), MIN_SLEEP)) count_ok = 0 cleared = set() try: for _ in xrange_six(self.pool_size): ctx = None try: ctx = self.queue.get_nowait() except (queue_six.Empty, AttributeError): break else: if ctx and ctx._rpc and id(ctx) not in cleared: try: try: ctx._rpc._reply_listener.queue_consumer.connection.drain_events( timeout=0.1) except socket.timeout: pass ctx._rpc._reply_listener.queue_consumer.connection.heartbeat_check( ) # rate=RATE except (ConnectionError, socket.error, IOError) as exc: _logger.info( "Heart beat failed. System will discard broken connection and replenish " "pool with a new connection, %s: %s", type(exc).__name__, exc.args[0]) ctx.__del__() ctx = ClusterRpcProxyPool.RpcContext( self, self.config) else: count_ok += 1 finally: if ctx is not None and self.queue is not None: self.queue.put_nowait(ctx) cleared.add(id(ctx)) elif ctx is not None: # unable to put it back, probaly due to system exit so better just delete the connection ctx.__del__() except Exception as exc: _logger.error("%s: %s", type(exc).__name__, exc.args[0]) # just log the error out without raise to keep the heartbeat thread going _logger.debug("Heart beat %d OK", count_ok)
def heartbeat_check(self): RATE = 2 while self.heartbeat and self.state == 'STARTED': time.sleep(self.heartbeat / abs(RATE)) count_ok = 0 cleared = list() for _ in xrange_six(self.pool_size): ctx = None try: ctx = self.queue.get_nowait() except queue_six.Empty: break else: if ctx._rpc and id(ctx) not in cleared: try: try: ctx._rpc._reply_listener.queue_consumer.connection.drain_events( timeout=0.1) except socket.timeout: pass ctx._rpc._reply_listener.queue_consumer.connection.heartbeat_check( ) # rate=RATE except (ConnectionError, socket.error) as exc: _logger.info( "Heart beat failed. System will auto recover broken connection: %s", str(exc)) ctx.__del__() ctx = ClusterRpcProxyPool.RpcContext( self, self.config) else: count_ok += 1 finally: if ctx is not None: self.queue.put_nowait(ctx) cleared.append(id(ctx)) _logger.debug("Heart beat %d OK", count_ok)