def start(self):
""" Populate pool with connections.
"""
self.queue = queue_six.Queue()
for i in xrange_six(self.pool_size):
ctx = ClusterRpcProxyPool.RpcContext(self, self.config)
self.queue.put(ctx)
def start(self):
""" Populate pool with connections.
"""
self.queue = queue_six.Queue()
for i in xrange_six(self.pool_size):
ctx = ClusterRpcProxyPool.RpcContext(self, self.config)
self.queue.put(ctx)
self.state = 'STARTED'
if self.heartbeat:
self._heartbeat_check_thread = Thread(target=self.heartbeat_check)
self._heartbeat_check_thread.start()
_logger.debug("Heart beat check thread started")
def _reload(self, num_of_worker=0):
""" Reload into pool's queue with number of new worker
:param num_of_worker:
:return:
"""
if num_of_worker <= 0:
num_of_worker = self.pool_size
count = 0
for i in xrange_six(num_of_worker):
if self.queue.full() is False:
ctx = ClusterRpcProxyPool.RpcContext(self, self.config)
self.queue.put_nowait(ctx)
count += 1
_logger.debug("Reload %d worker", count)
def heartbeat_check(self):
RATE = 2 + math.log(self.heartbeat, 30) if self.heartbeat > 30 else 2.
MIN_SLEEP = 3 # better sleep between 3 seconds, if this loop is running too frequent it may affect performance
while self.heartbeat and self.state == 'STARTED':
time.sleep(max(self.heartbeat / abs(RATE), MIN_SLEEP))
count_ok = 0
cleared = set()
try:
for _ in xrange_six(self.pool_size):
ctx = None
try:
ctx = self.queue.get_nowait()
except (queue_six.Empty, AttributeError):
break
else:
if ctx and ctx._rpc and id(ctx) not in cleared:
try:
try:
ctx._rpc._reply_listener.queue_consumer.connection.drain_events(
timeout=0.1)
except socket.timeout:
pass
ctx._rpc._reply_listener.queue_consumer.connection.heartbeat_check(
) # rate=RATE
except (ConnectionError, socket.error,
IOError) as exc:
_logger.info(
"Heart beat failed. System will discard broken connection and replenish "
"pool with a new connection, %s: %s",
type(exc).__name__, exc.args[0])
ctx.__del__()
ctx = ClusterRpcProxyPool.RpcContext(
self, self.config)
else:
count_ok += 1
finally:
if ctx is not None and self.queue is not None:
self.queue.put_nowait(ctx)
cleared.add(id(ctx))
elif ctx is not None:
# unable to put it back, probaly due to system exit so better just delete the connection
ctx.__del__()
except Exception as exc:
_logger.error("%s: %s", type(exc).__name__, exc.args[0])
# just log the error out without raise to keep the heartbeat thread going
_logger.debug("Heart beat %d OK", count_ok)
def heartbeat_check(self):
RATE = 2
while self.heartbeat and self.state == 'STARTED':
time.sleep(self.heartbeat / abs(RATE))
count_ok = 0
cleared = list()
for _ in xrange_six(self.pool_size):
ctx = None
try:
ctx = self.queue.get_nowait()
except queue_six.Empty:
break
else:
if ctx._rpc and id(ctx) not in cleared:
try:
try:
ctx._rpc._reply_listener.queue_consumer.connection.drain_events(
timeout=0.1)
except socket.timeout:
pass
ctx._rpc._reply_listener.queue_consumer.connection.heartbeat_check(
) # rate=RATE
except (ConnectionError, socket.error) as exc:
_logger.info(
"Heart beat failed. System will auto recover broken connection: %s",
str(exc))
ctx.__del__()
ctx = ClusterRpcProxyPool.RpcContext(
self, self.config)
else:
count_ok += 1
finally:
if ctx is not None:
self.queue.put_nowait(ctx)
cleared.append(id(ctx))
_logger.debug("Heart beat %d OK", count_ok)
