interval = INTERVAL_INIT
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
subscriber = subscriber_socket(context,poller)
worker = worker_socket(context, poller)
#cycles = 0
while True:
socks = dict(poller.poll(HEARTBEAT_INTERVAL * 1000))
if socks.get(subscriber) == zmq.POLLIN:
frames = subscriber.recv_multipart()
if not frames:
break
if len(frames) == 4:
ad_idx.dispatch_hander(worker,frames)
liveness = HEARTBEAT_LIVENESS
# Handle worker activity on backend
if socks.get(worker) == zmq.POLLIN:
# Get message
# - 3-part envelope + content -> request
# - 1-part HEARTBEAT -> heartbeat
frames = worker.recv_multipart()
if not frames:
break # Interrupted
if len(frames) == 4:
ad_idx.dispatch_hander(worker,frames)
liveness = HEARTBEAT_LIVENESS
# Simulate various problems, after a few cycles
interval = INTERVAL_INIT
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
subscriber = subscriber_socket(context, poller)
worker = worker_socket(context, poller)
#cycles = 0
while True:
socks = dict(poller.poll(HEARTBEAT_INTERVAL * 1000))
if socks.get(subscriber) == zmq.POLLIN:
frames = subscriber.recv_multipart()
if not frames:
break
if len(frames) == 4:
ad_idx.dispatch_hander(worker, frames)
liveness = HEARTBEAT_LIVENESS
# Handle worker activity on backend
if socks.get(worker) == zmq.POLLIN:
# Get message
# - 3-part envelope + content -> request
# - 1-part HEARTBEAT -> heartbeat
frames = worker.recv_multipart()
if not frames:
break # Interrupted
if len(frames) == 4:
ad_idx.dispatch_hander(worker, frames)
liveness = HEARTBEAT_LIVENESS
# Simulate various problems, after a few cycles
def work_hander(ix, task_queue, pid, has_sub=True):
ad_idx = ADIndex(ix, task_queue, pid)
context = zmq.Context(1)
poller = zmq.Poller()
liveness = HEARTBEAT_LIVENESS
interval = INTERVAL_INIT
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
subscriber = None
if has_sub:
subscriber = subscriber_socket(context, poller, pid)
worker = worker_socket(context, poller, pid)
#cycles = 0
while True:
socks = dict(poller.poll(HEARTBEAT_INTERVAL * 1000))
if has_sub and socks.get(subscriber) == zmq.POLLIN:
frames = subscriber.recv_multipart()
if not frames:
break
if len(frames) == 4:
ad_idx.dispatch_hander(worker, frames)
liveness = HEARTBEAT_LIVENESS
# Handle worker activity on backend
if socks.get(worker) == zmq.POLLIN:
# Get message
# - 3-part envelope + content -> request
# - 1-part HEARTBEAT -> heartbeat
frames = worker.recv_multipart()
if not frames:
break # Interrupted
if len(frames) > 4:
ad_idx.dispatch_hander(worker, frames)
liveness = HEARTBEAT_LIVENESS
# Simulate various problems, after a few cycles
#cycles += 1
#if cycles > 3 and randint(0, 5) == 0:
# print "I: Simulating a crash"
# break
#if cycles > 3 and randint(0, 5) == 0:
# print "I: Simulating CPU overload"
# time.sleep(3)
#print "I: Normal reply"
#worker.send_multipart(frames)
#liveness = HEARTBEAT_LIVENESS
#time.sleep(1) # Do some heavy work
elif len(frames) == 1 and frames[0] == PPP_HEARTBEAT:
# logger.info("I: Queue heartbeat")
liveness = HEARTBEAT_LIVENESS
else:
logger.info("E: Invalid message: %d %s" %
(len(frames), frames))
interval = INTERVAL_INIT
else:
liveness -= 1
if liveness == 0:
logger.info("W: Heartbeat failure, can't reach queue")
logger.info("W: Reconnecting in %0.2fs..." % interval)
time.sleep(interval)
if interval < INTERVAL_MAX:
interval *= 2
poller.unregister(worker)
worker.setsockopt(zmq.LINGER, 0)
worker.close()
worker = worker_socket(context, poller, pid)
if has_sub:
poller.unregister(subscriber)
subscriber.close()
subscriber = subscriber_socket(context, poller, pid)
liveness = HEARTBEAT_LIVENESS
if time.time() > heartbeat_at:
heartbeat_at = time.time() + HEARTBEAT_INTERVAL
# logger.info("I: Worker heartbeat")
worker.send(PPP_HEARTBEAT)