Example #1
0
    interval = INTERVAL_INIT

    heartbeat_at = time.time() + HEARTBEAT_INTERVAL

    subscriber = subscriber_socket(context,poller)

    worker = worker_socket(context, poller)
    #cycles = 0
    while True:
        socks = dict(poller.poll(HEARTBEAT_INTERVAL * 1000))
        if socks.get(subscriber) == zmq.POLLIN:
            frames = subscriber.recv_multipart()
            if not frames:
                break
            if len(frames) == 4:
                ad_idx.dispatch_hander(worker,frames)
                liveness = HEARTBEAT_LIVENESS

        # Handle worker activity on backend
        if socks.get(worker) == zmq.POLLIN:
            #  Get message
            #  - 3-part envelope + content -> request
            #  - 1-part HEARTBEAT -> heartbeat
            frames = worker.recv_multipart()
            if not frames:
                break # Interrupted

            if len(frames) == 4:
                ad_idx.dispatch_hander(worker,frames)
                liveness = HEARTBEAT_LIVENESS
                # Simulate various problems, after a few cycles
Example #2
0
    interval = INTERVAL_INIT

    heartbeat_at = time.time() + HEARTBEAT_INTERVAL

    subscriber = subscriber_socket(context, poller)

    worker = worker_socket(context, poller)
    #cycles = 0
    while True:
        socks = dict(poller.poll(HEARTBEAT_INTERVAL * 1000))
        if socks.get(subscriber) == zmq.POLLIN:
            frames = subscriber.recv_multipart()
            if not frames:
                break
            if len(frames) == 4:
                ad_idx.dispatch_hander(worker, frames)
                liveness = HEARTBEAT_LIVENESS

        # Handle worker activity on backend
        if socks.get(worker) == zmq.POLLIN:
            #  Get message
            #  - 3-part envelope + content -> request
            #  - 1-part HEARTBEAT -> heartbeat
            frames = worker.recv_multipart()
            if not frames:
                break  # Interrupted

            if len(frames) == 4:
                ad_idx.dispatch_hander(worker, frames)
                liveness = HEARTBEAT_LIVENESS
                # Simulate various problems, after a few cycles
Example #3
0
def work_hander(ix, task_queue, pid, has_sub=True):

    ad_idx = ADIndex(ix, task_queue, pid)

    context = zmq.Context(1)
    poller = zmq.Poller()

    liveness = HEARTBEAT_LIVENESS
    interval = INTERVAL_INIT

    heartbeat_at = time.time() + HEARTBEAT_INTERVAL

    subscriber = None

    if has_sub:
        subscriber = subscriber_socket(context, poller, pid)

    worker = worker_socket(context, poller, pid)
    #cycles = 0
    while True:
        socks = dict(poller.poll(HEARTBEAT_INTERVAL * 1000))
        if has_sub and socks.get(subscriber) == zmq.POLLIN:
            frames = subscriber.recv_multipart()
            if not frames:
                break
            if len(frames) == 4:
                ad_idx.dispatch_hander(worker, frames)
                liveness = HEARTBEAT_LIVENESS

        # Handle worker activity on backend
        if socks.get(worker) == zmq.POLLIN:
            #  Get message
            #  - 3-part envelope + content -> request
            #  - 1-part HEARTBEAT -> heartbeat
            frames = worker.recv_multipart()
            if not frames:
                break  # Interrupted

            if len(frames) > 4:
                ad_idx.dispatch_hander(worker, frames)
                liveness = HEARTBEAT_LIVENESS
                # Simulate various problems, after a few cycles
                #cycles += 1
                #if cycles > 3 and randint(0, 5) == 0:
                #    print "I: Simulating a crash"
                #    break
                #if cycles > 3 and randint(0, 5) == 0:
                #    print "I: Simulating CPU overload"
                #    time.sleep(3)
                #print "I: Normal reply"
                #worker.send_multipart(frames)
                #liveness = HEARTBEAT_LIVENESS
                #time.sleep(1)  # Do some heavy work
            elif len(frames) == 1 and frames[0] == PPP_HEARTBEAT:
                #                logger.info("I: Queue heartbeat")
                liveness = HEARTBEAT_LIVENESS
            else:
                logger.info("E: Invalid message: %d %s" %
                            (len(frames), frames))
            interval = INTERVAL_INIT
        else:
            liveness -= 1
            if liveness == 0:
                logger.info("W: Heartbeat failure, can't reach queue")
                logger.info("W: Reconnecting in %0.2fs..." % interval)
                time.sleep(interval)

                if interval < INTERVAL_MAX:
                    interval *= 2
                poller.unregister(worker)
                worker.setsockopt(zmq.LINGER, 0)
                worker.close()
                worker = worker_socket(context, poller, pid)

                if has_sub:
                    poller.unregister(subscriber)
                    subscriber.close()
                    subscriber = subscriber_socket(context, poller, pid)

                liveness = HEARTBEAT_LIVENESS
        if time.time() > heartbeat_at:
            heartbeat_at = time.time() + HEARTBEAT_INTERVAL
            #            logger.info("I: Worker heartbeat")
            worker.send(PPP_HEARTBEAT)