Ejemplo n.º 1
0
def run_worker(context):
    poller = Poller()

    liveness = HEARTBEAT_LIVENESS
    interval = INTERVAL_INIT

    heartbeat_at = time.time() + HEARTBEAT_INTERVAL

    worker = yield worker_socket(context, poller)
    cycles = 0
    while True:
        socks = yield poller.poll(HEARTBEAT_INTERVAL * 1000)
        socks = dict(socks)

        # Handle worker activity on backend
        if socks.get(worker) == zmq.POLLIN:
            #  Get message
            #  - 3-part envelope + content -> request
            #  - 1-part HEARTBEAT -> heartbeat
            frames = yield worker.recv_multipart()
            if not frames:
                break    # Interrupted

            if len(frames) == 3:
                # Simulate various problems, after a few cycles
                cycles += 1
                if cycles > 3 and randint(0, 5) == 0:
                    print("I: Simulating a crash")
                    break
                if cycles > 3 and randint(0, 5) == 0:
                    print("I: Simulating CPU overload")
                    yield gen.sleep(3)
                print("I: Normal reply")
                yield worker.send_multipart(frames)
                liveness = HEARTBEAT_LIVENESS
                yield gen.sleep(1)  # Do some heavy work
            elif len(frames) == 1 and frames[0] == PPP_HEARTBEAT:
                print("I: Queue heartbeat")
                liveness = HEARTBEAT_LIVENESS
            else:
                print("E: Invalid message: %s" % frames)
            interval = INTERVAL_INIT
        else:
            liveness -= 1
            if liveness == 0:
                print("W: Heartbeat failure, can't reach queue")
                print("W: Reconnecting in %0.2fs..." % interval)
                yield gen.sleep(interval)

                if interval < INTERVAL_MAX:
                    interval *= 2
                poller.unregister(worker)
                worker.setsockopt(zmq.LINGER, 0)
                worker.close()
                worker = yield worker_socket(context, poller)
                liveness = HEARTBEAT_LIVENESS
        if time.time() > heartbeat_at:
            heartbeat_at = time.time() + HEARTBEAT_INTERVAL
            print("I: Worker heartbeat")
            yield worker.send(PPP_HEARTBEAT)
Ejemplo n.º 2
0
def run():
    subscriber = Ctx.socket(zmq.SUB)
    subscriber.connect(Url)
    subscription = b"%03d" % 5
    subscriber.setsockopt(zmq.SUBSCRIBE, subscription)
    poller = Poller()
    poller.register(subscriber, zmq.POLLOUT)
    while True:
        topic, data = yield subscriber.recv_multipart()
        #assert topic == subscription
        print(data)
Ejemplo n.º 3
0
def run():
    subscriber = Ctx.socket(zmq.SUB)
    subscriber.connect(Url)
    subscription = b"%03d" % 5
    subscriber.setsockopt(zmq.SUBSCRIBE, subscription)
    poller = Poller()
    poller.register(subscriber, zmq.POLLOUT)
    while True:
        topic, data = yield subscriber.recv_multipart()
        #assert topic == subscription
        print(data)
Ejemplo n.º 4
0
def sender():
    """send a message every second"""
    tic = time.time()
    push = ctx.socket(zmq.PUSH)
    push.bind(url)
    poller = Poller()
    poller.register(push, zmq.POLLOUT)
    while True:
        print("sending")
        yield push.send_multipart([str(time.time() - tic).encode('ascii')])
        yield gen.sleep(1)
Ejemplo n.º 5
0
def publisher(port=8135):
    context = Context()
    pub_uuid = str(uuid.uuid4())
    pub = context.socket(zmq.PUB)
    pub.connect("tcp://localhost:%s" % port)
    poller = Poller()
    poller.register(pub, zmq.POLLOUT)
    while True:
        topic = 'heartbeat'
        utc = arrow.utcnow()
        raw = json.dumps({"timestamp":utc.timestamp, "uuid": pub_uuid})
        message = '{0} {1}'.format(topic, raw)
        yield pub.send(message)
        yield gen.sleep(1)
Ejemplo n.º 6
0
def receiver():
    """receive messages with poll and timeout"""
    pull = ctx.socket(zmq.PULL)
    pull.connect(url)
    poller = Poller()
    poller.register(pull, zmq.POLLIN)
    while True:
        events = yield poller.poll(timeout=500)
        if pull in dict(events):
            print("recving", events)
            msg = yield pull.recv_multipart()
            print('recvd', msg)
        else:
            print("nothing to recv")
Ejemplo n.º 7
0
def subscriber(port=8135):
    '''
        Bind Subscriber
    '''
    logging.warning("Binding SUB socket on port: {0}".format(port))
    context = Context()
    sub = context.socket(zmq.SUB)
    sub.bind("tcp://*:%s" % port)

    sub.setsockopt(zmq.SUBSCRIBE, " ")
    sub.setsockopt(zmq.SUBSCRIBE, "heartbeat")
    sub.setsockopt(zmq.SUBSCRIBE, "asterisk")
    sub.setsockopt(zmq.SUBSCRIBE, "logging")
    sub.setsockopt(zmq.SUBSCRIBE, "upload")
    sub.setsockopt(zmq.SUBSCRIBE, "beam")

    poller = Poller()
    poller.register(sub, zmq.POLLIN)

    http_client = _http_client.AsyncHTTPClient()

    while True:
        events = yield poller.poll(timeout=1000)
        if sub in dict(events):
            # receive raw msg from sub
            msg = yield sub.recv()
            # get topic and message
            topic = msg.split(' ')[0]
            message = ' '.join(msg.split(' ')[1:])
            # this make more sense directly on the beam
            # that why we're moving, still this sub_bind.py
            # clear some ideas directly.
            if topic.startswith('heartbeat'):
                print(topic, message)
            elif topic.startswith('asterisk'):
                print(topic, message)
            elif topic.startswith('logging'):
                print(topic, message)
            elif topic.startswith('upload'):
                print(topic, message)
            elif topic.startswith('beam'):
                print(topic, message)
            else:
                # let it crash
                logging.warning('let it crash')
                print(msg)
        else:
            #logging.warning('nothing receive')
            pass
Ejemplo n.º 8
0
 def __init__(self,
              url: str,
              logger,
              request_timeout: int = None,
              database: str = None):
     self._logger = logger
     self._database = database
     self._context = Context.instance()
     self._poller = Poller()
     self._request = self._context.socket(zmq.DEALER)
     self._request_timeout = request_timeout or 60
     self._rds_bus_url = url
     self._request.connect(self._rds_bus_url)
     self._request_dict = dict()
     self._io_loop = ioloop.IOLoop.current()
     self._io_loop.add_callback(self.start)
Ejemplo n.º 9
0
def run_queue(context):
    frontend = context.socket(zmq.ROUTER)  # ROUTER
    backend = context.socket(zmq.ROUTER)  # ROUTER
    frontend.bind(FRONT_END_ADDRESS)  # For clients
    backend.bind(BACK_END_ADDRESS)  # For workers
    poll_workers = Poller()
    poll_workers.register(backend, zmq.POLLIN)
    poll_both = Poller()
    poll_both.register(frontend, zmq.POLLIN)
    poll_both.register(backend, zmq.POLLIN)
    workers = WorkerQueue()
    heartbeat_at = time.time() + HEARTBEAT_INTERVAL
    while True:
        if len(workers.queue) > 0:
            poller = poll_both
        else:
            poller = poll_workers
        socks = yield poller.poll(HEARTBEAT_INTERVAL * 1000)
        socks = dict(socks)
        # Handle worker activity on backend
        if socks.get(backend) == zmq.POLLIN:
            # Use worker address for LRU routing
            frames = yield backend.recv_multipart()
            if not frames:
                break
            address = frames[0]
            workers.ready(Worker(address))
            # Validate control message, or return reply to client
            msg = frames[1:]
            if len(msg) == 1:
                if msg[0] not in (PPP_READY, PPP_HEARTBEAT):
                    print("E: Invalid message from worker: %s" % msg)
            else:
                yield frontend.send_multipart(msg)
            # Send heartbeats to idle workers if it's time
            if time.time() >= heartbeat_at:
                for worker in workers.queue:
                    msg = [worker, PPP_HEARTBEAT]
                    yield backend.send_multipart(msg)
                heartbeat_at = time.time() + HEARTBEAT_INTERVAL
        if socks.get(frontend) == zmq.POLLIN:
            frames = yield frontend.recv_multipart()
            if not frames:
                break
            frames.insert(0, next(workers))
            backend.send_multipart(frames)
        workers.purge()
Ejemplo n.º 10
0
def run_worker(context):
    # Socket to receive messages on
    receiver = context.socket(zmq.PULL)
    receiver.connect("tcp://localhost:5557")
    # Socket to send messages to
    sender = context.socket(zmq.PUSH)
    sender.connect("tcp://localhost:5558")
    # Socket for control input
    controller = context.socket(zmq.SUB)
    controller.connect("tcp://localhost:5559")
    controller.setsockopt(zmq.SUBSCRIBE, b"")
    # Process messages from receiver and controller
    poller = Poller()
    poller.register(receiver, zmq.POLLIN)
    poller.register(controller, zmq.POLLIN)
    # Process messages from both sockets
    while True:
        socks = yield poller.poll()
        socks = dict(socks)
        if socks.get(receiver) == zmq.POLLIN:
            message = yield receiver.recv()
            # Process task
            workload = int(message)  # Workload in msecs
            # Do the work
            yield gen.sleep(workload / 1000.0)
            # Send results to sink
            yield sender.send(message)
            # Simple progress indicator for the viewer
            sys.stdout.write(".")
            sys.stdout.flush()
        # Any waiting controller command acts as 'KILL'
        if socks.get(controller) == zmq.POLLIN:
            break
Ejemplo n.º 11
0
def run():
    print("Getting ready for hello world client.  Ctrl-C to exit.\n")
    socket = Ctx.socket(zmq.REP)
    socket.bind(Url)
    poller = Poller()
    poller.register(socket, zmq.POLLIN)
    while True:
        #  Wait for next request from client
        message = yield socket.recv()
        print("Received request: {}".format(message))
        #  Do some 'work'
        yield gen.sleep(1)
        #  Send reply back to client
        message = message.decode('utf-8')
        message = '{}, world'.format(message)
        message = message.encode('utf-8')
        print("Sending reply: {}".format(message))
        yield socket.send(message)
Ejemplo n.º 12
0
def run():
    print("Getting ready for hello world client.  Ctrl-C to exit.\n")
    socket = Ctx.socket(zmq.REP)
    socket.bind(Url)
    poller = Poller()
    poller.register(socket, zmq.POLLIN)
    while True:
        #  Wait for next request from client
        message = yield socket.recv()
        print("Received request: {}".format(message))
        #  Do some 'work'
        yield gen.sleep(1)
        #  Send reply back to client
        message = message.decode('utf-8')
        message = '{}, world'.format(message)
        message = message.encode('utf-8')
        print("Sending reply: {}".format(message))
        yield socket.send(message)
Ejemplo n.º 13
0
def run(ident):
    #  Socket to talk to server
    print("Connecting to hello world server.  Ctrl-C to exit early.\n")
    socket = Ctx.socket(zmq.REQ)
    socket.connect(Url)
    poller = Poller()
    poller.register(socket, zmq.POLLOUT)
    #  Do multiple requests, waiting each time for a response
    for request in range(10):
        message = '{} Hello {}'.format(ident, request)
        message = message.encode('utf-8')
        print("Sending message: {}".format(message))
        yield socket.send(message)
        #  Get the reply.
        message = yield socket.recv()
        print("Received reply: {}".format(message))
    print('exiting')
    raise gen.Return('nothing')
Ejemplo n.º 14
0
def run_worker():
    print('(run_worker) worker is starting')
    socket = Ctx.socket(zmq.DEALER)
    socket.connect(Backend_Url)
    poller = Poller()
    poller.register(socket, zmq.POLLIN)
    print('(run_worker) worker is waiting')
    while True:
        #  Wait for next request from client
        part1, part2, message = yield socket.recv_multipart()
        print("(run_worker) received: {}".format(message))
        #  Do some 'work'
        yield gen.sleep(1)
        #  Send reply back to client
        message = message.decode('utf-8')
        message = '{}, world'.format(message)
        message = message.encode('utf-8')
        print("(run_worker) sending: {}".format(message))
        yield socket.send_multipart([part1, part2, message])
        print("(run_worker) sent: {}".format(message))
Ejemplo n.º 15
0
def run_worker():
    print('(run_worker) worker is starting')
    socket = Ctx.socket(zmq.DEALER)
    socket.connect(Backend_Url)
    poller = Poller()
    poller.register(socket, zmq.POLLIN)
    print('(run_worker) worker is waiting')
    while True:
        #  Wait for next request from client
        part1, part2, message = yield socket.recv_multipart()
        print("(run_worker) received: {}".format(message))
        #  Do some 'work'
        yield gen.sleep(1)
        #  Send reply back to client
        message = message.decode('utf-8')
        message = '{}, world'.format(message)
        message = message.encode('utf-8')
        print("(run_worker) sending: {}".format(message))
        yield socket.send_multipart([part1, part2, message])
        print("(run_worker) sent: {}".format(message))
Ejemplo n.º 16
0
def run_client(context):
    print("I: Connecting to server...")
    client = context.socket(zmq.REQ)
    client.connect(SERVER_ENDPOINT)
    poll = Poller()
    poll.register(client, zmq.POLLIN)
    sequence = 0
    retries_left = REQUEST_RETRIES
    while retries_left:
        sequence += 1
        request = str(sequence)
        print("I: Sending (%s)" % request)
        yield client.send_string(request)
        expect_reply = True
        while expect_reply:
            socks = yield poll.poll(REQUEST_TIMEOUT)
            socks = dict(socks)
            if socks.get(client) == zmq.POLLIN:
                reply = yield client.recv()
                if not reply:
                    break
                if int(reply) == sequence:
                    print("I: Server replied OK (%s)" % reply)
                    retries_left = REQUEST_RETRIES
                    expect_reply = False
                else:
                    print("E: Malformed reply from server: %s" % reply)
            else:
                print("W: No response from server, retrying...")
                # Socket is confused. Close and remove it.
                print('W: confused')
                client.setsockopt(zmq.LINGER, 0)
                client.unbind(SERVER_ENDPOINT)
                #client.close()
                poll.unregister(client)
                retries_left -= 1
                if retries_left == 0:
                    print("E: Server seems to be offline, abandoning")
                    return
                print("I: Reconnecting and resending (%s)" % request)
                # Create new connection
                client = context.socket(zmq.REQ)
                client.connect(SERVER_ENDPOINT)
                poll.register(client, zmq.POLLIN)
                yield client.send_string(request)
Ejemplo n.º 17
0
def run(loop):
    """Server routine"""
    # Prepare our context and sockets
    # Socket to talk to clients
    clients = Ctx.socket(zmq.ROUTER)
    clients.bind(Url_client)
    workers = Ctx.socket(zmq.DEALER)
    workers.bind(Url_worker)
    # Start the workers
    # Caution: Do *not* use lambda to create the function call to the worker.
    #     lambda does not work correctly inside a for-statement.
    for idx in range(5):
        ident = 'worker {}'.format(idx)
        loop.add_callback(partial(worker_routine, ident))
    poller = Poller()
    poller.register(clients, zmq.POLLIN)
    poller.register(workers, zmq.POLLIN)
    print('mtserver ready for requests')
    while True:
        events = yield poller.poll()
        events = dict(events)
        if clients in events:
            message = yield clients.recv_multipart()
            printdbg('(run) received from client message_parts: {}'.format(
                message))
            client, empty, message = message[:3]
            printdbg('(run) received from client message: {}'.format(
                message))
            yield workers.send_multipart([client, b'', message])
            printdbg('(run) sent message to workers: {}'.format(message))
        elif workers in events:
            message = yield workers.recv_multipart()
            printdbg('(run) received from worker message_parts: {}'.format(
                message))
            client, empty, message = message[:3]
            printdbg('(run) received from worker message: {}'.format(
                message))
            yield clients.send_multipart([client, b'', message])
            printdbg('(run) sent message to clients: {}'.format(message))
Ejemplo n.º 18
0
def run_queue(context):
    frontend = context.socket(zmq.ROUTER)  # ROUTER
    backend = context.socket(zmq.ROUTER)  # ROUTER
    frontend.bind(FRONT_END_ADDRESS)  # For clients
    backend.bind(BACK_END_ADDRESS)  # For workers
    poll_workers = Poller()
    poll_workers.register(backend, zmq.POLLIN)
    poll_both = Poller()
    poll_both.register(frontend, zmq.POLLIN)
    poll_both.register(backend, zmq.POLLIN)
    workers = WorkerQueue()
    heartbeat_at = time.time() + HEARTBEAT_INTERVAL
    while True:
        if len(workers.queue) > 0:
            poller = poll_both
        else:
            poller = poll_workers
        socks = yield poller.poll(HEARTBEAT_INTERVAL * 1000)
        socks = dict(socks)
        # Handle worker activity on backend
        if socks.get(backend) == zmq.POLLIN:
            # Use worker address for LRU routing
            frames = yield backend.recv_multipart()
            if not frames:
                break
            address = frames[0]
            workers.ready(Worker(address))
            # Validate control message, or return reply to client
            msg = frames[1:]
            if len(msg) == 1:
                if msg[0] not in (PPP_READY, PPP_HEARTBEAT):
                    print("E: Invalid message from worker: %s" % msg)
            else:
                yield frontend.send_multipart(msg)
            # Send heartbeats to idle workers if it's time
            if time.time() >= heartbeat_at:
                for worker in workers.queue:
                    msg = [worker, PPP_HEARTBEAT]
                    yield backend.send_multipart(msg)
                heartbeat_at = time.time() + HEARTBEAT_INTERVAL
        if socks.get(frontend) == zmq.POLLIN:
            frames = yield frontend.recv_multipart()
            if not frames:
                break
            frames.insert(0, next(workers))
            backend.send_multipart(frames)
        workers.purge()
Ejemplo n.º 19
0
def run(loop):
    """Server routine"""
    # Prepare our context and sockets
    # Socket to talk to clients
    clients = Ctx.socket(zmq.ROUTER)
    clients.bind(Url_client)
    workers = Ctx.socket(zmq.DEALER)
    workers.bind(Url_worker)
    # Start the workers
    # Caution: Do *not* use lambda to create the function call to the worker.
    #     lambda does not work correctly inside a for-statement.
    for idx in range(5):
        ident = 'worker {}'.format(idx)
        loop.add_callback(partial(worker_routine, ident))
    poller = Poller()
    poller.register(clients, zmq.POLLIN)
    poller.register(workers, zmq.POLLIN)
    print('mtserver ready for requests')
    while True:
        events = yield poller.poll()
        events = dict(events)
        if clients in events:
            message = yield clients.recv_multipart()
            printdbg(
                '(run) received from client message_parts: {}'.format(message))
            client, empty, message = message[:3]
            printdbg('(run) received from client message: {}'.format(message))
            yield workers.send_multipart([client, b'', message])
            printdbg('(run) sent message to workers: {}'.format(message))
        elif workers in events:
            message = yield workers.recv_multipart()
            printdbg(
                '(run) received from worker message_parts: {}'.format(message))
            client, empty, message = message[:3]
            printdbg('(run) received from worker message: {}'.format(message))
            yield clients.send_multipart([client, b'', message])
            printdbg('(run) sent message to clients: {}'.format(message))
Ejemplo n.º 20
0
def run_client(context):
    print("I: Connecting to server...")
    client = context.socket(zmq.REQ)
    client.connect(SERVER_ENDPOINT)
    poll = Poller()
    poll.register(client, zmq.POLLIN)
    sequence = 0
    retries_left = REQUEST_RETRIES
    while retries_left:
        sequence += 1
        request = str(sequence)
        print("I: Sending (%s)" % request)
        yield client.send_string(request)
        expect_reply = True
        while expect_reply:
            socks = yield poll.poll(REQUEST_TIMEOUT)
            socks = dict(socks)
            if socks.get(client) == zmq.POLLIN:
                reply = yield client.recv()
                if not reply:
                    break
                if int(reply) == sequence:
                    print("I: Server replied OK (%s)" % reply)
                    retries_left = REQUEST_RETRIES
                    expect_reply = False
                else:
                    print("E: Malformed reply from server: %s" % reply)
            else:
                print("W: No response from server, retrying...")
                # Socket is confused. Close and remove it.
                print('W: confused')
                client.setsockopt(zmq.LINGER, 0)
                client.unbind(SERVER_ENDPOINT)
                #client.close()
                poll.unregister(client)
                retries_left -= 1
                if retries_left == 0:
                    print("E: Server seems to be offline, abandoning")
                    return
                print("I: Reconnecting and resending (%s)" % request)
                # Create new connection
                client = context.socket(zmq.REQ)
                client.connect(SERVER_ENDPOINT)
                poll.register(client, zmq.POLLIN)
                yield client.send_string(request)
Ejemplo n.º 21
0
def run_proxy(socket_from, socket_to):
    poller = Poller()
    poller.register(socket_from, zmq.POLLIN)
    poller.register(socket_to, zmq.POLLIN)
    while True:
        events = yield poller.poll()
        events = dict(events)
        if socket_from in events:
            msg = yield socket_from.recv_multipart()
            print('(run_proxy) received from frontend -- msg: {}'.format(msg))
            yield socket_to.send_multipart(msg)
            print('(run_proxy) sent to backend -- msg: {}'.format(msg))
        elif socket_to in events:
            msg = yield socket_to.recv_multipart()
            print('(run_proxy) received from backend -- msg: {}'.format(msg))
            yield socket_from.send_multipart(msg)
            print('(run_proxy) sent to frontend -- msg: {}'.format(msg))
Ejemplo n.º 22
0
def run_proxy(socket_from, socket_to):
    poller = Poller()
    poller.register(socket_from, zmq.POLLIN)
    poller.register(socket_to, zmq.POLLIN)
    while True:
        events = yield poller.poll()
        events = dict(events)
        if socket_from in events:
            msg = yield socket_from.recv_multipart()
            print('(run_proxy) received from frontend -- msg: {}'.format(msg))
            yield socket_to.send_multipart(msg)
            print('(run_proxy) sent to backend -- msg: {}'.format(msg))
        elif socket_to in events:
            msg = yield socket_to.recv_multipart()
            print('(run_proxy) received from backend -- msg: {}'.format(msg))
            yield socket_from.send_multipart(msg)
            print('(run_proxy) sent to frontend -- msg: {}'.format(msg))
Ejemplo n.º 23
0
def run_broker(context):
    # Prepare our context and sockets
    frontend = context.socket(zmq.ROUTER)
    backend = context.socket(zmq.DEALER)
    frontend.bind("tcp://*:5559")
    backend.bind("tcp://*:5560")
    # Initialize poll set
    poller = Poller()
    poller.register(frontend, zmq.POLLIN)
    poller.register(backend, zmq.POLLIN)
    # Switch messages between sockets
    while True:
        socks = yield poller.poll()
        socks = dict(socks)
        if socks.get(frontend) == zmq.POLLIN:
            message = yield frontend.recv_multipart()
            print('received from frontend: {}'.format(message))
            yield backend.send_multipart(message)
        if socks.get(backend) == zmq.POLLIN:
            message = yield backend.recv_multipart()
            print('received from backend: {}'.format(message))
            yield frontend.send_multipart(message)
Ejemplo n.º 24
0
def run_broker(context):
    # Prepare our context and sockets
    frontend = context.socket(zmq.ROUTER)
    backend = context.socket(zmq.DEALER)
    frontend.bind("tcp://*:5559")
    backend.bind("tcp://*:5560")
    # Initialize poll set
    poller = Poller()
    poller.register(frontend, zmq.POLLIN)
    poller.register(backend, zmq.POLLIN)
    # Switch messages between sockets
    while True:
        socks = yield poller.poll()
        socks = dict(socks)
        if socks.get(frontend) == zmq.POLLIN:
            message = yield frontend.recv_multipart()
            print('received from frontend: {}'.format(message))
            yield backend.send_multipart(message)
        if socks.get(backend) == zmq.POLLIN:
            message = yield backend.recv_multipart()
            print('received from backend: {}'.format(message))
            yield frontend.send_multipart(message)
Ejemplo n.º 25
0
def run_queue():
    context = Context(1)

    frontend = context.socket(zmq.ROUTER)    # ROUTER
    backend = context.socket(zmq.ROUTER)     # ROUTER
    frontend.bind("tcp://*:5555")            # For clients
    backend.bind("tcp://*:5556")             # For workers

    poll_workers = Poller()
    poll_workers.register(backend, zmq.POLLIN)

    poll_both = Poller()
    poll_both.register(frontend, zmq.POLLIN)
    poll_both.register(backend, zmq.POLLIN)

    workers = []

    while True:
        if workers:
            socks = yield poll_both.poll()
        else:
            socks = yield poll_workers.poll()
        socks = dict(socks)

        # Handle worker activity on backend
        if socks.get(backend) == zmq.POLLIN:
            # Use worker address for LRU routing
            msg = yield backend.recv_multipart()
            if not msg:
                break
            print('I: received msg: {}'.format(msg))
            address = msg[0]
            workers.append(address)

            # Everything after the second (delimiter) frame is reply
            reply = msg[2:]

            # Forward message to client if it's not a READY
            if reply[0] != LRU_READY:
                print('I: sending -- reply: {}'.format(reply))
                yield frontend.send_multipart(reply)
            else:
                print('I: received ready -- address: {}'.format(address))

        if socks.get(frontend) == zmq.POLLIN:
            #  Get client request, route to first available worker
            msg = yield frontend.recv_multipart()
            worker = workers.pop(0)
            request = [worker, b''] + msg
            print('I: sending -- worker: {}  msg: {}'.format(worker, msg))
            yield backend.send_multipart(request)
Ejemplo n.º 26
0
def run_broker(loop):
    """ main broker method """
    url_worker = "inproc://workers"
    url_client = "inproc://clients"
    client_nbr = NBR_CLIENTS * 3
    # Prepare our context and sockets
    context = Context()
    frontend = context.socket(zmq.ROUTER)
    frontend.bind(url_client)
    backend = context.socket(zmq.ROUTER)
    backend.bind(url_worker)
    # create workers and clients threads
    #worker_tasks = []
    for idx in range(NBR_WORKERS):
        loop.add_callback(partial(run_worker, url_worker, context, idx))
        #worker_tasks.append(task)
    #client_tasks = []
    for idx in range(NBR_CLIENTS):
        loop.add_callback(partial(run_client, url_client, context, idx))
        #client_tasks.append(task)
    # Logic of LRU loop
    # - Poll backend always, frontend only if 1+ worker ready
    # - If worker replies, queue worker as ready and forward reply
    # to client if necessary
    # - If client requests, pop next worker and send request to it
    # Queue of available workers
    available_workers = 0
    workers_list = []
    all_workers = set()
    # init poller
    poller = Poller()
    # Always poll for worker activity on backend
    poller.register(backend, zmq.POLLIN)
    # Poll front-end only if we have available workers
    poller.register(frontend, zmq.POLLIN)
    while True:
        socks = yield poller.poll()
        socks = dict(socks)
        # Handle worker activity on backend
        if (backend in socks and socks[backend] == zmq.POLLIN):
            # Queue worker address for LRU routing
            message = yield backend.recv_multipart()
            assert available_workers < NBR_WORKERS
            worker_addr = message[0]
            # add worker back to the list of workers
            available_workers += 1
            workers_list.append(worker_addr)
            all_workers.add(worker_addr)
            #   Second frame is empty
            empty = message[1]
            assert empty == b""
            # Third frame is READY or else a client reply address
            client_addr = message[2]
            # If client reply, send rest back to frontend
            if client_addr != b'READY':
                # Following frame is empty
                empty = message[3]
                assert empty == b""
                reply = message[4]
                yield frontend.send_multipart([client_addr, b"", reply])
                printdbg('(run_broker) to frontend -- reply: "{}"'.format(
                    reply))
                client_nbr -= 1
                if client_nbr == 0:
                    printdbg('(run_broker) exiting')
                    break   # Exit after N messages
        # poll on frontend only if workers are available
        if available_workers > 0:
            if (frontend in socks and socks[frontend] == zmq.POLLIN):
                # Now get next client request, route to LRU worker
                # Client request is [address][empty][request]
                response = yield frontend.recv_multipart()
                [client_addr, empty, request] = response
                assert empty == b""
                #  Dequeue and drop the next worker address
                available_workers += -1
                worker_id = workers_list.pop()
                yield backend.send_multipart(
                    [worker_id, b"", client_addr, b"", request])
                printdbg('(run_broker) to backend -- request: "{}"'.format(
                    request))
    #out of infinite loop: do some housekeeping
    printdbg('(run_broker) finishing')
    for worker_id in workers_list:
        yield backend.send_multipart([worker_id, b"", b"", b"", b"Stop"])
    printdbg('(run_broker) workers cancelled')
    yield gen.sleep(1)
    frontend.close()
    backend.close()
    #context.term()     # Caution: calling term() blocks.
    printdbg('(run_broker) returning')
    result = 'finished ok'
    raise gen.Return(result)
Ejemplo n.º 27
0
class TornadoRdsBusClient(object):
    ASC = 1
    DESC = -1

    def __init__(self,
                 url: str,
                 logger,
                 request_timeout: int = None,
                 database: str = None):
        self._logger = logger
        self._database = database
        self._context = Context.instance()
        self._poller = Poller()
        self._request = self._context.socket(zmq.DEALER)
        self._request_timeout = request_timeout or 60
        self._rds_bus_url = url
        self._request.connect(self._rds_bus_url)
        self._request_dict = dict()
        self._io_loop = ioloop.IOLoop.current()
        self._io_loop.add_callback(self.start)

    @classmethod
    def pack(cls,
             database: str,
             key: str,
             parameter: dict,
             is_query: bool = False,
             order_by: list = None,
             page_no: int = None,
             per_page: int = None,
             found_rows: bool = False):
        """
        打包请求数据
        :param database: RDS-BUS的数据库类名
        :param key: 数据库类所持有的实例名
        :param parameter: 参数字典
        :param is_query: 是否为查询操作
        :param order_by: 排序信息 [{"column": "字段名", "order": TornadoRdsBusClient.ASC/TornadoRdsBusClient.DESC}]
        :param page_no: 当前页(范围[0-n) n指第n页)
        :param per_page: 每页记录数
        :param found_rows: 是否统计总数
        :return:
        """
        if is_query:
            amount = int(per_page) if per_page else None
            offset = int(page_no) * amount if page_no else None
            limit = (dict(amount=amount, offset=offset) if offset else dict(
                amount=amount)) if amount else None
            result = dict(command="{}/{}".format(database, key),
                          data=dict(var=parameter,
                                    order_by=order_by,
                                    limit=limit,
                                    found_rows=found_rows))
        else:
            result = dict(command="{}/{}".format(database, key),
                          data=dict(var=parameter))
        return result

    @gen.coroutine
    def query(self,
              key: str,
              parameter: dict,
              order_by: list = None,
              page_no: int = None,
              per_page: int = None,
              found_rows: bool = False,
              database: str = None,
              execute: bool = True):
        """
        查询接口
        :param database: RDS-BUS的数据库类名
        :param key: 数据库类所持有的语句实例名
        :param parameter: 参数字典
        :param order_by: 排序信息 [{"column": "字段名", "order": TornadoRdsBusClient.ASC/TornadoRdsBusClient.DESC}]
        :param page_no: 当前页(范围[0-n) n指第n页)
        :param per_page: 每页记录数
        :param found_rows: 是否统计总数
        :param execute: 是否执行
        :return:
        """
        _database = database or self._database
        argument = self.pack(database=_database,
                             key=key,
                             parameter=parameter,
                             is_query=True,
                             order_by=order_by,
                             page_no=page_no,
                             per_page=per_page,
                             found_rows=found_rows)
        if execute:
            response = yield self._send(operation=OperationType.QUERY,
                                        argument=argument)
            result = RdsData(response)
        else:
            result = argument
        return result

    @gen.coroutine
    def insert(self,
               key: str,
               parameter: dict,
               database: str = None,
               execute: bool = True):
        """
        新增接口
        :param database: RDS-BUS的数据库类名
        :param key: 数据库类所持有的语句实例名
        :param parameter: 参数字典
        :param execute: 是否执行
        :return:
        """
        _database = database or self._database
        argument = self.pack(database=_database, key=key, parameter=parameter)
        if execute:
            response = yield self._send(operation=OperationType.INSERT,
                                        argument=argument)
            result = RdsData(response)
        else:
            result = argument
        return result

    @gen.coroutine
    def update(self,
               key: str,
               parameter: dict,
               database: str = None,
               execute: bool = True):
        """
        更新接口
        :param database: RDS-BUS的数据库类名
        :param key: 数据库类所持有的语句实例名
        :param parameter: 参数字典
        :param execute: 是否执行
        :return:
        """
        _database = database or self._database
        argument = self.pack(database=_database, key=key, parameter=parameter)
        if execute:
            response = yield self._send(operation=OperationType.UPDATE,
                                        argument=argument)
            result = RdsData(response)
        else:
            result = argument
        return result

    @gen.coroutine
    def delete(self,
               key: str,
               parameter: dict,
               database: str = None,
               execute: bool = False):
        """
        删除接口
        :param database: RDS-BUS的数据库类名
        :param key: 数据库类所持有的语句实例名
        :param parameter: 参数字典
        :param execute: 是否执行
        :return:
        """
        _database = database or self._database
        argument = self.pack(database=_database, key=key, parameter=parameter)
        if execute:
            response = yield self._send(operation=OperationType.DELETE,
                                        argument=argument)
            result = RdsData(response)
        else:
            result = argument
        return result

    @gen.coroutine
    def transaction(self, data: list, database: str = None):
        """
        事务接口
        :param database: RDS-BUS的数据库类名
        :param data: 操作列表
        :return:
        """
        _database = database or self._database
        result = yield self._send(
            operation=OperationType.TRANSACTION,
            argument=dict(command="{}/transaction".format(_database),
                          data=data))
        return RdsListData(result)

    @gen.coroutine
    def batch(self, data: list, database: str = None):
        """
        批量接口
        :param database: RDS-BUS的数据库类名
        :param data: 操作列表
        :return:
        """
        _database = database or self._database
        result = yield self._send(operation=OperationType.BATCH,
                                  argument=dict(
                                      command="{}/batch".format(_database),
                                      data=data))
        return RdsListData(result)

    @gen.coroutine
    def start(self):
        self._poller.register(self._request, zmq.POLLIN)
        while True:
            events = yield self._poller.poll()
            if self._request in dict(events):
                response = yield self._request.recv_json()
                self._logger.debug("received {}".format(response))
                if response["id"] in self._request_dict:
                    future = self._request_dict.pop(response["id"])
                    if HttpResult.is_duplicate_data_failure(response["code"]):
                        future.set_exception(
                            DuplicateDataException.new_exception(
                                response["desc"]))
                    elif HttpResult.is_failure(response["code"]):
                        future.set_exception(
                            CallServiceException(method="ZMQ",
                                                 url=self._rds_bus_url,
                                                 errmsg=response["desc"]))
                    else:
                        future.set_result(response["data"])
                else:
                    self._logger.warning(
                        "unknown response {}".format(response))

    def stop(self):
        self._poller.unregister(self._request)

    def shutdown(self):
        self.stop()
        self._request.close()

    def _send(self, operation, argument):
        """

        :param operation:
        :param argument:
        :return:
        """
        request_id = get_unique_id()
        self._request_dict[request_id] = Future()
        self._io_loop.call_later(60, self._session_timeout, request_id)
        self._request.send_multipart([
            json.dumps(
                dict(id=request_id,
                     operation=operation.value,
                     argument=argument)).encode("utf-8")
        ])
        return self._request_dict[request_id]

    def _session_timeout(self, request_id):
        if request_id in self._request_dict:
            future = self._request_dict.pop(request_id)
            future.set_exception(
                ServerTimeoutException(method="ZMQ", url=self._rds_bus_url))
Ejemplo n.º 28
0
def run_broker(loop):
    """ main broker method """
    url_worker = "inproc://workers"
    url_client = "inproc://clients"
    client_nbr = NBR_CLIENTS * 3
    # Prepare our context and sockets
    context = Context()
    frontend = context.socket(zmq.ROUTER)
    frontend.bind(url_client)
    backend = context.socket(zmq.ROUTER)
    backend.bind(url_worker)
    # create workers and clients threads
    # worker_tasks = []
    for idx in range(NBR_WORKERS):
        loop.add_callback(partial(run_worker, url_worker, context, idx))
        # worker_tasks.append(task)
    # client_tasks = []
    for idx in range(NBR_CLIENTS):
        loop.add_callback(partial(run_client, url_client, context, idx))
        # client_tasks.append(task)
    # Logic of LRU loop
    # - Poll backend always, frontend only if 1+ worker ready
    # - If worker replies, queue worker as ready and forward reply
    # to client if necessary
    # - If client requests, pop next worker and send request to it
    # Queue of available workers
    available_workers = 0
    workers_list = []
    all_workers = set()
    # init poller
    poller = Poller()
    # Always poll for worker activity on backend
    poller.register(backend, zmq.POLLIN)
    # Poll front-end only if we have available workers
    poller.register(frontend, zmq.POLLIN)
    while True:
        socks = yield poller.poll()
        socks = dict(socks)
        # Handle worker activity on backend
        if backend in socks and socks[backend] == zmq.POLLIN:
            # Queue worker address for LRU routing
            message = yield backend.recv_multipart()
            assert available_workers < NBR_WORKERS
            worker_addr = message[0]
            # add worker back to the list of workers
            available_workers += 1
            workers_list.append(worker_addr)
            all_workers.add(worker_addr)
            #   Second frame is empty
            empty = message[1]
            assert empty == b""
            # Third frame is READY or else a client reply address
            client_addr = message[2]
            # If client reply, send rest back to frontend
            if client_addr != b"READY":
                # Following frame is empty
                empty = message[3]
                assert empty == b""
                reply = message[4]
                yield frontend.send_multipart([client_addr, b"", reply])
                printdbg('(run_broker) to frontend -- reply: "{}"'.format(reply))
                client_nbr -= 1
                if client_nbr == 0:
                    printdbg("(run_broker) exiting")
                    break  # Exit after N messages
        # poll on frontend only if workers are available
        if available_workers > 0:
            if frontend in socks and socks[frontend] == zmq.POLLIN:
                # Now get next client request, route to LRU worker
                # Client request is [address][empty][request]
                response = yield frontend.recv_multipart()
                [client_addr, empty, request] = response
                assert empty == b""
                #  Dequeue and drop the next worker address
                available_workers += -1
                worker_id = workers_list.pop()
                yield backend.send_multipart([worker_id, b"", client_addr, b"", request])
                printdbg('(run_broker) to backend -- request: "{}"'.format(request))
    # out of infinite loop: do some housekeeping
    printdbg("(run_broker) finishing")
    for worker_id in workers_list:
        yield backend.send_multipart([worker_id, b"", b"", b"", b"Stop"])
    printdbg("(run_broker) workers cancelled")
    yield gen.sleep(1)
    frontend.close()
    backend.close()
    # context.term()     # Caution: calling term() blocks.
    printdbg("(run_broker) returning")
    result = "finished ok"
    raise gen.Return(result)