예제 #1
0
def go():
    """
    Reading from multiple sockets
    This version uses a simple recv loop
    """

    # create PULL socket. Connecs to task ventilator
    receiver = yield from aiozmq.create_zmq_stream(zmq.PULL, connect="tcp://localhost:5557")

    # create SUB socket. Connect to weather server
    subscriber = yield from aiozmq.create_zmq_stream(zmq.SUB, connect="tcp://localhost:5556")
    subscriber.transport.subscribe(b"10001")

    # Process messages from both sockets
    # We prioritize traffic from the task ventilator
    # TODO: Check if its possible to prioritize with asyncio
    while True:
        # Process any waiting tasks
        msg = yield from receiver.read()
        # process task
        print(*msg)

        # Process any waiting weather updates
        msg = yield from subscriber.read()
        # process weather update
        print(*msg)

        # No activity, so sleep for 1 msec
        yield from asyncio.sleep(0.001)
예제 #2
0
def go():
    """
    Task worker
    Connects PULL socket to tcp://localhost:5557
    Collects workloads from ventilator via that socket
    Connects PUSH socket to tcp://localhost:5558
    Sends results to sink via that socket
    """

    # create PULL socket to receive messages on
    receiver = yield from aiozmq.create_zmq_stream(zmq.PULL, connect='tcp://localhost:5557')

    # create PUSH socket to send messages on
    sender = yield from aiozmq.create_zmq_stream(zmq.PUSH, connect='tcp://localhost:5558')

    # Process tasks forever
    while True:
        s = yield from receiver.read()

        # Simple progress indicator for the viewer
        sys.stdout.write('.')
        sys.stdout.flush()

        # Do the work
        yield from asyncio.sleep(int(s[0]) * 0.001)

        # Send results to the sink
        sender.write([b''])
예제 #3
0
def go():
    """
    Reading from multiple sockets
    This version uses a simple recv loop
    """
    # TODO: Check if it make sense to use something similar to polling with asyncio
    # create PULL socket. Connecs to task ventilator
    receiver = yield from aiozmq.create_zmq_stream(
        zmq.PULL, connect='tcp://localhost:5557')

    # create SUB socket. Connect to weather server
    subscriber = yield from aiozmq.create_zmq_stream(
        zmq.SUB, connect='tcp://localhost:5556')
    subscriber.transport.subscribe(b'10001')

    # Process messages from both sockets
    # We prioritize traffic from the task ventilator
    # TODO: Check if its possible to prioritize with asyncio
    while True:
        # Process any waiting tasks
        msg = yield from receiver.read()
        # process task
        print(*msg)

        # Process any waiting weather updates
        msg = yield from subscriber.read()
        # process weather update
        print(*msg)

        # No activity, so sleep for 1 msec
        yield from asyncio.sleep(0.001)
예제 #4
0
def go():
    router = yield from aiozmq.create_zmq_stream(
        zmq.ROUTER,
        bind='tcp://127.0.0.1:*')
    addr = list(router.transport.bindings())[0]

    dealer = yield from aiozmq.create_zmq_stream(
        zmq.DEALER)

    yield from dealer.transport.enable_monitor()

    asyncio.Task(monitor_stream(dealer))

    yield from dealer.transport.connect(addr)

    for i in range(10):
        msg = (b'data', b'ask', str(i).encode('utf-8'))
        dealer.write(msg)
        data = yield from router.read()
        router.write(data)
        answer = yield from dealer.read()
        print(answer)

    router.close()
    dealer.close()
예제 #5
0
def run_tests(kernel_info):
    global stdout_reader_task, stderr_reader_task, stop_reading_streams
    # The scope of this method is same to the user's notebook session on the web browser.
    # kernel_sock should be mapped with AJAX calls.
    # stdout/stderr_sock should be mapped with WebSockets to asynchronously update cell output blocks.
    kernel_sock = yield from aiozmq.create_zmq_stream(
        zmq.REQ, connect=kernel_info['agent_sock'], loop=loop)
    stdout_sock = yield from aiozmq.create_zmq_stream(
        zmq.SUB, connect=kernel_info['stdout_sock'], loop=loop)
    stderr_sock = yield from aiozmq.create_zmq_stream(
        zmq.SUB, connect=kernel_info['stderr_sock'], loop=loop)
    stop_reading_streams = False
    stdout_reader_task = asyncio. async (handle_out_stream(
        stdout_sock, 'stdout'),
                                         loop=loop)
    stderr_reader_task = asyncio. async (handle_out_stream(
        stderr_sock, 'stderr'),
                                         loop=loop)
    yield from asyncio.sleep(0.01)
    c = 'a = 123\nprint(a)'
    yield from run_command(kernel_sock, stdout_sock, stderr_sock, 1, c)
    c = 'a += 1\nprint(a)'
    yield from run_command(kernel_sock, stdout_sock, stderr_sock, 2, c)
    c = 'def sum(a,b):\n  return a+b'
    yield from run_command(kernel_sock, stdout_sock, stderr_sock, 3, c)
    c = 'import sys\nprint(sum(a, 456), file=sys.stderr)'
    yield from run_command(kernel_sock, stdout_sock, stderr_sock, 4, c)
    c = 'raise RuntimeError("test")'
    yield from run_command(kernel_sock, stdout_sock, stderr_sock, 5, c)
    stop_reading_streams = True
    stdout_reader_task.cancel()
    stderr_reader_task.cancel()
    stdout_sock.close()
    stderr_sock.close()

    c = 'print(sum(5,6))\nprint(sum(-5,-3), file=sys.stderr)'
    stdout, stderr = yield from run_command(kernel_sock,
                                            stdout_sock,
                                            stderr_sock,
                                            6,
                                            c,
                                            redirect_output=True)
    print(Fore.GREEN + '[6]' + Fore.RESET, stdout)
    print(Fore.YELLOW + '[6]' + Fore.RESET, stderr, file=sys.stderr)

    kernel_sock.close()

    api_sock = yield from aiozmq.create_zmq_stream(
        zmq.REQ, connect='tcp://127.0.0.1:5001', loop=loop)
    req = Message(
        ('action', ManagerRequestTypes.DESTROY),
        ('kernel_id', kernel_info.kernel_id),
        ('body', ''),
    )
    api_sock.write([req.encode()])
    resp_data = yield from api_sock.read()
    resp = Message.decode(resp_data[0])
    assert resp['reply'] == ManagerResponseTypes.SUCCESS
    api_sock.close()
예제 #6
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:*',
                loop=self.loop)

            self.assertEqual(100, s1._event_queue.maxlen)
예제 #7
0
    def send(self,
             node_id,
             message,
             message_type="message",
             wait_response=True):
        peer_info = self.directory[node_id]

        address = peer_info['address']
        port = peer_info['server_port']

        address = 'tcp://%(address)s:%(port)s' % locals()
        request_socket = yield from aiozmq.create_zmq_stream(zmq.DEALER,
                                                             connect=address,
                                                             loop=self.loop)

        log_info = (message_type, json.dumps(message), address)
        self.logger.info('Send %s/%s to %s' % log_info)
        message = (message_type.encode('utf-8'),
                   json.dumps(message).encode('utf-8'))
        request_socket.write(message)

        if wait_response:
            message_type, message = yield from request_socket.read()
            request_socket.close()
            assert message_type.decode('utf-8') == 'message'
            return json.loads(message.decode('utf-8'))

        yield from request_socket.drain()
        request_socket.close()
예제 #8
0
    def start(self):

        # Pub
        self.pub = yield from aiozmq.create_zmq_stream(
            zmq.PUB,
            bind="tcp://*:*",
            loop=self.loop
        )

        # Server
        self.server, self.server_t = yield from aiozmq.create_zmq_connection(
            lambda: ServerProtocol(self.process_message, self.loop),
            zmq.ROUTER, bind="tcp://*:*",
            loop=self.loop
        )
        self.server_t.transport.setsockopt(zmq.IDENTITY, self.node_id.encode('utf-8'))

        # Sub
        self.sub, _ = yield from aiozmq.create_zmq_connection(
            lambda: SubProtocol(self.process_event, self.loop),
            zmq.SUB,
            loop=self.loop
        )

        yield from super(ZeroMQMedium, self).start()
예제 #9
0
def go():
    """
    Task sink
    Binds PULL socket to tcp://localhost:5558
    Collects results from workers via that socket
    """

    # create PULL socket to receive messages on
    receiver = yield from aiozmq.create_zmq_stream(zmq.PULL, bind='tcp://*:5558')

    # Wait for start of batch
    s = yield from receiver.read()

    # Start our clock now
    tstart = time.time()

    # Process 100 confirmations
    total_msec = 0
    for task_nbr in range(100):
        s = yield from receiver.read()
        sys.stdout.write(':') if task_nbr % 10 == 0 else sys.stdout.write('.')
        sys.stdout.flush()

    # Calculate and report duration of batch
    tend = time.time()
    print('Total elapsed time: {} msec'.format((tend - tstart) * 1000))
예제 #10
0
def go():
    """
    Task sink
    Binds PULL socket to tcp://localhost:5558
    Collects results from workers via that socket
    """

    # create PULL socket to receive messages on
    receiver = yield from aiozmq.create_zmq_stream(zmq.PULL,
                                                   bind='tcp://*:5558')

    # Wait for start of batch
    s = yield from receiver.read()

    # Start our clock now
    tstart = time.time()

    # Process 100 confirmations
    total_msec = 0
    for task_nbr in range(100):
        s = yield from receiver.read()
        sys.stdout.write(':') if task_nbr % 10 == 0 else sys.stdout.write('.')
        sys.stdout.flush()

    # Calculate and report duration of batch
    tend = time.time()
    print('Total elapsed time: {} msec'.format((tend - tstart) * 1000))
예제 #11
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:*',
                loop=self.loop)

            self.assertIsNone(s1.exception())
예제 #12
0
def create_kernel():
    api_sock = yield from aiozmq.create_zmq_stream(
        zmq.REQ, connect='tcp://127.0.0.1:5001', loop=loop)

    # Test if ping works.
    req_id = generate_uuid()
    req = Message(
        ('action', ManagerRequestTypes.PING),
        ('kernel_id', ''),
        ('body', req_id),
    )
    api_sock.write([req.encode()])
    resp_data = yield from api_sock.read()
    resp = Message.decode(resp_data[0])
    assert resp['reply'] == ManagerResponseTypes.PONG
    assert resp['body'] == req['body']

    # Create a kernel instance.
    req = Message(
        ('action', ManagerRequestTypes.CREATE),
        ('kernel_id', ''),
    )
    api_sock.write([req.encode()])
    resp_data = yield from api_sock.read()
    resp = Message.decode(resp_data[0])
    assert resp['reply'] == ManagerResponseTypes.SUCCESS
    api_sock.close()
    return resp['kernel_id'], resp['body']
예제 #13
0
파일: zmq.py 프로젝트: lablup/logger
def init(loop, opts, rec_avail_ev):
    topo = Topologies[opts['topology']].value
    server = loop.run_until_complete(
            aiozmq.create_zmq_stream(topo, bind=opts['endpoint'], loop=loop))
    if topo == zmq.SUB:
        server.transport.setsockopt(zmq.SUBSCRIBE, b'')
    asyncio.ensure_future(zmq_fetcher(loop, server, rec_avail_ev), loop=loop)
예제 #14
0
        def go():
            addr = 'tcp://127.0.0.1:{}'.format(port)
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.ROUTER,
                bind=addr,
                loop=self.loop)

            @asyncio.coroutine
            def f(s, events):
                try:
                    while True:
                        event = yield from s.read_event()
                        events.append(event)
                except aiozmq.ZmqStreamClosed:
                    pass

            s2 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                loop=self.loop)

            events = []
            t = asyncio.async(f(s2, events), loop=self.loop)

            yield from s2.transport.enable_monitor()
            yield from s2.transport.connect(addr)
            yield from s2.transport.disconnect(addr)
            yield from s2.transport.connect(addr)

            s2.write([b'request'])
            req = yield from s1.read()
            self.assertEqual([mock.ANY, b'request'], req)
            s1.write([req[0], b'answer'])
            answer = yield from s2.read()
            self.assertEqual([b'answer'], answer)

            s2.close()
            s1.close()

            yield from t

            # Confirm that the events received by the monitor were valid.
            self.assertGreater(len(events), 0)
            while len(events):
                event = events.pop()
                self.assertIsInstance(event, SocketEvent)
                self.assertIn(event.event, ZMQ_EVENTS)
예제 #15
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s1._protocol.pause_writing()
            s1.close()
예제 #16
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s2 = yield from aiozmq.create_zmq_stream(
                zmq.ROUTER,
                connect='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s1.write([b'request'])
            req = yield from s2.read()
            self.assertEqual([mock.ANY, b'request'], req)
            s2.write([req[0], b'answer'])
            answer = yield from s1.read()
            self.assertEqual([b'answer'], answer)
예제 #17
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s1._protocol.pause_writing()
            s1.close()
예제 #18
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s2 = yield from aiozmq.create_zmq_stream(
                zmq.ROUTER,
                connect='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            self.assertFalse(s2.at_closing())
            s2.close()
            s1.write([b'request'])
            with self.assertRaises(aiozmq.ZmqStreamClosed):
                yield from s2.read()
            self.assertTrue(s2.at_closing())
예제 #19
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s2 = yield from aiozmq.create_zmq_stream(
                zmq.ROUTER,
                connect='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s1.write([b'request'])
            req = yield from s2.read()
            self.assertEqual([mock.ANY, b'request'], req)
            s2.write([req[0], b'answer'])
            answer = yield from s1.read()
            self.assertEqual([b'answer'], answer)
예제 #20
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s2 = yield from aiozmq.create_zmq_stream(
                zmq.ROUTER,
                connect='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            self.assertFalse(s2.at_closing())
            s2.close()
            s1.write([b'request'])
            with self.assertRaises(aiozmq.ZmqStreamClosed):
                yield from s2.read()
            self.assertTrue(s2.at_closing())
예제 #21
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:*',
                loop=self.loop)

            self.assertIsInstance(s1.get_extra_info('zmq_socket'),
                                  zmq.Socket)
예제 #22
0
 def client():
     dealer = yield from aiozmq.create_zmq_stream(zmq.DEALER,
                                                  connect=options.addr)
     data = read_data()
     dealer.write(data)
     print("Async client write: {!r}".format(data))
     echo = yield from dealer.read()
     print("Async client read: {!r}".format(echo))
     stop.set_result(None)
예제 #23
0
 def client():
     dealer = yield from aiozmq.create_zmq_stream(
         zmq.DEALER, connect=options.addr)
     data = read_data()
     dealer.write(data)
     print("Async client write: {!r}".format(data))
     echo = yield from dealer.read()
     print("Async client read: {!r}".format(echo))
     stop.set_result(None)
예제 #24
0
def go():
    """
    Simple message queuing broker
    Same as request-reply broker but using QUEUE device
    """

    # Create ROUTER socket. Socket facing clients
    frontend = yield from aiozmq.create_zmq_stream(zmq.ROUTER,
                                                   bind='tcp://*:5559')

    # create DEALER socket. Socket facing services
    backend = yield from aiozmq.create_zmq_stream(zmq.DEALER,
                                                  bind='tcp://*:5560')

    # create QUEUE device
    #TODO: not sure that this is the best way to do it
    zmq.device(zmq.QUEUE, frontend.transport._zmq_sock,
               backend.transport._zmq_sock)
예제 #25
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(zmq.DEALER,
                                                     bind='tcp://127.0.0.1:*',
                                                     loop=self.loop)

            with self.assertRaises(ValueError):
                s1.set_read_buffer_limits(high=1, low=2)

            s1.close()
예제 #26
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:*',
                loop=self.loop)

            with self.assertRaises(ValueError):
                s1.set_read_buffer_limits(high=1, low=2)

            s1.close()
예제 #27
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(zmq.DEALER,
                                                     bind='tcp://127.0.0.1:*',
                                                     loop=self.loop)

            self.assertIsInstance(s1.transport, aiozmq.ZmqTransport)
            s1.close()
            with self.assertRaises(aiozmq.ZmqStreamClosed):
                yield from s1.read()
            self.assertIsNone(s1.transport)
예제 #28
0
def go():
    router = yield from aiozmq.create_zmq_stream(
        zmq.ROUTER,
        bind='tcp://127.0.0.1:*')

    addr = list(router.transport.bindings())[0]
    dealer = yield from aiozmq.create_zmq_stream(
        zmq.DEALER,
        connect=addr)

    for i in range(10):
        msg = (b'data', b'ask', str(i).encode('utf-8'))
        dealer.write(msg)
        data = yield from router.read()
        router.write(data)
        answer = yield from dealer.read()
        print(answer)
    dealer.close()
    router.close()
예제 #29
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(zmq.DEALER,
                                                     bind='tcp://127.0.0.1:*',
                                                     loop=self.loop)

            s1.set_read_buffer_limits(high=60)
            self.assertEqual(15, s1._low_water)
            self.assertEqual(60, s1._high_water)

            s1.close()
예제 #30
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(zmq.DEALER,
                                                     bind='tcp://127.0.0.1:*',
                                                     loop=self.loop)

            exc = RuntimeError('some exc')
            s1.set_exception(exc)
            self.assertIs(exc, s1.exception())

            with self.assertRaisesRegex(RuntimeError, 'some exc'):
                yield from s1.read()
예제 #31
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s2 = yield from aiozmq.create_zmq_stream(
                zmq.ROUTER,
                connect='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s2.set_read_buffer_limits(high=5)
            s1.write([b'request'])

            yield from asyncio.sleep(0.01, loop=self.loop)
            self.assertTrue(s2._paused)

            msg = yield from s2.read()
            self.assertEqual([mock.ANY, b'request'], msg)
            self.assertFalse(s2._paused)
예제 #32
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s2 = yield from aiozmq.create_zmq_stream(
                zmq.ROUTER,
                connect='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s2.set_read_buffer_limits(high=5)
            s1.write([b'request'])

            yield from asyncio.sleep(0.01, loop=self.loop)
            self.assertTrue(s2._paused)

            msg = yield from s2.read()
            self.assertEqual([mock.ANY, b'request'], msg)
            self.assertFalse(s2._paused)
예제 #33
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s1.close()
            yield from asyncio.sleep(0, loop=self.loop)

            with self.assertRaises(ConnectionResetError):
                yield from s1.drain()
예제 #34
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            s1.close()
            yield from asyncio.sleep(0, loop=self.loop)

            with self.assertRaises(ConnectionResetError):
                yield from s1.drain()
예제 #35
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:*',
                loop=self.loop)

            s1.set_read_buffer_limits(high=60)
            self.assertEqual(15, s1._low_water)
            self.assertEqual(60, s1._high_water)

            s1.close()
예제 #36
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:*',
                loop=self.loop)

            self.assertIsInstance(s1.transport, aiozmq.ZmqTransport)
            s1.close()
            with self.assertRaises(aiozmq.ZmqStreamClosed):
                yield from s1.read()
            self.assertIsNone(s1.transport)
예제 #37
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:*',
                loop=self.loop)

            exc = RuntimeError('some exc')
            s1.set_exception(exc)
            self.assertIs(exc, s1.exception())

            with self.assertRaisesRegex(RuntimeError, 'some exc'):
                yield from s1.read()
예제 #38
0
 def server():
     router = yield from aiozmq.create_zmq_stream(
         zmq.ROUTER, bind=options.addr)
     while True:
         try:
             data = yield from router.read()
         except asyncio.CancelledError:
             break
         print("Async server read: {!r}".format(data))
         router.write(data)
         print("Async server write: {!r}".format(data))
     router.close()
예제 #39
0
 def server():
     router = yield from aiozmq.create_zmq_stream(zmq.ROUTER,
                                                  bind=options.addr)
     while True:
         try:
             data = yield from router.read()
         except asyncio.CancelledError:
             break
         print("Async server read: {!r}".format(data))
         router.write(data)
         print("Async server write: {!r}".format(data))
     router.close()
예제 #40
0
def go():
    """
    Simple request-reply broker
    """

    # Create ROUTER socket
    frontend = yield from aiozmq.create_zmq_stream(zmq.ROUTER, bind='tcp://*:5559')

    # Create DEALER socket
    backend = yield from aiozmq.create_zmq_stream(zmq.DEALER, bind='tcp://*:5560')

    # TODO: Check for the closest thing to a Poller with asyncio

    # Switch messages between sockets
    while True:
        # Check frontend
        message = yield from frontend.read()
        backend.write(message)

        # Check backend
        message = yield from backend.read()
        frontend.write(message)
예제 #41
0
def main():
    global kernel_ip_override

    def manager_args(parser):
        parser.add('--manager-port', env_var='BACKEND.AI_MANAGER_PORT', type=port_no, default=5001,
                   help='The TCP port number where the legacy manager listens on. '
                        '(default: 5001)')

    config = load_config(extra_args_func=manager_args)
    init_logger(config)

    def handle_signal(loop, term_ev):
        if term_ev.is_set():
            log.warning('Forced shutdown!')
            sys.exit(1)
        else:
            term_ev.set()
            loop.stop()

    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
    loop = asyncio.get_event_loop()

    log.info('Backend.Ai Manager {}'.format(__version__))

    server = loop.run_until_complete(
        aiozmq.create_zmq_stream(zmq.REP, bind='tcp://*:{0}'.format(config.manager_port),
                                 loop=loop))

    my_ip = loop.run_until_complete(get_instance_ip())
    log.info(_f('serving on tcp://{0}:{1}', my_ip, config.manager_port))
    log.info(_f('using redis on tcp://{0}:{1}', *config.redis_addr))
    kernel_ip_override = config.kernel_ip_override

    registry = InstanceRegistry(config.redis_addr, loop=loop)
    loop.run_until_complete(registry.init())

    term_ev = asyncio.Event()
    term_barrier = AsyncBarrier(3)
    loop.add_signal_handler(signal.SIGINT, handle_signal, loop, term_ev)
    loop.add_signal_handler(signal.SIGTERM, handle_signal, loop, term_ev)
    asyncio.ensure_future(handle_api(loop, term_ev, term_barrier, server, registry))
    asyncio.ensure_future(handle_notifications(loop, term_ev, term_barrier, registry))
    try:
        loop.run_forever()
        # interrupted
        server.close()
        loop.run_until_complete(graceful_shutdown(loop, term_barrier))
        loop.run_until_complete(asyncio.sleep(0.1))
    finally:
        loop.close()
        log.info('exit.')
예제 #42
0
def go():
    """
    Task ventilator
    Binds PUSH socket to tcp://localhost:5557
    Sends batch of tasks to workers via that socket
    """

    # create PUSH socket to send messages on
    sender = yield from aiozmq.create_zmq_stream(zmq.PUSH, bind='tcp://*:5557')

    # create PUSH socket with direct access to the sink:
    # used to syncronize start of batch
    sink = yield from aiozmq.create_zmq_stream(zmq.PUSH,
                                               connect='tcp://localhost:5558')

    print('Press Enter when the workers are ready: ')
    _ = input()
    print('Sending tasks to workers...')

    # The first message is "0" and signals start of batch
    sink.write([b'0'])

    # Initialize random number generator
    random.seed()

    # Send 100 tasks
    total_msec = 0
    for task_nbr in range(100):
        # Random workload from 1 to 100 msecs
        workload = random.randint(1, 100)
        total_msec += workload

        sender.write([bytes('{}'.format(workload), 'utf-8')])

    print('Total expected cost: {} msec'.format(total_msec))

    # Give ZMQ time to deliver
    yield from sender.drain()
예제 #43
0
 def go():
     s1 = yield from aiozmq.create_zmq_stream(
         zmq.REP,
         bind='tcp://127.0.0.1:{}'.format(port),
         loop=self.loop)
     handler = mock.Mock()
     self.loop.set_exception_handler(handler)
     s1.write([b'data'])
     with self.assertRaises(OSError) as ctx:
         yield from s1.read()
     check_errno(zmq.EFSM, ctx.exception)
     with self.assertRaises(OSError) as ctx2:
         yield from s1.drain()
     check_errno(zmq.EFSM, ctx2.exception)
예제 #44
0
def go():
    """
    Request-reply service
    Connects REP socket to tcp://localhost:5560
    Expects "Hello" from client, replies with "World"
    """

    # create REP socket
    socket = yield from aiozmq.create_zmq_stream(zmq.REP, connect='tcp://localhost:5560')

    while True:
        message = yield from socket.read()
        print('Received request: {}'.format(*message))
        socket.write([b'World'])
예제 #45
0
 def go():
     s1 = yield from aiozmq.create_zmq_stream(
         zmq.REP,
         bind='tcp://127.0.0.1:{}'.format(port),
         loop=self.loop)
     handler = mock.Mock()
     self.loop.set_exception_handler(handler)
     s1.write([b'data'])
     with self.assertRaises(OSError) as ctx:
         yield from s1.read()
     check_errno(zmq.EFSM, ctx.exception)
     with self.assertRaises(OSError) as ctx2:
         yield from s1.drain()
     check_errno(zmq.EFSM, ctx2.exception)
예제 #46
0
def go():
    """
    Task ventilator
    Binds PUSH socket to tcp://localhost:5557
    Sends batch of tasks to workers via that socket
    """

    # create PUSH socket to send messages on
    sender = yield from aiozmq.create_zmq_stream(zmq.PUSH, bind='tcp://*:5557')

    # create PUSH socket with direct access to the sink:
    # used to syncronize start of batch
    sink = yield from aiozmq.create_zmq_stream(zmq.PUSH, connect='tcp://localhost:5558')

    print('Press Enter when the workers are ready: ')
    _ = input()
    print('Sending tasks to workers...')

    # The first message is "0" and signals start of batch
    sink.write([b'0'])

    # Initialize random number generator
    random.seed()

    # Send 100 tasks
    total_msec = 0
    for task_nbr in range(100):
        # Random workload from 1 to 100 msecs
        workload = random.randint(1, 100)
        total_msec += workload

        sender.write([bytes('{}'.format(workload), 'utf-8')])

    print('Total expected cost: {} msec'.format(total_msec))

    # Give ZMQ time to deliver
    yield from sender.drain()
예제 #47
0
def go():
    """
    Simple request-reply broker
    """

    # Create ROUTER socket
    frontend = yield from aiozmq.create_zmq_stream(zmq.ROUTER,
                                                   bind='tcp://*:5559')

    # Create DEALER socket
    backend = yield from aiozmq.create_zmq_stream(zmq.DEALER,
                                                  bind='tcp://*:5560')

    # TODO: Check for the closest thing to a Poller with asyncio

    # Switch messages between sockets
    while True:
        # Check frontend
        message = yield from frontend.read()
        backend.write(message)

        # Check backend
        message = yield from backend.read()
        frontend.write(message)
예제 #48
0
def go():
    """
    Request-reply service
    Connects REP socket to tcp://localhost:5560
    Expects "Hello" from client, replies with "World"
    """

    # create REP socket
    socket = yield from aiozmq.create_zmq_stream(
        zmq.REP, connect='tcp://localhost:5560')

    while True:
        message = yield from socket.read()
        print('Received request: {}'.format(*message))
        socket.write([b'World'])
예제 #49
0
def go():
    """
    Request-reply client
    Connects REQ socket to tcp://localhost:5559
    Sends "Hello" to server, expects "World" back
    """

    # Create REQ socket
    socket = yield from aiozmq.create_zmq_stream(zmq.REQ, connect='tcp://localhost:5559')

    # Do 10 requests, waiting each time for a response
    for request in range(1, 11):
        socket.write([b'Hello'])
        message = yield from socket.read()
        print('Received reply {} from [{}]'.format(request, *message))
예제 #50
0
def go():
    """
    Request-reply client
    Connects REQ socket to tcp://localhost:5559
    Sends "Hello" to server, expects "World" back
    """

    # Create REQ socket
    socket = yield from aiozmq.create_zmq_stream(
        zmq.REQ, connect='tcp://localhost:5559')

    # Do 10 requests, waiting each time for a response
    for request in range(1, 11):
        socket.write([b'Hello'])
        message = yield from socket.read()
        print('Received reply {} from [{}]'.format(request, *message))
예제 #51
0
def go():
    """
    Weather update server
    Binds PUB socket to tcp://*:5556
    Publishes random weather updates
    """

    # create PUB socket
    pub = yield from aiozmq.create_zmq_stream(zmq.PUB, bind='tcp://*:5556')

    while True:
        zipcode = randrange(1, 100000)
        temperature = randrange(-80, 135)
        relhumidity = randrange(10, 60)

        pub.write([bytes("{} {} {}".format(zipcode, temperature, relhumidity), 'utf-8')])
예제 #52
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(zmq.DEALER,
                                                     bind='tcp://127.0.0.1:*',
                                                     loop=self.loop)

            def f():
                yield from s1.read()

            t1 = ensure_future(f(), loop=self.loop)
            # to run f() up to yield from
            yield from asyncio.sleep(0.001, loop=self.loop)

            with self.assertRaises(RuntimeError):
                yield from s1.read()

            t1.cancel()
예제 #53
0
def go():
    """
    Hello World client in Python
    Connects REQ socket to tcp://localhost:5555
    Sends "Hello" to server, expects "World" back
    """

    # create REQ socket and connect to server
    req = yield from aiozmq.create_zmq_stream(zmq.REQ, connect='tcp://localhost:5555')

    for request in range(10):
        print('Sending request {}...'.format(request))
        req.write([b"Hello"])

        # get the reply
        message = yield from req.read()
        print('Received reply {} [{}]'.format(request, *message))
예제 #54
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:*',
                loop=self.loop)

            def f():
                yield from s1.read()

            t1 = asyncio.async(f(), loop=self.loop)
            # to run f() up to yield from
            yield from asyncio.sleep(0.001, loop=self.loop)

            with self.assertRaises(RuntimeError):
                yield from s1.read()

            t1.cancel()
예제 #55
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            def f():
                yield from s1.read()

            t1 = asyncio.async(f(), loop=self.loop)
            # to run f() up to yield from
            yield from asyncio.sleep(0.001, loop=self.loop)

            s1.close()
            yield from asyncio.sleep(0.001, loop=self.loop)

            with self.assertRaises(aiozmq.ZmqStreamClosed):
                t1.result()
예제 #56
0
        def go():
            s1 = yield from aiozmq.create_zmq_stream(
                zmq.DEALER,
                bind='tcp://127.0.0.1:{}'.format(port),
                loop=self.loop)

            self.assertFalse(s1._paused)
            s1._protocol.pause_writing()

            @asyncio.coroutine
            def f():
                yield from s1.drain()

            fut = asyncio.async(f(), loop=self.loop)
            yield from asyncio.sleep(0.01, loop=self.loop)

            s1.close()
            yield from fut