Пример #1
0
def client_task(ctx, pipe):
    dealer = ctx.socket(zmq.DEALER)
    socket_set_hwm(dealer, 1)
    dealer.connect("tcp://127.0.0.1:6000")
    total = 0  # Total bytes received
    chunks = 0  # Total chunks received
    while True:
        # ask for next chunk
        yield dealer.send_multipart(
            [b"fetch", b"%i" % total,
             b"%i" % CHUNK_SIZE])
        try:
            chunk = yield dealer.recv()
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return  # shutting down, quit
            else:
                raise
        chunks += 1
        size = len(chunk)
        total += size
        if size < CHUNK_SIZE:
            break  # Last chunk received; exit
    yield dealer.send_multipart([
        b"finish",
        b"-1",
        b"-1",
    ])
    message = "client received %i chunks, %i bytes" % (chunks, total)
    print(message)
    yield pipe.send(b"OK")
    print('(client) finished')
    raise gen.Return(('client', message))
Пример #2
0
def client_task(ctx, pipe):
    dealer = ctx.socket(zmq.DEALER)
    socket_set_hwm(dealer, 1)
    dealer.connect("tcp://127.0.0.1:6000")
    total = 0       # Total bytes received
    chunks = 0      # Total chunks received
    while True:
        # ask for next chunk
        yield from dealer.send_multipart([
            b"fetch",
            b"%i" % total,
            b"%i" % CHUNK_SIZE
        ])
        try:
            chunk = yield from dealer.recv()
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return   # shutting down, quit
            else:
                raise
        chunks += 1
        size = len(chunk)
        total += size
        if size < CHUNK_SIZE:
            break   # Last chunk received; exit
    yield from dealer.send_multipart([
        b"finish",
        b"-1",
        b"-1",
    ])
    message = "client received %i chunks, %i bytes" % (chunks, total)
    print(message)
    yield from pipe.send(b"OK")
    print('(client) finished')
    return ('client', message)
Пример #3
0
def client_thread(ctx, pipe):
    dealer = ctx.socket(zmq.DEALER)
    socket_set_hwm(dealer, 1)
    dealer.connect("tcp://127.0.0.1:6000")

    total = 0       # Total bytes received
    chunks = 0      # Total chunks received

    while True:
        # ask for next chunk
        dealer.send_multipart([
            b"fetch",
            b"%i" % total,
            b"%i" % CHUNK_SIZE
        ])

        try:
            chunk = dealer.recv()
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return   # shutting down, quit
            else:
                raise

        chunks += 1
        size = len(chunk)
        total += size
        if size < CHUNK_SIZE:
            break   # Last chunk received; exit

    print ("%i chunks received, %i bytes" % (chunks, total))
    pipe.send(b"OK")
Пример #4
0
def server_thread(ctx):
    file = open("testdata", "rb")

    router = ctx.socket(zmq.ROUTER)
    socket_set_hwm(router, PIPELINE)
    router.bind("tcp://*:6000")

    while True:
        # First frame in each message is the sender identity
        # Second frame is "fetch" command
        try:
            msg = router.recv_multipart()
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return   # shutting down, quit
            else:
                raise

        identity, command, offset_str, chunksz_str = msg

        assert command == b"fetch"

        offset = int(offset_str)
        chunksz = int(chunksz_str)

        # Read chunk of data from file
        file.seek(offset, os.SEEK_SET)
        data = file.read(chunksz)

        # Send resulting chunk to client
        router.send_multipart([identity, data])
Пример #5
0
def server_thread(ctx):
    file = open("testdata", "rb")

    router = ctx.socket(zmq.ROUTER)

    # Default HWM is 1000, which will drop messages here
    # since we send more than 1,000 chunks of test data,
    # so set an infinite HWM as a simple, stupid solution:
    socket_set_hwm(router, 0)
    router.bind("tcp://*:6000")

    while True:
        # First frame in each message is the sender identity
        # Second frame is "fetch" command
        try:
            identity, command = router.recv_multipart()
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return   # shutting down, quit
            else:
                raise

        assert command == b"fetch"

        while True:
            data = file.read(CHUNK_SIZE)
            router.send_multipart([identity, data])
            if not data:
                break
Пример #6
0
def server_task(ctx):
    file = open("testdata", "r")
    router = ctx.socket(zmq.ROUTER)
    # Default HWM is 1000, which will drop messages here
    # since we send more than 1,000 chunks of test data,
    # so set an infinite HWM as a simple, stupid solution:
    socket_set_hwm(router, 0)
    router.bind("tcp://*:6000")
    count = 0
    while True:
        # First frame in each message is the sender identity
        # Second frame is "fetch" command
        try:
            identity, command = yield router.recv_multipart()
            break
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return   # shutting down, quit
            else:
                raise
    assert command == b"fetch"
    while True:
        data = file.read(CHUNK_SIZE)
        data = data.encode('utf-8')
        yield router.send_multipart([identity, data])
        if not data:
            break
        count += 1
    message = 'server sent {} chunks'.format(count)
    print(message)
    print('(server_task) finished')
    raise gen.Return(('server', message))
Пример #7
0
def server_thread(ctx):
    file = open("testdata", "r")

    router = ctx.socket(zmq.ROUTER)

    # Default HWM is 1000, which will drop messages here
    # since we send more than 1,000 chunks of test data,
    # so set an infinite HWM as a simple, stupid solution:
    socket_set_hwm(router, 0)
    router.bind("tcp://*:6000")

    while True:
        # First frame in each message is the sender identity
        # Second frame is "fetch" command
        try:
            identity, command = router.recv_multipart()
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return  # shutting down, quit
            else:
                raise

        assert command == b"fetch"

        while True:
            data = file.read(CHUNK_SIZE)
            router.send_multipart([identity, data])
            if not data:
                break
Пример #8
0
def server_task(ctx):
    file = open("testdata", "r")
    router = ctx.socket(zmq.ROUTER)
    # Default HWM is 1000, which will drop messages here
    # since we send more than 1,000 chunks of test data,
    # so set an infinite HWM as a simple, stupid solution:
    socket_set_hwm(router, 0)
    router.bind("tcp://*:6000")
    count = 0
    while True:
        # First frame in each message is the sender identity
        # Second frame is "fetch" command
        try:
            identity, command = yield router.recv_multipart()
            break
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return  # shutting down, quit
            else:
                raise
    assert command == b"fetch"
    while True:
        data = file.read(CHUNK_SIZE)
        data = data.encode('utf-8')
        yield router.send_multipart([identity, data])
        if not data:
            break
        count += 1
    message = 'server sent {} chunks'.format(count)
    print(message)
    print('(server_task) finished')
    raise gen.Return(('server', message))
Пример #9
0
def server_task(ctx):
    file = open("testdata", "r")
    router = ctx.socket(zmq.ROUTER)
    socket_set_hwm(router, PIPELINE)
    router.bind("tcp://*:6000")
    count = 0
    total = 0
    while True:
        # First frame in each message is the sender identity
        # Second frame is "fetch" command
        try:
            msg = yield router.recv_multipart()
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return   # shutting down, quit
            else:
                raise
        identity, command, offset_str, chunksz_str = msg
        if command == b"finish":
            break
        assert command == b"fetch"
        offset = int(offset_str)
        chunksz = int(chunksz_str)
        # Read chunk of data from file
        file.seek(offset, os.SEEK_SET)
        data = file.read(chunksz)
        data = data.encode('utf-8')
        # Send resulting chunk to client
        yield router.send_multipart([identity, data])
        print('(server) sent chunk: {} size: {}'.format(count, len(data)))
        count += 1
        total += len(data)
    message = 'server sent {} chunks, {} bytes'.format(count, total)
    print('(server) finished')
    raise gen.Return(('server', message))
Пример #10
0
def client_thread(ctx, pipe):
    dealer = ctx.socket(zmq.DEALER)
    socket_set_hwm(dealer, 1)
    dealer.connect("tcp://127.0.0.1:6000")

    total = 0  # Total bytes received
    chunks = 0  # Total chunks received

    while True:
        # ask for next chunk
        dealer.send_multipart([b"fetch", b"%i" % total, b"%i" % CHUNK_SIZE])

        try:
            chunk = dealer.recv()
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return  # shutting down, quit
            else:
                raise

        chunks += 1
        size = len(chunk)
        total += size
        if size < CHUNK_SIZE:
            break  # Last chunk received; exit

    print("%i chunks received, %i bytes" % (chunks, total))
    pipe.send(b"%s" % chunks)
Пример #11
0
def server_task(ctx):
    print('(server) starting')
    infile = open("testdata", "r")
    print('(server) setup 1')
    router = ctx.socket(zmq.ROUTER)
    # Default HWM is 1000, which will drop messages here
    # since we send more than 1,000 chunks of test data,
    # so set an infinite HWM as a simple, stupid solution:
    print('(server) setup 2')
    socket_set_hwm(router, 0)
    router.bind("tcp://*:6000")
    print('(server) after setup')
    count = 0
    total = 0
    while True:
        # First frame in each message is the sender identity
        # Second frame is "fetch" command
        print('(server) waiting for request')
        try:
            identity, command = yield from router.recv_multipart()
            break
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return   # shutting down, quit
            else:
                raise
    print('(server) received request.  command: {}'.format(command))
    assert command == b"fetch"
    while True:
        data = infile.read(CHUNK_SIZE)
        data = data.encode('utf-8')
        yield from router.send_multipart([identity, data])
        print('(server) sent chunk.  length: {}'.format(len(data)))
        if not data:
            break
        count += 1
        total += len(data)
    message = 'server sent {} chunks, {} bytes'.format(count, total)
    print(message)
    print('(server_task) finished')
    return ('server', message)
Пример #12
0
def server_task(ctx):
    print('(server) starting')
    infile = open("testdata", "r")
    print('(server) setup 1')
    router = ctx.socket(zmq.ROUTER)
    # Default HWM is 1000, which will drop messages here
    # since we send more than 1,000 chunks of test data,
    # so set an infinite HWM as a simple, stupid solution:
    print('(server) setup 2')
    socket_set_hwm(router, 0)
    router.bind("tcp://*:6000")
    print('(server) after setup')
    count = 0
    total = 0
    while True:
        # First frame in each message is the sender identity
        # Second frame is "fetch" command
        print('(server) waiting for request')
        try:
            identity, command = yield from router.recv_multipart()
            break
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return  # shutting down, quit
            else:
                raise
    print('(server) received request.  command: {}'.format(command))
    assert command == b"fetch"
    while True:
        data = infile.read(CHUNK_SIZE)
        data = data.encode('utf-8')
        yield from router.send_multipart([identity, data])
        print('(server) sent chunk.  length: {}'.format(len(data)))
        if not data:
            break
        count += 1
        total += len(data)
    message = 'server sent {} chunks, {} bytes'.format(count, total)
    print(message)
    print('(server_task) finished')
    return ('server', message)
Пример #13
0
def client_task(ctx, pipe):
    dealer = ctx.socket(zmq.DEALER)
    socket_set_hwm(dealer, PIPELINE)
    dealer.connect("tcp://127.0.0.1:6000")
    credit = PIPELINE   # Up to PIPELINE chunks in transit
    total = 0           # Total bytes received
    chunks = 0          # Total chunks received
    offset = 0          # Offset of next chunk request
    while True:
        while credit:
            # ask for next chunk
            yield from dealer.send_multipart([
                b"fetch",
                b"%i" % total,
                b"%i" % CHUNK_SIZE,
            ])
            offset += CHUNK_SIZE
            credit -= 1
        try:
            chunk = yield from dealer.recv()
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return   # shutting down, quit
            else:
                raise
        chunks += 1
        credit += 1
        size = len(chunk)
        total += size
        if size < CHUNK_SIZE:
            break   # Last chunk received; exit
    yield from dealer.send_multipart([
        b"finish",
        b"-1",
        b"-1",
    ])
    message = "client received %i chunks, %i bytes" % (chunks, total)
    print(message)
    print('(client) finished')
    yield from pipe.send(b"OK")
    return ('client', message)
Пример #14
0
def client_thread(ctx, pipe):
    dealer = ctx.socket(zmq.DEALER)
    socket_set_hwm(dealer, PIPELINE)
    dealer.connect("tcp://127.0.0.1:6000")

    credit = PIPELINE   # Up to PIPELINE chunks in transit

    total = 0           # Total bytes received
    chunks = 0          # Total chunks received
    offset = 0          # Offset of next chunk request

    while True:
        while credit:
            # ask for next chunk
            dealer.send_multipart([
                b"fetch",
                b"%i" % offset,
                b"%i" % CHUNK_SIZE,
            ])

            offset += CHUNK_SIZE
            credit -= 1

        try:
            chunk = dealer.recv()
        except zmq.ZMQError as e:
            if e.errno == zmq.ETERM:
                return   # shutting down, quit
            else:
                raise

        chunks += 1
        credit += 1
        size = len(chunk)
        total += size
        if size < CHUNK_SIZE:
            break   # Last chunk received; exit

    print ("%i chunks received, %i bytes" % (chunks, total))
    pipe.send(b"OK")
Пример #15
0
        for address in expired:
            print "W: Idle worker expired: %s" % address
            self.queue.pop(address, None)
            alive_workers.pop(address, None)

    def next(self):
        address, worker = self.queue.popitem(False)
        return address

context = zmq.Context(1)

frontend = context.socket(zmq.ROUTER) # ROUTER
backend = context.socket(zmq.ROUTER)  # ROUTER
info_sock = context.socket(zmq.REP)

socket_set_hwm(frontend, 5)
socket_set_hwm(backend, 5)

frontend.bind("tcp://*:5559") # For clients
backend.bind("tcp://*:5560")  # For workers
info_sock.bind("tcp://*:5411") # for debugging internal state info

poll_workers = zmq.Poller()
poll_workers.register(backend, zmq.POLLIN)

poll_both = zmq.Poller()
poll_both.register(frontend, zmq.POLLIN)
poll_both.register(backend, zmq.POLLIN)
poll_both.register(info_sock, zmq.POLLIN)

workers = WorkerQueue()