def client_task(ctx, pipe):
dealer = ctx.socket(zmq.DEALER)
socket_set_hwm(dealer, 1)
dealer.connect("tcp://127.0.0.1:6000")
total = 0 # Total bytes received
chunks = 0 # Total chunks received
while True:
# ask for next chunk
yield dealer.send_multipart(
[b"fetch", b"%i" % total,
b"%i" % CHUNK_SIZE])
try:
chunk = yield dealer.recv()
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
chunks += 1
size = len(chunk)
total += size
if size < CHUNK_SIZE:
break # Last chunk received; exit
yield dealer.send_multipart([
b"finish",
b"-1",
b"-1",
])
message = "client received %i chunks, %i bytes" % (chunks, total)
print(message)
yield pipe.send(b"OK")
print('(client) finished')
raise gen.Return(('client', message))
def client_task(ctx, pipe):
dealer = ctx.socket(zmq.DEALER)
socket_set_hwm(dealer, 1)
dealer.connect("tcp://127.0.0.1:6000")
total = 0 # Total bytes received
chunks = 0 # Total chunks received
while True:
# ask for next chunk
yield from dealer.send_multipart([
b"fetch",
b"%i" % total,
b"%i" % CHUNK_SIZE
])
try:
chunk = yield from dealer.recv()
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
chunks += 1
size = len(chunk)
total += size
if size < CHUNK_SIZE:
break # Last chunk received; exit
yield from dealer.send_multipart([
b"finish",
b"-1",
b"-1",
])
message = "client received %i chunks, %i bytes" % (chunks, total)
print(message)
yield from pipe.send(b"OK")
print('(client) finished')
return ('client', message)
def client_thread(ctx, pipe):
dealer = ctx.socket(zmq.DEALER)
socket_set_hwm(dealer, 1)
dealer.connect("tcp://127.0.0.1:6000")
total = 0 # Total bytes received
chunks = 0 # Total chunks received
while True:
# ask for next chunk
dealer.send_multipart([
b"fetch",
b"%i" % total,
b"%i" % CHUNK_SIZE
])
try:
chunk = dealer.recv()
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
chunks += 1
size = len(chunk)
total += size
if size < CHUNK_SIZE:
break # Last chunk received; exit
print ("%i chunks received, %i bytes" % (chunks, total))
pipe.send(b"OK")
def server_thread(ctx):
file = open("testdata", "rb")
router = ctx.socket(zmq.ROUTER)
socket_set_hwm(router, PIPELINE)
router.bind("tcp://*:6000")
while True:
# First frame in each message is the sender identity
# Second frame is "fetch" command
try:
msg = router.recv_multipart()
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
identity, command, offset_str, chunksz_str = msg
assert command == b"fetch"
offset = int(offset_str)
chunksz = int(chunksz_str)
# Read chunk of data from file
file.seek(offset, os.SEEK_SET)
data = file.read(chunksz)
# Send resulting chunk to client
router.send_multipart([identity, data])
def server_thread(ctx):
file = open("testdata", "rb")
router = ctx.socket(zmq.ROUTER)
# Default HWM is 1000, which will drop messages here
# since we send more than 1,000 chunks of test data,
# so set an infinite HWM as a simple, stupid solution:
socket_set_hwm(router, 0)
router.bind("tcp://*:6000")
while True:
# First frame in each message is the sender identity
# Second frame is "fetch" command
try:
identity, command = router.recv_multipart()
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
assert command == b"fetch"
while True:
data = file.read(CHUNK_SIZE)
router.send_multipart([identity, data])
if not data:
break
def server_task(ctx):
file = open("testdata", "r")
router = ctx.socket(zmq.ROUTER)
# Default HWM is 1000, which will drop messages here
# since we send more than 1,000 chunks of test data,
# so set an infinite HWM as a simple, stupid solution:
socket_set_hwm(router, 0)
router.bind("tcp://*:6000")
count = 0
while True:
# First frame in each message is the sender identity
# Second frame is "fetch" command
try:
identity, command = yield router.recv_multipart()
break
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
assert command == b"fetch"
while True:
data = file.read(CHUNK_SIZE)
data = data.encode('utf-8')
yield router.send_multipart([identity, data])
if not data:
break
count += 1
message = 'server sent {} chunks'.format(count)
print(message)
print('(server_task) finished')
raise gen.Return(('server', message))
def server_thread(ctx):
file = open("testdata", "r")
router = ctx.socket(zmq.ROUTER)
# Default HWM is 1000, which will drop messages here
# since we send more than 1,000 chunks of test data,
# so set an infinite HWM as a simple, stupid solution:
socket_set_hwm(router, 0)
router.bind("tcp://*:6000")
while True:
# First frame in each message is the sender identity
# Second frame is "fetch" command
try:
identity, command = router.recv_multipart()
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
assert command == b"fetch"
while True:
data = file.read(CHUNK_SIZE)
router.send_multipart([identity, data])
if not data:
break
def server_task(ctx):
file = open("testdata", "r")
router = ctx.socket(zmq.ROUTER)
# Default HWM is 1000, which will drop messages here
# since we send more than 1,000 chunks of test data,
# so set an infinite HWM as a simple, stupid solution:
socket_set_hwm(router, 0)
router.bind("tcp://*:6000")
count = 0
while True:
# First frame in each message is the sender identity
# Second frame is "fetch" command
try:
identity, command = yield router.recv_multipart()
break
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
assert command == b"fetch"
while True:
data = file.read(CHUNK_SIZE)
data = data.encode('utf-8')
yield router.send_multipart([identity, data])
if not data:
break
count += 1
message = 'server sent {} chunks'.format(count)
print(message)
print('(server_task) finished')
raise gen.Return(('server', message))
def server_task(ctx):
file = open("testdata", "r")
router = ctx.socket(zmq.ROUTER)
socket_set_hwm(router, PIPELINE)
router.bind("tcp://*:6000")
count = 0
total = 0
while True:
# First frame in each message is the sender identity
# Second frame is "fetch" command
try:
msg = yield router.recv_multipart()
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
identity, command, offset_str, chunksz_str = msg
if command == b"finish":
break
assert command == b"fetch"
offset = int(offset_str)
chunksz = int(chunksz_str)
# Read chunk of data from file
file.seek(offset, os.SEEK_SET)
data = file.read(chunksz)
data = data.encode('utf-8')
# Send resulting chunk to client
yield router.send_multipart([identity, data])
print('(server) sent chunk: {} size: {}'.format(count, len(data)))
count += 1
total += len(data)
message = 'server sent {} chunks, {} bytes'.format(count, total)
print('(server) finished')
raise gen.Return(('server', message))
def client_thread(ctx, pipe):
dealer = ctx.socket(zmq.DEALER)
socket_set_hwm(dealer, 1)
dealer.connect("tcp://127.0.0.1:6000")
total = 0 # Total bytes received
chunks = 0 # Total chunks received
while True:
# ask for next chunk
dealer.send_multipart([b"fetch", b"%i" % total, b"%i" % CHUNK_SIZE])
try:
chunk = dealer.recv()
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
chunks += 1
size = len(chunk)
total += size
if size < CHUNK_SIZE:
break # Last chunk received; exit
print("%i chunks received, %i bytes" % (chunks, total))
pipe.send(b"%s" % chunks)
def server_task(ctx):
print('(server) starting')
infile = open("testdata", "r")
print('(server) setup 1')
router = ctx.socket(zmq.ROUTER)
# Default HWM is 1000, which will drop messages here
# since we send more than 1,000 chunks of test data,
# so set an infinite HWM as a simple, stupid solution:
print('(server) setup 2')
socket_set_hwm(router, 0)
router.bind("tcp://*:6000")
print('(server) after setup')
count = 0
total = 0
while True:
# First frame in each message is the sender identity
# Second frame is "fetch" command
print('(server) waiting for request')
try:
identity, command = yield from router.recv_multipart()
break
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
print('(server) received request. command: {}'.format(command))
assert command == b"fetch"
while True:
data = infile.read(CHUNK_SIZE)
data = data.encode('utf-8')
yield from router.send_multipart([identity, data])
print('(server) sent chunk. length: {}'.format(len(data)))
if not data:
break
count += 1
total += len(data)
message = 'server sent {} chunks, {} bytes'.format(count, total)
print(message)
print('(server_task) finished')
return ('server', message)
def server_task(ctx):
print('(server) starting')
infile = open("testdata", "r")
print('(server) setup 1')
router = ctx.socket(zmq.ROUTER)
# Default HWM is 1000, which will drop messages here
# since we send more than 1,000 chunks of test data,
# so set an infinite HWM as a simple, stupid solution:
print('(server) setup 2')
socket_set_hwm(router, 0)
router.bind("tcp://*:6000")
print('(server) after setup')
count = 0
total = 0
while True:
# First frame in each message is the sender identity
# Second frame is "fetch" command
print('(server) waiting for request')
try:
identity, command = yield from router.recv_multipart()
break
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
print('(server) received request. command: {}'.format(command))
assert command == b"fetch"
while True:
data = infile.read(CHUNK_SIZE)
data = data.encode('utf-8')
yield from router.send_multipart([identity, data])
print('(server) sent chunk. length: {}'.format(len(data)))
if not data:
break
count += 1
total += len(data)
message = 'server sent {} chunks, {} bytes'.format(count, total)
print(message)
print('(server_task) finished')
return ('server', message)
def client_task(ctx, pipe):
dealer = ctx.socket(zmq.DEALER)
socket_set_hwm(dealer, PIPELINE)
dealer.connect("tcp://127.0.0.1:6000")
credit = PIPELINE # Up to PIPELINE chunks in transit
total = 0 # Total bytes received
chunks = 0 # Total chunks received
offset = 0 # Offset of next chunk request
while True:
while credit:
# ask for next chunk
yield from dealer.send_multipart([
b"fetch",
b"%i" % total,
b"%i" % CHUNK_SIZE,
])
offset += CHUNK_SIZE
credit -= 1
try:
chunk = yield from dealer.recv()
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
chunks += 1
credit += 1
size = len(chunk)
total += size
if size < CHUNK_SIZE:
break # Last chunk received; exit
yield from dealer.send_multipart([
b"finish",
b"-1",
b"-1",
])
message = "client received %i chunks, %i bytes" % (chunks, total)
print(message)
print('(client) finished')
yield from pipe.send(b"OK")
return ('client', message)
def client_thread(ctx, pipe):
dealer = ctx.socket(zmq.DEALER)
socket_set_hwm(dealer, PIPELINE)
dealer.connect("tcp://127.0.0.1:6000")
credit = PIPELINE # Up to PIPELINE chunks in transit
total = 0 # Total bytes received
chunks = 0 # Total chunks received
offset = 0 # Offset of next chunk request
while True:
while credit:
# ask for next chunk
dealer.send_multipart([
b"fetch",
b"%i" % offset,
b"%i" % CHUNK_SIZE,
])
offset += CHUNK_SIZE
credit -= 1
try:
chunk = dealer.recv()
except zmq.ZMQError as e:
if e.errno == zmq.ETERM:
return # shutting down, quit
else:
raise
chunks += 1
credit += 1
size = len(chunk)
total += size
if size < CHUNK_SIZE:
break # Last chunk received; exit
print ("%i chunks received, %i bytes" % (chunks, total))
pipe.send(b"OK")
for address in expired:
print "W: Idle worker expired: %s" % address
self.queue.pop(address, None)
alive_workers.pop(address, None)
def next(self):
address, worker = self.queue.popitem(False)
return address
context = zmq.Context(1)
frontend = context.socket(zmq.ROUTER) # ROUTER
backend = context.socket(zmq.ROUTER) # ROUTER
info_sock = context.socket(zmq.REP)
socket_set_hwm(frontend, 5)
socket_set_hwm(backend, 5)
frontend.bind("tcp://*:5559") # For clients
backend.bind("tcp://*:5560") # For workers
info_sock.bind("tcp://*:5411") # for debugging internal state info
poll_workers = zmq.Poller()
poll_workers.register(backend, zmq.POLLIN)
poll_both = zmq.Poller()
poll_both.register(frontend, zmq.POLLIN)
poll_both.register(backend, zmq.POLLIN)
poll_both.register(info_sock, zmq.POLLIN)
workers = WorkerQueue()
