def run(self, n_workers, callback=DummyCallback(), device=zmq.device):
loadbal_callback = type(callback)()
loadbal_addr = self.run_device(callback=loadbal_callback,
device=device)
# We need the load balancer to be bound before continuing.
loadbal_callback.wait()
with callback.catch_exceptions():
server = Server(loadbal_addr,
self.registry,
connect=True,
context=self.context)
server_threads = []
server_callbacks = []
run_logger.debug("Spawning {0} workers", n_workers)
for i in xrange(n_workers):
server_callback = type(callback)()
server_thread = server_callback.spawn(
server.run, kwargs={'callback': server_callback})
server_threads.append(server_thread)
server_callbacks.append(server_callback)
run_logger.debug("Spawned {0} workers", n_workers)
# Wait for all callbacks to complete before triggering ours.
server_sockets = [
server_callback.wait() for server_callback in server_callbacks
]
callback.send(server_sockets)
return loadbal_addr, server_threads
def run(self, n_workers, callback=DummyCallback(), daemon=True):
loadbal_callback = type(callback)()
loadbal_addr = self.run_device(callback=loadbal_callback,
daemon=daemon)
# We need the load balancer to be bound before continuing.
loadbal_callback.wait()
with callback.catch_exceptions():
server = Server(loadbal_addr,
self.registry,
connect=True,
context=self.context)
server_threads = []
server_callbacks = []
for i in xrange(n_workers):
server_callback = type(callback)()
server_thread = threading.Thread(
target=server.run, kwargs={'callback': server_callback})
server_thread.daemon = daemon
server_thread.start()
server_threads.append(server_thread)
server_callbacks.append(server_callback)
# Wait for all callbacks to complete before triggering ours.
server_sockets = [
server_callback.wait() for server_callback in server_callbacks
]
callback.send(server_sockets)
return loadbal_addr, server_threads
def run(self, die_after=None, callback=DummyCallback()):
"""
Run the worker, optionally dying after a number of requests.
:param int die_after:
Die after processing a set number of messages (default: continue
forever).
:param callback:
A :class:`~zrpc.concurrency.Callback` which will be called with the
socket once it has been successfully connected or bound.
"""
with callback.catch_exceptions():
socket = self.context.socket(zmq.REP)
if self.connect:
run_logger.debug("Replying to requests from {0!r}", self.addr)
socket.connect(self.addr)
else:
run_logger.debug("Listening for requests on {0!r}", self.addr)
socket.bind(self.addr)
callback.send(socket)
iterator = die_after and repeat(None, die_after) or repeat(None)
with nested(run_logger.catch_exceptions(), closing(socket)):
try:
for _ in iterator:
message = BSON(socket.recv()).decode()
socket.send(BSON.encode(self.process_message(message)))
except zmq.ZMQError, exc:
if exc.errno == zmq.ETERM:
pass
else:
raise
def run_device(self, callback=DummyCallback(), device=zmq.device):
"""
Run the ZMQ queue device in a background thread.
The return value of this function will be the address of the device's
output socket (an `inproc://`-type address). In order to kill the
background thread, you need to terminate the context the device is
attached to.
"""
output_addr = 'inproc://%s' % uuid.uuid4()
loadbal = LoadBalancer(self.addr, output_addr, context=self.context)
loadbal_thread = callback.spawn(loadbal.run,
kwargs={
'callback': callback,
'device': device
})
return output_addr
def run_device(self, daemon=True, callback=DummyCallback()):
"""
Run the ZMQ queue device in a background thread.
The return value of this function will be the address of the device's
output socket (an `inproc://`-type address). In order to kill the
background thread, you need to terminate the context the device is
attached to.
"""
output_addr = 'inproc://%s' % uuid.uuid4()
loadbal = LoadBalancer(self.addr, output_addr, context=self.context)
loadbal_thread = threading.Thread(target=loadbal.run,
kwargs={'callback': callback})
loadbal_thread.daemon = daemon
loadbal_thread.start()
return output_addr
def run(self, callback=DummyCallback(), device=zmq.device):
"""
Run the load balancer.
:param callback:
A :class:`~zrpc.concurrency.Callback` which will be called with the
input and output sockets once they have been successfully connected
or bound.
"""
with callback.catch_exceptions():
logger.debug("Listening for requests on {0!r}", self.input)
input_socket = self.get_socket(zmq.XREP)
input_socket.bind(self.input)
output_socket = self.get_socket(zmq.XREQ)
if hasattr(self.output, '__iter__'):
logger.debug("Connecting to {0} workers", len(self.output))
for node in self.output:
output_socket.connect(node)
else:
logger.debug("Listening for workers on {0!r}", self.output)
output_socket.bind(self.output)
callback.send((input_socket, output_socket))
with nested(run_logger.catch_exceptions(),
closing(input_socket),
closing(output_socket)):
try:
device(zmq.QUEUE, input_socket, output_socket)
except zmq.ZMQError, exc:
if exc.errno in (0, zmq.ETERM, zmq.EAGAIN):
run_logger.info("Context was terminated, shutting down")
else:
raise
except KeyboardInterrupt:
run_logger.info("SIGINT received, shutting down")
def run(self, callback=DummyCallback()):
"""
Run the load balancer.
:param callback:
A :class:`~zrpc.concurrency.Callback` which will be called with the
input and output sockets once they have been successfully connected
or bound.
"""
with callback.catch_exceptions():
logger.debug("Listening for requests on {0!r}", self.input)
input_socket = self.context.socket(zmq.XREP)
input_socket.bind(self.input)
output_socket = self.context.socket(zmq.XREQ)
if hasattr(self.output, '__iter__'):
logger.debug("Connecting to {0} workers", len(self.output))
for node in self.output:
output_socket.connect(node)
else:
logger.debug("Listening for workers on {0!r}", self.output)
output_socket.bind(self.output)
callback.send((input_socket, output_socket))
with nested(logger.catch_exceptions(),
closing(input_socket),
closing(output_socket)):
try:
zmq.device(zmq.QUEUE, input_socket, output_socket)
except zmq.ZMQError, exc:
if exc.errno == zmq.ETERM:
pass
else:
raise