def handle_service_message(self):
"""Handles incoming service messages from supervisor socket"""
try:
serialized_request = self.remote_control_socket.recv_multipart(
flags=zmq.NOBLOCK)[0]
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
return
instruction = ServiceMessage.loads(serialized_request)[0]
try:
response = self.instructions[instruction]()
except KeyError:
errors_logger.exception("%s instruction not recognized by worker" %
instruction)
return
self.remote_control_socket.send_multipart(
[ServiceMessage.dumps(response)], flags=zmq.NOBLOCK)
# If halt instruction succedded, raise HaltException
# so the worker event loop knows it has to stop
if instruction == WORKER_HALT and int(response) == SUCCESS_STATUS:
raise HaltException()
return
def command(self, instruction,
workers_ids=None, max_retries=3,
timeout=None):
"""Command an action to workers.
An optional list of workers ids can be provided
as an argument, in order to restrain the command
to specific workers.
"""
workers_ids = workers_ids or self.workers.iterkeys()
timeout = timeout or self.timeout
responses = []
for worker_id in workers_ids:
if worker_id in self.workers:
worker_socket = self.workers[worker_id]['socket']
request = ServiceMessage.dumps(instruction)
self.socket.send_multipart([worker_socket, request], flags=zmq.NOBLOCK)
retried = 0
while retried <= max_retries:
sockets = dict(self.poller.poll(self.timeout))
if sockets and sockets.get(self.socket) == zmq.POLLIN:
serialized_response = self.socket.recv_multipart(flags=zmq.NOBLOCK)[1]
responses.append(ServiceMessage.loads(serialized_response))
break
else:
retried += 1
if retried == max_retries:
err_msg = "Instruction %s sent to %s failed. Retried %d times"
errors_logger.error(err_msg % (instruction, worker_id, retried))
return responses
def wire_sockets(self):
"""Connects the worker sockets to their endpoints, and sends
alive signal to the supervisor"""
self.backend_socket.connect('inproc://backend')
self.remote_control_socket.connect('inproc://supervisor')
self.remote_control_socket.send_multipart(
[ServiceMessage.dumps(self.uid)])
def command(self,
instruction,
workers_ids=None,
max_retries=3,
timeout=None):
"""Command an action to workers.
An optional list of workers ids can be provided
as an argument, in order to restrain the command
to specific workers.
"""
workers_ids = workers_ids or self.workers.iterkeys()
timeout = timeout or self.timeout
responses = []
for worker_id in workers_ids:
if worker_id in self.workers:
worker_socket = self.workers[worker_id]['socket']
request = ServiceMessage.dumps(instruction)
self.socket.send_multipart([worker_socket, request],
flags=zmq.NOBLOCK)
retried = 0
while retried <= max_retries:
sockets = dict(self.poller.poll(self.timeout))
if sockets and sockets.get(self.socket) == zmq.POLLIN:
serialized_response = self.socket.recv_multipart(
flags=zmq.NOBLOCK)[1]
responses.append(
ServiceMessage.loads(serialized_response))
break
else:
retried += 1
if retried == max_retries:
err_msg = "Instruction %s sent to %s failed. Retried %d times"
errors_logger.error(err_msg %
(instruction, worker_id, retried))
return responses
def handle_service_message(self):
"""Handles incoming service messages from supervisor socket"""
try:
serialized_request = self.remote_control_socket.recv_multipart(flags=zmq.NOBLOCK)[0]
except zmq.ZMQError as e:
if e.errno == zmq.EAGAIN:
return
instruction = ServiceMessage.loads(serialized_request)[0]
try:
response = self.instructions[instruction]()
except KeyError:
errors_logger.exception("%s instruction not recognized by worker" % instruction)
return
self.remote_control_socket.send_multipart([ServiceMessage.dumps(response)], flags=zmq.NOBLOCK)
# If halt instruction succedded, raise HaltException
# so the worker event loop knows it has to stop
if instruction == WORKER_HALT and int(response) == SUCCESS_STATUS:
raise HaltException()
return
def init_workers(self, count):
"""Starts `count` workers.
Awaits for their id to be received (blocking), and
registers their socket id and thread reference
"""
pos = 0
while pos < count:
# Start a worker
worker = Worker(self.zmq_context, self.databases_store)
worker.start()
socket_id, response = self.socket.recv_multipart()
worker_id = ServiceMessage.loads(response)[0]
self.workers[worker_id]['socket'] = socket_id
self.workers[worker_id]['thread'] = worker
pos += 1
def init_workers(self, count):
"""Starts `count` workers.
Awaits for their id to be received (blocking), and
registers their socket id and thread reference
"""
pos = 0
while pos < count:
# Start a worker
worker = Worker(self.zmq_context, self.databases_store)
worker.start()
socket_id, response = self.socket.recv_multipart()
worker_id = ServiceMessage.loads(response)[0]
self.workers[worker_id]['socket'] = socket_id
self.workers[worker_id]['thread'] = worker
pos += 1
def wire_sockets(self):
"""Connects the worker sockets to their endpoints, and sends
alive signal to the supervisor"""
self.backend_socket.connect("inproc://backend")
self.remote_control_socket.connect("inproc://supervisor")
self.remote_control_socket.send_multipart([ServiceMessage.dumps(self.uid)])