def handle_service_message(self): """Handles incoming service messages from supervisor socket""" try: serialized_request = self.remote_control_socket.recv_multipart( flags=zmq.NOBLOCK)[0] except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: return instruction = ServiceMessage.loads(serialized_request)[0] try: response = self.instructions[instruction]() except KeyError: errors_logger.exception("%s instruction not recognized by worker" % instruction) return self.remote_control_socket.send_multipart( [ServiceMessage.dumps(response)], flags=zmq.NOBLOCK) # If halt instruction succedded, raise HaltException # so the worker event loop knows it has to stop if instruction == WORKER_HALT and int(response) == SUCCESS_STATUS: raise HaltException() return
def command(self, instruction, workers_ids=None, max_retries=3, timeout=None): """Command an action to workers. An optional list of workers ids can be provided as an argument, in order to restrain the command to specific workers. """ workers_ids = workers_ids or self.workers.iterkeys() timeout = timeout or self.timeout responses = [] for worker_id in workers_ids: if worker_id in self.workers: worker_socket = self.workers[worker_id]['socket'] request = ServiceMessage.dumps(instruction) self.socket.send_multipart([worker_socket, request], flags=zmq.NOBLOCK) retried = 0 while retried <= max_retries: sockets = dict(self.poller.poll(self.timeout)) if sockets and sockets.get(self.socket) == zmq.POLLIN: serialized_response = self.socket.recv_multipart(flags=zmq.NOBLOCK)[1] responses.append(ServiceMessage.loads(serialized_response)) break else: retried += 1 if retried == max_retries: err_msg = "Instruction %s sent to %s failed. Retried %d times" errors_logger.error(err_msg % (instruction, worker_id, retried)) return responses
def init_workers(self, count): """Starts `count` workers. Awaits for their id to be received (blocking), and registers their socket id and thread reference """ pos = 0 while pos < count: # Start a worker worker = Worker(self.zmq_context, self.databases_store) worker.start() socket_id, response = self.socket.recv_multipart() worker_id = ServiceMessage.loads(response)[0] self.workers[worker_id]['socket'] = socket_id self.workers[worker_id]['thread'] = worker pos += 1
def command(self, instruction, workers_ids=None, max_retries=3, timeout=None): """Command an action to workers. An optional list of workers ids can be provided as an argument, in order to restrain the command to specific workers. """ workers_ids = workers_ids or self.workers.iterkeys() timeout = timeout or self.timeout responses = [] for worker_id in workers_ids: if worker_id in self.workers: worker_socket = self.workers[worker_id]['socket'] request = ServiceMessage.dumps(instruction) self.socket.send_multipart([worker_socket, request], flags=zmq.NOBLOCK) retried = 0 while retried <= max_retries: sockets = dict(self.poller.poll(self.timeout)) if sockets and sockets.get(self.socket) == zmq.POLLIN: serialized_response = self.socket.recv_multipart( flags=zmq.NOBLOCK)[1] responses.append( ServiceMessage.loads(serialized_response)) break else: retried += 1 if retried == max_retries: err_msg = "Instruction %s sent to %s failed. Retried %d times" errors_logger.error(err_msg % (instruction, worker_id, retried)) return responses
def handle_service_message(self): """Handles incoming service messages from supervisor socket""" try: serialized_request = self.remote_control_socket.recv_multipart(flags=zmq.NOBLOCK)[0] except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: return instruction = ServiceMessage.loads(serialized_request)[0] try: response = self.instructions[instruction]() except KeyError: errors_logger.exception("%s instruction not recognized by worker" % instruction) return self.remote_control_socket.send_multipart([ServiceMessage.dumps(response)], flags=zmq.NOBLOCK) # If halt instruction succedded, raise HaltException # so the worker event loop knows it has to stop if instruction == WORKER_HALT and int(response) == SUCCESS_STATUS: raise HaltException() return