def communicate(self, conn): self.connections.add(conn) while True: # Read a message on the socket. msg = yield cw._readmsg(conn) if msg is None: break if isinstance(msg, cw.TaskMessage): self.queued_tasks.append((msg, conn)) elif isinstance(msg, cw.ResultMessage): client = self.active_tasks.pop(msg.jobid) self.idle_workers.append(conn) if client in self.connections: # Ensure client has not disappeared. yield cw._sendmsg(client, msg) elif isinstance(msg, cw.WorkerRegisterMessage): self.idle_workers.append(conn) self._show_workers() elif isinstance(msg, cw.WorkerDepartMessage): self.idle_workers.remove(conn) self._show_workers() else: assert False # Dispatch as many waiting tasks as we can. while self.queued_tasks and self.idle_workers: task_message, client = self.queued_tasks.pop(0) if client in self.connections: worker = self.idle_workers.pop(0) self.active_tasks[task_message.jobid] = client yield cw._sendmsg(worker, task_message) self.connections.remove(conn)
def communicate(self): conn = yield bluelet.connect(self.host, self.port) yield cw._sendmsg(conn, cw.WorkerRegisterMessage()) connected = True try: while True: # Get a task from the master. msg = yield cw._readmsg(conn) if msg is None: print('connection to master closed') connected = False break assert isinstance(msg, cw.TaskMessage) try: with chdir(msg.cwd): func = cw.func_deser(msg.func_blob) args = cw.slow_deser(msg.args_blob) kwargs = cw.slow_deser(msg.kwargs_blob) res = func(*args, **kwargs) except: res = format_remote_exc() response = cw.ResultMessage(msg.jobid, False, cw.slow_ser(res)) else: response = cw.ResultMessage(msg.jobid, True, cw.slow_ser(res)) yield cw._sendmsg(conn, response) finally: if connected: yield cw._sendmsg(conn, cw.WorkerDepartMessage())
def send_job(self, jobid, func, *args, **kwargs): task = cw.TaskMessage( jobid, cw.func_ser(func), cw.slow_ser(args), cw.slow_ser(kwargs), os.getcwd(), ) yield cw._sendmsg(self.conn, task)
def send_job(self, jobid, func, *args, **kwargs): task = cw.TaskMessage( jobid, cw.func_ser(func), cw.slow_ser(args), cw.slow_ser(kwargs), os.getcwd(), sys.path, ) yield cw._sendmsg(self.conn, task)
def communicate(self): conn = yield bluelet.connect(self.host, self.port) yield cw._sendmsg(conn, cw.WorkerRegisterMessage()) connected = True try: while True: # Get a task from the master. msg = yield cw._readmsg(conn) if msg is None: print('connection to master closed') connected = False break assert isinstance(msg, cw.TaskMessage) try: with chdir(msg.cwd): func = cw.func_deser(msg.func_blob) args = cw.slow_deser(msg.args_blob) kwargs = cw.slow_deser(msg.kwargs_blob) res = func(*args, **kwargs) except Exception as exception: # Check if the function tells us to shut down this worker. # We can't just catch the exception normally - # for some reason between serialization things get # fouled up. if type(exception).__name__ == 'ShutdownSignal': res = 'shutdown signal received' response = cw.ResultMessage( msg.jobid, True, cw.slow_ser(res), True) yield cw._sendmsg(conn, response) break res = format_remote_exc() response = cw.ResultMessage(msg.jobid, False, cw.slow_ser(res), False) else: response = cw.ResultMessage(msg.jobid, True, cw.slow_ser(res), False) yield cw._sendmsg(conn, response) finally: if connected: yield cw._sendmsg(conn, cw.WorkerDepartMessage())