def fail_unreachable(self, msg_id, why=error.ImpossibleDependency): """a task has become unreachable, send a reply with an ImpossibleDependency error.""" if msg_id not in self.queue_map: self.log.error("task %r already failed!", msg_id) return job = self.queue_map.pop(msg_id) # lazy-delete from the queue job.removed = True for mid in job.dependents: if mid in self.graph: self.graph[mid].remove(msg_id) try: raise why() except: content = error.wrap_exception() self.log.debug("task %r failing as unreachable with: %s", msg_id, content['ename']) self.all_done.add(msg_id) self.all_failed.add(msg_id) msg = self.session.send( self.client_stream, 'apply_reply', content, parent=job.header, ident=job.idents, ) self.session.send(self.mon_stream, msg, ident=[b'outtask'] + job.idents) self.update_graph(msg_id, success=False)
def handle_stranded_tasks(self, engine): """Deal with jobs resident in an engine that died.""" lost = self.pending[engine] for msg_id in list(lost.keys()): if msg_id not in lost: # prevent double-handling of messages continue raw_msg = lost[msg_id].raw_msg idents, msg = self.session.feed_identities(raw_msg, copy=False) parent = self.session.unpack(msg[1].bytes) idents = [engine, idents[0]] # build fake error reply try: raise error.EngineError( "Engine %r died while running task %r" % (engine, msg_id)) except: content = error.wrap_exception() # build fake metadata md = dict(status=u'error', engine=engine.decode('ascii'), date=util.utcnow()) msg = self.session.msg('apply_reply', content, parent=parent, metadata=md) raw_reply = list( map(zmq.Message, self.session.serialize(msg, ident=idents))) # and dispatch it self.dispatch_result(raw_reply) # finally scrub completed/failed lists self.completed.pop(engine) self.failed.pop(engine)
def handle_stranded_tasks(self, engine): """Deal with jobs resident in an engine that died.""" lost = self.pending[engine] for msg_id in lost.keys(): if msg_id not in self.pending[engine]: # prevent double-handling of messages continue raw_msg = lost[msg_id].raw_msg idents,msg = self.session.feed_identities(raw_msg, copy=False) parent = self.session.unpack(msg[1].bytes) idents = [engine, idents[0]] # build fake error reply try: raise error.EngineError("Engine %r died while running task %r"%(engine, msg_id)) except: content = error.wrap_exception() # build fake metadata md = dict( status=u'error', engine=engine.decode('ascii'), date=util.utcnow(), ) msg = self.session.msg('apply_reply', content, parent=parent, metadata=md) raw_reply = list(map(zmq.Message, self.session.serialize(msg, ident=idents))) # and dispatch it self.dispatch_result(raw_reply) # finally scrub completed/failed lists self.completed.pop(engine) self.failed.pop(engine)
def fail_unreachable(self, msg_id, why=error.ImpossibleDependency): """a task has become unreachable, send a reply with an ImpossibleDependency error.""" if msg_id not in self.queue_map: self.log.error("task %r already failed!", msg_id) return job = self.queue_map.pop(msg_id) # lazy-delete from the queue job.removed = True for mid in job.dependents: if mid in self.graph: self.graph[mid].remove(msg_id) try: raise why() except: content = error.wrap_exception() self.log.debug("task %r failing as unreachable with: %s", msg_id, content['ename']) self.all_done.add(msg_id) self.all_failed.add(msg_id) msg = self.session.send(self.client_stream, 'apply_reply', content, parent=job.header, ident=job.idents) self.session.send(self.mon_stream, msg, ident=[b'outtask']+job.idents) self.update_graph(msg_id, success=False)
def dispatch_control(self, stream, raw_msg): """Dispatch message from the control scheduler If we have a handler registered""" try: idents, msg_frames = self.session.feed_identities(raw_msg) except Exception as e: self.log.error(f"Bad control message: {raw_msg}", exc_info=True) return try: msg = self.session.deserialize(msg_frames, content=True) except Exception: content = error.wrap_exception() self.log.error("Bad control message: %r", msg_frames, exc_info=True) return msg_type = msg['header']['msg_type'] if msg_type.endswith("_request"): reply_type = msg_type[-len("_request"):] else: reply_type = "error" self.log.debug(f"Client {idents[-1]} requested {msg_type}") handler = self.control_handlers.get(msg_type, None) if handler is None: # don't have an intercept handler, relay original message to parent self.log.debug(f"Relaying {msg_type} {msg['header']['msg_id']}") self.parent_stream.send_multipart(raw_msg) return try: content = handler(msg['content']) except Exception: content = error.wrap_exception() self.log.error("Error handling request: %r", msg_type, exc_info=True) self.session.send(stream, reply_type, ident=idents, content=content, parent=msg)