コード例 #1
0
ファイル: task_scheduler.py プロジェクト: tomoboy/ipyparallel
    def fail_unreachable(self, msg_id, why=error.ImpossibleDependency):
        """a task has become unreachable, send a reply with an ImpossibleDependency
        error."""
        if msg_id not in self.queue_map:
            self.log.error("task %r already failed!", msg_id)
            return
        job = self.queue_map.pop(msg_id)
        # lazy-delete from the queue
        job.removed = True
        for mid in job.dependents:
            if mid in self.graph:
                self.graph[mid].remove(msg_id)

        try:
            raise why()
        except:
            content = error.wrap_exception()
        self.log.debug("task %r failing as unreachable with: %s", msg_id,
                       content['ename'])

        self.all_done.add(msg_id)
        self.all_failed.add(msg_id)

        msg = self.session.send(
            self.client_stream,
            'apply_reply',
            content,
            parent=job.header,
            ident=job.idents,
        )
        self.session.send(self.mon_stream,
                          msg,
                          ident=[b'outtask'] + job.idents)

        self.update_graph(msg_id, success=False)
コード例 #2
0
ファイル: task_scheduler.py プロジェクト: tomoboy/ipyparallel
    def handle_stranded_tasks(self, engine):
        """Deal with jobs resident in an engine that died."""
        lost = self.pending[engine]
        for msg_id in list(lost.keys()):
            if msg_id not in lost:
                # prevent double-handling of messages
                continue

            raw_msg = lost[msg_id].raw_msg
            idents, msg = self.session.feed_identities(raw_msg, copy=False)
            parent = self.session.unpack(msg[1].bytes)
            idents = [engine, idents[0]]

            # build fake error reply
            try:
                raise error.EngineError(
                    "Engine %r died while running task %r" % (engine, msg_id))
            except:
                content = error.wrap_exception()
            # build fake metadata
            md = dict(status=u'error',
                      engine=engine.decode('ascii'),
                      date=util.utcnow())
            msg = self.session.msg('apply_reply',
                                   content,
                                   parent=parent,
                                   metadata=md)
            raw_reply = list(
                map(zmq.Message, self.session.serialize(msg, ident=idents)))
            # and dispatch it
            self.dispatch_result(raw_reply)

        # finally scrub completed/failed lists
        self.completed.pop(engine)
        self.failed.pop(engine)
コード例 #3
0
ファイル: scheduler.py プロジェクト: Ssekhar2017/ipyparallel
    def handle_stranded_tasks(self, engine):
        """Deal with jobs resident in an engine that died."""
        lost = self.pending[engine]
        for msg_id in lost.keys():
            if msg_id not in self.pending[engine]:
                # prevent double-handling of messages
                continue

            raw_msg = lost[msg_id].raw_msg
            idents,msg = self.session.feed_identities(raw_msg, copy=False)
            parent = self.session.unpack(msg[1].bytes)
            idents = [engine, idents[0]]

            # build fake error reply
            try:
                raise error.EngineError("Engine %r died while running task %r"%(engine, msg_id))
            except:
                content = error.wrap_exception()
            # build fake metadata
            md = dict(
                status=u'error',
                engine=engine.decode('ascii'),
                date=util.utcnow(),
            )
            msg = self.session.msg('apply_reply', content, parent=parent, metadata=md)
            raw_reply = list(map(zmq.Message, self.session.serialize(msg, ident=idents)))
            # and dispatch it
            self.dispatch_result(raw_reply)

        # finally scrub completed/failed lists
        self.completed.pop(engine)
        self.failed.pop(engine)
コード例 #4
0
ファイル: scheduler.py プロジェクト: Ssekhar2017/ipyparallel
    def fail_unreachable(self, msg_id, why=error.ImpossibleDependency):
        """a task has become unreachable, send a reply with an ImpossibleDependency
        error."""
        if msg_id not in self.queue_map:
            self.log.error("task %r already failed!", msg_id)
            return
        job = self.queue_map.pop(msg_id)
        # lazy-delete from the queue
        job.removed = True
        for mid in job.dependents:
            if mid in self.graph:
                self.graph[mid].remove(msg_id)

        try:
            raise why()
        except:
            content = error.wrap_exception()
        self.log.debug("task %r failing as unreachable with: %s", msg_id, content['ename'])

        self.all_done.add(msg_id)
        self.all_failed.add(msg_id)

        msg = self.session.send(self.client_stream, 'apply_reply', content,
                                                parent=job.header, ident=job.idents)
        self.session.send(self.mon_stream, msg, ident=[b'outtask']+job.idents)

        self.update_graph(msg_id, success=False)
コード例 #5
0
    def dispatch_control(self, stream, raw_msg):
        """Dispatch message from the control scheduler

        If we have a handler registered"""
        try:
            idents, msg_frames = self.session.feed_identities(raw_msg)
        except Exception as e:
            self.log.error(f"Bad control message: {raw_msg}", exc_info=True)
            return

        try:
            msg = self.session.deserialize(msg_frames, content=True)
        except Exception:
            content = error.wrap_exception()
            self.log.error("Bad control message: %r",
                           msg_frames,
                           exc_info=True)
            return

        msg_type = msg['header']['msg_type']
        if msg_type.endswith("_request"):
            reply_type = msg_type[-len("_request"):]
        else:
            reply_type = "error"
        self.log.debug(f"Client {idents[-1]} requested {msg_type}")

        handler = self.control_handlers.get(msg_type, None)
        if handler is None:
            # don't have an intercept handler, relay original message to parent
            self.log.debug(f"Relaying {msg_type} {msg['header']['msg_id']}")
            self.parent_stream.send_multipart(raw_msg)
            return

        try:
            content = handler(msg['content'])
        except Exception:
            content = error.wrap_exception()
            self.log.error("Error handling request: %r",
                           msg_type,
                           exc_info=True)

        self.session.send(stream,
                          reply_type,
                          ident=idents,
                          content=content,
                          parent=msg)