예제 #1
0
    def _on_rpc_update_graph(self, task_id, peer_addr, state, is_final):
        pck = self._by_task_id.get(task_id)

        logging.debug('_on_rpc_update_graph: task_id=%s, pck_id=%s; status=%s; is_final=%s' % (
            task_id,
            pck and pck.id,
            GraphState.str(state['state']),
            is_final
        ))

        if not pck:
            raise WrongTaskIdError('No packet for sbx:%s' % task_id)

        with pck._lock:
            # FIXME
            # pck: connect
            # pck: write
            # rem: enter _on_rpc_update_graph
            # pck: CRASHED
            # pck: Sandbox' task FAILURE
            # rem: _on_task_status_change (enter + exit)
            # rem: _on_rpc_update_graph with self.lock <-- OOPS

            assert pck._state not in [
                RemotePacketState.CREATING,
                RemotePacketState.TASK_FIN_WAIT,
                RemotePacketState.FINISHED,
                RemotePacketState.FETCHING_RESOURCE_LIST,
                RemotePacketState.FETCHING_FINAL_UPDATE
            ], "_on_rpc_update_graph in %s state" % pck._state

            if pck._state in [RemotePacketState.STARTING,
                              RemotePacketState.CHECKING_START_ERROR,
                              RemotePacketState.STARTED]:
                if pck._state != RemotePacketState.STARTED:
                    pck._set_state(RemotePacketState.STARTED, '_on_rpc_update_graph')
                    pck._drop_sched_if_need()
                    assert not pck._peer_addr
                #else:
                    #assert not pck._sched # stop may be scheduled for example

                if not pck._peer_addr:
                    pck._peer_addr = peer_addr
                    logging.debug('SET pck._peer_addr = %s for %s' % (peer_addr, pck))

                    if pck._target_stop_mode:
                        if is_final:
                            pck._sent_stop_mode = pck._target_stop_mode # FIXME
                        else:
                            self._start_packet_stop(pck)

            if pck._target_stop_mode != StopMode.CANCEL:
                pck._update_graph(state, is_final)

            if is_final:
                if pck._target_stop_mode == StopMode.CANCEL \
                    or state['state'] == GraphState.SUCCESSFULL:
                    self._mark_task_fin_wait(pck, '_on_rpc_update_graph(SUCCESSFULL)')
                else:
                    pass # XXX WAITING for TaskStateGroups.TERMINATED
예제 #2
0
    def _update_state(self):
        new = self._calc_state()
        if new == self.state:
            return

        self.state = new
        logging.debug('SandboxJobGraphExecutorProxy.state => %s' % GraphState.str(self.state))
        self._ops.on_state_change()
예제 #3
0
    def _do_on_packet_terminated(self):
        def on_stop():
            self._prev_task_id = self._remote_packet._sandbox_task_id
            self._remote_packet = None
            self._remote_state = None
            self._remote_time_wait_deadline = None

        assert not self.time_wait_deadline and not self.time_wait_sched

        r = self._remote_packet

        self.stopping = False

        if self.cancelled:
            self.cancelled = False
            on_stop()
            return

        # TODO Handle Exception packet state
        # FIXME Rollback history/Status to prev state

        logging.debug('state for SandboxJobGraphExecutorProxy == %s' \
            % None if r._final_state is None else GraphState.str(r._final_state))

        if r._error:
            self._error = r._error

            if r._is_error_permanent:
                self._tries = 0

            elif self._tries:
                retry_idx = self.MAX_TRY_COUNT - self._tries - 1

                delay = self.RETRY_INTERVALS[retry_idx] \
                    if retry_idx < len(self.RETRY_INTERVALS) \
                    else self.RETRY_INTERVALS[-1]

                logging.debug("%s failed. Will retry after %s seconds" % (r, delay))

                self.time_wait_deadline = time.time() + delay

                self._schedule_time_wait_stop()

        elif r._final_state == GraphState.TIME_WAIT:
            self.time_wait_deadline = r._last_update['nearest_retry_deadline']
            self._schedule_time_wait_stop()

        elif r._final_state == GraphState.SUCCESSFULL:
            self.result = True

        elif r._final_state == GraphState.ERROR:
            self.result = False

        self._prev_snapshot_resource_id = r._result_snapshot_resource_id \
            if r._final_state != GraphState.SUCCESSFULL else None

        on_stop()
예제 #4
0
    def _update_graph(self, update, is_final):
        assert self._target_stop_mode != StopMode.CANCEL

        self._last_update = update # TODO

        succeed_jobs = set(map(int, update['succeed_jobs']))

        new_succeed_jobs = succeed_jobs - self._succeeded_jobs
        self._succeeded_jobs = succeed_jobs

        if is_final:
            self._final_state = update['state']

            logging.debug('%s._final_state = %s' % (self, GraphState.str(self._final_state)))

        self._ops._on_sandbox_packet_update(update, new_succeed_jobs, is_final)