Пример #1
0
    def execute(self, job, timeout=None, log_exceptions=True):
        """Runs the job

        Options:

        - **job**: Job to be performed. Can be a :class:`Job`
          instance or a string. If it's a string a :class:`Job` instance
          will be automatically created out of it.
        - **timeout**: maximum allowed time for a job to run.
          If not provided, uses the one defined in the constructor.

        If the job fails after the timeout, raises a :class:`TimeoutError`.

        This method is thread-safe and uses a lock. If you need to execute a
        lot of jobs simultaneously on a broker, use the :class:`Pool` class.

        """
        if timeout is None:
            timeout = self.timeout_max_overflow

        try:
            duration, res = timed(self.debug)(self._execute)(job, timeout)
        except Exception:
            # logged, connector replaced.
            if log_exceptions:
                logger.exception('Failed to execute the job.')
            raise

        if 'error' in res:
            raise ValueError(res['error'])

        return res['result']
Пример #2
0
    def execute(self, job, timeout=None, log_exceptions=True):
        """Runs the job

        Options:

        - **job**: Job to be performed. Can be a :class:`Job`
          instance or a string. If it's a string a :class:`Job` instance
          will be automatically created out of it.
        - **timeout**: maximum allowed time for a job to run.
          If not provided, uses the one defined in the constructor.

        If the job fails after the timeout, raises a :class:`TimeoutError`.

        This method is thread-safe and uses a lock. If you need to execute a
        lot of jobs simultaneously on a broker, use the :class:`Pool` class.

        """
        if timeout is None:
            timeout = self.timeout_max_overflow

        try:
            duration, res = timed(self.debug)(self._execute)(job, timeout)
        except Exception:
            # logged, connector replaced.
            if log_exceptions:
                logger.exception('Failed to execute the job.')
            raise

        if 'error' in res:
            raise ValueError(res['error'])

        return res['result']
Пример #3
0
    def execute(self, job, timeout=None, extract=True, log_exceptions=True):
        """Runs the job

        Options:

        - **job**: Job to be performed. Can be a :class:`Job`
          instance or a string. If it's a string a :class:`Job` instance
          will be automatically created out of it.
        - **timeout**: maximum allowed time for a job to run.
          If not provided, uses the one defined in the constructor.

        If the job fails after the timeout, raises a :class:`TimeoutError`.

        This method is thread-safe and uses a lock. If you need to execute a
        lot of jobs simultaneously on a broker, use the :class:`Pool` class.

        """
        if timeout is None:
            timeout = self.timeout_max_overflow

        try:
            duration, res = timed(self.debug)(self._execute)(job, timeout,
                                                             extract)

            # XXX unify
            if isinstance(res, str):
                return json.loads(res)['result']

            worker_pid, res, data = res

            # if we overflowed we want to increment the counter
            # if not we reset it
            if duration * 1000 > self.timeout:
                self.timeout_counters[worker_pid] += 1

                # XXX well, we have the result but we want to timeout
                # nevertheless because that's been too much overflow
                if self.timeout_counters[worker_pid] > self.timeout_overflows:
                    raise TimeoutError(timeout / 1000)
            else:
                self.timeout_counters[worker_pid] = 0

            if not res:
                if data == 'No worker':
                    raise NoWorkerError()
                raise ExecutionError(data)
        except Exception:
            # logged, connector replaced.
            if log_exceptions:
                logger.exception('Failed to execute the job.')
            raise

        res = json.loads(data)
        if 'error' in res:
            raise ValueError(res['error'])
        return res['result']
Пример #4
0
    def _kill_worker(self, proc):
        pid = proc.pid
        logger.debug('%d final termination' % proc.pid)

        if proc.poll() is None:
            logger.debug('Calling kill on %d' % proc.pid)
            try:
                proc.kill()
            except OSError:
                logger.exception('Cannot kill %d' % pid)
Пример #5
0
    def _handle_recv_front(self, msg, tentative=0):
        """front => back

        All commands starting with CTRL_ are sent to the controller.
        """
        target = msg[0]

        try:
            data = json.loads(msg[-1])
        except ValueError:
            exc = "Invalid JSON received."
            logger.exception(exc)
            self.send_json(target, {"error": exc})
            return

        cmd = data["command"]

        # a command handled by the controller
        if cmd.startswith("CTRL_"):
            cmd = cmd[len("CTRL_") :]
            logger.debug("calling %s" % cmd)
            try:
                res = self.ctrl.run_command(cmd, msg, data)
            except Exception, e:
                logger.debug("Failed")
                exc_type, exc_value, exc_traceback = sys.exc_info()
                exc = traceback.format_tb(exc_traceback)
                exc.insert(0, str(e))
                self.send_json(target, {"error": exc})
            else:
                # sending back a synchronous result if needed.
                if res is not None:
                    logger.debug("sync success %s" % str(res))
                    self.send_json(target, res)
                else:
                    logger.debug("async success")
Пример #6
0
    def _handle_recv_front(self, msg, tentative=0):
        """front => back

        All commands starting with CTRL_ are sent to the controller.
        """
        target = msg[0]

        try:
            data = json.loads(msg[-1])
        except ValueError:
            exc = 'Invalid JSON received.'
            logger.exception(exc)
            self.send_json(target, {'error': exc})
            return

        cmd = data['command']

        # a command handled by the controller
        if cmd.startswith('CTRL_'):
            cmd = cmd[len('CTRL_'):]
            logger.debug('calling %s' % cmd)
            try:
                res = self.ctrl.run_command(cmd, msg, data)
            except Exception, e:
                logger.debug('Failed')
                exc_type, exc_value, exc_traceback = sys.exc_info()
                exc = traceback.format_tb(exc_traceback)
                exc.insert(0, str(e))
                self.send_json(target, {'error': exc})
            else:
                # sending back a synchronous result if needed.
                if res is not None:
                    logger.debug('sync success %s' % str(res))
                    self.send_json(target, res)
                else:
                    logger.debug('async success')