def execute(self, job, timeout=None, log_exceptions=True):
"""Runs the job
Options:
- **job**: Job to be performed. Can be a :class:`Job`
instance or a string. If it's a string a :class:`Job` instance
will be automatically created out of it.
- **timeout**: maximum allowed time for a job to run.
If not provided, uses the one defined in the constructor.
If the job fails after the timeout, raises a :class:`TimeoutError`.
This method is thread-safe and uses a lock. If you need to execute a
lot of jobs simultaneously on a broker, use the :class:`Pool` class.
"""
if timeout is None:
timeout = self.timeout_max_overflow
try:
duration, res = timed(self.debug)(self._execute)(job, timeout)
except Exception:
# logged, connector replaced.
if log_exceptions:
logger.exception('Failed to execute the job.')
raise
if 'error' in res:
raise ValueError(res['error'])
return res['result']
def execute(self, job, timeout=None, log_exceptions=True):
"""Runs the job
Options:
- **job**: Job to be performed. Can be a :class:`Job`
instance or a string. If it's a string a :class:`Job` instance
will be automatically created out of it.
- **timeout**: maximum allowed time for a job to run.
If not provided, uses the one defined in the constructor.
If the job fails after the timeout, raises a :class:`TimeoutError`.
This method is thread-safe and uses a lock. If you need to execute a
lot of jobs simultaneously on a broker, use the :class:`Pool` class.
"""
if timeout is None:
timeout = self.timeout_max_overflow
try:
duration, res = timed(self.debug)(self._execute)(job, timeout)
except Exception:
# logged, connector replaced.
if log_exceptions:
logger.exception('Failed to execute the job.')
raise
if 'error' in res:
raise ValueError(res['error'])
return res['result']
def execute(self, job, timeout=None, extract=True, log_exceptions=True):
"""Runs the job
Options:
- **job**: Job to be performed. Can be a :class:`Job`
instance or a string. If it's a string a :class:`Job` instance
will be automatically created out of it.
- **timeout**: maximum allowed time for a job to run.
If not provided, uses the one defined in the constructor.
If the job fails after the timeout, raises a :class:`TimeoutError`.
This method is thread-safe and uses a lock. If you need to execute a
lot of jobs simultaneously on a broker, use the :class:`Pool` class.
"""
if timeout is None:
timeout = self.timeout_max_overflow
try:
duration, res = timed(self.debug)(self._execute)(job, timeout,
extract)
# XXX unify
if isinstance(res, str):
return json.loads(res)['result']
worker_pid, res, data = res
# if we overflowed we want to increment the counter
# if not we reset it
if duration * 1000 > self.timeout:
self.timeout_counters[worker_pid] += 1
# XXX well, we have the result but we want to timeout
# nevertheless because that's been too much overflow
if self.timeout_counters[worker_pid] > self.timeout_overflows:
raise TimeoutError(timeout / 1000)
else:
self.timeout_counters[worker_pid] = 0
if not res:
if data == 'No worker':
raise NoWorkerError()
raise ExecutionError(data)
except Exception:
# logged, connector replaced.
if log_exceptions:
logger.exception('Failed to execute the job.')
raise
res = json.loads(data)
if 'error' in res:
raise ValueError(res['error'])
return res['result']
def _kill_worker(self, proc):
pid = proc.pid
logger.debug('%d final termination' % proc.pid)
if proc.poll() is None:
logger.debug('Calling kill on %d' % proc.pid)
try:
proc.kill()
except OSError:
logger.exception('Cannot kill %d' % pid)
def _handle_recv_front(self, msg, tentative=0):
"""front => back
All commands starting with CTRL_ are sent to the controller.
"""
target = msg[0]
try:
data = json.loads(msg[-1])
except ValueError:
exc = "Invalid JSON received."
logger.exception(exc)
self.send_json(target, {"error": exc})
return
cmd = data["command"]
# a command handled by the controller
if cmd.startswith("CTRL_"):
cmd = cmd[len("CTRL_") :]
logger.debug("calling %s" % cmd)
try:
res = self.ctrl.run_command(cmd, msg, data)
except Exception, e:
logger.debug("Failed")
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
self.send_json(target, {"error": exc})
else:
# sending back a synchronous result if needed.
if res is not None:
logger.debug("sync success %s" % str(res))
self.send_json(target, res)
else:
logger.debug("async success")
def _handle_recv_front(self, msg, tentative=0):
"""front => back
All commands starting with CTRL_ are sent to the controller.
"""
target = msg[0]
try:
data = json.loads(msg[-1])
except ValueError:
exc = 'Invalid JSON received.'
logger.exception(exc)
self.send_json(target, {'error': exc})
return
cmd = data['command']
# a command handled by the controller
if cmd.startswith('CTRL_'):
cmd = cmd[len('CTRL_'):]
logger.debug('calling %s' % cmd)
try:
res = self.ctrl.run_command(cmd, msg, data)
except Exception, e:
logger.debug('Failed')
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
self.send_json(target, {'error': exc})
else:
# sending back a synchronous result if needed.
if res is not None:
logger.debug('sync success %s' % str(res))
self.send_json(target, res)
else:
logger.debug('async success')