def _handle_recv_back(self, msg):
# do the job and send the result
if self.debug:
logger.debug('Job received')
target = timed()(self.target)
else:
target = self.target
duration = -1
# results are sent with a PID:OK: or a PID:ERROR prefix
try:
with self.timer.run_job():
res = target(Job.load_from_string(msg[0]))
# did we timout ?
if self.timer.timed_out:
# let's dump the last
for line in self.timer.last_dump:
logger.error(line)
if self.debug:
duration, res = res
res = '%d:OK:%s' % (self.pid, res)
except Exception, e:
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = '%d:ERROR:%s' % (self.pid, '\n'.join(exc))
logger.error(res)
def execute(self, job, timeout=None):
"""Runs the job
Options:
- **job**: Job to be performed. Can be a :class:`Job`
instance or a string. If it's a string a :class:`Job` instance
will be automatically created out of it.
- **timeout**: maximum allowed time for a job to run.
If not provided, uses the one defined in the constructor.
If the job fails after the timeout, raises a :class:`TimeoutError`.
This method is thread-safe and uses a lock. If you need to execute a
lot of jobs simultaneously on a broker, use the :class:`Pool` class.
"""
if timeout is None:
timeout = self.timeout_max_overflow
try:
duration, res = timed(self.debug)(self._execute)(job, timeout)
worker_pid, res, data = res
# if we overflowed we want to increment the counter
# if not we reset it
if duration * 1000 > self.timeout:
self.timeout_counters[worker_pid] += 1
# XXX well, we have the result but we want to timeout
# nevertheless because that's been too much overflow
if self.timeout_counters[worker_pid] > self.timeout_overflows:
raise TimeoutError(timeout / 1000)
else:
self.timeout_counters[worker_pid] = 0
if not res:
if data == 'No worker':
raise NoWorkerError()
raise ExecutionError(data)
except Exception:
# logged, connector replaced.
logger.exception('Failed to execute the job.')
raise
return data
def _handle_recv_back(self, msg):
# do the job and send the result
if self.debug:
logger.debug("Job received")
target = timed()(self.target)
else:
target = self.target
duration = -1
# results are sent with a PID:OK: or a PID:ERROR prefix
riemann_message = {
"host": socket.gethostname(),
"service": "powerhose-worker",
"metric": 1,
"state": "ok",
"tags": ["running"],
}
try:
with self.timer.run_job():
self.riemann.send(riemann_message)
res = target(Job.load_from_string(msg[0]))
# did we timout ?
if self.timer.timed_out:
# let's dump the last
riemann_message["state"] = "error"
riemann_message["tags"] = ["timeouts"]
self.riemann.send(riemann_message)
for line in self.timer.last_dump:
logger.error(line)
if self.debug:
duration, res = res
res = "%d:OK:%s" % (self.pid, res)
except Exception, e:
riemann_message["description"] = str(e)
riemann_message["state"] = "error"
riemann_message["tags"] = ["faults"]
exc_type, exc_value, exc_traceback = sys.exc_info()
exc = traceback.format_tb(exc_traceback)
exc.insert(0, str(e))
res = "%d:ERROR:%s" % (self.pid, "\n".join(exc))
logger.error(res)
