def on_failure(self, exc_info):
"""The handler used if the task raised an exception."""
state.task_ready(self)
if self.task.acks_late:
self.acknowledge()
self.send_event("task-failed",
uuid=self.task_id,
exception=repr(exc_info.exception),
traceback=exc_info.traceback)
context = {
"hostname": self.hostname,
"id": self.task_id,
"name": self.task_name,
"exc": repr(exc_info.exception),
"traceback": unicode(exc_info.traceback, 'utf-8'),
"args": self.args,
"kwargs": self.kwargs
}
self.logger.error(self.error_msg.strip() % context)
task_obj = tasks.get(self.task_name, object)
self.send_error_email(task_obj,
context,
exc_info.exception,
whitelist=conf.CELERY_TASK_ERROR_WHITELIST,
enabled=conf.CELERY_SEND_TASK_ERROR_EMAILS)
def on_failure(self, exc_info):
"""The handler used if the task raised an exception."""
state.task_ready(self)
if self.task.acks_late:
self.acknowledge()
self.send_event("task-failed", uuid=self.task_id,
exception=repr(exc_info.exception),
traceback=exc_info.traceback)
# This is a special case as the process would not have had
# time to write the result.
if isinstance(exc_info.exception, WorkerLostError):
self.task.backend.mark_as_failure(self.task_id,
exc_info.exception)
context = {"hostname": self.hostname,
"id": self.task_id,
"name": self.task_name,
"exc": repr(exc_info.exception),
"traceback": unicode(exc_info.traceback, 'utf-8'),
"args": self.args,
"kwargs": self.kwargs}
self.logger.error(self.error_msg.strip() % context)
task_obj = tasks.get(self.task_name, object)
self.send_error_email(task_obj, context, exc_info.exception,
enabled=task_obj.send_error_emails,
whitelist=task_obj.error_whitelist)
def on_failure(self, exc_info):
"""The handler used if the task raised an exception."""
state.task_ready(self)
if self.task.acks_late:
self.acknowledge()
self.send_event("task-failed", uuid=self.task_id,
exception=repr(exc_info.exception),
traceback=exc_info.traceback)
context = {"hostname": self.hostname,
"id": self.task_id,
"name": self.task_name,
"exc": repr(exc_info.exception),
"traceback": unicode(exc_info.traceback, 'utf-8'),
"args": self.args,
"kwargs": self.kwargs}
self.logger.error(self.error_msg.strip() % context)
task_obj = tasks.get(self.task_name, object)
send_error_email = conf.CELERY_SEND_TASK_ERROR_EMAILS and not \
task_obj.disable_error_emails and not any(
isinstance(exc_info.exception, whexc)
for whexc in
conf.CELERY_TASK_ERROR_WHITELIST)
if send_error_email:
subject = self.email_subject.strip() % context
body = self.email_body.strip() % context
mail_admins(subject, body, fail_silently=True)
def test_ready(self, requests=[SimpleReq('foo'), SimpleReq('bar')]):
for request in requests:
state.task_accepted(request)
assert len(state.active_requests) == 2
for request in requests:
state.task_ready(request)
assert len(state.active_requests) == 0
def test_ready(self, requests=[SimpleReq('foo'), SimpleReq('bar')]):
for request in requests:
state.task_accepted(request)
self.assertEqual(len(state.active_requests), 2)
for request in requests:
state.task_ready(request)
self.assertEqual(len(state.active_requests), 0)
def on_timeout(self, soft):
state.task_ready(self)
if soft:
self.logger.warning("Soft time limit exceeded for %s[%s]" % (
self.task_name, self.task_id))
else:
self.logger.error("Hard time limit exceeded for %s[%s]" % (
self.task_name, self.task_id))
def test_ready(self, requests=[SimpleReq('foo'),
SimpleReq('bar')]):
for request in requests:
state.task_accepted(request)
assert len(state.active_requests) == 2
for request in requests:
state.task_ready(request)
assert len(state.active_requests) == 0
def test_ready(self, requests=[SimpleReq('foo'),
SimpleReq('bar')]):
for request in requests:
state.task_accepted(request)
self.assertEqual(len(state.active_requests), 2)
for request in requests:
state.task_ready(request)
self.assertEqual(len(state.active_requests), 0)
def on_timeout(self, soft):
state.task_ready(self)
if soft:
self.logger.warning("Soft time limit exceeded for %s[%s]" %
(self.task_name, self.task_id))
else:
self.logger.error("Hard time limit exceeded for %s[%s]" %
(self.task_name, self.task_id))
def on_timeout(self, soft):
state.task_ready(self)
if soft:
self.logger.warning("Soft time limit exceeded for %s[%s]" % (
self.task_name, self.task_id))
exc = SoftTimeLimitExceeded()
else:
self.logger.error("Hard time limit exceeded for %s[%s]" % (
self.task_name, self.task_id))
exc = TimeLimitExceeded()
self.task.backend.mark_as_failure(self.task_id, exc)
def on_timeout(self, soft):
state.task_ready(self)
if soft:
self.logger.warning("Soft time limit exceeded for %s[%s]" % (
self.task_name, self.task_id))
exc = SoftTimeLimitExceeded()
else:
self.logger.error("Hard time limit exceeded for %s[%s]" % (
self.task_name, self.task_id))
exc = TimeLimitExceeded()
if self._store_errors:
self.task.backend.mark_as_failure(self.task_id, exc)
def on_success(self, ret_value):
"""The handler used if the task was successfully processed (
without raising an exception)."""
state.task_ready(self.task_name)
if self.task.acks_late:
self.acknowledge()
runtime = time.time() - self.time_start
self.send_event("task-succeeded", uuid=self.task_id, result=repr(ret_value), runtime=runtime)
msg = self.success_msg.strip() % {"id": self.task_id, "name": self.task_name, "return_value": ret_value}
self.logger.info(msg)
def execute(self, request, pool, loglevel, logfile):
if not self._pool: # just take pool from first task.
self._pool = pool
state.task_ready(request) # immediately remove from worker state.
self._buffer.put(request)
if self._tref is None: # first request starts flush timer.
self._tref = timer2.apply_interval(self.flush_interval * 1000,
self._do_flush)
if not self._count() % self.flush_every:
self._do_flush()
def execute(self, request, pool, loglevel, logfile):
if not self._pool: # just take pool from first task.
self._pool = pool
state.task_ready(request) # immediately remove from worker state.
self._buffer.put(request)
if self._tref is None: # first request starts flush timer.
self._tref = timer2.apply_interval(self.flush_interval * 1000,
self._do_flush)
if not self._count() % self.flush_every:
self._do_flush()
def on_timeout(self, soft):
"""Handler called if the task times out."""
state.task_ready(self)
if soft:
self.logger.warning("Soft time limit exceeded for %s[%s]" % (
self.task_name, self.task_id))
exc = SoftTimeLimitExceeded()
else:
self.logger.error("Hard time limit exceeded for %s[%s]" % (
self.task_name, self.task_id))
exc = TimeLimitExceeded()
if self._store_errors:
self.task.backend.mark_as_failure(self.task_id, exc)
def on_failure(self, exc_info):
"""Handler called if the task raised an exception."""
state.task_ready(self)
if self.task.acks_late:
self.acknowledge()
if isinstance(exc_info.exception, RetryTaskError):
return self.on_retry(exc_info)
# This is a special case as the process would not have had
# time to write the result.
if isinstance(exc_info.exception, WorkerLostError):
if self._store_errors:
self.task.backend.mark_as_failure(self.task_id,
exc_info.exception)
self.send_event("task-failed",
uuid=self.task_id,
exception=safe_repr(exc_info.exception),
traceback=safe_str(exc_info.traceback))
context = {
"hostname": self.hostname,
"id": self.task_id,
"name": self.task_name,
"exc": safe_repr(exc_info.exception),
"traceback": safe_str(exc_info.traceback),
"args": self.args,
"kwargs": self.kwargs
}
log_with_extra(self.logger,
logging.ERROR,
self.error_msg.strip() % context,
exc_info=exc_info,
extra={
"data": {
"hostname": self.hostname,
"id": self.task_id,
"name": self.task_name
}
})
task_obj = tasks.get(self.task_name, object)
self.send_error_email(task_obj,
context,
exc_info.exception,
enabled=task_obj.send_error_emails,
whitelist=task_obj.error_whitelist)
def on_timeout(self, soft, timeout):
"""Handler called if the task times out."""
state.task_ready(self)
if soft:
self.logger.warning("Soft time limit (%ss) exceeded for %s[%s]" %
(timeout, self.task_name, self.task_id))
exc = SoftTimeLimitExceeded(timeout)
else:
self.logger.error("Hard time limit (%ss) exceeded for %s[%s]" %
(timeout, self.task_name, self.task_id))
exc = TimeLimitExceeded(timeout)
if self._store_errors:
self.task.backend.mark_as_failure(self.task_id, exc)
def on_success(self, ret_value):
"""Handler called if the task was successfully processed."""
state.task_ready(self)
if self.task.acks_late:
self.acknowledge()
runtime = self.time_start and (time.time() - self.time_start) or 0
self.send_event("task-succeeded", uuid=self.task_id,
result=repr(ret_value), runtime=runtime)
self.logger.info(self.success_msg.strip() % {
"id": self.task_id,
"name": self.task_name,
"return_value": self.repr_result(ret_value),
"runtime": runtime})
def test_revoke_terminate(self):
request = Mock()
request.id = tid = uuid()
state = self.create_state()
state.consumer = Mock()
worker_state.task_reserved(request)
try:
r = control.revoke(state, tid, terminate=True)
assert tid in revoked
assert request.terminate.call_count
assert 'terminate:' in r['ok']
# unknown task id only revokes
r = control.revoke(state, uuid(), terminate=True)
assert 'tasks unknown' in r['ok']
finally:
worker_state.task_ready(request)
def test_revoke_terminate(self):
request = Mock()
request.id = tid = uuid()
state = self.create_state()
state.consumer = Mock()
worker_state.task_reserved(request)
try:
r = control.revoke(state, tid, terminate=True)
self.assertIn(tid, revoked)
self.assertTrue(request.terminate.call_count)
self.assertIn('terminate:', r['ok'])
# unknown task id only revokes
r = control.revoke(state, uuid(), terminate=True)
self.assertIn('tasks unknown', r['ok'])
finally:
worker_state.task_ready(request)
def test_revoke_terminate(self):
request = Mock()
request.id = tid = uuid()
state = self.create_state()
state.consumer = Mock()
worker_state.task_reserved(request)
try:
r = control.revoke(state, tid, terminate=True)
self.assertIn(tid, revoked)
self.assertTrue(request.terminate.call_count)
self.assertIn('terminate:', r['ok'])
# unknown task id only revokes
r = control.revoke(state, uuid(), terminate=True)
self.assertIn('tasks unknown', r['ok'])
finally:
worker_state.task_ready(request)
def test_revoke_terminate(self):
request = Mock()
request.id = tid = uuid()
state = self.create_state()
state.consumer = Mock()
worker_state.task_reserved(request)
try:
r = control.revoke(state, tid, terminate=True)
assert tid in revoked
assert request.terminate.call_count
assert 'terminate:' in r['ok']
# unknown task id only revokes
r = control.revoke(state, uuid(), terminate=True)
assert 'tasks unknown' in r['ok']
finally:
worker_state.task_ready(request)
def on_success(self, ret_value):
"""The handler used if the task was successfully processed (
without raising an exception)."""
state.task_ready(self)
if self.task.acks_late:
self.acknowledge()
runtime = time.time() - self.time_start
self.send_event("task-succeeded", uuid=self.task_id,
result=repr(ret_value), runtime=runtime)
msg = self.success_msg.strip() % {
"id": self.task_id,
"name": self.task_name,
"return_value": self.repr_result(ret_value),
"runtime": runtime}
self.logger.info(msg)
def test_no_negative_scale(self):
total_num_processes = []
worker = Mock(name='worker')
x = autoscale.Autoscaler(self.pool, 10, 3, worker=worker)
x.body() # the body func scales up or down
_keep = [Mock(name=f'req{i}') for i in range(35)]
for req in _keep:
state.task_reserved(req)
x.body()
total_num_processes.append(self.pool.num_processes)
for req in _keep:
state.task_ready(req)
x.body()
total_num_processes.append(self.pool.num_processes)
assert all(x.min_concurrency <= i <= x.max_concurrency
for i in total_num_processes)
def test_no_negative_scale(self):
total_num_processes = []
worker = Mock(name='worker')
x = autoscale.Autoscaler(self.pool, 10, 3, worker=worker)
x.body() # the body func scales up or down
_keep = [Mock(name='req{0}'.format(i)) for i in range(35)]
for req in _keep:
state.task_reserved(req)
x.body()
total_num_processes.append(self.pool.num_processes)
for req in _keep:
state.task_ready(req)
x.body()
total_num_processes.append(self.pool.num_processes)
assert all(x.min_concurrency <= i <= x.max_concurrency
for i in total_num_processes)
def on_success(self, ret_value):
"""Handler called if the task was successfully processed."""
state.task_ready(self)
if self.task.acks_late:
self.acknowledge()
runtime = self.time_start and (time.time() - self.time_start) or 0
self.send_event("task-succeeded",
uuid=self.task_id,
result=safe_repr(ret_value),
runtime=runtime)
self.logger.info(
self.success_msg.strip() % {
"id": self.task_id,
"name": self.task_name,
"return_value": self.repr_result(ret_value),
"runtime": runtime
})
def on_failure(self, exc_info):
"""Handler called if the task raised an exception."""
state.task_ready(self)
if self.task.acks_late:
self.acknowledge()
if isinstance(exc_info.exception, RetryTaskError):
return self.on_retry(exc_info)
# This is a special case as the process would not have had
# time to write the result.
if isinstance(exc_info.exception, WorkerLostError):
if self._store_errors:
self.task.backend.mark_as_failure(self.task_id,
exc_info.exception)
self.send_event("task-failed", uuid=self.task_id,
exception=repr(exc_info.exception),
traceback=exc_info.traceback)
context = {"hostname": self.hostname,
"id": self.task_id,
"name": self.task_name,
"exc": repr(exc_info.exception),
"traceback": unicode(exc_info.traceback,
sys.getfilesystemencoding()),
"args": self.args,
"kwargs": self.kwargs}
log_with_extra(self.logger, logging.ERROR,
self.error_msg.strip() % context,
exc_info=exc_info,
extra={"data": {"hostname": self.hostname,
"id": self.task_id,
"name": self.task_name}})
task_obj = tasks.get(self.task_name, object)
self.send_error_email(task_obj, context, exc_info.exception,
enabled=task_obj.send_error_emails,
whitelist=task_obj.error_whitelist)
def on_failure(self, exc_info):
"""The handler used if the task raised an exception."""
state.task_ready(self)
if self.task.acks_late:
self.acknowledge()
self.send_event("task-failed", uuid=self.task_id,
exception=repr(exc_info.exception),
traceback=exc_info.traceback)
context = {"hostname": self.hostname,
"id": self.task_id,
"name": self.task_name,
"exc": repr(exc_info.exception),
"traceback": unicode(exc_info.traceback, 'utf-8'),
"args": self.args,
"kwargs": self.kwargs}
self.logger.error(self.error_msg.strip() % context)
task_obj = tasks.get(self.task_name, object)
self.send_error_email(task_obj, context, exc_info.exception,
whitelist=conf.CELERY_TASK_ERROR_WHITELIST,
enabled=conf.CELERY_SEND_TASK_ERROR_EMAILS)
