def on_failure(self, exc_info): """The handler used if the task raised an exception.""" state.task_ready(self) if self.task.acks_late: self.acknowledge() self.send_event("task-failed", uuid=self.task_id, exception=repr(exc_info.exception), traceback=exc_info.traceback) context = { "hostname": self.hostname, "id": self.task_id, "name": self.task_name, "exc": repr(exc_info.exception), "traceback": unicode(exc_info.traceback, 'utf-8'), "args": self.args, "kwargs": self.kwargs } self.logger.error(self.error_msg.strip() % context) task_obj = tasks.get(self.task_name, object) self.send_error_email(task_obj, context, exc_info.exception, whitelist=conf.CELERY_TASK_ERROR_WHITELIST, enabled=conf.CELERY_SEND_TASK_ERROR_EMAILS)
def on_failure(self, exc_info): """The handler used if the task raised an exception.""" state.task_ready(self) if self.task.acks_late: self.acknowledge() self.send_event("task-failed", uuid=self.task_id, exception=repr(exc_info.exception), traceback=exc_info.traceback) # This is a special case as the process would not have had # time to write the result. if isinstance(exc_info.exception, WorkerLostError): self.task.backend.mark_as_failure(self.task_id, exc_info.exception) context = {"hostname": self.hostname, "id": self.task_id, "name": self.task_name, "exc": repr(exc_info.exception), "traceback": unicode(exc_info.traceback, 'utf-8'), "args": self.args, "kwargs": self.kwargs} self.logger.error(self.error_msg.strip() % context) task_obj = tasks.get(self.task_name, object) self.send_error_email(task_obj, context, exc_info.exception, enabled=task_obj.send_error_emails, whitelist=task_obj.error_whitelist)
def on_failure(self, exc_info): """The handler used if the task raised an exception.""" state.task_ready(self) if self.task.acks_late: self.acknowledge() self.send_event("task-failed", uuid=self.task_id, exception=repr(exc_info.exception), traceback=exc_info.traceback) context = {"hostname": self.hostname, "id": self.task_id, "name": self.task_name, "exc": repr(exc_info.exception), "traceback": unicode(exc_info.traceback, 'utf-8'), "args": self.args, "kwargs": self.kwargs} self.logger.error(self.error_msg.strip() % context) task_obj = tasks.get(self.task_name, object) send_error_email = conf.CELERY_SEND_TASK_ERROR_EMAILS and not \ task_obj.disable_error_emails and not any( isinstance(exc_info.exception, whexc) for whexc in conf.CELERY_TASK_ERROR_WHITELIST) if send_error_email: subject = self.email_subject.strip() % context body = self.email_body.strip() % context mail_admins(subject, body, fail_silently=True)
def test_ready(self, requests=[SimpleReq('foo'), SimpleReq('bar')]): for request in requests: state.task_accepted(request) assert len(state.active_requests) == 2 for request in requests: state.task_ready(request) assert len(state.active_requests) == 0
def test_ready(self, requests=[SimpleReq('foo'), SimpleReq('bar')]): for request in requests: state.task_accepted(request) self.assertEqual(len(state.active_requests), 2) for request in requests: state.task_ready(request) self.assertEqual(len(state.active_requests), 0)
def on_timeout(self, soft): state.task_ready(self) if soft: self.logger.warning("Soft time limit exceeded for %s[%s]" % ( self.task_name, self.task_id)) else: self.logger.error("Hard time limit exceeded for %s[%s]" % ( self.task_name, self.task_id))
def on_timeout(self, soft): state.task_ready(self) if soft: self.logger.warning("Soft time limit exceeded for %s[%s]" % (self.task_name, self.task_id)) else: self.logger.error("Hard time limit exceeded for %s[%s]" % (self.task_name, self.task_id))
def on_timeout(self, soft): state.task_ready(self) if soft: self.logger.warning("Soft time limit exceeded for %s[%s]" % ( self.task_name, self.task_id)) exc = SoftTimeLimitExceeded() else: self.logger.error("Hard time limit exceeded for %s[%s]" % ( self.task_name, self.task_id)) exc = TimeLimitExceeded() self.task.backend.mark_as_failure(self.task_id, exc)
def on_timeout(self, soft): state.task_ready(self) if soft: self.logger.warning("Soft time limit exceeded for %s[%s]" % ( self.task_name, self.task_id)) exc = SoftTimeLimitExceeded() else: self.logger.error("Hard time limit exceeded for %s[%s]" % ( self.task_name, self.task_id)) exc = TimeLimitExceeded() if self._store_errors: self.task.backend.mark_as_failure(self.task_id, exc)
def on_success(self, ret_value): """The handler used if the task was successfully processed ( without raising an exception).""" state.task_ready(self.task_name) if self.task.acks_late: self.acknowledge() runtime = time.time() - self.time_start self.send_event("task-succeeded", uuid=self.task_id, result=repr(ret_value), runtime=runtime) msg = self.success_msg.strip() % {"id": self.task_id, "name": self.task_name, "return_value": ret_value} self.logger.info(msg)
def execute(self, request, pool, loglevel, logfile): if not self._pool: # just take pool from first task. self._pool = pool state.task_ready(request) # immediately remove from worker state. self._buffer.put(request) if self._tref is None: # first request starts flush timer. self._tref = timer2.apply_interval(self.flush_interval * 1000, self._do_flush) if not self._count() % self.flush_every: self._do_flush()
def on_timeout(self, soft): """Handler called if the task times out.""" state.task_ready(self) if soft: self.logger.warning("Soft time limit exceeded for %s[%s]" % ( self.task_name, self.task_id)) exc = SoftTimeLimitExceeded() else: self.logger.error("Hard time limit exceeded for %s[%s]" % ( self.task_name, self.task_id)) exc = TimeLimitExceeded() if self._store_errors: self.task.backend.mark_as_failure(self.task_id, exc)
def on_failure(self, exc_info): """Handler called if the task raised an exception.""" state.task_ready(self) if self.task.acks_late: self.acknowledge() if isinstance(exc_info.exception, RetryTaskError): return self.on_retry(exc_info) # This is a special case as the process would not have had # time to write the result. if isinstance(exc_info.exception, WorkerLostError): if self._store_errors: self.task.backend.mark_as_failure(self.task_id, exc_info.exception) self.send_event("task-failed", uuid=self.task_id, exception=safe_repr(exc_info.exception), traceback=safe_str(exc_info.traceback)) context = { "hostname": self.hostname, "id": self.task_id, "name": self.task_name, "exc": safe_repr(exc_info.exception), "traceback": safe_str(exc_info.traceback), "args": self.args, "kwargs": self.kwargs } log_with_extra(self.logger, logging.ERROR, self.error_msg.strip() % context, exc_info=exc_info, extra={ "data": { "hostname": self.hostname, "id": self.task_id, "name": self.task_name } }) task_obj = tasks.get(self.task_name, object) self.send_error_email(task_obj, context, exc_info.exception, enabled=task_obj.send_error_emails, whitelist=task_obj.error_whitelist)
def on_timeout(self, soft, timeout): """Handler called if the task times out.""" state.task_ready(self) if soft: self.logger.warning("Soft time limit (%ss) exceeded for %s[%s]" % (timeout, self.task_name, self.task_id)) exc = SoftTimeLimitExceeded(timeout) else: self.logger.error("Hard time limit (%ss) exceeded for %s[%s]" % (timeout, self.task_name, self.task_id)) exc = TimeLimitExceeded(timeout) if self._store_errors: self.task.backend.mark_as_failure(self.task_id, exc)
def on_success(self, ret_value): """Handler called if the task was successfully processed.""" state.task_ready(self) if self.task.acks_late: self.acknowledge() runtime = self.time_start and (time.time() - self.time_start) or 0 self.send_event("task-succeeded", uuid=self.task_id, result=repr(ret_value), runtime=runtime) self.logger.info(self.success_msg.strip() % { "id": self.task_id, "name": self.task_name, "return_value": self.repr_result(ret_value), "runtime": runtime})
def test_revoke_terminate(self): request = Mock() request.id = tid = uuid() state = self.create_state() state.consumer = Mock() worker_state.task_reserved(request) try: r = control.revoke(state, tid, terminate=True) assert tid in revoked assert request.terminate.call_count assert 'terminate:' in r['ok'] # unknown task id only revokes r = control.revoke(state, uuid(), terminate=True) assert 'tasks unknown' in r['ok'] finally: worker_state.task_ready(request)
def test_revoke_terminate(self): request = Mock() request.id = tid = uuid() state = self.create_state() state.consumer = Mock() worker_state.task_reserved(request) try: r = control.revoke(state, tid, terminate=True) self.assertIn(tid, revoked) self.assertTrue(request.terminate.call_count) self.assertIn('terminate:', r['ok']) # unknown task id only revokes r = control.revoke(state, uuid(), terminate=True) self.assertIn('tasks unknown', r['ok']) finally: worker_state.task_ready(request)
def on_success(self, ret_value): """The handler used if the task was successfully processed ( without raising an exception).""" state.task_ready(self) if self.task.acks_late: self.acknowledge() runtime = time.time() - self.time_start self.send_event("task-succeeded", uuid=self.task_id, result=repr(ret_value), runtime=runtime) msg = self.success_msg.strip() % { "id": self.task_id, "name": self.task_name, "return_value": self.repr_result(ret_value), "runtime": runtime} self.logger.info(msg)
def test_no_negative_scale(self): total_num_processes = [] worker = Mock(name='worker') x = autoscale.Autoscaler(self.pool, 10, 3, worker=worker) x.body() # the body func scales up or down _keep = [Mock(name=f'req{i}') for i in range(35)] for req in _keep: state.task_reserved(req) x.body() total_num_processes.append(self.pool.num_processes) for req in _keep: state.task_ready(req) x.body() total_num_processes.append(self.pool.num_processes) assert all(x.min_concurrency <= i <= x.max_concurrency for i in total_num_processes)
def test_no_negative_scale(self): total_num_processes = [] worker = Mock(name='worker') x = autoscale.Autoscaler(self.pool, 10, 3, worker=worker) x.body() # the body func scales up or down _keep = [Mock(name='req{0}'.format(i)) for i in range(35)] for req in _keep: state.task_reserved(req) x.body() total_num_processes.append(self.pool.num_processes) for req in _keep: state.task_ready(req) x.body() total_num_processes.append(self.pool.num_processes) assert all(x.min_concurrency <= i <= x.max_concurrency for i in total_num_processes)
def on_success(self, ret_value): """Handler called if the task was successfully processed.""" state.task_ready(self) if self.task.acks_late: self.acknowledge() runtime = self.time_start and (time.time() - self.time_start) or 0 self.send_event("task-succeeded", uuid=self.task_id, result=safe_repr(ret_value), runtime=runtime) self.logger.info( self.success_msg.strip() % { "id": self.task_id, "name": self.task_name, "return_value": self.repr_result(ret_value), "runtime": runtime })
def on_failure(self, exc_info): """Handler called if the task raised an exception.""" state.task_ready(self) if self.task.acks_late: self.acknowledge() if isinstance(exc_info.exception, RetryTaskError): return self.on_retry(exc_info) # This is a special case as the process would not have had # time to write the result. if isinstance(exc_info.exception, WorkerLostError): if self._store_errors: self.task.backend.mark_as_failure(self.task_id, exc_info.exception) self.send_event("task-failed", uuid=self.task_id, exception=repr(exc_info.exception), traceback=exc_info.traceback) context = {"hostname": self.hostname, "id": self.task_id, "name": self.task_name, "exc": repr(exc_info.exception), "traceback": unicode(exc_info.traceback, sys.getfilesystemencoding()), "args": self.args, "kwargs": self.kwargs} log_with_extra(self.logger, logging.ERROR, self.error_msg.strip() % context, exc_info=exc_info, extra={"data": {"hostname": self.hostname, "id": self.task_id, "name": self.task_name}}) task_obj = tasks.get(self.task_name, object) self.send_error_email(task_obj, context, exc_info.exception, enabled=task_obj.send_error_emails, whitelist=task_obj.error_whitelist)
def on_failure(self, exc_info): """The handler used if the task raised an exception.""" state.task_ready(self) if self.task.acks_late: self.acknowledge() self.send_event("task-failed", uuid=self.task_id, exception=repr(exc_info.exception), traceback=exc_info.traceback) context = {"hostname": self.hostname, "id": self.task_id, "name": self.task_name, "exc": repr(exc_info.exception), "traceback": unicode(exc_info.traceback, 'utf-8'), "args": self.args, "kwargs": self.kwargs} self.logger.error(self.error_msg.strip() % context) task_obj = tasks.get(self.task_name, object) self.send_error_email(task_obj, context, exc_info.exception, whitelist=conf.CELERY_TASK_ERROR_WHITELIST, enabled=conf.CELERY_SEND_TASK_ERROR_EMAILS)