def handle_failure(self, task, req, store_errors=True, call_errbacks=True):
"""Handle exception."""
_, _, tb = sys.exc_info()
try:
exc = self.retval
# make sure we only send pickleable exceptions back to parent.
einfo = ExceptionInfo()
einfo.exception = get_pickleable_exception(einfo.exception)
einfo.type = get_pickleable_etype(einfo.type)
task.backend.mark_as_failure(
req.id,
exc,
einfo.traceback,
request=req,
store_result=store_errors,
call_errbacks=call_errbacks,
)
task.on_failure(exc, req.id, req.args, req.kwargs, einfo)
signals.task_failure.send(sender=task,
task_id=req.id,
exception=exc,
args=req.args,
kwargs=req.kwargs,
traceback=tb,
einfo=einfo)
self._log_error(task, req, einfo)
return einfo
finally:
del tb
def handle_failure(self, task, req, store_errors=True, call_errbacks=True):
"""Handle exception."""
_, _, tb = sys.exc_info()
try:
exc = self.retval
# make sure we only send pickleable exceptions back to parent.
einfo = ExceptionInfo()
einfo.exception = get_pickleable_exception(einfo.exception)
einfo.type = get_pickleable_etype(einfo.type)
task.backend.mark_as_failure(
req.id, exc, einfo.traceback,
request=req, store_result=store_errors,
call_errbacks=call_errbacks,
)
task.on_failure(exc, req.id, req.args, req.kwargs, einfo)
signals.task_failure.send(sender=task, task_id=req.id,
exception=exc, args=req.args,
kwargs=req.kwargs,
traceback=tb,
einfo=einfo)
self._log_error(task, req, einfo)
return einfo
finally:
del(tb)
def handle_failure(self, task, store_errors=True):
"""Handle exception."""
req = task.request
type_, _, tb = sys.exc_info()
try:
exc = self.retval
einfo = ExceptionInfo()
einfo.exception = get_pickleable_exception(einfo.exception)
einfo.type = get_pickleable_etype(einfo.type)
if store_errors:
task.backend.mark_as_failure(
req.id,
exc,
einfo.traceback,
request=req,
)
task.on_failure(exc, req.id, req.args, req.kwargs, einfo)
signals.task_failure.send(sender=task,
task_id=req.id,
exception=exc,
args=req.args,
kwargs=req.kwargs,
traceback=tb,
einfo=einfo)
return einfo
finally:
del (tb)
def clean_up_failure_task(result={},
export_provider_task_uids=[],
run_uid=None,
run_dir=None,
worker=None,
*args,
**kwargs):
"""
Used to close tasks in a failed chain.
If a task fails or is canceled, it all of the uid will be passed here and the failed object will be found and propagated,
to the subsequent tasks in the chain. Additionally they will be finalized to ensure that the run finishes.
"""
from eventkit_cloud.tasks.models import ExportProviderTask, ExportTaskException
from billiard.einfo import ExceptionInfo
task_status = None
incomplete_export_provider_task = None
for export_provider_task_uid in export_provider_task_uids:
export_provider_task = ExportProviderTask.objects.get(
uid=export_provider_task_uid)
for export_task in export_provider_task.tasks.all():
if TaskStates[
export_task.status] in TaskStates.get_incomplete_states():
if not task_status:
task_status = export_task.status
incomplete_export_provider_task = export_provider_task.name
else:
if task_status:
export_task.status = task_status
try:
raise CancelException(
message=
"{0} could not complete because it depends on {1}".
format(export_provider_task.name,
incomplete_export_provider_task))
except CancelException as ce:
einfo = ExceptionInfo()
einfo.exception = ce
ExportTaskException.objects.create(
task=export_task, exception=cPickle.dumps(einfo))
export_task.save()
finalize_export_provider_task.si(
run_uid=run_uid,
export_provider_task_uid=export_provider_task_uid,
worker=worker).set(queue=worker, routing_key=worker).apply_async(
interval=1,
max_retries=10,
queue=worker,
routing_key=worker,
priority=TaskPriority.FINALIZE_PROVIDER.value)
return result
def handle_failure(self, task, store_errors=True):
"""Handle exception."""
req = task.request
type_, _, tb = sys.exc_info()
try:
exc = self.retval
einfo = ExceptionInfo()
einfo.exception = get_pickleable_exception(einfo.exception)
einfo.type = get_pickleable_etype(einfo.type)
if store_errors:
task.backend.mark_as_failure(req.id, exc, einfo.traceback, request=req)
task.on_failure(exc, req.id, req.args, req.kwargs, einfo)
signals.task_failure.send(
sender=task, task_id=req.id, exception=exc, args=req.args, kwargs=req.kwargs, traceback=tb, einfo=einfo
)
return einfo
finally:
del (tb)
def on_success(self, retval, task_id, args, kwargs):
try:
logger.info(
'my task success and taskid is {} ,retval is{} ,args is{}.kwargs id {}'
.format(task_id, retval, args, kwargs))
# 如果执行成功，且有下一步，则执行下一步
if self.do_success(retval, task_id, args,
kwargs) and kwargs.get('next_task_kwargs'):
for next_task_kwarg in kwargs['next_task_kwargs']:
with session_scope() as ss:
from worker.run_task import run_celery_task
run_celery_task(session=ss, **next_task_kwarg)
except Exception as e:
einfo = ExceptionInfo()
einfo.exception = get_pickleable_exception(einfo.exception)
einfo.type = get_pickleable_etype(einfo.type)
self.on_failure(e, task_id, args, kwargs, einfo)
def _signal_internal_error(task, uuid, args, kwargs, request, exc):
"""Send a special `internal_error` signal to the app for outside body errors."""
try:
_, _, tb = sys.exc_info()
einfo = ExceptionInfo()
einfo.exception = get_pickleable_exception(einfo.exception)
einfo.type = get_pickleable_etype(einfo.type)
signals.task_internal_error.send(
sender=task,
task_id=uuid,
args=args,
kwargs=kwargs,
request=request,
exception=exc,
traceback=tb,
einfo=einfo,
)
finally:
del tb
def cancel_export_provider_task(result={},
export_provider_task_uid=None,
canceling_user=None):
"""
Cancels an ExportProviderTask and terminates each subtasks execution.
"""
from ..tasks.models import ExportProviderTask, ExportTaskException, ExportTaskResult
from ..tasks.exceptions import CancelException
from billiard.einfo import ExceptionInfo
from datetime import datetime, timedelta
export_provider_task = ExportProviderTask.objects.filter(
uid=export_provider_task_uid).first()
if not export_provider_task:
result['result'] = False
return result
export_tasks = export_provider_task.tasks.all()
# Loop through both the tasks in the ExportProviderTask model, as well as the Task Chain in celery
for export_task in export_tasks.filter(~Q(
status=TaskStates.CANCELED.value)):
export_task.status = TaskStates.CANCELED.value
export_task.cancel_user = canceling_user
export_task.save()
# This part is to populate the UI with the cancel message. If a different mechanism is incorporated
# to pass task information to the users, then it may make sense to replace this.
try:
raise CancelException(task_name=export_provider_task.name,
user_name=canceling_user)
except CancelException as ce:
einfo = ExceptionInfo()
einfo.exception = ce
ExportTaskException.objects.create(task=export_task,
exception=cPickle.dumps(einfo))
# Remove the ExportTaskResult, which will clean up the files.
task_result = ExportTaskResult.objects.filter(task=export_task).first()
if task_result:
task_result.delete()
if export_task.pid and export_task.worker:
kill_task.apply_async(
kwargs={
"task_pid": export_task.pid,
"celery_uid": export_task.celery_uid
},
queue="{0}.cancel".format(export_task.worker),
priority=TaskPriority.CANCEL.value,
routing_key="{0}.cancel".format(export_task.worker))
export_provider_task.status = TaskStates.CANCELED.value
export_provider_task.save()
# Because the task is revoked the follow on is never run... if using revoke this is required, if using kill,
# this can probably be removed as the task will simply fail and the follow on task from the task_factory will
# pick up the task.
run_uid = export_provider_task.run.uid
worker = export_provider_task.tasks.first().worker
# Because we don't care about the files in a canceled task the stage dir can be the run dir,
# which will be cleaned up in final steps.
stage_dir = os.path.join(settings.EXPORT_STAGING_ROOT.rstrip('\/'),
str(run_uid))
finalize_export_provider_task.si(
run_uid=run_uid,
stage_dir=stage_dir,
export_provider_task_uid=export_provider_task_uid,
worker=worker).set(queue=worker, routing_key=worker).apply_async(
interval=1,
max_retries=10,
expires=datetime.now() + timedelta(days=2),
priority=TaskPriority.FINALIZE_PROVIDER.value,
routing_key=worker,
queue=worker)
return result
