def prepare_tuple(subtuple, tuple_type):
from django_celery_results.models import TaskResult
compute_plan_id = None
worker_queue = f"{settings.LEDGER['name']}.worker"
if 'computePlanID' in subtuple and subtuple['computePlanID']:
compute_plan_id = subtuple['computePlanID']
flresults = TaskResult.objects.filter(
task_name='substrapp.tasks.tasks.compute_task',
result__icontains=f'"computePlanID": "{compute_plan_id}"')
if flresults and flresults.count() > 0:
worker_queue = json.loads(
flresults.first().as_dict()['result'])['worker']
try:
log_start_tuple(tuple_type, subtuple['key'])
except LedgerStatusError as e:
# Do not log_fail_tuple in this case, because prepare_tuple task are not unique
# in case of multiple instances of substra backend running for the same organisation
# So prepare_tuple tasks are ignored if it cannot log_start_tuple
logging.exception(e)
raise Ignore()
try:
compute_task.apply_async((tuple_type, subtuple, compute_plan_id),
queue=worker_queue)
except Exception as e:
error_code = compute_error_code(e)
logging.error(error_code, exc_info=True)
log_fail_tuple(tuple_type, subtuple['key'], error_code)
def test_exception_handler(self):
# Python exception in system
try:
1 / 0
except Exception as e:
error_code = compute_error_code(e)
value_error_code, _ = get_exception_code(ZeroDivisionError)
self.assertIn(f'00-01-{value_error_code}', error_code)
# Python exception in docker
try:
client = docker.from_env()
client.containers.run("python:3.6", ['python3', '-c', 'print(KO)'],
remove=True)
except Exception as e:
error_code = compute_error_code(e)
container_error_code, _ = get_exception_code(NameError)
self.assertIn(f'01-01-{container_error_code}', error_code)
def on_failure(self, exc, task_id, args, kwargs, einfo):
tuple_type, subtuple, compute_plan_id = self.split_args(args)
try:
error_code = compute_error_code(exc)
logging.error(error_code, exc_info=True)
log_fail_tuple(tuple_type, subtuple['key'], error_code)
except LedgerError as e:
logging.exception(e)
try:
try_remove_local_folder(subtuple, compute_plan_id)
except Exception as e:
logging.exception(e)
def on_failure(self, exc, task_id, args, kwargs, einfo):
from django.db import close_old_connections
close_old_connections()
channel_name, tuple_type, subtuple, compute_plan_key = self.split_args(
args)
try:
error_code = compute_error_code(exc)
# Do not show traceback if it's a container error as we already see them in
# container log
type_exc = type(exc)
type_value = str(type_exc).split("'")[1]
logger.error(
f'Failed compute task: {tuple_type} {subtuple["key"]} {error_code} - {type_value}'
)
log_fail_tuple(channel_name, tuple_type, subtuple['key'],
error_code)
except LedgerError as e:
logger.exception(e)