def _task_status(self, task_id, event_status, tt):
if tt == TT_MGMT and task_id not in self.last_tasks:
self.log('Ignoring mgmt task %s, because we never called it', task_id, level=DEBUG)
return
task = cq.AsyncResult(task_id)
if get_callback(task):
self.log('Ignoring task %s, because it has a callback', task_id, level=DEBUG)
return
parent_task_id = is_callback(task)
if parent_task_id:
_task_id = task_id
task_id = parent_task_id
self.log('Changing task %s to %s, because it is a callback', _task_id, task_id, level=DEBUG)
elif is_logtask(task):
self.log('Ignoring task %s, because it is a logtask without caller', task_id)
return
# Try to get the apiview information from cache
apiview = None
# noinspection PyBroadException
try:
apiview = cache.get('sio-' + task_id)
except Exception:
pass
if not apiview:
apiview = {}
res = self._api_task_status(task_id)
self.log('Task %s %s', task_id, event_status)
self.emit('task_status', res.data, apiview)
def task_log(task_id,
msg,
vm=None,
obj=None,
user=None,
api_view=None,
task_status=None,
task_result=None,
owner=None,
time=None,
detail=None,
update_user_tasks=True,
dc_id=None,
**kwargs):
"""
Create task log read by the user. Also maintain dictionary of active tasks for specific VM.
"""
task_id = str(task_id)
task_prefix = task_prefix_from_task_id(task_id)
dc_id = dc_id or int(
task_prefix[4]) # Specifying a custom dc_id is usually not a good idea
update_tasks = False
# When? (time)
if not time:
time = timezone.now()
# Who? (username, task creator)
if user and not isinstance(user, string_types):
username = user.username
user_id = user.id
else:
user = None
user_id = int(task_prefix[0])
if user_id == int(cq.conf.ERIGONES_TASK_USER):
username = cq.conf.ERIGONES_TASK_USERNAME
else:
try:
username = User.objects.get(id=user_id).username
except User.DoesNotExist:
username = '******'
# What? (status and result)
if not task_status or not task_result:
# Do not follow callback here, because we are called from pending callbacks
task = cq.AsyncResult(task_id)
if not task_status:
task_status = task.state
if not task_result:
task_result = task.result
# Object? (Vm, Node, ...)
if vm:
obj = vm
if obj and isinstance(obj, (tuple, list)):
object_name = obj[0]
object_alias = obj[1]
object_pk = obj[2]
content_type = ContentType.objects.get_for_model(obj[3])
object_type = content_type.model
obj = None
elif obj and not isinstance(obj, string_types):
object_name = obj.log_name
object_alias = obj.log_alias
object_pk = obj.pk
content_type = ContentType.objects.get_for_model(obj)
object_type = content_type.model
if update_user_tasks and hasattr(obj, 'tasks'):
update_tasks = True
else:
if obj:
object_name = str(obj)
else:
object_name = ''
object_alias = ''
object_pk = ''
obj = None
content_type = None
object_type = None
if object_pk is None:
object_pk = ''
else:
object_pk = str(object_pk)
# Detail?
if not detail:
if task_result:
# noinspection PyBroadException
try:
detail = task_result['detail']
except:
# noinspection PyBroadException
try:
detail = task_result['message']
except:
detail = ''
if detail is None:
detail = ''
# Owner?
if not owner:
if obj:
# noinspection PyBroadException
try:
owner = obj.owner
except:
owner = user
elif user:
owner = user
# noinspection PyBroadException
try:
owner_id = owner.id
except:
owner_id = int(task_prefix[2])
if task_status == states.STARTED:
task_status = states.PENDING
try: # Log!
log({
'dc_id': dc_id,
'time': time,
'task': task_id,
'status': task_status,
'task_type': task_type_from_task_prefix(task_prefix),
'flag': task_flag_from_task_msg(msg),
'user_id': user_id,
'username': username,
'owner_id': owner_id,
'object_pk': object_pk,
'object_type': object_type,
'object_name': object_name,
'object_alias': object_alias,
'content_type': content_type,
'msg': msg,
'detail': detail,
})
finally:
if update_tasks: # Save task info into UserTasks cache after the task was logged
try:
if task_status in TASK_STATE_PENDING:
obj.tasks_add(task_id,
api_view,
msg=msg,
time=time.isoformat())
return True
else:
obj.tasks_del(task_id)
except Exception as e:
logger.exception(e)
logger.error(
'Got exception (%s) when updating task %s for %s.', e,
task_id, obj)
return None
def inner(task_obj, result, task_id, *args, **kwargs):
try: # Just in case, there will be probably no callback information at this point
del result['meta']['callback']
except KeyError:
pass
result['meta']['caller'] = task_id
if meta_kwargs:
result['meta'].update(meta_kwargs)
# Issue #chili-512
# + skipping checking of parent task status when task is being retried
if check_parent_status and not task_obj.request.retries:
cb_name = fun.__name__
logger.debug(
'Waiting for parent task %s status, before running %s',
task_id, cb_name)
timer = 0
while timer < TASK_PARENT_STATUS_MAXWAIT:
ar = cq.AsyncResult(task_id)
if ar.ready():
logger.info(
'Parent task %s has finished with status=%s. Running %s',
task_id, ar.status, cb_name)
break
timer += 1
logger.warning(
'Whoa! Parent task %s has not finished yet with status=%s. Waiting 1 second (%d), '
'before running %s', task_id, ar.status, timer,
cb_name)
sleep(1.0)
else:
logger.error('Task %s is not ready. Running %s anyway :(',
task_id, cb_name)
try:
return fun(result, task_id, *args, **kwargs)
except OPERATIONAL_ERRORS as exc:
raise exc # Caught by que.mgmt.MgmtCallbackTask
except Exception as e:
logger.exception(e)
logger.error('Task %s failed', task_id)
if not isinstance(e, TaskException):
e = TaskException(result, str(e))
if log_exception or update_user_tasks:
if e.obj is None:
try:
obj = get_vms_object(kwargs)
except ObjectDoesNotExist:
obj = None
# noinspection PyProtectedMember
_UserTasksModel._tasks_del(
task_id) # Always remove user task
else:
obj = e.obj
if log_exception:
msg = e.result['meta'].get('msg', '')
# Also removes user task in task_log
task_log(task_id,
msg,
obj=obj,
task_status=states.FAILURE,
task_result=e.result)
elif obj: # update_user_tasks
obj.tasks_del(task_id)
if error_fun:
try:
error_fun(result,
task_id,
task_exception=e,
*args,
**kwargs)
except Exception as ex:
logger.exception(ex)
raise e
finally:
cb = UserCallback(task_id).load()
if cb:
logger.debug('Creating task for UserCallback[%s]: %s',
task_id, cb)
from api.task.tasks import task_user_callback_cb # Circular import
task_user_callback_cb.call(task_id, cb, **kwargs)