def create_task(self, application, title, flag=None):
task = Task(backend=self.backend, pid=self.pid, title=title,
application=application)
if flag:
task.flag = flag
task.save()
self.add_logger(task)
return task
def clean_failed_remote_tasks(self, local_backend):
name = 'remote_tasks'
if not self.can_clean(name):
return
# look for tasks locked at backends that did not ack itself to the
# database for at least 600 seconds
timestamp = datetime.now() - timedelta(seconds=600)
backends = BackendServer.objects(**{
'id__ne': local_backend.id,
'is_enabled__ne': False,
'worker_ping__%s__lte' % self.name: timestamp
})
if backends:
log.debug(_("{len} non responsive backends: {names}").format(
len=len(backends), names=[b.name for b in backends]))
for task in Task.objects(backend__in=backends,
status=TaskStatus.running,
date_created__lte=timestamp):
log.warning(_("Task '{name}' with id {tid} is locked on "
"backend {backend}, but it didn't send any "
"pings for 10 minutes, marking as "
"failed").format(
name=task.title, tid=task.safe_id,
backend=task.backend.name))
task.update(set__status=TaskStatus.failed,
set__date_finished=datetime.now())
self.last_clean[name] = datetime.now()
def clean_failed_tasks(self, backend):
name = 'local_tasks'
if not self.can_clean(name):
return
for task in Task.objects(backend=backend,
status=TaskStatus.running):
if not is_pid_running(task.pid):
log.warning(_(
"Found failed task, marking as failed (id: {id}, app: "
"{name})").format(id=task.safe_id,
name=task.application.name))
task.update(set__status=TaskStatus.failed,
set__date_finished=datetime.now())
self.last_clean[name] = datetime.now()
def messages(self, request, **kwargs):
self.method_check(request, allowed=['get'])
try:
offset = int(request.GET.get('offset', 0))
except:
offset = 0
task = Task.objects(**self.remove_api_resource_names(kwargs)).first()
messages = []
for msg in task.messages[int(offset):]:
messages.append({
'timestamp': msg.timestamp.isoformat(),
'level': logging.getLevelName(msg.level),
'message': msg.message,
})
return self.create_response(request, messages)
def clean_orphaned_locks(self):
name = 'orphaned_locks'
if not self.can_clean(name):
return
timestamp = datetime.now() - timedelta(seconds=60)
for lock in FlagLock.objects(
backend__exists=False, flag__in=SINGLE_SHOT_FLAGS,
date_created__lte=timestamp):
if len(Task.objects(
Q(application=lock.application, flag=lock.flag) & (
Q(status=TaskStatus.running) |
Q(date_finished__gte=(
datetime.now() - timedelta(seconds=30)))))) == 0:
log.warning(_("Found stale lock, removing (app: "
"{name})").format(name=lock.application.name))
self.reset_pending_state(lock)
lock.delete()
self.last_clean[name] = datetime.now()
def get_queryset(self):
return Task.objects(application__in=self.request.user.applications)
def tasks(self):
"""
List of all tasks for this application.
"""
return Task.objects(application=self)
