def clean_failed_remote_tasks(self, local_backend): name = 'remote_tasks' if not self.can_clean(name): return # look for tasks locked at backends that did not ack itself to the # database for at least 600 seconds timestamp = datetime.now() - timedelta(seconds=600) backends = BackendServer.objects(**{ 'id__ne': local_backend.id, 'is_enabled__ne': False, 'worker_ping__%s__lte' % self.name: timestamp }) if backends: log.debug(_("{len} non responsive backends: {names}").format( len=len(backends), names=[b.name for b in backends])) for task in Task.objects(backend__in=backends, status=TaskStatus.running, date_created__lte=timestamp): log.warning(_("Task '{name}' with id {tid} is locked on " "backend {backend}, but it didn't send any " "pings for 10 minutes, marking as " "failed").format( name=task.title, tid=task.safe_id, backend=task.backend.name)) task.update(set__status=TaskStatus.failed, set__date_finished=datetime.now()) self.last_clean[name] = datetime.now()
def clean_failed_tasks(self, backend): name = 'local_tasks' if not self.can_clean(name): return for task in Task.objects(backend=backend, status=TaskStatus.running): if not is_pid_running(task.pid): log.warning(_( "Found failed task, marking as failed (id: {id}, app: " "{name})").format(id=task.safe_id, name=task.application.name)) task.update(set__status=TaskStatus.failed, set__date_finished=datetime.now()) self.last_clean[name] = datetime.now()
def messages(self, request, **kwargs): self.method_check(request, allowed=['get']) try: offset = int(request.GET.get('offset', 0)) except: offset = 0 task = Task.objects(**self.remove_api_resource_names(kwargs)).first() messages = [] for msg in task.messages[int(offset):]: messages.append({ 'timestamp': msg.timestamp.isoformat(), 'level': logging.getLevelName(msg.level), 'message': msg.message, }) return self.create_response(request, messages)
def clean_orphaned_locks(self): name = 'orphaned_locks' if not self.can_clean(name): return timestamp = datetime.now() - timedelta(seconds=60) for lock in FlagLock.objects( backend__exists=False, flag__in=SINGLE_SHOT_FLAGS, date_created__lte=timestamp): if len(Task.objects( Q(application=lock.application, flag=lock.flag) & ( Q(status=TaskStatus.running) | Q(date_finished__gte=( datetime.now() - timedelta(seconds=30)))))) == 0: log.warning(_("Found stale lock, removing (app: " "{name})").format(name=lock.application.name)) self.reset_pending_state(lock) lock.delete() self.last_clean[name] = datetime.now()
def get_queryset(self): return Task.objects(application__in=self.request.user.applications)
def tasks(self): """ List of all tasks for this application. """ return Task.objects(application=self)