def unlock_flag(self, flag): if flag.name in SINGLE_SHOT_FLAGS: lock = FlagLock.objects(application=flag.application, flag=flag.name, backend__exists=False).first() else: lock = FlagLock.objects(application=flag.application, flag=flag.name, backend=self.backend).first() if lock: lock.delete()
def find_flag(self): if not self.mule_flags: raise RuntimeError(_('No flags set for mule')) flag = self.flag_filter().first() if flag: kwargs = {} if flag.name in SINGLE_SHOT_FLAGS: flag.update(set__pending=False) else: kwargs['backend'] = self.backend lock = FlagLock(application=flag.application, flag=flag.name, pid=self.pid, **kwargs) try: lock.save() except NotUniqueError: return return flag
def clean_failed_locks(self, backend): name = 'local_locks' if not self.can_clean(name): return for lock in FlagLock.objects(backend=backend): if not is_pid_running(lock.pid): log.warning(_("Found stale lock, removing (app: {name}, pid: " "{pid})").format(name=lock.application.name, pid=lock.pid)) self.reset_pending_state(lock) lock.delete() self.last_clean[name] = datetime.now()
def flag_filter(self): single_shot_flags = [] multi_show_flags = [] for flag in self.mule_flags: if flag in SINGLE_SHOT_FLAGS: single_shot_flags.append(flag) else: multi_show_flags.append(flag) return ApplicationFlag.objects( ( Q(pending__ne=False) & Q(name__in=single_shot_flags) & Q(application__nin=FlagLock.objects( flag__in=single_shot_flags).distinct('application')) ) | ( Q(pending_backends=self.backend) & Q(name__in=multi_show_flags) & Q(application__nin=FlagLock.objects( flag__in=multi_show_flags, backend=self.backend).distinct('application')) ) )
def clean_orphaned_locks(self): name = 'orphaned_locks' if not self.can_clean(name): return timestamp = datetime.now() - timedelta(seconds=60) for lock in FlagLock.objects( backend__exists=False, flag__in=SINGLE_SHOT_FLAGS, date_created__lte=timestamp): if len(Task.objects( Q(application=lock.application, flag=lock.flag) & ( Q(status=TaskStatus.running) | Q(date_finished__gte=( datetime.now() - timedelta(seconds=30)))))) == 0: log.warning(_("Found stale lock, removing (app: " "{name})").format(name=lock.application.name)) self.reset_pending_state(lock) lock.delete() self.last_clean[name] = datetime.now()
def clean_failed_remote_locks(self, local_backend): name = 'remote_tasks' if not self.can_clean(name): return # look for tasks locked at backends that did not ack itself to the # database for at least 600 seconds timestamp = datetime.now() - timedelta(seconds=600) backends = BackendServer.objects(**{ 'id__ne': local_backend.id, 'is_enabled__ne': False, 'worker_ping__%s__lte' % self.name: timestamp }) if backends: log.debug(_("{len} non responsive backends: {names}").format( len=len(backends), names=[b.name for b in backends])) for lock in FlagLock.objects(backend__in=backends, date_created__lte=timestamp): log.warning(_("Found old lock on backend {backend}, but it " "didn't send any pings for 10 minutes, " "removing").format(backend=lock.backend)) self.reset_pending_state(lock) lock.delete() self.last_clean[name] = datetime.now()