def unlock_flag(self, flag):
if flag.name in SINGLE_SHOT_FLAGS:
lock = FlagLock.objects(application=flag.application,
flag=flag.name,
backend__exists=False).first()
else:
lock = FlagLock.objects(application=flag.application,
flag=flag.name,
backend=self.backend).first()
if lock:
lock.delete()
def find_flag(self):
if not self.mule_flags:
raise RuntimeError(_('No flags set for mule'))
flag = self.flag_filter().first()
if flag:
kwargs = {}
if flag.name in SINGLE_SHOT_FLAGS:
flag.update(set__pending=False)
else:
kwargs['backend'] = self.backend
lock = FlagLock(application=flag.application, flag=flag.name,
pid=self.pid, **kwargs)
try:
lock.save()
except NotUniqueError:
return
return flag
def clean_failed_locks(self, backend):
name = 'local_locks'
if not self.can_clean(name):
return
for lock in FlagLock.objects(backend=backend):
if not is_pid_running(lock.pid):
log.warning(_("Found stale lock, removing (app: {name}, pid: "
"{pid})").format(name=lock.application.name,
pid=lock.pid))
self.reset_pending_state(lock)
lock.delete()
self.last_clean[name] = datetime.now()
def flag_filter(self):
single_shot_flags = []
multi_show_flags = []
for flag in self.mule_flags:
if flag in SINGLE_SHOT_FLAGS:
single_shot_flags.append(flag)
else:
multi_show_flags.append(flag)
return ApplicationFlag.objects(
(
Q(pending__ne=False) &
Q(name__in=single_shot_flags) &
Q(application__nin=FlagLock.objects(
flag__in=single_shot_flags).distinct('application'))
) | (
Q(pending_backends=self.backend) &
Q(name__in=multi_show_flags) &
Q(application__nin=FlagLock.objects(
flag__in=multi_show_flags,
backend=self.backend).distinct('application'))
)
)
def clean_orphaned_locks(self):
name = 'orphaned_locks'
if not self.can_clean(name):
return
timestamp = datetime.now() - timedelta(seconds=60)
for lock in FlagLock.objects(
backend__exists=False, flag__in=SINGLE_SHOT_FLAGS,
date_created__lte=timestamp):
if len(Task.objects(
Q(application=lock.application, flag=lock.flag) & (
Q(status=TaskStatus.running) |
Q(date_finished__gte=(
datetime.now() - timedelta(seconds=30)))))) == 0:
log.warning(_("Found stale lock, removing (app: "
"{name})").format(name=lock.application.name))
self.reset_pending_state(lock)
lock.delete()
self.last_clean[name] = datetime.now()
def clean_failed_remote_locks(self, local_backend):
name = 'remote_tasks'
if not self.can_clean(name):
return
# look for tasks locked at backends that did not ack itself to the
# database for at least 600 seconds
timestamp = datetime.now() - timedelta(seconds=600)
backends = BackendServer.objects(**{
'id__ne': local_backend.id,
'is_enabled__ne': False,
'worker_ping__%s__lte' % self.name: timestamp
})
if backends:
log.debug(_("{len} non responsive backends: {names}").format(
len=len(backends), names=[b.name for b in backends]))
for lock in FlagLock.objects(backend__in=backends,
date_created__lte=timestamp):
log.warning(_("Found old lock on backend {backend}, but it "
"didn't send any pings for 10 minutes, "
"removing").format(backend=lock.backend))
self.reset_pending_state(lock)
lock.delete()
self.last_clean[name] = datetime.now()