def add_logger(self, task):
self.log_handler = MongoLogHandler(task)
root_logger = logging.getLogger()
for handler in root_logger.handlers:
self.log_handlers_level[handler] = handler.level
handler.level = logging.ERROR
root_logger.addHandler(self.log_handler)
class MuleCommand(NoArgsCommand):
mule_name = _('Mule')
mule_flags = []
option_list = NoArgsCommand.option_list + (
make_option('--task-limit', dest='task_limit', type=int, default=0,
help=_('Exit after processing given number of tasks '
'(default is no limit)')),
make_option('--ping-disabled', action="store_true",
dest='ping_disabled', default=False,
help=_('Disable health check pings')),
make_option('--ping-interval', dest='ping_interval', type=int,
default=60, help=_('Health check ping interval (default is'
' 60 seconds)')),
)
def __init__(self, *args, **kwargs):
super(MuleCommand, self).__init__(*args, **kwargs)
self.backend_helper = MuleBackendHelper(
self.mule_name.replace(' ', ''))
self.backend = self.backend_helper.backend
self.task_helper = MuleTaskHelper(self.mule_name.replace(' ', ''))
self.is_exiting = False
self.tasks_done = 0
self.task_limit = 0
self.cleanup()
self.pid = getpid()
self.log_handler = None
self.log_handlers_level = {}
# FIXME capture all logs and prefix with self.logger
def cleanup(self):
self.app_name = _('N/A')
def add_logger(self, task):
self.log_handler = MongoLogHandler(task)
root_logger = logging.getLogger()
for handler in root_logger.handlers:
self.log_handlers_level[handler] = handler.level
handler.level = logging.ERROR
root_logger.addHandler(self.log_handler)
def remove_logger(self):
if self.log_handler:
self.log_handler.flush()
root_logger = logging.getLogger()
for handler in root_logger.handlers:
level = self.log_handlers_level.get(handler)
if level is not None:
handler.level = level
if self.log_handler:
root_logger.removeHandler(self.log_handler)
def mark_exiting(self, *args):
log.info(_("Shutting down, waiting for current task to finish"))
self.is_exiting = True
def create_task(self, application, title, flag=None):
task = Task(backend=self.backend, pid=self.pid, title=title,
application=application)
if flag:
task.flag = flag
task.save()
self.add_logger(task)
return task
def fail_task(self, task):
self.remove_logger()
self.cleanup()
task.update(set__status=TaskStatus.failed,
set__date_finished=datetime.now())
log.error(_("Task failed for {name} [{id}]").format(
name=task.application.name, id=task.application.safe_id))
raise MuleTaskFailed
def mark_task_successful(self, task):
task.update(set__status=TaskStatus.successful, set__progress=100,
set__date_finished=datetime.now())
def task_completed(self):
self.tasks_done += 1
log.info(_("Task completed, [done: {tasks_done}, limit: "
"{task_limit}]").format(tasks_done=self.tasks_done,
task_limit=self.task_limit))
def handle_noargs(self, **options):
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP,
signal.SIGQUIT]:
signal.signal(sig, self.mark_exiting)
if not options['ping_disabled']:
self.backend_helper.start_pinger(interval=options['ping_interval'])
self.task_limit = options['task_limit']
log.info(_("{name} ready, waiting for tasks (limit: "
"{task_limit})").format(name=self.mule_name,
task_limit=self.task_limit))
while True:
if self.task_limit and self.tasks_done >= self.task_limit:
log.info(_('Task limit reached {task_limit}, exiting').format(
task_limit=self.task_limit))
self.is_exiting = True
if self.is_exiting:
self.backend_helper.stop_pinger()
return
self.task_helper.clean(self.backend)
self.handle_task()
sleep(1)
def handle_task(self):
flag = self.find_flag()
if flag:
failed = False
try:
self.handle_flag(flag)
except MuleTaskFailed:
failed = True
finally:
self.unlock_flag(flag)
self.remove_logger()
self.task_completed()
if failed:
return True
if flag.name in SINGLE_SHOT_FLAGS:
ApplicationFlag.objects(application=flag.application,
name=flag.name,
pending=False).delete()
else:
flag.update(pull__pending_backends=self.backend)
ApplicationFlag.objects(application=flag.application,
name=flag.name,
pending_backends__size=0).delete()
self.cleanup()
return True
return False
def handle_flag(self, flag):
raise NotImplementedError
def fail_flag(self, flag, task):
flag.delete()
self.fail_task(task)
def flag_filter(self):
single_shot_flags = []
multi_show_flags = []
for flag in self.mule_flags:
if flag in SINGLE_SHOT_FLAGS:
single_shot_flags.append(flag)
else:
multi_show_flags.append(flag)
return ApplicationFlag.objects(
(
Q(pending__ne=False) &
Q(name__in=single_shot_flags) &
Q(application__nin=FlagLock.objects(
flag__in=single_shot_flags).distinct('application'))
) | (
Q(pending_backends=self.backend) &
Q(name__in=multi_show_flags) &
Q(application__nin=FlagLock.objects(
flag__in=multi_show_flags,
backend=self.backend).distinct('application'))
)
)
def find_flag(self):
if not self.mule_flags:
raise RuntimeError(_('No flags set for mule'))
flag = self.flag_filter().first()
if flag:
kwargs = {}
if flag.name in SINGLE_SHOT_FLAGS:
flag.update(set__pending=False)
else:
kwargs['backend'] = self.backend
lock = FlagLock(application=flag.application, flag=flag.name,
pid=self.pid, **kwargs)
try:
lock.save()
except NotUniqueError:
return
return flag
def unlock_flag(self, flag):
if flag.name in SINGLE_SHOT_FLAGS:
lock = FlagLock.objects(application=flag.application,
flag=flag.name,
backend__exists=False).first()
else:
lock = FlagLock.objects(application=flag.application,
flag=flag.name,
backend=self.backend).first()
if lock:
lock.delete()
