def create_scheduler_task_history(self, task, last_run, next_run,
scheduled_task_id):
obj = SchedulerTaskHistory(name=task.name,
last_run=last_run,
next_run=next_run,
scheduled_task_id=scheduled_task_id)
yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH,
task.name.encode('utf-8'),
obj.serialize())
def _cleanup_scheduled_history(self):
# Clean hash table in redis for task with very old last-run and without next_run
log.info("Run cleanup task for table Scheduled History")
tasks_history = yield from self.connection.hgetall(
SETTINGS.SCHEDULER_HISTORY_HASH)
for f in tasks_history:
key, value = yield from f
# Iterate over all tasks in history and deserialize
try:
task_history = SchedulerTaskHistory.deserialize(value)
except (pickle.UnpicklingError, EOFError, TypeError, ImportError):
log.error(
'Cannot deserialize SchedulerTaskHistory for {}'.format(
key),
exc_info=True)
continue
if not task_history.next_run and (
int(now()) - task_history.last_run) > (
SETTINGS.SCHEDULED_HISTORY_CLEANUP_MAX_TTL * 1000):
# task is too old, remove it
log.info(
'Cleanup for Scheduled History table. Remove task, name={}'
.format(task_history.name))
yield from self.connection.hdel(
SETTINGS.SCHEDULER_HISTORY_HASH, [key])
self.current_loop.call_later(SETTINGS.SCHEDULED_HISTORY_CLEANUP_PERIOD,
self._create_asyncio_task,
self._cleanup_scheduled_history)
def _load_scheduler_tasks_history(self):
""" Load list of scheduled tasks tasks run times """
# Load run history for scheduled tasks
tasks_history = yield from self.connection.hgetall(
SETTINGS.SCHEDULER_HISTORY_HASH)
new_keys = set()
for f in tasks_history:
key, value = yield from f
key = key.decode('utf-8')
new_keys.add(key)
# Iterate over all tasks in history
if key in self.scheduler_tasks:
# Is task still in crontab?
# Deserialize
try:
task_history = SchedulerTaskHistory.deserialize(value)
except (pickle.UnpicklingError, EOFError, TypeError,
ImportError):
log.error('Cannot deserialize SchedulerTaskHistory for {}'.
format(key),
exc_info=True)
continue
self.scheduler_tasks_history[key].update(
dict(last_run=task_history.last_run,
next_run=task_history.next_run,
scheduled_task_id=task_history.scheduled_task_id))
for key in set(self.scheduler_tasks_history.keys()) - new_keys:
del self.scheduler_tasks_history[key]
def _check_expired_tasks(self):
time_now = int(now())
if time_now - self._ttl_check_last_run < 1000: # 1000 = 1sec
return
self._ttl_check_last_run = time_now
TTL = SETTINGS.WORKER_TASK_TIMEOUT * 1000
for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items():
scheduled_task = self.scheduler_tasks.get(scheduled_task_name)
if (scheduled_task_history.get('next_run')
and scheduled_task_history.get('scheduled_task_id')
and (time_now - scheduled_task_history.get('next_run')) > (scheduled_task.get('ttl') or SETTINGS.WORKER_TASK_TIMEOUT)*1000):
task_id = scheduled_task_history.get('scheduled_task_id')
log.info('Fix broken task id={}, name={}'.format(task_id, scheduled_task_name))
# Get task object from redis key
key = SETTINGS.TASK_STORAGE_KEY.format(scheduled_task_history.get('scheduled_task_id')).encode('utf-8')
task_obj = yield from self.connection.get(key)
# Deserialize task object
try:
if not task_obj:
raise TypeError()
task = Task.deserialize(task_obj)
if task.status != Task.SUCCESSFUL:
# Update task object status
task = task._replace(status=Task.FAILED)
# Set new status to redis
yield from self.connection.set(key, task.serialize(), expire=SETTINGS.TASK_STORAGE_EXPIRE)
except TypeError as ex:
task = None
log.error("Wrong task id={}".format(scheduled_task_history.get('scheduled_task_id')), exc_info=True)
yield from self.connection.delete([key])
# Publish message about finish (FAILED)
if task:
yield from self.connection.publish(SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), task.status.encode('utf-8'))
else:
yield from self.connection.publish(SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), Task.FAILED.encode('utf-8'))
# Update scheduler information
# Store next_run in scheduled
try:
task_scheduler_obj = yield from self.connection.hget(SETTINGS.SCHEDULER_HISTORY_HASH, scheduled_task_name.encode('utf-8'))
task_scheduler = SchedulerTaskHistory.deserialize(task_scheduler_obj)
if task and task.status == Task.SUCCESSFUL:
scheduled_task_history['last_run'] = scheduled_task_history.get('next_run', 0)
scheduled_task_history['next_run'] = 0
task_scheduler = task_scheduler._replace(last_run=task_scheduler.next_run, next_run=0, scheduled_task_id=None)
else:
scheduled_task_history['next_run'] = 0
scheduled_task_history['scheduled_task_id'] = None
task_scheduler = task_scheduler._replace(next_run=0, scheduled_task_id=None)
yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8'), task_scheduler.serialize())
except:
# We lost SCHEDULER_HISTORY_HASH in db
if task and task.status == Task.SUCCESSFUL:
scheduled_task_history['last_run'] = scheduled_task_history.get('next_run', 0)
scheduled_task_history['next_run'] = 0
else:
scheduled_task_history['next_run'] = 0
scheduled_task_history['scheduled_task_id'] = None
def _reload_config_tasks_list(self):
""" Load list of tasks, details """
time_now = int(now())
if time_now - self._ttl_reload_config_last_run < 1000: # 1000 = 1sec
return
self._ttl_reload_config_last_run = time_now
config_version = self.config.get_config_version()
if config_version != self.config_version:
log.info('Changes in actions list, update.')
new_scheduler_tasks = self.config.get_scheduled_actions()
new_keys = set(new_scheduler_tasks.keys()) - set(self.scheduler_tasks.keys())
deleted_keys = set(self.scheduler_tasks.keys()) - set(new_scheduler_tasks.keys())
if new_keys or deleted_keys:
log.info('New actions list, new_keys={}, deleted_keys={}'.format(new_keys, deleted_keys))
self.scheduler_tasks = new_scheduler_tasks
yield from self._load_scheduler_tasks_history()
# Check scheduler_tasks_history here, please
# Возможно, интервал запуска изменился с длинного на короткий
# А у нас уже next_run стоит далеко в будущем
for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items():
# Смотри все таски, для которых сохранена инфорамция по шедулингу
if scheduled_task_history.get('next_run', 0): # and (scheduled_task_name in self.scheduler_tasks):
# Если есть запланированный таск
if scheduled_task_name in self.scheduler_tasks:
# Если у таска осталось расписание
possible_next_run = datetime_to_timestamp(self._get_next_run_time(scheduled_task_name, self.scheduler_tasks[scheduled_task_name], int(now())))
else:
# У таска не осталось расписания, next_run надо привести к 0 и больше ничего не делать
possible_next_run = 0
if scheduled_task_history.get('next_run', 0) != possible_next_run:
# Cancel scheduled task
# Reset next_run
task_id = scheduled_task_history.get('scheduled_task_id')
log.info('Schedule changed for task with id={}, name={}, reschedule next_task'.format(task_id, scheduled_task_name))
key = SETTINGS.TASK_STORAGE_KEY.format(task_id).encode('utf-8')
task_obj = yield from self.connection.delete([key])
scheduled_task_history['next_run'] = 0
scheduled_task_history['scheduled_task_id'] = 0
try:
task_scheduler_obj = yield from self.connection.hget(SETTINGS.SCHEDULER_HISTORY_HASH, scheduled_task_name.encode('utf-8'))
task_scheduler = SchedulerTaskHistory.deserialize(task_scheduler_obj)
task_scheduler = task_scheduler._replace(next_run=0, scheduled_task_id=None)
yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8'), task_scheduler.serialize())
except:
log.error('Broken SchedulerTaskHistory object for task id={}, delete it'.format(scheduled_task_name))
yield from self.connection.hdel(SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8'))
# Удалился какой-то таск? Удалим его из мониторинга выполнения
for key in deleted_keys:
if key in self.scheduler_tasks_history:
del self.scheduler_tasks_history[key]
self.config_version = config_version
def _cleanup_task(self, task):
""" clenaup_task -- Task for cleanup completed tasks from redis queue.
:param task: `sensors.models.Task` instance
:return: None
"""
log.debug("_cleanup_task task_id={}".format(task.id))
# Remove task from inprogress queue
# XXX may be transaction here?
cnt1 = yield from self.connection.lrem(SETTINGS.INPROGRESS_QUEUE,
value=task.bid())
# Remove task from sorted set
cnt2 = yield from self.connection.zrem(SETTINGS.INPROGRESS_TASKS_SET,
[task.bid()])
# Update scheduler information
# Store next_run in scheduled
task_scheduler_obj = yield from self.connection.hget(
SETTINGS.SCHEDULER_HISTORY_HASH, task.name.encode('utf-8'))
try:
task_scheduler = SchedulerTaskHistory.deserialize(
task_scheduler_obj)
except (pickle.UnpicklingError, EOFError, TypeError, ImportError):
task_scheduler = None
if task_scheduler and task_scheduler.scheduled_task_id == task.id:
#if task.status == Task.SUCCESSFUL:
# # Update last_run only on success
# last_run = datetime_to_timestamp(task.run_at)
#else:
# # If task failed, do not update last_run (last_run is about SUCCESSFUL task exectuion)
# last_run = task_scheduler.last_run
last_run = datetime_to_timestamp(task.run_at)
task_scheduler = task_scheduler._replace(last_run=last_run,
next_run=0,
scheduled_task_id=None)
yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH,
task.name.encode('utf-8'),
task_scheduler.serialize())
# Publish message about finish
yield from self.connection.publish(
SETTINGS.TASK_CHANNEL.format(task.id).encode('utf-8'),
task.status.encode('utf-8'))
log.debug('Publish message about task {} to {}'.format(
task.id, SETTINGS.TASK_CHANNEL.format(task.id)))
log.debug("_cleanup_task lrem result {}".format(cnt1))
log.debug("_cleanup_task zrem result {}".format(cnt2))
# Ping scheduler
yield from self._ping_scheduler(task)
def _cleanup_scheduled_history(self):
# Clean hash table in redis for task with very old last-run and without next_run
log.info("Run cleanup task for table Scheduled History")
tasks_history = yield from self.connection.hgetall(SETTINGS.SCHEDULER_HISTORY_HASH)
for f in tasks_history:
key, value = yield from f
# Iterate over all tasks in history and deserialize
try:
task_history = SchedulerTaskHistory.deserialize(value)
except (pickle.UnpicklingError, EOFError, TypeError, ImportError):
log.error('Cannot deserialize SchedulerTaskHistory for {}'.format(key), exc_info=True)
continue
if not task_history.next_run and (int(now()) - task_history.last_run) > (SETTINGS.SCHEDULED_HISTORY_CLEANUP_MAX_TTL * 1000):
# task is too old, remove it
log.info('Cleanup for Scheduled History table. Remove task, name={}'.format(task_history.name))
yield from self.connection.hdel(SETTINGS.SCHEDULER_HISTORY_HASH, [key])
self.current_loop.call_later(SETTINGS.SCHEDULED_HISTORY_CLEANUP_PERIOD, self._create_asyncio_task, self._cleanup_scheduled_history)
def _cleanup_task(self, task):
""" clenaup_task -- Task for cleanup completed tasks from redis queue.
:param task: `sensors.models.Task` instance
:return: None
"""
log.debug("_cleanup_task task_id={}".format(task.id))
# Remove task from inprogress queue
# XXX may be transaction here?
cnt1 = yield from self.connection.lrem(SETTINGS.INPROGRESS_QUEUE, value=task.bid())
# Remove task from sorted set
cnt2 = yield from self.connection.zrem(SETTINGS.INPROGRESS_TASKS_SET, [task.bid()])
# Update scheduler information
# Store next_run in scheduled
task_scheduler_obj = yield from self.connection.hget(SETTINGS.SCHEDULER_HISTORY_HASH, task.name.encode('utf-8'))
try:
task_scheduler = SchedulerTaskHistory.deserialize(task_scheduler_obj)
except (pickle.UnpicklingError, EOFError, TypeError, ImportError):
task_scheduler = None
if task_scheduler and task_scheduler.scheduled_task_id == task.id:
#if task.status == Task.SUCCESSFUL:
# # Update last_run only on success
# last_run = datetime_to_timestamp(task.run_at)
#else:
# # If task failed, do not update last_run (last_run is about SUCCESSFUL task exectuion)
# last_run = task_scheduler.last_run
last_run = datetime_to_timestamp(task.run_at)
task_scheduler = task_scheduler._replace(last_run=last_run, next_run=0, scheduled_task_id=None)
yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH, task.name.encode('utf-8'), task_scheduler.serialize())
# Publish message about finish
yield from self.connection.publish(SETTINGS.TASK_CHANNEL.format(task.id).encode('utf-8'), task.status.encode('utf-8'))
log.debug('Publish message about task {} to {}'.format(task.id, SETTINGS.TASK_CHANNEL.format(task.id)))
log.debug("_cleanup_task lrem result {}".format(cnt1))
log.debug("_cleanup_task zrem result {}".format(cnt2))
# Ping scheduler
yield from self._ping_scheduler(task)
def _load_scheduler_tasks_history(self):
""" Load list of scheduled tasks tasks run times """
# Load run history for scheduled tasks
tasks_history = yield from self.connection.hgetall(SETTINGS.SCHEDULER_HISTORY_HASH)
new_keys = set()
for f in tasks_history:
key, value = yield from f
key = key.decode('utf-8')
new_keys.add(key)
# Iterate over all tasks in history
if key in self.scheduler_tasks:
# Is task still in crontab?
# Deserialize
try:
task_history = SchedulerTaskHistory.deserialize(value)
except (pickle.UnpicklingError, EOFError, TypeError, ImportError):
log.error('Cannot deserialize SchedulerTaskHistory for {}'.format(key), exc_info=True)
continue
self.scheduler_tasks_history[key].update(dict(last_run=task_history.last_run,
next_run=task_history.next_run,
scheduled_task_id=task_history.scheduled_task_id))
for key in set(self.scheduler_tasks_history.keys()) - new_keys:
del self.scheduler_tasks_history[key]
def _check_expired_tasks(self):
time_now = int(now())
if time_now - self._ttl_check_last_run < 1000: # 1000 = 1sec
return
self._ttl_check_last_run = time_now
TTL = SETTINGS.WORKER_TASK_TIMEOUT * 1000
for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items(
):
scheduled_task = self.scheduler_tasks.get(scheduled_task_name)
if (scheduled_task_history.get('next_run')
and scheduled_task_history.get('scheduled_task_id')
and (time_now - scheduled_task_history.get('next_run')) >
(scheduled_task.get('ttl') or SETTINGS.WORKER_TASK_TIMEOUT) *
1000):
task_id = scheduled_task_history.get('scheduled_task_id')
log.info('Fix broken task id={}, name={}'.format(
task_id, scheduled_task_name))
# Get task object from redis key
key = SETTINGS.TASK_STORAGE_KEY.format(
scheduled_task_history.get('scheduled_task_id')).encode(
'utf-8')
task_obj = yield from self.connection.get(key)
# Deserialize task object
try:
if not task_obj:
raise TypeError()
task = Task.deserialize(task_obj)
if task.status != Task.SUCCESSFUL:
# Update task object status
task = task._replace(status=Task.FAILED)
# Set new status to redis
yield from self.connection.set(
key,
task.serialize(),
expire=SETTINGS.TASK_STORAGE_EXPIRE)
except TypeError as ex:
task = None
log.error("Wrong task id={}".format(
scheduled_task_history.get('scheduled_task_id')),
exc_info=True)
yield from self.connection.delete([key])
# Publish message about finish (FAILED)
if task:
yield from self.connection.publish(
SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'),
task.status.encode('utf-8'))
else:
yield from self.connection.publish(
SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'),
Task.FAILED.encode('utf-8'))
# Update scheduler information
# Store next_run in scheduled
try:
task_scheduler_obj = yield from self.connection.hget(
SETTINGS.SCHEDULER_HISTORY_HASH,
scheduled_task_name.encode('utf-8'))
task_scheduler = SchedulerTaskHistory.deserialize(
task_scheduler_obj)
if task and task.status == Task.SUCCESSFUL:
scheduled_task_history[
'last_run'] = scheduled_task_history.get(
'next_run', 0)
scheduled_task_history['next_run'] = 0
task_scheduler = task_scheduler._replace(
last_run=task_scheduler.next_run,
next_run=0,
scheduled_task_id=None)
else:
scheduled_task_history['next_run'] = 0
scheduled_task_history['scheduled_task_id'] = None
task_scheduler = task_scheduler._replace(
next_run=0, scheduled_task_id=None)
yield from self.connection.hset(
SETTINGS.SCHEDULER_HISTORY_HASH,
task_scheduler.name.encode('utf-8'),
task_scheduler.serialize())
except:
# We lost SCHEDULER_HISTORY_HASH in db
if task and task.status == Task.SUCCESSFUL:
scheduled_task_history[
'last_run'] = scheduled_task_history.get(
'next_run', 0)
scheduled_task_history['next_run'] = 0
else:
scheduled_task_history['next_run'] = 0
scheduled_task_history['scheduled_task_id'] = None
def _reload_config_tasks_list(self):
""" Load list of tasks, details """
time_now = int(now())
if time_now - self._ttl_reload_config_last_run < 1000: # 1000 = 1sec
return
self._ttl_reload_config_last_run = time_now
config_version = self.config.get_config_version()
if config_version != self.config_version:
log.info('Changes in actions list, update.')
new_scheduler_tasks = self.config.get_scheduled_actions()
new_keys = set(new_scheduler_tasks.keys()) - set(
self.scheduler_tasks.keys())
deleted_keys = set(self.scheduler_tasks.keys()) - set(
new_scheduler_tasks.keys())
if new_keys or deleted_keys:
log.info(
'New actions list, new_keys={}, deleted_keys={}'.format(
new_keys, deleted_keys))
self.scheduler_tasks = new_scheduler_tasks
yield from self._load_scheduler_tasks_history()
# Check scheduler_tasks_history here, please
# Возможно, интервал запуска изменился с длинного на короткий
# А у нас уже next_run стоит далеко в будущем
for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items(
):
# Смотри все таски, для которых сохранена инфорамция по шедулингу
if scheduled_task_history.get(
'next_run', 0
): # and (scheduled_task_name in self.scheduler_tasks):
# Если есть запланированный таск
if scheduled_task_name in self.scheduler_tasks:
# Если у таска осталось расписание
possible_next_run = datetime_to_timestamp(
self._get_next_run_time(
scheduled_task_name,
self.scheduler_tasks[scheduled_task_name],
int(now())))
else:
# У таска не осталось расписания, next_run надо привести к 0 и больше ничего не делать
possible_next_run = 0
if scheduled_task_history.get('next_run',
0) != possible_next_run:
# Cancel scheduled task
# Reset next_run
task_id = scheduled_task_history.get(
'scheduled_task_id')
log.info(
'Schedule changed for task with id={}, name={}, reschedule next_task'
.format(task_id, scheduled_task_name))
key = SETTINGS.TASK_STORAGE_KEY.format(task_id).encode(
'utf-8')
task_obj = yield from self.connection.delete([key])
scheduled_task_history['next_run'] = 0
scheduled_task_history['scheduled_task_id'] = 0
try:
task_scheduler_obj = yield from self.connection.hget(
SETTINGS.SCHEDULER_HISTORY_HASH,
scheduled_task_name.encode('utf-8'))
task_scheduler = SchedulerTaskHistory.deserialize(
task_scheduler_obj)
task_scheduler = task_scheduler._replace(
next_run=0, scheduled_task_id=None)
yield from self.connection.hset(
SETTINGS.SCHEDULER_HISTORY_HASH,
task_scheduler.name.encode('utf-8'),
task_scheduler.serialize())
except:
log.error(
'Broken SchedulerTaskHistory object for task id={}, delete it'
.format(scheduled_task_name))
yield from self.connection.hdel(
SETTINGS.SCHEDULER_HISTORY_HASH,
task_scheduler.name.encode('utf-8'))
# Удалился какой-то таск? Удалим его из мониторинга выполнения
for key in deleted_keys:
if key in self.scheduler_tasks_history:
del self.scheduler_tasks_history[key]
self.config_version = config_version
def create_scheduler_task_history(self, task, last_run, next_run, scheduled_task_id):
obj = SchedulerTaskHistory(name=task.name, last_run=last_run, next_run=next_run, scheduled_task_id=scheduled_task_id)
yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH, task.name.encode('utf-8'), obj.serialize())
