def _cleanup_scheduled_history(self): # Clean hash table in redis for task with very old last-run and without next_run log.info("Run cleanup task for table Scheduled History") tasks_history = yield from self.connection.hgetall( SETTINGS.SCHEDULER_HISTORY_HASH) for f in tasks_history: key, value = yield from f # Iterate over all tasks in history and deserialize try: task_history = SchedulerTaskHistory.deserialize(value) except (pickle.UnpicklingError, EOFError, TypeError, ImportError): log.error( 'Cannot deserialize SchedulerTaskHistory for {}'.format( key), exc_info=True) continue if not task_history.next_run and ( int(now()) - task_history.last_run) > ( SETTINGS.SCHEDULED_HISTORY_CLEANUP_MAX_TTL * 1000): # task is too old, remove it log.info( 'Cleanup for Scheduled History table. Remove task, name={}' .format(task_history.name)) yield from self.connection.hdel( SETTINGS.SCHEDULER_HISTORY_HASH, [key]) self.current_loop.call_later(SETTINGS.SCHEDULED_HISTORY_CLEANUP_PERIOD, self._create_asyncio_task, self._cleanup_scheduled_history)
def _load_scheduler_tasks_history(self): """ Load list of scheduled tasks tasks run times """ # Load run history for scheduled tasks tasks_history = yield from self.connection.hgetall( SETTINGS.SCHEDULER_HISTORY_HASH) new_keys = set() for f in tasks_history: key, value = yield from f key = key.decode('utf-8') new_keys.add(key) # Iterate over all tasks in history if key in self.scheduler_tasks: # Is task still in crontab? # Deserialize try: task_history = SchedulerTaskHistory.deserialize(value) except (pickle.UnpicklingError, EOFError, TypeError, ImportError): log.error('Cannot deserialize SchedulerTaskHistory for {}'. format(key), exc_info=True) continue self.scheduler_tasks_history[key].update( dict(last_run=task_history.last_run, next_run=task_history.next_run, scheduled_task_id=task_history.scheduled_task_id)) for key in set(self.scheduler_tasks_history.keys()) - new_keys: del self.scheduler_tasks_history[key]
def _check_expired_tasks(self): time_now = int(now()) if time_now - self._ttl_check_last_run < 1000: # 1000 = 1sec return self._ttl_check_last_run = time_now TTL = SETTINGS.WORKER_TASK_TIMEOUT * 1000 for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items(): scheduled_task = self.scheduler_tasks.get(scheduled_task_name) if (scheduled_task_history.get('next_run') and scheduled_task_history.get('scheduled_task_id') and (time_now - scheduled_task_history.get('next_run')) > (scheduled_task.get('ttl') or SETTINGS.WORKER_TASK_TIMEOUT)*1000): task_id = scheduled_task_history.get('scheduled_task_id') log.info('Fix broken task id={}, name={}'.format(task_id, scheduled_task_name)) # Get task object from redis key key = SETTINGS.TASK_STORAGE_KEY.format(scheduled_task_history.get('scheduled_task_id')).encode('utf-8') task_obj = yield from self.connection.get(key) # Deserialize task object try: if not task_obj: raise TypeError() task = Task.deserialize(task_obj) if task.status != Task.SUCCESSFUL: # Update task object status task = task._replace(status=Task.FAILED) # Set new status to redis yield from self.connection.set(key, task.serialize(), expire=SETTINGS.TASK_STORAGE_EXPIRE) except TypeError as ex: task = None log.error("Wrong task id={}".format(scheduled_task_history.get('scheduled_task_id')), exc_info=True) yield from self.connection.delete([key]) # Publish message about finish (FAILED) if task: yield from self.connection.publish(SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), task.status.encode('utf-8')) else: yield from self.connection.publish(SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), Task.FAILED.encode('utf-8')) # Update scheduler information # Store next_run in scheduled try: task_scheduler_obj = yield from self.connection.hget(SETTINGS.SCHEDULER_HISTORY_HASH, scheduled_task_name.encode('utf-8')) task_scheduler = SchedulerTaskHistory.deserialize(task_scheduler_obj) if task and task.status == Task.SUCCESSFUL: scheduled_task_history['last_run'] = scheduled_task_history.get('next_run', 0) scheduled_task_history['next_run'] = 0 task_scheduler = task_scheduler._replace(last_run=task_scheduler.next_run, next_run=0, scheduled_task_id=None) else: scheduled_task_history['next_run'] = 0 scheduled_task_history['scheduled_task_id'] = None task_scheduler = task_scheduler._replace(next_run=0, scheduled_task_id=None) yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8'), task_scheduler.serialize()) except: # We lost SCHEDULER_HISTORY_HASH in db if task and task.status == Task.SUCCESSFUL: scheduled_task_history['last_run'] = scheduled_task_history.get('next_run', 0) scheduled_task_history['next_run'] = 0 else: scheduled_task_history['next_run'] = 0 scheduled_task_history['scheduled_task_id'] = None
def _reload_config_tasks_list(self): """ Load list of tasks, details """ time_now = int(now()) if time_now - self._ttl_reload_config_last_run < 1000: # 1000 = 1sec return self._ttl_reload_config_last_run = time_now config_version = self.config.get_config_version() if config_version != self.config_version: log.info('Changes in actions list, update.') new_scheduler_tasks = self.config.get_scheduled_actions() new_keys = set(new_scheduler_tasks.keys()) - set(self.scheduler_tasks.keys()) deleted_keys = set(self.scheduler_tasks.keys()) - set(new_scheduler_tasks.keys()) if new_keys or deleted_keys: log.info('New actions list, new_keys={}, deleted_keys={}'.format(new_keys, deleted_keys)) self.scheduler_tasks = new_scheduler_tasks yield from self._load_scheduler_tasks_history() # Check scheduler_tasks_history here, please # Возможно, интервал запуска изменился с длинного на короткий # А у нас уже next_run стоит далеко в будущем for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items(): # Смотри все таски, для которых сохранена инфорамция по шедулингу if scheduled_task_history.get('next_run', 0): # and (scheduled_task_name in self.scheduler_tasks): # Если есть запланированный таск if scheduled_task_name in self.scheduler_tasks: # Если у таска осталось расписание possible_next_run = datetime_to_timestamp(self._get_next_run_time(scheduled_task_name, self.scheduler_tasks[scheduled_task_name], int(now()))) else: # У таска не осталось расписания, next_run надо привести к 0 и больше ничего не делать possible_next_run = 0 if scheduled_task_history.get('next_run', 0) != possible_next_run: # Cancel scheduled task # Reset next_run task_id = scheduled_task_history.get('scheduled_task_id') log.info('Schedule changed for task with id={}, name={}, reschedule next_task'.format(task_id, scheduled_task_name)) key = SETTINGS.TASK_STORAGE_KEY.format(task_id).encode('utf-8') task_obj = yield from self.connection.delete([key]) scheduled_task_history['next_run'] = 0 scheduled_task_history['scheduled_task_id'] = 0 try: task_scheduler_obj = yield from self.connection.hget(SETTINGS.SCHEDULER_HISTORY_HASH, scheduled_task_name.encode('utf-8')) task_scheduler = SchedulerTaskHistory.deserialize(task_scheduler_obj) task_scheduler = task_scheduler._replace(next_run=0, scheduled_task_id=None) yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8'), task_scheduler.serialize()) except: log.error('Broken SchedulerTaskHistory object for task id={}, delete it'.format(scheduled_task_name)) yield from self.connection.hdel(SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8')) # Удалился какой-то таск? Удалим его из мониторинга выполнения for key in deleted_keys: if key in self.scheduler_tasks_history: del self.scheduler_tasks_history[key] self.config_version = config_version
def _cleanup_task(self, task): """ clenaup_task -- Task for cleanup completed tasks from redis queue. :param task: `sensors.models.Task` instance :return: None """ log.debug("_cleanup_task task_id={}".format(task.id)) # Remove task from inprogress queue # XXX may be transaction here? cnt1 = yield from self.connection.lrem(SETTINGS.INPROGRESS_QUEUE, value=task.bid()) # Remove task from sorted set cnt2 = yield from self.connection.zrem(SETTINGS.INPROGRESS_TASKS_SET, [task.bid()]) # Update scheduler information # Store next_run in scheduled task_scheduler_obj = yield from self.connection.hget( SETTINGS.SCHEDULER_HISTORY_HASH, task.name.encode('utf-8')) try: task_scheduler = SchedulerTaskHistory.deserialize( task_scheduler_obj) except (pickle.UnpicklingError, EOFError, TypeError, ImportError): task_scheduler = None if task_scheduler and task_scheduler.scheduled_task_id == task.id: #if task.status == Task.SUCCESSFUL: # # Update last_run only on success # last_run = datetime_to_timestamp(task.run_at) #else: # # If task failed, do not update last_run (last_run is about SUCCESSFUL task exectuion) # last_run = task_scheduler.last_run last_run = datetime_to_timestamp(task.run_at) task_scheduler = task_scheduler._replace(last_run=last_run, next_run=0, scheduled_task_id=None) yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH, task.name.encode('utf-8'), task_scheduler.serialize()) # Publish message about finish yield from self.connection.publish( SETTINGS.TASK_CHANNEL.format(task.id).encode('utf-8'), task.status.encode('utf-8')) log.debug('Publish message about task {} to {}'.format( task.id, SETTINGS.TASK_CHANNEL.format(task.id))) log.debug("_cleanup_task lrem result {}".format(cnt1)) log.debug("_cleanup_task zrem result {}".format(cnt2)) # Ping scheduler yield from self._ping_scheduler(task)
def _cleanup_scheduled_history(self): # Clean hash table in redis for task with very old last-run and without next_run log.info("Run cleanup task for table Scheduled History") tasks_history = yield from self.connection.hgetall(SETTINGS.SCHEDULER_HISTORY_HASH) for f in tasks_history: key, value = yield from f # Iterate over all tasks in history and deserialize try: task_history = SchedulerTaskHistory.deserialize(value) except (pickle.UnpicklingError, EOFError, TypeError, ImportError): log.error('Cannot deserialize SchedulerTaskHistory for {}'.format(key), exc_info=True) continue if not task_history.next_run and (int(now()) - task_history.last_run) > (SETTINGS.SCHEDULED_HISTORY_CLEANUP_MAX_TTL * 1000): # task is too old, remove it log.info('Cleanup for Scheduled History table. Remove task, name={}'.format(task_history.name)) yield from self.connection.hdel(SETTINGS.SCHEDULER_HISTORY_HASH, [key]) self.current_loop.call_later(SETTINGS.SCHEDULED_HISTORY_CLEANUP_PERIOD, self._create_asyncio_task, self._cleanup_scheduled_history)
def _cleanup_task(self, task): """ clenaup_task -- Task for cleanup completed tasks from redis queue. :param task: `sensors.models.Task` instance :return: None """ log.debug("_cleanup_task task_id={}".format(task.id)) # Remove task from inprogress queue # XXX may be transaction here? cnt1 = yield from self.connection.lrem(SETTINGS.INPROGRESS_QUEUE, value=task.bid()) # Remove task from sorted set cnt2 = yield from self.connection.zrem(SETTINGS.INPROGRESS_TASKS_SET, [task.bid()]) # Update scheduler information # Store next_run in scheduled task_scheduler_obj = yield from self.connection.hget(SETTINGS.SCHEDULER_HISTORY_HASH, task.name.encode('utf-8')) try: task_scheduler = SchedulerTaskHistory.deserialize(task_scheduler_obj) except (pickle.UnpicklingError, EOFError, TypeError, ImportError): task_scheduler = None if task_scheduler and task_scheduler.scheduled_task_id == task.id: #if task.status == Task.SUCCESSFUL: # # Update last_run only on success # last_run = datetime_to_timestamp(task.run_at) #else: # # If task failed, do not update last_run (last_run is about SUCCESSFUL task exectuion) # last_run = task_scheduler.last_run last_run = datetime_to_timestamp(task.run_at) task_scheduler = task_scheduler._replace(last_run=last_run, next_run=0, scheduled_task_id=None) yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH, task.name.encode('utf-8'), task_scheduler.serialize()) # Publish message about finish yield from self.connection.publish(SETTINGS.TASK_CHANNEL.format(task.id).encode('utf-8'), task.status.encode('utf-8')) log.debug('Publish message about task {} to {}'.format(task.id, SETTINGS.TASK_CHANNEL.format(task.id))) log.debug("_cleanup_task lrem result {}".format(cnt1)) log.debug("_cleanup_task zrem result {}".format(cnt2)) # Ping scheduler yield from self._ping_scheduler(task)
def _load_scheduler_tasks_history(self): """ Load list of scheduled tasks tasks run times """ # Load run history for scheduled tasks tasks_history = yield from self.connection.hgetall(SETTINGS.SCHEDULER_HISTORY_HASH) new_keys = set() for f in tasks_history: key, value = yield from f key = key.decode('utf-8') new_keys.add(key) # Iterate over all tasks in history if key in self.scheduler_tasks: # Is task still in crontab? # Deserialize try: task_history = SchedulerTaskHistory.deserialize(value) except (pickle.UnpicklingError, EOFError, TypeError, ImportError): log.error('Cannot deserialize SchedulerTaskHistory for {}'.format(key), exc_info=True) continue self.scheduler_tasks_history[key].update(dict(last_run=task_history.last_run, next_run=task_history.next_run, scheduled_task_id=task_history.scheduled_task_id)) for key in set(self.scheduler_tasks_history.keys()) - new_keys: del self.scheduler_tasks_history[key]
def _check_expired_tasks(self): time_now = int(now()) if time_now - self._ttl_check_last_run < 1000: # 1000 = 1sec return self._ttl_check_last_run = time_now TTL = SETTINGS.WORKER_TASK_TIMEOUT * 1000 for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items( ): scheduled_task = self.scheduler_tasks.get(scheduled_task_name) if (scheduled_task_history.get('next_run') and scheduled_task_history.get('scheduled_task_id') and (time_now - scheduled_task_history.get('next_run')) > (scheduled_task.get('ttl') or SETTINGS.WORKER_TASK_TIMEOUT) * 1000): task_id = scheduled_task_history.get('scheduled_task_id') log.info('Fix broken task id={}, name={}'.format( task_id, scheduled_task_name)) # Get task object from redis key key = SETTINGS.TASK_STORAGE_KEY.format( scheduled_task_history.get('scheduled_task_id')).encode( 'utf-8') task_obj = yield from self.connection.get(key) # Deserialize task object try: if not task_obj: raise TypeError() task = Task.deserialize(task_obj) if task.status != Task.SUCCESSFUL: # Update task object status task = task._replace(status=Task.FAILED) # Set new status to redis yield from self.connection.set( key, task.serialize(), expire=SETTINGS.TASK_STORAGE_EXPIRE) except TypeError as ex: task = None log.error("Wrong task id={}".format( scheduled_task_history.get('scheduled_task_id')), exc_info=True) yield from self.connection.delete([key]) # Publish message about finish (FAILED) if task: yield from self.connection.publish( SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), task.status.encode('utf-8')) else: yield from self.connection.publish( SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), Task.FAILED.encode('utf-8')) # Update scheduler information # Store next_run in scheduled try: task_scheduler_obj = yield from self.connection.hget( SETTINGS.SCHEDULER_HISTORY_HASH, scheduled_task_name.encode('utf-8')) task_scheduler = SchedulerTaskHistory.deserialize( task_scheduler_obj) if task and task.status == Task.SUCCESSFUL: scheduled_task_history[ 'last_run'] = scheduled_task_history.get( 'next_run', 0) scheduled_task_history['next_run'] = 0 task_scheduler = task_scheduler._replace( last_run=task_scheduler.next_run, next_run=0, scheduled_task_id=None) else: scheduled_task_history['next_run'] = 0 scheduled_task_history['scheduled_task_id'] = None task_scheduler = task_scheduler._replace( next_run=0, scheduled_task_id=None) yield from self.connection.hset( SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8'), task_scheduler.serialize()) except: # We lost SCHEDULER_HISTORY_HASH in db if task and task.status == Task.SUCCESSFUL: scheduled_task_history[ 'last_run'] = scheduled_task_history.get( 'next_run', 0) scheduled_task_history['next_run'] = 0 else: scheduled_task_history['next_run'] = 0 scheduled_task_history['scheduled_task_id'] = None
def _reload_config_tasks_list(self): """ Load list of tasks, details """ time_now = int(now()) if time_now - self._ttl_reload_config_last_run < 1000: # 1000 = 1sec return self._ttl_reload_config_last_run = time_now config_version = self.config.get_config_version() if config_version != self.config_version: log.info('Changes in actions list, update.') new_scheduler_tasks = self.config.get_scheduled_actions() new_keys = set(new_scheduler_tasks.keys()) - set( self.scheduler_tasks.keys()) deleted_keys = set(self.scheduler_tasks.keys()) - set( new_scheduler_tasks.keys()) if new_keys or deleted_keys: log.info( 'New actions list, new_keys={}, deleted_keys={}'.format( new_keys, deleted_keys)) self.scheduler_tasks = new_scheduler_tasks yield from self._load_scheduler_tasks_history() # Check scheduler_tasks_history here, please # Возможно, интервал запуска изменился с длинного на короткий # А у нас уже next_run стоит далеко в будущем for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items( ): # Смотри все таски, для которых сохранена инфорамция по шедулингу if scheduled_task_history.get( 'next_run', 0 ): # and (scheduled_task_name in self.scheduler_tasks): # Если есть запланированный таск if scheduled_task_name in self.scheduler_tasks: # Если у таска осталось расписание possible_next_run = datetime_to_timestamp( self._get_next_run_time( scheduled_task_name, self.scheduler_tasks[scheduled_task_name], int(now()))) else: # У таска не осталось расписания, next_run надо привести к 0 и больше ничего не делать possible_next_run = 0 if scheduled_task_history.get('next_run', 0) != possible_next_run: # Cancel scheduled task # Reset next_run task_id = scheduled_task_history.get( 'scheduled_task_id') log.info( 'Schedule changed for task with id={}, name={}, reschedule next_task' .format(task_id, scheduled_task_name)) key = SETTINGS.TASK_STORAGE_KEY.format(task_id).encode( 'utf-8') task_obj = yield from self.connection.delete([key]) scheduled_task_history['next_run'] = 0 scheduled_task_history['scheduled_task_id'] = 0 try: task_scheduler_obj = yield from self.connection.hget( SETTINGS.SCHEDULER_HISTORY_HASH, scheduled_task_name.encode('utf-8')) task_scheduler = SchedulerTaskHistory.deserialize( task_scheduler_obj) task_scheduler = task_scheduler._replace( next_run=0, scheduled_task_id=None) yield from self.connection.hset( SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8'), task_scheduler.serialize()) except: log.error( 'Broken SchedulerTaskHistory object for task id={}, delete it' .format(scheduled_task_name)) yield from self.connection.hdel( SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8')) # Удалился какой-то таск? Удалим его из мониторинга выполнения for key in deleted_keys: if key in self.scheduler_tasks_history: del self.scheduler_tasks_history[key] self.config_version = config_version