def _check_expired_tasks(self): time_now = int(now()) if time_now - self._ttl_check_last_run < 1000: # 1000 = 1sec return self._ttl_check_last_run = time_now TTL = SETTINGS.WORKER_TASK_TIMEOUT * 1000 for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items(): scheduled_task = self.scheduler_tasks.get(scheduled_task_name) if (scheduled_task_history.get('next_run') and scheduled_task_history.get('scheduled_task_id') and (time_now - scheduled_task_history.get('next_run')) > (scheduled_task.get('ttl') or SETTINGS.WORKER_TASK_TIMEOUT)*1000): task_id = scheduled_task_history.get('scheduled_task_id') log.info('Fix broken task id={}, name={}'.format(task_id, scheduled_task_name)) # Get task object from redis key key = SETTINGS.TASK_STORAGE_KEY.format(scheduled_task_history.get('scheduled_task_id')).encode('utf-8') task_obj = yield from self.connection.get(key) # Deserialize task object try: if not task_obj: raise TypeError() task = Task.deserialize(task_obj) if task.status != Task.SUCCESSFUL: # Update task object status task = task._replace(status=Task.FAILED) # Set new status to redis yield from self.connection.set(key, task.serialize(), expire=SETTINGS.TASK_STORAGE_EXPIRE) except TypeError as ex: task = None log.error("Wrong task id={}".format(scheduled_task_history.get('scheduled_task_id')), exc_info=True) yield from self.connection.delete([key]) # Publish message about finish (FAILED) if task: yield from self.connection.publish(SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), task.status.encode('utf-8')) else: yield from self.connection.publish(SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), Task.FAILED.encode('utf-8')) # Update scheduler information # Store next_run in scheduled try: task_scheduler_obj = yield from self.connection.hget(SETTINGS.SCHEDULER_HISTORY_HASH, scheduled_task_name.encode('utf-8')) task_scheduler = SchedulerTaskHistory.deserialize(task_scheduler_obj) if task and task.status == Task.SUCCESSFUL: scheduled_task_history['last_run'] = scheduled_task_history.get('next_run', 0) scheduled_task_history['next_run'] = 0 task_scheduler = task_scheduler._replace(last_run=task_scheduler.next_run, next_run=0, scheduled_task_id=None) else: scheduled_task_history['next_run'] = 0 scheduled_task_history['scheduled_task_id'] = None task_scheduler = task_scheduler._replace(next_run=0, scheduled_task_id=None) yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8'), task_scheduler.serialize()) except: # We lost SCHEDULER_HISTORY_HASH in db if task and task.status == Task.SUCCESSFUL: scheduled_task_history['last_run'] = scheduled_task_history.get('next_run', 0) scheduled_task_history['next_run'] = 0 else: scheduled_task_history['next_run'] = 0 scheduled_task_history['scheduled_task_id'] = None
def _deserialize_task(self, raw_task): try: task_obj = yield from self.connection.get(SETTINGS.TASK_STORAGE_KEY.format(raw_task.decode('utf-8')).encode('utf-8')) if not task_obj: raise TypeError() task = Task.deserialize(task_obj) log.info("Got new task id={}, type={}, status={}".format(task.id, task.type, task.status)) if not task.status == Task.SCHEDULED: log.error("Wrong status={} for task id={}, type={}; Should be SCHEDULED".format(task.status, task.id, task.type)) return return task except TypeError as ex: log.error("Wrong task_id {}".format(raw_task)) return except (pickle.UnpicklingError, EOFError, TypeError, ImportError): yield from self.connection.lrem(SETTINGS.INPROGRESS_QUEUE, value=raw_task) log.error("Wrong message in queue", exc_info=True) return
def _deserialize_task(self, raw_task): try: task_obj = yield from self.connection.get( SETTINGS.TASK_STORAGE_KEY.format( raw_task.decode('utf-8')).encode('utf-8')) if not task_obj: raise TypeError() task = Task.deserialize(task_obj) log.info("Got new task id={}, type={}, status={}".format( task.id, task.type, task.status)) if not task.status == Task.SCHEDULED: log.error( "Wrong status={} for task id={}, type={}; Should be SCHEDULED" .format(task.status, task.id, task.type)) return return task except TypeError as ex: log.error("Wrong task_id {}".format(raw_task)) return except (pickle.UnpicklingError, EOFError, TypeError, ImportError): yield from self.connection.lrem(SETTINGS.INPROGRESS_QUEUE, value=raw_task) log.error("Wrong message in queue", exc_info=True) return
def _check_expired_tasks(self): time_now = int(now()) if time_now - self._ttl_check_last_run < 1000: # 1000 = 1sec return self._ttl_check_last_run = time_now TTL = SETTINGS.WORKER_TASK_TIMEOUT * 1000 for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items( ): scheduled_task = self.scheduler_tasks.get(scheduled_task_name) if (scheduled_task_history.get('next_run') and scheduled_task_history.get('scheduled_task_id') and (time_now - scheduled_task_history.get('next_run')) > (scheduled_task.get('ttl') or SETTINGS.WORKER_TASK_TIMEOUT) * 1000): task_id = scheduled_task_history.get('scheduled_task_id') log.info('Fix broken task id={}, name={}'.format( task_id, scheduled_task_name)) # Get task object from redis key key = SETTINGS.TASK_STORAGE_KEY.format( scheduled_task_history.get('scheduled_task_id')).encode( 'utf-8') task_obj = yield from self.connection.get(key) # Deserialize task object try: if not task_obj: raise TypeError() task = Task.deserialize(task_obj) if task.status != Task.SUCCESSFUL: # Update task object status task = task._replace(status=Task.FAILED) # Set new status to redis yield from self.connection.set( key, task.serialize(), expire=SETTINGS.TASK_STORAGE_EXPIRE) except TypeError as ex: task = None log.error("Wrong task id={}".format( scheduled_task_history.get('scheduled_task_id')), exc_info=True) yield from self.connection.delete([key]) # Publish message about finish (FAILED) if task: yield from self.connection.publish( SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), task.status.encode('utf-8')) else: yield from self.connection.publish( SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), Task.FAILED.encode('utf-8')) # Update scheduler information # Store next_run in scheduled try: task_scheduler_obj = yield from self.connection.hget( SETTINGS.SCHEDULER_HISTORY_HASH, scheduled_task_name.encode('utf-8')) task_scheduler = SchedulerTaskHistory.deserialize( task_scheduler_obj) if task and task.status == Task.SUCCESSFUL: scheduled_task_history[ 'last_run'] = scheduled_task_history.get( 'next_run', 0) scheduled_task_history['next_run'] = 0 task_scheduler = task_scheduler._replace( last_run=task_scheduler.next_run, next_run=0, scheduled_task_id=None) else: scheduled_task_history['next_run'] = 0 scheduled_task_history['scheduled_task_id'] = None task_scheduler = task_scheduler._replace( next_run=0, scheduled_task_id=None) yield from self.connection.hset( SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8'), task_scheduler.serialize()) except: # We lost SCHEDULER_HISTORY_HASH in db if task and task.status == Task.SUCCESSFUL: scheduled_task_history[ 'last_run'] = scheduled_task_history.get( 'next_run', 0) scheduled_task_history['next_run'] = 0 else: scheduled_task_history['next_run'] = 0 scheduled_task_history['scheduled_task_id'] = None