Ejemplo n.º 1
0
    def _check_expired_tasks(self):
        time_now = int(now())
        if time_now - self._ttl_check_last_run < 1000:  # 1000 = 1sec
            return
        self._ttl_check_last_run = time_now

        TTL = SETTINGS.WORKER_TASK_TIMEOUT * 1000
        for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items():
            scheduled_task = self.scheduler_tasks.get(scheduled_task_name)
            if (scheduled_task_history.get('next_run')
              and scheduled_task_history.get('scheduled_task_id')
              and (time_now - scheduled_task_history.get('next_run')) > (scheduled_task.get('ttl') or SETTINGS.WORKER_TASK_TIMEOUT)*1000):
                task_id = scheduled_task_history.get('scheduled_task_id')
                log.info('Fix broken task id={}, name={}'.format(task_id, scheduled_task_name))
                # Get task object from redis key
                key = SETTINGS.TASK_STORAGE_KEY.format(scheduled_task_history.get('scheduled_task_id')).encode('utf-8')
                task_obj = yield from self.connection.get(key)
                # Deserialize task object
                try:
                    if not task_obj:
                        raise TypeError()
                    task = Task.deserialize(task_obj)
                    if task.status != Task.SUCCESSFUL:
                        # Update task object status
                        task = task._replace(status=Task.FAILED)
                        # Set new status to redis
                        yield from self.connection.set(key, task.serialize(), expire=SETTINGS.TASK_STORAGE_EXPIRE)
                except TypeError as ex:
                    task = None
                    log.error("Wrong task id={}".format(scheduled_task_history.get('scheduled_task_id')), exc_info=True)
                    yield from self.connection.delete([key])

                # Publish message about finish (FAILED)
                if task:
                    yield from self.connection.publish(SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), task.status.encode('utf-8'))
                else:
                    yield from self.connection.publish(SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'), Task.FAILED.encode('utf-8'))

                # Update scheduler information
                # Store next_run in scheduled
                try:
                    task_scheduler_obj = yield from self.connection.hget(SETTINGS.SCHEDULER_HISTORY_HASH, scheduled_task_name.encode('utf-8'))
                    task_scheduler = SchedulerTaskHistory.deserialize(task_scheduler_obj)
                    if task and task.status == Task.SUCCESSFUL:
                        scheduled_task_history['last_run'] = scheduled_task_history.get('next_run', 0)
                        scheduled_task_history['next_run'] = 0
                        task_scheduler = task_scheduler._replace(last_run=task_scheduler.next_run, next_run=0, scheduled_task_id=None)
                    else:
                        scheduled_task_history['next_run'] = 0
                        scheduled_task_history['scheduled_task_id'] = None
                        task_scheduler = task_scheduler._replace(next_run=0, scheduled_task_id=None)
                    yield from self.connection.hset(SETTINGS.SCHEDULER_HISTORY_HASH, task_scheduler.name.encode('utf-8'), task_scheduler.serialize())
                except:
                    # We lost SCHEDULER_HISTORY_HASH in db
                    if task and task.status == Task.SUCCESSFUL:
                        scheduled_task_history['last_run'] = scheduled_task_history.get('next_run', 0)
                        scheduled_task_history['next_run'] = 0
                    else:
                        scheduled_task_history['next_run'] = 0
                        scheduled_task_history['scheduled_task_id'] = None
Ejemplo n.º 2
0
 def _deserialize_task(self, raw_task):
     try:
         task_obj = yield from self.connection.get(SETTINGS.TASK_STORAGE_KEY.format(raw_task.decode('utf-8')).encode('utf-8'))
         if not task_obj:
             raise TypeError()
         task = Task.deserialize(task_obj)
         log.info("Got new task id={}, type={}, status={}".format(task.id, task.type, task.status))
         if not task.status == Task.SCHEDULED:
             log.error("Wrong status={} for task id={}, type={}; Should be SCHEDULED".format(task.status, task.id, task.type))
             return
         return task
     except TypeError as ex:
         log.error("Wrong task_id {}".format(raw_task))
         return
     except (pickle.UnpicklingError, EOFError, TypeError, ImportError):
         yield from self.connection.lrem(SETTINGS.INPROGRESS_QUEUE, value=raw_task)
         log.error("Wrong message in queue", exc_info=True)
         return
Ejemplo n.º 3
0
 def _deserialize_task(self, raw_task):
     try:
         task_obj = yield from self.connection.get(
             SETTINGS.TASK_STORAGE_KEY.format(
                 raw_task.decode('utf-8')).encode('utf-8'))
         if not task_obj:
             raise TypeError()
         task = Task.deserialize(task_obj)
         log.info("Got new task id={}, type={}, status={}".format(
             task.id, task.type, task.status))
         if not task.status == Task.SCHEDULED:
             log.error(
                 "Wrong status={} for task id={}, type={}; Should be SCHEDULED"
                 .format(task.status, task.id, task.type))
             return
         return task
     except TypeError as ex:
         log.error("Wrong task_id {}".format(raw_task))
         return
     except (pickle.UnpicklingError, EOFError, TypeError, ImportError):
         yield from self.connection.lrem(SETTINGS.INPROGRESS_QUEUE,
                                         value=raw_task)
         log.error("Wrong message in queue", exc_info=True)
         return
Ejemplo n.º 4
0
    def _check_expired_tasks(self):
        time_now = int(now())
        if time_now - self._ttl_check_last_run < 1000:  # 1000 = 1sec
            return
        self._ttl_check_last_run = time_now

        TTL = SETTINGS.WORKER_TASK_TIMEOUT * 1000
        for scheduled_task_name, scheduled_task_history in self.scheduler_tasks_history.items(
        ):
            scheduled_task = self.scheduler_tasks.get(scheduled_task_name)
            if (scheduled_task_history.get('next_run')
                    and scheduled_task_history.get('scheduled_task_id')
                    and (time_now - scheduled_task_history.get('next_run')) >
                (scheduled_task.get('ttl') or SETTINGS.WORKER_TASK_TIMEOUT) *
                    1000):
                task_id = scheduled_task_history.get('scheduled_task_id')
                log.info('Fix broken task id={}, name={}'.format(
                    task_id, scheduled_task_name))
                # Get task object from redis key
                key = SETTINGS.TASK_STORAGE_KEY.format(
                    scheduled_task_history.get('scheduled_task_id')).encode(
                        'utf-8')
                task_obj = yield from self.connection.get(key)
                # Deserialize task object
                try:
                    if not task_obj:
                        raise TypeError()
                    task = Task.deserialize(task_obj)
                    if task.status != Task.SUCCESSFUL:
                        # Update task object status
                        task = task._replace(status=Task.FAILED)
                        # Set new status to redis
                        yield from self.connection.set(
                            key,
                            task.serialize(),
                            expire=SETTINGS.TASK_STORAGE_EXPIRE)
                except TypeError as ex:
                    task = None
                    log.error("Wrong task id={}".format(
                        scheduled_task_history.get('scheduled_task_id')),
                              exc_info=True)
                    yield from self.connection.delete([key])

                # Publish message about finish (FAILED)
                if task:
                    yield from self.connection.publish(
                        SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'),
                        task.status.encode('utf-8'))
                else:
                    yield from self.connection.publish(
                        SETTINGS.TASK_CHANNEL.format(task_id).encode('utf-8'),
                        Task.FAILED.encode('utf-8'))

                # Update scheduler information
                # Store next_run in scheduled
                try:
                    task_scheduler_obj = yield from self.connection.hget(
                        SETTINGS.SCHEDULER_HISTORY_HASH,
                        scheduled_task_name.encode('utf-8'))
                    task_scheduler = SchedulerTaskHistory.deserialize(
                        task_scheduler_obj)
                    if task and task.status == Task.SUCCESSFUL:
                        scheduled_task_history[
                            'last_run'] = scheduled_task_history.get(
                                'next_run', 0)
                        scheduled_task_history['next_run'] = 0
                        task_scheduler = task_scheduler._replace(
                            last_run=task_scheduler.next_run,
                            next_run=0,
                            scheduled_task_id=None)
                    else:
                        scheduled_task_history['next_run'] = 0
                        scheduled_task_history['scheduled_task_id'] = None
                        task_scheduler = task_scheduler._replace(
                            next_run=0, scheduled_task_id=None)
                    yield from self.connection.hset(
                        SETTINGS.SCHEDULER_HISTORY_HASH,
                        task_scheduler.name.encode('utf-8'),
                        task_scheduler.serialize())
                except:
                    # We lost SCHEDULER_HISTORY_HASH in db
                    if task and task.status == Task.SUCCESSFUL:
                        scheduled_task_history[
                            'last_run'] = scheduled_task_history.get(
                                'next_run', 0)
                        scheduled_task_history['next_run'] = 0
                    else:
                        scheduled_task_history['next_run'] = 0
                        scheduled_task_history['scheduled_task_id'] = None