Beispiel #1
0
 def __init__(self, *args, **kwargs):
     AbstractDaemon.__init__(self, *args, **kwargs)
     self.timer = MultiTimer()
     self.set = set()
Beispiel #2
0
class Scheduler(AbstractDaemon):
    """Scheduling process for handling Schedules.
    
    Takes unclaimed Schedules from the database and adds their next
    instance to a timer.  At the appropriate time, the instance is
    added to its queue and the Schedule is updated.
    
    Idea: Split this up into two threads, one which continuously handles
    already claimed schedules, the other which periodically polls the DB
    for new schedules.
    
    """
    class Meta:
        app_label = 'core'
        db_table = 'norc_scheduler'
    
    objects = QuerySetManager()
    
    class QuerySet(AbstractDaemon.QuerySet):
        """Custom manager/query set for Scheduler."""
        
        def undead(self):
            """Schedulers that are active but the heart isn't beating."""
            cutoff = datetime.utcnow() - timedelta(seconds=HEARTBEAT_FAILED)
            return self.status_in("active").filter(heartbeat__lt=cutoff)
    
    # All the statuses Schedulers can have.  See constants.py.
    VALID_STATUSES = [
        Status.CREATED,
        Status.RUNNING,
        Status.PAUSED,
        Status.ENDED,
        Status.ERROR,
    ]
    
    VALID_REQUESTS = [
        Request.STOP,
        Request.KILL,
        Request.PAUSE,
        Request.RESUME,
        Request.RELOAD,
    ]
    
    # The status of this scheduler.
    status = PositiveSmallIntegerField(default=Status.CREATED,
        choices=[(s, Status.name(s)) for s in VALID_STATUSES])
    
    # A state-change request.
    request = PositiveSmallIntegerField(null=True,
        choices=[(r, Request.name(r)) for r in VALID_REQUESTS])
    
    def __init__(self, *args, **kwargs):
        AbstractDaemon.__init__(self, *args, **kwargs)
        self.timer = MultiTimer()
        self.set = set()
    
    def start(self):
        """Starts the Scheduler."""
        # Temporary check until multiple schedulers is supported fully.
        if Scheduler.objects.alive().count() > 0:
            print "Cannot run more than one scheduler at a time."
            return
        AbstractDaemon.start(self)
    
    def run(self):
        """Main run loop of the Scheduler."""
        self.timer.start()
        
        while not Status.is_final(self.status):
            if self.request:
                self.handle_request()
            
            if self.status == Status.RUNNING:
                # Clean up orphaned schedules and undead schedulers.
                # Schedule.objects.orphaned().update(scheduler=None)
                # CronSchedule.objects.orphaned().update(scheduler=None)
                
                cron = CronSchedule.objects.unclaimed()[:SCHEDULER_LIMIT]
                simple = Schedule.objects.unclaimed()[:SCHEDULER_LIMIT]
                for schedule in itertools.chain(cron, simple):
                    self.log.info('Claiming %s.' % schedule)
                    schedule.scheduler = self
                    schedule.save()
                    self.add(schedule)
            if not Status.is_final(self.status):
                self.wait()
                self.request = Scheduler.objects.get(pk=self.pk).request
    
    def wait(self):
        """Waits on the flag."""
        AbstractDaemon.wait(self, SCHEDULER_PERIOD)
    
    def clean_up(self):
        self.timer.cancel()
        self.timer.join()
        cron = self.cronschedules.all()
        simple = self.schedules.all()
        claimed_count = cron.count() + simple.count()
        if claimed_count > 0:
            self.log.info("Cleaning up %s schedules." % claimed_count)
            cron.update(scheduler=None)
            simple.update(scheduler=None)
    
    def handle_request(self):
        """Called when a request is found."""
        
        # Clear request immediately.
        request = self.request
        self.request = None
        self.save()
        
        self.log.info("Request received: %s" % Request.name(request))
        
        if request == Request.PAUSE:
            self.set_status(Status.PAUSED)
        
        elif request == Request.RESUME:
            if self.status != Status.PAUSED:
                self.log.info("Must be paused to resume; clearing request.")
            else:
                self.set_status(Status.RUNNING)
        
        elif request == Request.STOP:
            self.set_status(Status.ENDED)
        
        elif request == Request.KILL:
            self.set_status(Status.KILLED)
        
        elif request == Request.RELOAD:
            changed = MultiQuerySet(Schedule, CronSchedule)
            changed = changed.objects.unfinished.filter(
                changed=True, scheduler=self)
            for item in self.timer.tasks:
                s = item[2][0]
                if s in changed:
                    self.log.info("Removing outdated: %s" % s)
                    self.timer.tasks.remove(item)
                    self.set.remove(s)
                s = type(s).objects.get(pk=s.pk)
            for s in changed:
                self.log.info("Adding updated: %s" % s)
                self.add(s)
            changed.update(changed=False)
    
    def add(self, schedule):
        """Adds the schedule to the timer."""
        try:
            if schedule in self.set:
                self.log.error("%s has already been added to this Scheduler." %
                    schedule)
                return
            self.log.debug('Adding %s to timer for %s.' %
                (schedule, schedule.next))
            self.timer.add_task(schedule.next, self._enqueue, [schedule])
            self.set.add(schedule)
        except:
            self.log.error(
                "Invalid schedule %s found, deleting." % schedule)
            schedule.soft_delete()
    
    def _enqueue(self, schedule):
        """Called by the timer to add an instance to the queue."""
        updated_schedule = get_object(type(schedule), pk=schedule.pk)
        self.set.remove(schedule)
        if updated_schedule == None or updated_schedule.deleted:
            self.log.info('%s was removed.' % schedule)
            if updated_schedule != None:
                updated_schedule.scheduler = None
                updated_schedule.save()
            return
        schedule = updated_schedule
        
        if not schedule.scheduler == self:
            self.log.info("%s is no longer tied to this Scheduler." %
                schedule)
            # self.set.remove(schedule)
            return
        instance = Instance.objects.create(
            task=schedule.task, schedule=schedule)
        self.log.info('Enqueuing %s.' % instance)
        schedule.queue.push(instance)
        schedule.enqueued()
        if not schedule.finished():
            self.add(schedule)
        else:
            schedule.scheduler = None
            schedule.save()
    
    @property
    def log_path(self):
        return 'schedulers/scheduler-%s' % self.id