def handle_request(self): """Called when a request is found.""" # Clear request immediately. request = self.request self.request = None self.save() self.log.info("Request received: %s" % Request.name(request)) if request == Request.PAUSE: self.set_status(Status.PAUSED) elif request == Request.RESUME: if self.status not in (Status.PAUSED, Status.SUSPENDED): self.log.info("Must be paused or suspended to resume; " + "clearing request.") else: self.set_status(Status.RUNNING) elif request == Request.STOP: self.set_status(Status.STOPPING) elif request == Request.KILL: # for p in self.processes.values(): # p.terminate() for pid, p in self.processes.iteritems(): self.log.info("Killing process for %s." % p.instance) os.kill(pid, signal.SIGTERM) self.set_status(Status.KILLED)
def handle_request(self): """Called when a request is found.""" # Clear request immediately. request = self.request self.request = None self.save() self.log.info("Request received: %s" % Request.name(request)) if request == Request.PAUSE: self.set_status(Status.PAUSED) elif request == Request.RESUME: if self.status != Status.PAUSED: self.log.info("Must be paused to resume; clearing request.") else: self.set_status(Status.RUNNING) elif request == Request.STOP: self.set_status(Status.STOPPING) elif request == Request.KILL: # for p in self.processes.values(): # p.terminate() for pid, p in self.processes.iteritems(): self.log.info("Killing process for %s." % p.instance) os.kill(pid, signal.SIGTERM) self.set_status(Status.KILLED)
def make_request(self, request): """This method is how the request field should always be set.""" assert request in self.VALID_REQUESTS, "Invalid request: " + \ "\"%s\" (%s)" % (Request.name(request), request) if not Status.is_final(self.status): self.request = request self.save() self.flag.set() return True else: return False
def handle_request(self): """Called when a request is found.""" # Clear request immediately. request = self.request self.request = None self.save() self.log.info("Request received: %s" % Request.name(request)) if request == Request.PAUSE: self.set_status(Status.PAUSED) elif request == Request.RESUME: if self.status != Status.PAUSED: self.log.info("Must be paused to resume; clearing request.") else: self.set_status(Status.RUNNING) elif request == Request.STOP: self.set_status(Status.ENDED) elif request == Request.KILL: self.set_status(Status.KILLED) elif request == Request.RELOAD: changed = MultiQuerySet(Schedule, CronSchedule) changed = changed.objects.unfinished.filter( changed=True, scheduler=self) for item in self.timer.tasks: s = item[2][0] if s in changed: self.log.info("Removing outdated: %s" % s) self.timer.tasks.remove(item) self.set.remove(s) s = type(s).objects.get(pk=s.pk) for s in changed: self.log.info("Adding updated: %s" % s) self.add(s) changed.update(changed=False)
def handle_request(self): """Called when a request is found.""" # Clear request immediately. request = self.request self.request = None self.save() self.log.info("Request received: %s" % Request.name(request)) if request == Request.PAUSE: self.set_status(Status.PAUSED) elif request == Request.RESUME: if self.status != Status.PAUSED: self.log.info("Must be paused to resume; clearing request.") else: self.set_status(Status.RUNNING) elif request == Request.STOP: self.set_status(Status.ENDED) elif request == Request.KILL: self.set_status(Status.KILLED) elif request == Request.RELOAD: changed = MultiQuerySet(Schedule, CronSchedule) changed = changed.objects.unfinished.filter(changed=True, scheduler=self) for item in self.timer.tasks: s = item[2][0] if s in changed: self.log.info("Removing outdated: %s" % s) self.timer.tasks.remove(item) self.set.remove(s) s = type(s).objects.get(pk=s.pk) for s in changed: self.log.info("Adding updated: %s" % s) self.add(s) changed.update(changed=False)
class Executor(AbstractDaemon): """Executors are responsible for the running of instances. Executors have a single queue that they pull instances from. There can (and in many cases should) be more than one Executor running for a single queue. """ class Meta: app_label = 'core' db_table = 'norc_executor' objects = QuerySetManager() class QuerySet(AbstractDaemon.QuerySet): def for_queue(self, q): """Executors pulling from the given queue.""" return self.filter( queue_id=q.id, queue_type=ContentType.objects.get_for_model(q).id) @property def instances(self): """A custom implementation of the Django related manager pattern.""" return MultiQuerySet( *[i.objects.filter(executor=self.pk) for i in INSTANCE_MODELS]) # All the statuses executors can have. See constants.py. VALID_STATUSES = [ Status.CREATED, Status.RUNNING, Status.PAUSED, Status.STOPPING, Status.ENDED, Status.ERROR, Status.KILLED, Status.SUSPENDED, ] VALID_REQUESTS = [ Request.STOP, Request.KILL, Request.PAUSE, Request.RESUME, ] # The status of this executor. status = PositiveSmallIntegerField(default=Status.CREATED, choices=[(s, Status.name(s)) for s in VALID_STATUSES]) # A state-change request. request = PositiveSmallIntegerField(null=True, choices=[(r, Request.name(r)) for r in VALID_REQUESTS]) # The queue this executor draws task instances from. queue_type = ForeignKey(ContentType) queue_id = PositiveIntegerField() queue = GenericForeignKey('queue_type', 'queue_id') # The number of things that can be run concurrently. concurrent = IntegerField() @property def alive(self): return self.status == Status.RUNNING and self.heartbeat > \ datetime.utcnow() - timedelta(seconds=HEARTBEAT_FAILED) def __init__(self, *args, **kwargs): AbstractDaemon.__init__(self, *args, **kwargs) self.processes = {} def run(self): """Core executor function.""" if settings.BACKUP_SYSTEM: self.pool = ThreadPool(self.concurrent + 1) self.log.info("%s is now running on host %s." % (self, self.host)) if self.log.debug_on: self.resource_reporter = Thread(target=self.report_resources) self.resource_reporter.daemon = True self.resource_reporter.start() # Main loop. while not Status.is_final(self.status): if self.request: self.handle_request() if self.status == Status.RUNNING: while len(self.processes) < self.concurrent: # self.log.debug("Popping instance...") instance = self.queue.pop() if instance: # self.log.debug("Popped %s" % instance) self.start_instance(instance) else: # self.log.debug("No instance in queue.") break elif self.status == Status.STOPPING and len(self.processes) == 0: self.set_status(Status.ENDED) self.save(safe=True) # Clean up completed tasks before iterating. for pid, p in self.processes.items()[:]: p.poll() self.log.debug("Checking pid %s: return code %s." % (pid, p.returncode)) if not p.returncode == None: i = type(p.instance).objects.get(pk=p.instance.pk) if i.status == Status.CREATED: self.log.info( ("%s fail to initialize properly; " + "entering suspension to avoid more errors.") % i) self.set_status(Status.SUSPENDED) self.save() if not Status.is_final(i.status): self.log.info(("%s ended with invalid " + "status %s, changing to ERROR.") % (i, Status.name(i.status))) i.status = Status.ERROR i.save() self.log.info("%s ended with status %s." % (i, Status.name(i.status))) del self.processes[pid] if settings.BACKUP_SYSTEM: self.pool.queueTask(self.backup_instance_log, [i]) if not Status.is_final(self.status): self.wait(EXECUTOR_PERIOD) self.request = Executor.objects.get(pk=self.pk).request def clean_up(self): if settings.BACKUP_SYSTEM: self.pool.joinAll() def report_resources(self): while not Status.is_final(self.status): time.sleep(10) rself = resource.getrusage(resource.RUSAGE_SELF) self.log.debug(rself) rchildren = resource.getrusage(resource.RUSAGE_CHILDREN) self.log.debug(rchildren) def start_instance(self, instance): """Starts a given instance in a new process.""" instance.executor = self instance.save() self.log.info("Starting %s..." % instance) # p = Process(target=self.execute, args=[instance.start]) # p.start() ct = ContentType.objects.get_for_model(instance) f = make_log(instance.log_path).file p = Popen('norc_taskrunner --ct_pk %s --target_pk %s' % (ct.pk, instance.pk), stdout=f, stderr=STDOUT, shell=True) p.instance = instance self.processes[p.pid] = p # This should be used in 2.6, but with subprocess it's not possible. # def execute(self, func): # """Calls a function, then sets the flag after its execution.""" # try: # func() # finally: # self.flag.set() def handle_request(self): """Called when a request is found.""" # Clear request immediately. request = self.request self.request = None self.save() self.log.info("Request received: %s" % Request.name(request)) if request == Request.PAUSE: self.set_status(Status.PAUSED) elif request == Request.RESUME: if self.status not in (Status.PAUSED, Status.SUSPENDED): self.log.info("Must be paused or suspended to resume; " + "clearing request.") else: self.set_status(Status.RUNNING) elif request == Request.STOP: self.set_status(Status.STOPPING) elif request == Request.KILL: # for p in self.processes.values(): # p.terminate() for pid, p in self.processes.iteritems(): self.log.info("Killing process for %s." % p.instance) os.kill(pid, signal.SIGTERM) self.set_status(Status.KILLED) def backup_instance_log(self, instance): self.log.info("Attempting upload of log for %s..." % instance) if backup_log(instance.log_path): self.log.info("Completed upload of log for %s." % instance) else: self.log.info("Failed to upload log for %s." % instance) @property def log_path(self): return 'executors/executor-%s' % self.id
def main(): usage = "norc_control [executor | scheduler | host] <id | host> " + \ "--[stop | kill | pause | resume | reload | handle] [--wait]" def bad_args(message): print message print usage sys.exit(2) parser = OptionParser(usage) parser.add_option("-s", "--stop", action="store_true", default=False, help="Send a stop request.") parser.add_option("-k", "--kill", action="store_true", default=False, help="Send a kill request.") parser.add_option("-p", "--pause", action="store_true", default=False, help="Send a pause request.") parser.add_option("-u", "--resume", action="store_true", default=False, help="Send an resume request.") parser.add_option("-r", "--reload", action="store_true", default=False, help="Send an reload request to a Scheduler.") parser.add_option("--handle", action="store_true", default=False, help="Change the object's status to HANDLED.") parser.add_option("-f", "--force", action="store_true", default=False, help="Force the request to be made..") parser.add_option("-w", "--wait", action="store_true", default=False, help="Wait until the request has been responded to.") options, args = parser.parse_args() if len(args) != 2: bad_args("Invalid number of arguments.") requests = filter(lambda a: getattr(options, a.lower()), Request.NAMES.values()) if len(requests) + (1 if options.handle else 0) != 1: bad_args("Must request exactly one action.") if not options.handle: request = requests[0] req = getattr(Request, request) cls = None if args[0] in EXECUTOR_KEYWORDS: cls = Executor elif args[0] in SCHEDULER_KEYWORDS: cls = Scheduler elif args[0] in HOST_KEYWORDS: if options.handle: bad_args("Can't perform handle operation on multiple daemons.") daemons = MultiQuerySet(Executor, Scheduler).objects.all() daemons = daemons.filter(host=args[1]).status_in("active") if not options.force: daemons = daemons.filter(request=None) for d in daemons: if req in d.VALID_REQUESTS: d.make_request(req) print "%s was sent a %s request." % (d, request) if options.wait: _wait(daemons, req) else: bad_args("Invalid keyword '%s'." % args[0]) if cls: name = cls.__name__ try: obj_id = int(args[1]) except ValueError: bad_args("Invalid id '%s'; must be an integer." % args[1]) try: d = cls.objects.get(id=obj_id) except cls.DoesNotExist: print "Could not find a(n) %s with id=%s" % (name, obj_id) else: if options.handle: if controls.handle(d): print "The error state of %s was marked as handled." % d else: print "%s isn't in an error state." % d elif Status.is_final(d.status) and not options.force: print "%s is already in a final state." % d elif d.request == None or options.force: d.make_request(req) print "%s was sent a %s request." % (d, request) if options.wait: _wait([d], req) else: print "%s already has request %s." % \ (d, Request.name(d.request))
class Scheduler(AbstractDaemon): """Scheduling process for handling Schedules. Takes unclaimed Schedules from the database and adds their next instance to a timer. At the appropriate time, the instance is added to its queue and the Schedule is updated. Idea: Split this up into two threads, one which continuously handles already claimed schedules, the other which periodically polls the DB for new schedules. """ class Meta: app_label = 'core' db_table = 'norc_scheduler' objects = QuerySetManager() class QuerySet(AbstractDaemon.QuerySet): """Custom manager/query set for Scheduler.""" def undead(self): """Schedulers that are active but the heart isn't beating.""" cutoff = datetime.utcnow() - timedelta(seconds=HEARTBEAT_FAILED) return self.status_in("active").filter(heartbeat__lt=cutoff) # All the statuses Schedulers can have. See constants.py. VALID_STATUSES = [ Status.CREATED, Status.RUNNING, Status.PAUSED, Status.ENDED, Status.ERROR, ] VALID_REQUESTS = [ Request.STOP, Request.KILL, Request.PAUSE, Request.RESUME, Request.RELOAD, ] # The status of this scheduler. status = PositiveSmallIntegerField(default=Status.CREATED, choices=[(s, Status.name(s)) for s in VALID_STATUSES]) # A state-change request. request = PositiveSmallIntegerField(null=True, choices=[(r, Request.name(r)) for r in VALID_REQUESTS]) def __init__(self, *args, **kwargs): AbstractDaemon.__init__(self, *args, **kwargs) self.timer = MultiTimer() self.set = set() def start(self): """Starts the Scheduler.""" # Temporary check until multiple schedulers is supported fully. if Scheduler.objects.alive().count() > 0: print "Cannot run more than one scheduler at a time." return AbstractDaemon.start(self) def run(self): """Main run loop of the Scheduler.""" self.timer.start() while not Status.is_final(self.status): if self.request: self.handle_request() if self.status == Status.RUNNING: # Clean up orphaned schedules and undead schedulers. # Schedule.objects.orphaned().update(scheduler=None) # CronSchedule.objects.orphaned().update(scheduler=None) cron = CronSchedule.objects.unclaimed()[:SCHEDULER_LIMIT] simple = Schedule.objects.unclaimed()[:SCHEDULER_LIMIT] for schedule in itertools.chain(cron, simple): self.log.info('Claiming %s.' % schedule) schedule.scheduler = self schedule.save() self.add(schedule) if not Status.is_final(self.status): self.wait() self.request = Scheduler.objects.get(pk=self.pk).request def wait(self): """Waits on the flag.""" AbstractDaemon.wait(self, SCHEDULER_PERIOD) def clean_up(self): self.timer.cancel() self.timer.join() cron = self.cronschedules.all() simple = self.schedules.all() claimed_count = cron.count() + simple.count() if claimed_count > 0: self.log.info("Cleaning up %s schedules." % claimed_count) cron.update(scheduler=None) simple.update(scheduler=None) def handle_request(self): """Called when a request is found.""" # Clear request immediately. request = self.request self.request = None self.save() self.log.info("Request received: %s" % Request.name(request)) if request == Request.PAUSE: self.set_status(Status.PAUSED) elif request == Request.RESUME: if self.status != Status.PAUSED: self.log.info("Must be paused to resume; clearing request.") else: self.set_status(Status.RUNNING) elif request == Request.STOP: self.set_status(Status.ENDED) elif request == Request.KILL: self.set_status(Status.KILLED) elif request == Request.RELOAD: changed = MultiQuerySet(Schedule, CronSchedule) changed = changed.objects.unfinished.filter(changed=True, scheduler=self) for item in self.timer.tasks: s = item[2][0] if s in changed: self.log.info("Removing outdated: %s" % s) self.timer.tasks.remove(item) self.set.remove(s) s = type(s).objects.get(pk=s.pk) for s in changed: self.log.info("Adding updated: %s" % s) self.add(s) changed.update(changed=False) def add(self, schedule): """Adds the schedule to the timer.""" try: if schedule in self.set: self.log.error("%s has already been added to this Scheduler." % schedule) return self.log.debug('Adding %s to timer for %s.' % (schedule, schedule.next)) self.timer.add_task(schedule.next, self._enqueue, [schedule]) self.set.add(schedule) except: self.log.error("Invalid schedule %s found, deleting." % schedule) schedule.soft_delete() def _enqueue(self, schedule): """Called by the timer to add an instance to the queue.""" updated_schedule = get_object(type(schedule), pk=schedule.pk) self.set.remove(schedule) if updated_schedule == None or updated_schedule.deleted: self.log.info('%s was removed.' % schedule) if updated_schedule != None: updated_schedule.scheduler = None updated_schedule.save() return schedule = updated_schedule if not schedule.scheduler == self: self.log.info("%s is no longer tied to this Scheduler." % schedule) # self.set.remove(schedule) return instance = Instance.objects.create(task=schedule.task, schedule=schedule) self.log.info('Enqueuing %s.' % instance) schedule.queue.push(instance) schedule.enqueued() if not schedule.finished(): self.add(schedule) else: schedule.scheduler = None schedule.save() @property def log_path(self): return 'schedulers/scheduler-%s' % self.id