def run(self):
"""Core executor function."""
if settings.BACKUP_SYSTEM:
self.pool = ThreadPool(self.concurrent + 1)
self.log.info("%s is now running on host %s." % (self, self.host))
if self.log.debug_on:
self.resource_reporter = Thread(target=self.report_resources)
self.resource_reporter.daemon = True
self.resource_reporter.start()
# Main loop.
while not Status.is_final(self.status):
if self.request:
self.handle_request()
if self.status == Status.RUNNING:
while len(self.processes) < self.concurrent:
# self.log.debug("Popping instance...")
instance = self.queue.pop()
if instance:
# self.log.debug("Popped %s" % instance)
self.start_instance(instance)
else:
# self.log.debug("No instance in queue.")
break
elif self.status == Status.STOPPING and len(self.processes) == 0:
self.set_status(Status.ENDED)
self.save(safe=True)
# Clean up completed tasks before iterating.
for pid, p in self.processes.items()[:]:
p.poll()
self.log.debug("Checking pid %s: return code %s." %
(pid, p.returncode))
if not p.returncode == None:
i = type(p.instance).objects.get(pk=p.instance.pk)
if i.status == Status.CREATED:
self.log.info(
("%s fail to initialize properly; " +
"entering suspension to avoid more errors.") % i)
self.set_status(Status.SUSPENDED)
self.save()
if not Status.is_final(i.status):
self.log.info(("%s ended with invalid " +
"status %s, changing to ERROR.") %
(i, Status.name(i.status)))
i.status = Status.ERROR
i.save()
self.log.info("%s ended with status %s." %
(i, Status.name(i.status)))
del self.processes[pid]
if settings.BACKUP_SYSTEM:
self.pool.queueTask(self.backup_instance_log, [i])
if not Status.is_final(self.status):
self.wait(EXECUTOR_PERIOD)
self.request = Executor.objects.get(pk=self.pk).request
