def stop(self):
from chroma_core.services.job_scheduler.agent_rpc import AgentRpc
super(Service, self).stop()
# Guard against trying to stop after child threads are created, but before they are started
self._children_started.wait()
AgentRpc.shutdown()
self.log.info("Stopping...")
self._rpc_thread.stop()
self._queue_thread.stop()
self._progress_thread.stop()
self._mail_alerts_thread.stop()
self.log.info("Joining...")
self._rpc_thread.join()
self._queue_thread.join()
self._job_scheduler.join_run_threads()
self._progress_thread.join()
self._mail_alerts_thread.join()
self.log.info("Complete.")
self._complete.set()
def run(self):
from chroma_core.services.job_scheduler.job_scheduler import JobScheduler
from chroma_core.services.job_scheduler.job_scheduler_client import JobSchedulerRpc
from chroma_core.services.job_scheduler.agent_rpc import AgentRpc
super(Service, self).run()
# Cancel anything that's left behind from a previous run
for command in Command.objects.filter(complete=False):
command.completed(True, True)
Job.objects.filter(~Q(state="complete")).update(state="complete", cancelled=True)
self._job_scheduler = JobScheduler()
self._queue_thread = ServiceThread(QueueHandler(self._job_scheduler))
self._rpc_thread = ServiceThread(JobSchedulerRpc(self._job_scheduler))
self._progress_thread = ServiceThread(self._job_scheduler.progress)
AgentRpc.start()
self._queue_thread.start()
self._rpc_thread.start()
self._progress_thread.start()
self._children_started.set()
self._mail_alerts_thread = MailAlerts(settings.EMAIL_SENDER, settings.EMAIL_SUBJECT_PREFIX, settings.EMAIL_HOST)
self._mail_alerts_thread.start()
self._complete.wait()
self.log.info("Cancelling outstanding jobs...")
for job in Job.objects.filter(~Q(state="complete")).order_by("-id"):
self._job_scheduler.cancel_job(job.id)
def invoke_agent(self, host, command, args = {}):
"""
Wrapper around AgentRpc.call which provides logging
"""
from chroma_core.services.job_scheduler.agent_rpc import AgentRpc, AgentException
job_log.info("invoke_agent on agent %s %s %s" % (host, command, args))
try:
result, action_state = AgentRpc.call(host.fqdn, command, args, self._cancel_event)
self._log_subprocesses(action_state.subprocesses)
return result
except AgentException as e:
self._log_subprocesses(e.subprocesses)
raise