def run(self, params):
additional_timeout = params.get("timeout", 300)
stats = {
"requeued": 0,
"started": 0
}
# There shouldn't be that much "started" jobs so we can quite safely
# iterate over them.
fields = {"_id": 1, "datestarted": 1, "queue": 1, "path": 1, "retry_count": 1}
for job_data in connections.mongodb_jobs.mrq_jobs.find(
{"status": "started"}, projection=fields):
job = Job(job_data["_id"])
job.set_data(job_data)
stats["started"] += 1
expire_date = datetime.datetime.utcnow(
) - datetime.timedelta(seconds=job.timeout + additional_timeout)
if job_data["datestarted"] < expire_date:
log.debug("Requeueing job %s" % job.id)
job.requeue()
stats["requeued"] += 1
return stats
def do_orchestrate(self, group):
""" Manage the desired workers of *all* the agents in the given group """
log.debug("Starting orchestration run for worker group %s" %
group["_id"])
agents = self.fetch_worker_group_agents(group)
# Evaluate what workers are currently, rightfully there. They won't be touched.
for agent in agents:
desired_workers = self.get_desired_workers_for_agent(group, agent)
agent["new_desired_workers"] = []
agent["new_desired_workers"] = desired_workers
for agent in agents:
if sorted(agent["new_desired_workers"]) != sorted(
agent.get("desired_workers", [])):
connections.mongodb_jobs.mrq_agents.update_one(
{"_id": agent["_id"]}, {
"$set": {
"desired_workers": agent["new_desired_workers"]
}
})
# Remember the date of the last successful orchestration (will be reported)
self.dateorchestrated = datetime.datetime.utcnow()
log.debug("Orchestration finished.")
