def run(self, params): additional_timeout = params.get("timeout", 300) stats = { "requeued": 0, "started": 0 } # There shouldn't be that much "started" jobs so we can quite safely # iterate over them. fields = {"_id": 1, "datestarted": 1, "queue": 1, "path": 1, "retry_count": 1} for job_data in connections.mongodb_jobs.mrq_jobs.find( {"status": "started"}, projection=fields): job = Job(job_data["_id"]) job.set_data(job_data) stats["started"] += 1 expire_date = datetime.datetime.utcnow( ) - datetime.timedelta(seconds=job.timeout + additional_timeout) if job_data["datestarted"] < expire_date: log.debug("Requeueing job %s" % job.id) job.requeue() stats["requeued"] += 1 return stats
def do_orchestrate(self, group): """ Manage the desired workers of *all* the agents in the given group """ log.debug("Starting orchestration run for worker group %s" % group["_id"]) agents = self.fetch_worker_group_agents(group) # Evaluate what workers are currently, rightfully there. They won't be touched. for agent in agents: desired_workers = self.get_desired_workers_for_agent(group, agent) agent["new_desired_workers"] = [] agent["new_desired_workers"] = desired_workers for agent in agents: if sorted(agent["new_desired_workers"]) != sorted( agent.get("desired_workers", [])): connections.mongodb_jobs.mrq_agents.update_one( {"_id": agent["_id"]}, { "$set": { "desired_workers": agent["new_desired_workers"] } }) # Remember the date of the last successful orchestration (will be reported) self.dateorchestrated = datetime.datetime.utcnow() log.debug("Orchestration finished.")