async def update_queue(self):
"""
Send waiting jobs to available agents
"""
# For now, round-robin
not_found_for_agent = []
while len(self._available_agents) > 0 and len(self._waiting_jobs) > 0:
agent_addr = self._available_agents.pop(0)
# Find first job that can be run on this agent
found = False
client_addr, job_id, typestr, job_msg = None, None, None, None
for (client_addr, job_id,
typestr), job_msg in self._waiting_jobs.items():
if typestr == "batch" and job_msg.container_name in self._batch_containers_on_agent[
agent_addr]:
found = True
break
elif typestr == "grade" and job_msg.environment in self._containers_on_agent[
agent_addr]:
found = True
break
if not found:
self._logger.debug("Nothing to do for agent %s", agent_addr)
not_found_for_agent.append(agent_addr)
continue
# Remove the job from the queue
del self._waiting_jobs[(client_addr, job_id, typestr)]
if typestr == "grade" and isinstance(job_msg, ClientNewJob):
job_id = (client_addr, job_msg.job_id)
self._job_running[job_id] = (agent_addr, job_msg, time.time())
self._logger.info("Sending job %s %s to agent %s", client_addr,
job_msg.job_id, agent_addr)
await ZMQUtils.send_with_addr(
self._agent_socket, agent_addr,
BackendNewJob(job_id, job_msg.course_id, job_msg.task_id,
job_msg.inputdata, job_msg.environment,
job_msg.enable_network, job_msg.time_limit,
job_msg.hard_time_limit, job_msg.mem_limit,
job_msg.debug))
elif typestr == "batch":
job_id = (client_addr, job_msg.job_id)
self._batch_job_running[job_id] = (agent_addr, job_msg,
time.time())
self._logger.info("Sending batch job %s %s to agent %s",
client_addr, job_msg.job_id, agent_addr)
await ZMQUtils.send_with_addr(
self._agent_socket, agent_addr,
BackendNewBatchJob(job_id, job_msg.container_name,
job_msg.input_data))
# Do not forget to add again for which we did not find jobs to do
self._available_agents += not_found_for_agent
async def update_queue(self):
"""
Send waiting jobs to available agents
"""
available_agents = list(
self._available_agents) # do a copy to avoid bad things
# Loop on available agents to maximize running jobs, and break if priority queue empty
for agent_addr in available_agents:
if self._waiting_jobs_pq.empty():
break # nothing to do
try:
job = None
while job is None:
# keep the object, do not unzip it directly! It's sometimes modified when a job is killed.
topics = [(*env, False) for env in
self._registered_agents[agent_addr].environments]
if self._registered_agents[agent_addr].ssh_allowed:
topics += [
(*env, True) for env in
self._registered_agents[agent_addr].environments
]
job = self._waiting_jobs_pq.get(topics)
priority, insert_time, client_addr, job_id, job_msg = job
# Killed job, removing it from the mapping
if not job_msg:
del self._waiting_jobs[job_id]
job = None # repeat the while loop. we need a job
except queue.Empty:
continue # skip agent, nothing to do!
# We have found a job, let's remove the agent from the available list
self._available_agents.remove(agent_addr)
# Remove the job from the queue
del self._waiting_jobs[job_id]
# Send the job to agent
self._job_running[job_id] = RunningJob(agent_addr, client_addr,
job_msg, time.time())
self._logger.info("Sending job %s %s to agent %s", client_addr,
job_id, agent_addr)
await ZMQUtils.send_with_addr(
self._agent_socket, agent_addr,
BackendNewJob(job_id, job_msg.course_id, job_msg.task_id,
job_msg.task_problems, job_msg.inputdata,
job_msg.environment_type, job_msg.environment,
job_msg.environment_parameters, job_msg.debug))
async def update_queue(self):
"""
Send waiting jobs to available agents
"""
# Loop on available agents to maximize running jobs, and break if priority queue empty
for i in range(0, len(self._available_agents)):
if self._waiting_jobs_pq.empty():
break
priority, insert_time, client_addr, job_id, job_msg = self._waiting_jobs_pq.get(
)
# Killed job, removing it from the mapping
if not job_msg:
del self._waiting_jobs[(client_addr, job_id)]
continue
# Find agents that can run this job
possible_agents = list(
set(self._containers[job_msg.environment][2]).intersection(
set(self._available_agents)))
# No agent available, put job back to queue with lower priority
if not possible_agents:
job = (priority + 1, insert_time, client_addr, job_id, job_msg)
self._waiting_jobs_pq.put(job)
self._logger.warning(
"No agent for job id %s, putting it back in the queue.",
job_id)
continue
# Agent chosen, removing it from availability list
agent_addr = possible_agents[0]
self._available_agents.remove(agent_addr)
# Remove the job from the queue
del self._waiting_jobs[(client_addr, job_id)]
# Send the job to agent
job_id = (client_addr, job_msg.job_id)
self._job_running[job_id] = (agent_addr, job_msg, time.time())
self._logger.info("Sending job %s %s to agent %s", client_addr,
job_msg.job_id, agent_addr)
await ZMQUtils.send_with_addr(
self._agent_socket, agent_addr,
BackendNewJob(job_id, job_msg.course_id, job_msg.task_id,
job_msg.inputdata, job_msg.environment,
job_msg.enable_network, job_msg.time_limit,
job_msg.hard_time_limit, job_msg.mem_limit,
job_msg.debug))
async def update_queue(self):
"""
Send waiting jobs to available agents
"""
jobs_ignored = []
available_agents = list(
self._available_agents) # do a copy to avoid bad things
# Loop on available agents to maximize running jobs, and break if priority queue empty
for agent_addr in available_agents:
if self._waiting_jobs_pq.empty():
break # nothing to do
try:
job = None
while job is None:
# keep the object, do not unzip it directly! It's sometimes modified when a job is killed.
job = self._waiting_jobs_pq.get(
self._registered_agents[agent_addr]
["environments"].keys())
priority, insert_time, client_addr, job_id, job_msg = job
# Killed job, removing it from the mapping
if not job_msg:
del self._waiting_jobs[(client_addr, job_id)]
job = None # repeat the while loop. we need a job
except queue.Empty:
continue # skip agent, nothing to do!
# We have found a job, let's remove the agent from the available list
self._available_agents.remove(agent_addr)
# Remove the job from the queue
del self._waiting_jobs[(client_addr, job_id)]
# Send the job to agent
job_id = (client_addr, job_msg.job_id)
self._job_running[job_id] = (agent_addr, job_msg, time.time())
self._logger.info("Sending job %s %s to agent %s", client_addr,
job_msg.job_id, agent_addr)
await ZMQUtils.send_with_addr(
self._agent_socket, agent_addr,
BackendNewJob(job_id, job_msg.course_id, job_msg.task_id,
job_msg.inputdata, job_msg.environment,
job_msg.environment_parameters, job_msg.debug))
# Let's not forget to add again the ignored jobs to the PQ.
for entry in jobs_ignored:
self._waiting_jobs_pq.put(entry)