def _mark_agent_done(self, agent_info: AgentInfo) -> None:
"""
Handle marking an agent as done, and telling the frontend agent
that they have successfully completed their task.
If the agent is in a final non-successful status, or already
told of partner disconnect, skip
"""
if agent_info.agent.db_status in AgentState.complete() + [
AgentState.STATUS_PARTNER_DISCONNECT
]:
return
send_packet = Packet(
packet_type=PACKET_TYPE_UPDATE_AGENT_STATUS,
sender_id=SYSTEM_CHANNEL_ID,
receiver_id=agent_info.agent.get_agent_id(),
data={
"status": "completed",
"state": {
"done_text":
"You have completed this task. Please submit.",
"task_done": True,
},
},
)
channel_info = self.channels[agent_info.used_channel_id]
channel_info.channel.send(send_packet)
def _handle_updated_agent_status(self, status_map: Dict[str, str]):
"""
Handle updating the local statuses for agents based on
the previously reported agent statuses.
Takes as input a mapping from agent_id to server-side status
"""
for agent_id, status in status_map.items():
if status not in AgentState.valid():
logger.warning(
f"Invalid status for agent {agent_id}: {status}")
continue
if agent_id not in self.agents:
# no longer tracking agent
continue
agent = self.agents[agent_id].agent
db_status = agent.get_status()
if agent.has_updated_status.is_set():
continue # Incoming info may be stale if we have new info to send
if status == AgentState.STATUS_NONE:
# Stale or reconnect, send a status update
self._send_status_update(self.agents[agent_id])
continue
if status != db_status:
if db_status in AgentState.complete():
logger.info(
f"Got updated status {status} when already final: {agent.db_status}"
)
continue
elif status == AgentState.STATUS_COMPLETED:
continue # COMPLETED can only be marked locally
agent.update_status(status)
pass
def _launch_and_run_assignment(
self,
assignment: "Assignment",
agent_infos: List["AgentInfo"],
task_runner: "TaskRunner",
):
"""Launch a thread to supervise the completion of an assignment"""
try:
tracked_agents: List["Agent"] = []
for a in agent_infos:
assert isinstance(
a.agent, Agent
), f"Can launch assignments for Agents, not OnboardingAgents, got {a.agent}"
tracked_agents.append(a.agent)
task_runner.launch_assignment(assignment, tracked_agents)
for agent_info in agent_infos:
self._mark_agent_done(agent_info)
# Wait for agents to be complete
for agent_info in agent_infos:
agent = agent_info.agent
if agent.get_status() not in AgentState.complete():
if not agent.did_submit.is_set():
# Wait for a submit to occur
# TODO(#94) make submit timeout configurable
agent.has_action.wait(timeout=300)
agent.act()
agent.mark_done()
except Exception as e:
logger.exception(f"Cleaning up assignment: {e}", exc_info=True)
task_runner.cleanup_assignment(assignment)
finally:
task_run = task_runner.task_run
for unit in assignment.get_units():
task_run.clear_reservation(unit)
def update_status(self, new_status: str) -> None:
"""Update the database status of this agent, and
possibly send a message to the frontend agent informing
them of this update"""
if self.db_status == new_status:
return # Noop, this is already the case
logger.debug(f"Updating {self} to {new_status}")
if self.db_status in AgentState.complete():
logger.info(f"Updating {self} from final status to {new_status}")
old_status = self.db_status
self.db.update_agent(self.db_id, status=new_status)
self.db_status = new_status
self.has_updated_status.set()
if new_status in [
AgentState.STATUS_RETURNED,
AgentState.STATUS_DISCONNECT,
AgentState.STATUS_TIMEOUT,
]:
# Disconnect statuses should free any pending acts
self.has_action.set()
self.did_submit.set()
if old_status == AgentState.STATUS_WAITING:
# Waiting agents' unit can be reassigned, as no work
# has been done yet.
self.get_unit().clear_assigned_agent()
def mark_done(self) -> None:
"""
Take any required step with the crowd_provider to ensure that
the worker can submit their work and be marked as complete via
a call to get_status
"""
if self.get_status() not in AgentState.complete():
self.db.update_agent(
agent_id=self.db_id, status=AgentState.STATUS_COMPLETED
)
def update_status(self, new_status: str) -> None:
"""Update the database status of this agent, and
possibly send a message to the frontend agent informing
them of this update"""
if self.db_status == new_status:
return # Noop, this is already the case
logger.debug(f"Updating {self} to {new_status}")
if self.db_status in AgentState.complete():
logger.info(f"Updating {self} from final status to {new_status}")
old_status = self.db_status
self.db.update_agent(self.db_id, status=new_status)
self.db_status = new_status
if self._associated_live_run is not None:
live_run = self.get_live_run()
live_run.loop_wrap.execute_coro(
live_run.worker_pool.push_status_update(self))
if new_status in [
AgentState.STATUS_RETURNED,
AgentState.STATUS_DISCONNECT,
AgentState.STATUS_TIMEOUT,
]:
# Disconnect statuses should free any pending acts
self.has_live_update.set()
self.did_submit.set()
if old_status == AgentState.STATUS_WAITING:
# Waiting agents' unit can be reassigned, as no work
# has been done yet.
unit = self.get_unit()
logger.debug(
f"Clearing {self} from {unit} for update to {new_status}")
unit.clear_assigned_agent()
# Metrics changes
ACTIVE_AGENT_STATUSES.labels(status=old_status,
agent_type="main").dec()
ACTIVE_AGENT_STATUSES.labels(status=new_status,
agent_type="main").inc()
if (old_status not in AgentState.complete()
and new_status in AgentState.complete()):
ACTIVE_WORKERS.labels(worker_id=self.worker_id,
agent_type="main").dec()
def update_status(self, new_status: str) -> None:
"""Update the database status of this agent, and
possibly send a message to the frontend agent informing
them of this update"""
if self.db_status == new_status:
return # Noop, this is already the case
logger.debug(f"Updating {self} to {new_status}")
if self.db_status in AgentState.complete():
logger.info(f"Updating {self} from final status to {new_status}")
old_status = self.db_status
self.db.update_onboarding_agent(self.db_id, status=new_status)
self.db_status = new_status
if self._associated_live_run is not None:
if new_status not in [
AgentState.STATUS_APPROVED,
AgentState.STATUS_REJECTED,
]:
live_run = self.get_live_run()
live_run.loop_wrap.execute_coro(
live_run.worker_pool.push_status_update(self))
if new_status in [
AgentState.STATUS_RETURNED, AgentState.STATUS_DISCONNECT
]:
# Disconnect statuses should free any pending acts
self.has_live_update.set()
self.did_submit.set()
# Metrics changes
ACTIVE_AGENT_STATUSES.labels(status=old_status,
agent_type="onboarding").dec()
ACTIVE_AGENT_STATUSES.labels(status=new_status,
agent_type="onboarding").inc()
if (old_status not in AgentState.complete()
and new_status in AgentState.complete()):
ACTIVE_WORKERS.labels(worker_id=self.worker_id,
agent_type="onboarding").dec()
def get_status(self) -> str:
"""Get the status of this agent in their work on their unit"""
if self.db_status not in AgentState.complete():
row = self.db.get_onboarding_agent(self.db_id)
if row["status"] != self.db_status:
if row["status"] in [
AgentState.STATUS_RETURNED,
AgentState.STATUS_DISCONNECT,
]:
# Disconnect statuses should free any pending acts
self.has_action.set()
self.has_updated_status.set()
self.db_status = row["status"]
return self.db_status
def get_status(self) -> str:
"""Get the status of this agent in their work on their unit"""
if self.db_status not in AgentState.complete():
row = self.db.get_agent(self.db_id)
if row["status"] != self.db_status:
if row["status"] in [
AgentState.STATUS_RETURNED,
AgentState.STATUS_DISCONNECT,
]:
# Disconnect statuses should free any pending acts
self.has_live_update.set()
if self._associated_live_run is not None:
live_run = self.get_live_run()
live_run.loop_wrap.execute_coro(
live_run.worker_pool.push_status_update(self))
self.db_status = row["status"]
return self.db_status
def update_status(self, new_status: str) -> None:
"""Update the database status of this agent, and
possibly send a message to the frontend agent informing
them of this update"""
if self.db_status == new_status:
return # Noop, this is already the case
if self.db_status in AgentState.complete():
print(f"Updating a final status, was {self.db_status} "
f"and want to set to {new_status}")
self.db.update_onboarding_agent(self.db_id, status=new_status)
self.db_status = new_status
self.has_updated_status.set()
if new_status in [
AgentState.STATUS_RETURNED, AgentState.STATUS_DISCONNECT
]:
# Disconnect statuses should free any pending acts
self.has_action.set()
self.did_submit.set()
def _launch_and_run_unit(self, unit: "Unit", agent_info: "AgentInfo",
task_runner: "TaskRunner"):
"""Launch a thread to supervise the completion of an assignment"""
try:
agent = agent_info.agent
assert isinstance(
agent, Agent
), f"Can launch units for Agents, not OnboardingAgents, got {agent}"
task_runner.launch_unit(unit, agent)
if agent.get_status() not in AgentState.complete():
self._mark_agent_done(agent_info)
if not agent.did_submit.is_set():
# Wait for a submit to occur
# TODO(#94) make submit timeout configurable
agent.has_action.wait(timeout=300)
agent.act()
agent.mark_done()
except Exception as e:
logger.exception(f"Cleaning up unit: {e}", exc_info=True)
task_runner.cleanup_unit(unit)
finally:
task_runner.task_run.clear_reservation(unit)