def get_unit(self) -> "Unit":
"""
Return the Unit that this agent is working on.
"""
if self._unit is None:
from mephisto.data_model.unit import Unit
self._unit = Unit.get(self.db, self.unit_id)
return self._unit
def test_unit_fails(self) -> None:
"""Ensure units fail to be created or loaded under failure conditions"""
assert self.db is not None, "No db initialized"
db: MephistoDB = self.db
# Cant get non-existent entry
with self.assertRaises(EntryDoesNotExistException):
unit = Unit.get(db, self.get_fake_id("Unit"))
assignment_id = get_test_assignment(db)
assignment = Assignment.get(db, assignment_id)
unit_index = 0
pay_amount = 15.0
provider_type = PROVIDER_TYPE
# Can't use invalid assignment_id name
with self.assertRaises(EntryDoesNotExistException):
unit_id = db.new_unit(
assignment.task_id,
assignment.task_run_id,
assignment.requester_id,
self.get_fake_id("Assignment"),
unit_index,
pay_amount,
provider_type,
assignment.sandbox,
)
unit_id = db.new_unit(
assignment.task_id,
assignment.task_run_id,
assignment.requester_id,
assignment.db_id,
unit_index,
pay_amount,
provider_type,
assignment.sandbox,
)
# Can't create same unit again
with self.assertRaises(EntryAlreadyExistsException):
unit_id = db.new_unit(
assignment.task_id,
assignment.task_run_id,
assignment.requester_id,
assignment.db_id,
unit_index,
pay_amount,
provider_type,
assignment.sandbox,
)
# Ensure no units were created
units = db.find_units()
self.assertEqual(len(units), 1)
def test_unit(self) -> None:
"""Test creation and querying of units"""
assert self.db is not None, "No db initialized"
db: MephistoDB = self.db
# Check creation and retrieval of a unit
assignment_id = get_test_assignment(db)
assignment = Assignment.get(db, assignment_id)
unit_index = 0
pay_amount = 15.0
provider_type = PROVIDER_TYPE
unit_id = db.new_unit(
assignment.task_id,
assignment.task_run_id,
assignment.requester_id,
assignment.db_id,
unit_index,
pay_amount,
provider_type,
assignment.sandbox,
)
self.assertIsNotNone(unit_id)
self.assertTrue(isinstance(unit_id, str))
unit_row = db.get_unit(unit_id)
self.assertEqual(unit_row["assignment_id"], assignment_id)
self.assertEqual(unit_row["pay_amount"], pay_amount)
self.assertEqual(unit_row["status"], AssignmentState.CREATED)
unit = Unit.get(db, unit_id)
self.assertEqual(unit.assignment_id, assignment_id)
# Check finding for units
units = db.find_units()
self.assertEqual(len(units), 1)
self.assertTrue(isinstance(units[0], Unit))
self.assertEqual(units[0].db_id, unit_id)
self.assertEqual(units[0].assignment_id, assignment_id)
self.assertEqual(units[0].pay_amount, pay_amount)
# Check finding for specific units
units = db.find_units(assignment_id=assignment_id)
self.assertEqual(len(units), 1)
self.assertTrue(isinstance(units[0], Unit))
self.assertEqual(units[0].db_id, unit_id)
self.assertEqual(units[0].assignment_id, assignment_id)
self.assertEqual(units[0].pay_amount, pay_amount)
units = db.find_units(assignment_id=self.get_fake_id("Assignment"))
self.assertEqual(len(units), 0)
def test_agent(self) -> None:
"""Test creation and querying of agents"""
assert self.db is not None, "No db initialized"
db: MephistoDB = self.db
# Check creation and retrieval of a agent
worker_name, worker_id = get_test_worker(db)
unit_id = get_test_unit(db)
unit = Unit.get(db, unit_id)
agent_id = db.new_agent(
worker_id,
unit_id,
unit.task_id,
unit.task_run_id,
unit.assignment_id,
unit.task_type,
unit.provider_type,
)
self.assertIsNotNone(agent_id)
self.assertTrue(isinstance(agent_id, str))
agent_row = db.get_agent(agent_id)
self.assertEqual(agent_row["worker_id"], worker_id)
self.assertEqual(agent_row["unit_id"], unit_id)
self.assertEqual(agent_row["status"], AgentState.STATUS_NONE)
# ensure the unit is assigned now
units = db.find_units(status=AssignmentState.ASSIGNED)
self.assertEqual(len(units), 1)
agent = Agent.get(db, agent_id)
self.assertEqual(agent.worker_id, worker_id)
# Check finding for agents
agents = db.find_agents()
self.assertEqual(len(agents), 1)
self.assertTrue(isinstance(agents[0], Agent))
self.assertEqual(agents[0].db_id, agent_id)
self.assertEqual(agents[0].worker_id, worker_id)
# Check finding for specific agents
agents = db.find_agents(worker_id=worker_id)
self.assertEqual(len(agents), 1)
self.assertTrue(isinstance(agents[0], Agent))
self.assertEqual(agents[0].db_id, agent_id)
self.assertEqual(agents[0].worker_id, worker_id)
agents = db.find_agents(worker_id=self.get_fake_id("Worker"))
self.assertEqual(len(agents), 0)
def get_mturk_ids_from_unit_id(db, unit_id: str) -> Dict[str, Optional[str]]:
"""
Find the relevant mturk ids from the given mephisto unit id
"""
mturk_unit = Unit.get(db, unit_id)
assignment_id = mturk_unit.get_mturk_assignment_id()
hit_id = mturk_unit.get_mturk_hit_id()
agent = mturk_unit.get_assigned_agent()
worker_id = None
if agent is not None:
worker_id = agent.get_worker().get_mturk_worker_id()
return {
"assignment_id": assignment_id,
"hit_id": hit_id,
"worker_id": worker_id
}
def test_agent_fails(self) -> None:
"""Ensure agents fail to be created or loaded under failure conditions"""
assert self.db is not None, "No db initialized"
db: MephistoDB = self.db
# Cant get non-existent entry
with self.assertRaises(EntryDoesNotExistException):
agent = Agent.get(db, self.get_fake_id("Agent"))
unit_id = get_test_unit(db)
worker_name, worker_id = get_test_worker(db)
unit = Unit.get(db, unit_id)
# Can't use invalid worker id
with self.assertRaises(EntryDoesNotExistException):
agent_id = db.new_agent(
self.get_fake_id("Worker"),
unit_id,
unit.task_id,
unit.task_run_id,
unit.assignment_id,
unit.task_type,
unit.provider_type,
)
# Can't use invalid unit id
with self.assertRaises(EntryDoesNotExistException):
agent_id = db.new_agent(
worker_id,
self.get_fake_id("Unit"),
unit.task_id,
unit.task_run_id,
unit.assignment_id,
unit.task_type,
unit.provider_type,
)
# Ensure no agents were created
agents = db.find_agents()
self.assertEqual(len(agents), 0)
def format_for_printing_data(data):
global db
# Custom tasks can define methods for how to display their data in a relevant way
worker_name = Worker.get(db, data["worker_id"]).worker_name
contents = data["data"]
duration = contents["times"]["task_end"] - contents["times"]["task_start"]
metadata_string = (
f"Worker: {worker_name}\nUnit: {data['unit_id']}\n"
f"Duration: {int(duration)}\nStatus: {data['status']}\n")
inputs = contents["inputs"]
inputs_string = f"Character: {inputs['character_name']}\nDescription: {inputs['character_description']}\n"
outputs = contents["outputs"]
output_string = f" Rating: {outputs['rating']}\n"
found_files = outputs.get("files")
if found_files is not None:
file_dir = Unit.get(
db, data["unit_id"]).get_assigned_agent().get_data_dir()
output_string += f" Files: {found_files}\n"
output_string += f" File directory {file_dir}\n"
else:
output_string += f" Files: No files attached\n"
return f"-------------------\n{metadata_string}{inputs_string}{output_string}"
def get_submitted_data():
try:
task_run_ids = request.args.getlist("task_run_id")
task_names = request.args.getlist("task_name")
assignment_ids = request.args.getlist("assignment_id")
unit_ids = request.args.getlist("unit_ids")
statuses = request.args.getlist("status")
db = app.extensions["db"]
units = []
assignments = []
assert len(
task_names) == 0, "Searching via task names not yet supported"
task_runs = [
TaskRun.get(db, task_run_id) for task_run_id in task_run_ids
]
for task_run in task_runs:
assignments += task_run.get_assignments()
assignments += [
Assignment.get(db, assignment_id)
for assignment_id in assignment_ids
]
if len(statuses) == 0:
statuses = [
AssignmentState.COMPLETED,
AssignmentState.ACCEPTED,
AssignmentState.REJECTED,
]
filtered_assignments = [
a for a in assignments if a.get_status() in statuses
]
for assignment in assignments:
units += assignment.get_units()
units += [Unit.get(db, unit_id) for unit_id in unit_ids]
all_unit_data = []
for unit in units:
unit_data = {
"assignment_id": unit.assignment_id,
"task_run_id": unit.task_run_id,
"status": unit.db_status,
"unit_id": unit.db_id,
"worker_id": unit.worker_id,
"data": None,
}
agent = unit.get_assigned_agent()
if agent is not None:
unit_data["data"] = agent.state.get_data()
unit_data["worker_id"] = agent.worker_id
all_unit_data.append(unit_data)
print(all_unit_data)
return jsonify({"success": True, "units": all_unit_data})
except Exception as e:
import traceback
traceback.print_exc()
return jsonify({"success": False, "msg": str(e)})
def main():
"""
Script to launch makeup tasks for workers that
can't be bonused via other avenues.
Creates a task for a worker, qualifying them directly,
and marks as a soft_rejected HIT for the given task name.
"""
db = LocalMephistoDB()
task_name = input(
"Please enter a task name for bookkeeping. This task name will be tied to "
"the additional spend granted through this script, and should be the same "
"as the task you originally launched that you now need to compensate for:\n>> "
)
tasks = db.find_tasks(task_name=task_name)
if len(tasks) == 0:
print("No tasks found with the given name...")
all_tasks = db.find_tasks()
all_names = set([t.task_name for t in all_tasks])
print(
f"Choose an existing task of {all_names} to use this functionality."
)
print(f"Compensation hits must be tied to an existing task")
return 0
task = tasks[0]
req_name = input(
"Please enter an MTurkRequester name to use to bonus from:\n>> ")
requesters = db.find_requesters(requester_name=req_name)
if len(requesters) == 0:
print("Could not find a requester by that name...")
return 0
requester = requesters[0]
client = requester._get_client(requester._requester_name)
print(
"You can now enter a worker id, amount, and reason for as many compensation tasks "
"as you want to launch for this.")
compensation_hits = []
amount = None
reason = None
while True:
worker_id = input(
"Enter a worker id to compensate. Leave blank to move on to launching: \n>> "
).strip()
if len(worker_id) == 0:
break
prev_amount = "" if amount is None else f" (leave blank for ${amount})"
next_amount = input(
f"Enter the amount in dollars to pay out in this compensation task{prev_amount}:\n>> $"
)
amount = float(next_amount) if len(
next_amount.strip()) != 0 else amount
assert amount is not None, "Amount can not be left blank"
prev_reason = "" if reason is None else f" (leave blank for '{reason}'"
next_reason = input(
f"Provide reason for launching this compensation task. This will be sent to the worker{prev_reason}:\n>> "
)
reason = next_reason if len(next_reason.strip()) != 0 else reason
assert reason is not None, "Reason can not be left blank"
compensation_hits.append({
"worker_id": worker_id,
"amount": amount,
"reason": reason,
})
if len(compensation_hits) == 0:
print("No compensation details provided, exiting")
return 0
print(f"You entered the following tasks:\n{compensation_hits}")
input("Input anything to confirm and continue...")
# Iterate through and launch tasks
for comp_dict in compensation_hits:
# Create the MTurk qualification for this specific worker
worker_id = comp_dict["worker_id"]
qual_name = f"compensation-for-{worker_id}-on-{task_name}"
print(f"Creating qualification for {worker_id}: {qual_name}....")
qualification = make_qualification_dict(qual_name, QUAL_EXISTS, None)
qual_map = requester.datastore.get_qualification_mapping(qual_name)
if qual_map is None:
qualification[
"QualificationTypeId"] = requester._create_new_mturk_qualification(
qual_name)
else:
qualification["QualificationTypeId"] = qual_map[
"mturk_qualification_id"]
give_worker_qualification(client, worker_id,
qualification["QualificationTypeId"])
# Create the task run for this HIT
print(f"Creating task run and data model components for this HIT")
config = build_task_config(comp_dict, requester)
init_params = OmegaConf.to_yaml(OmegaConf.structured(config))
new_run_id = db.new_task_run(
task.db_id,
requester.db_id,
json.dumps(init_params),
requester.provider_type,
"mock",
requester.is_sandbox(),
)
task_run = TaskRun.get(db, new_run_id)
# Create an assignment, unit, agent, and mark as assigned
# Assignment creation
task_args = task_run.get_task_args()
assignment_id = db.new_assignment(
task_run.task_id,
task_run.db_id,
task_run.requester_id,
task_run.task_type,
task_run.provider_type,
task_run.sandbox,
)
data = InitializationData({}, [{}])
assignment = Assignment.get(db, assignment_id)
assignment.write_assignment_data(data)
# Unit creation
unit_id = db.new_unit(
task_run.task_id,
task_run.db_id,
task_run.requester_id,
assignment_id,
COMPENSATION_UNIT_INDEX,
task_args.task_reward,
task_run.provider_type,
task_run.task_type,
task_run.sandbox,
)
compensation_unit = Unit.get(db, unit_id)
print(f"Created {task_run}, {assignment}, and {compensation_unit}...")
# Set up HIT type
hit_type_id = create_hit_type(
client,
task_run.get_task_args(),
[qualification],
auto_approve_delay=30,
skip_locale_qual=True,
)
# Create the task on MTurk, email the worker
print("Creating and deploying task on MTurk")
duration = 60 * 60 * 24
run_id = task_run.db_id
hit_link, hit_id, response = create_compensation_hit_with_hit_type(
client, comp_dict["reason"], hit_type_id)
requester.datastore.new_hit(hit_id, hit_link, duration, task_run.db_id)
print("Sending email to worker...")
result = email_worker(
client,
worker_id,
"Compensation HIT Launched",
("Hello Worker,\n We've launched a compensation hit for a task that you've worked on "
f"for us in the past. The reason supplied for this task was: {reason}. This task is "
f"only doable by you, and should reward ${comp_dict['amount']}. Thanks for being a valued "
"contributor to our tasks, and for allowing us to try and resolve the issue.\n\n"
f"Your task can be accessed at the following link: {hit_link}."),
)
if not result[0]:
print(
f"Email send failed, for reason {result[1]}\n"
f"Please send {hit_link} to {worker_id} yourself if they reached out about this issue."
)
# Mark the agent as soft_rejected, such that we've "paid" it
compensation_unit.set_db_status(AssignmentState.SOFT_REJECTED)