def get_test_assignment(self) -> Assignment:
"""Create a test assignment for self.task_run using mock agents"""
task_run = self.task_run
assignment_id = self.db.new_assignment(
task_run.task_id,
task_run.db_id,
task_run.requester_id,
task_run.task_type,
task_run.provider_type,
)
assign = Assignment.get(self.db, assignment_id)
unit_id = self.db.new_unit(
task_run.task_id,
task_run.db_id,
task_run.requester_id,
assignment_id,
0,
0,
task_run.provider_type,
task_run.task_type,
)
unit = MockUnit.get(self.db, unit_id)
worker_id = self.db.new_worker("MOCK_TEST_WORKER", MOCK_PROVIDER_TYPE)
worker = MockWorker.get(self.db, worker_id)
agent_id = self.db.new_agent(
worker.db_id,
unit_id,
task_run.task_id,
task_run.db_id,
assignment_id,
task_run.task_type,
task_run.provider_type,
)
Agent = MockAgent.get(self.db, agent_id)
return assign
def get_assignment(self) -> "Assignment":
"""
Return the assignment that this Unit is part of.
"""
if self.__assignment is None:
from mephisto.data_model.assignment import Assignment
self.__assignment = Assignment.get(self.db, self.assignment_id)
return self.__assignment
def get_assignment(self) -> "Assignment":
"""Return the assignment this agent is working on"""
if self._assignment is None:
if self._unit is not None:
self._assignment = self._unit.get_assignment()
else:
from mephisto.data_model.assignment import Assignment
self._assignment = Assignment.get(self.db, self.assignment_id)
return self._assignment
def test_unit_fails(self) -> None:
"""Ensure units fail to be created or loaded under failure conditions"""
assert self.db is not None, "No db initialized"
db: MephistoDB = self.db
# Cant get non-existent entry
with self.assertRaises(EntryDoesNotExistException):
unit = Unit.get(db, self.get_fake_id("Unit"))
assignment_id = get_test_assignment(db)
assignment = Assignment.get(db, assignment_id)
unit_index = 0
pay_amount = 15.0
provider_type = PROVIDER_TYPE
# Can't use invalid assignment_id name
with self.assertRaises(EntryDoesNotExistException):
unit_id = db.new_unit(
assignment.task_id,
assignment.task_run_id,
assignment.requester_id,
self.get_fake_id("Assignment"),
unit_index,
pay_amount,
provider_type,
assignment.sandbox,
)
unit_id = db.new_unit(
assignment.task_id,
assignment.task_run_id,
assignment.requester_id,
assignment.db_id,
unit_index,
pay_amount,
provider_type,
assignment.sandbox,
)
# Can't create same unit again
with self.assertRaises(EntryAlreadyExistsException):
unit_id = db.new_unit(
assignment.task_id,
assignment.task_run_id,
assignment.requester_id,
assignment.db_id,
unit_index,
pay_amount,
provider_type,
assignment.sandbox,
)
# Ensure no units were created
units = db.find_units()
self.assertEqual(len(units), 1)
def test_unit(self) -> None:
"""Test creation and querying of units"""
assert self.db is not None, "No db initialized"
db: MephistoDB = self.db
# Check creation and retrieval of a unit
assignment_id = get_test_assignment(db)
assignment = Assignment.get(db, assignment_id)
unit_index = 0
pay_amount = 15.0
provider_type = PROVIDER_TYPE
unit_id = db.new_unit(
assignment.task_id,
assignment.task_run_id,
assignment.requester_id,
assignment.db_id,
unit_index,
pay_amount,
provider_type,
assignment.sandbox,
)
self.assertIsNotNone(unit_id)
self.assertTrue(isinstance(unit_id, str))
unit_row = db.get_unit(unit_id)
self.assertEqual(unit_row["assignment_id"], assignment_id)
self.assertEqual(unit_row["pay_amount"], pay_amount)
self.assertEqual(unit_row["status"], AssignmentState.CREATED)
unit = Unit.get(db, unit_id)
self.assertEqual(unit.assignment_id, assignment_id)
# Check finding for units
units = db.find_units()
self.assertEqual(len(units), 1)
self.assertTrue(isinstance(units[0], Unit))
self.assertEqual(units[0].db_id, unit_id)
self.assertEqual(units[0].assignment_id, assignment_id)
self.assertEqual(units[0].pay_amount, pay_amount)
# Check finding for specific units
units = db.find_units(assignment_id=assignment_id)
self.assertEqual(len(units), 1)
self.assertTrue(isinstance(units[0], Unit))
self.assertEqual(units[0].db_id, unit_id)
self.assertEqual(units[0].assignment_id, assignment_id)
self.assertEqual(units[0].pay_amount, pay_amount)
units = db.find_units(assignment_id=self.get_fake_id("Assignment"))
self.assertEqual(len(units), 0)
def test_assignment(self) -> None:
"""Test creation and querying of assignments"""
assert self.db is not None, "No db initialized"
db: MephistoDB = self.db
task_run_id = get_test_task_run(db)
task_run = TaskRun.get(db, task_run_id)
# Check creation and retrieval of an assignment
assignment_id = db.new_assignment(
task_run.task_id,
task_run_id,
task_run.requester_id,
task_run.task_type,
task_run.provider_type,
task_run.sandbox,
)
self.assertIsNotNone(assignment_id)
self.assertTrue(isinstance(assignment_id, str))
assignment_row = db.get_assignment(assignment_id)
self.assertEqual(assignment_row["task_run_id"], task_run_id)
assignment = Assignment.get(db, assignment_id)
self.assertEqual(assignment.task_run_id, task_run_id)
# Check finding for assignments
assignments = db.find_assignments()
self.assertEqual(len(assignments), 1)
self.assertTrue(isinstance(assignments[0], Assignment))
self.assertEqual(assignments[0].db_id, assignment_id)
self.assertEqual(assignments[0].task_run_id, task_run_id)
# Check finding for specific assignments
assignments = db.find_assignments(task_run_id=task_run_id)
self.assertEqual(len(assignments), 1)
self.assertTrue(isinstance(assignments[0], Assignment))
self.assertEqual(assignments[0].db_id, assignment_id)
self.assertEqual(assignments[0].task_run_id, task_run_id)
assignments = db.find_assignments(task_run_id=self.get_fake_id("Assignment"))
self.assertEqual(len(assignments), 0)
def _create_single_assignment(self, assignment_data) -> None:
"""Create a single assignment in the database using its read assignment_data"""
task_run = self.task_run
task_args = task_run.get_task_args()
assignment_id = self.db.new_assignment(
task_run.task_id,
task_run.db_id,
task_run.requester_id,
task_run.task_type,
task_run.provider_type,
task_run.sandbox,
)
assignment = Assignment.get(self.db, assignment_id)
assignment.write_assignment_data(assignment_data)
self.assignments.append(assignment)
unit_count = len(assignment_data.unit_data)
for unit_idx in range(unit_count):
unit = self.UnitClass.new(self.db, assignment, unit_idx,
task_args.task_reward)
self.units.append(unit)
with self.unlaunched_units_access_condition:
self.unlaunched_units[unit.db_id] = unit
def launch_evaluation_unit(self, unit_data: Dict[str, Any],
unit_type_index: int) -> "Unit":
"""Launch a specific evaluation unit, used for quality control"""
assert (self.launch_url is not None
), "Cannot launch an evaluation unit before launching others"
task_run = self.task_run
task_args = task_run.get_task_args()
assignment_id = self.db.new_assignment(
task_run.task_id,
task_run.db_id,
task_run.requester_id,
task_run.task_type,
task_run.provider_type,
task_run.sandbox,
)
data = InitializationData(unit_data, [{}])
assignment = Assignment.get(self.db, assignment_id)
assignment.write_assignment_data(data)
self.assignments.append(assignment)
evaluation_unit = self.UnitClass.new(self.db, assignment,
unit_type_index,
task_args.task_reward)
evaluation_unit.launch(self.launch_url)
return evaluation_unit
def get_submitted_data():
try:
task_run_ids = request.args.getlist("task_run_id")
task_names = request.args.getlist("task_name")
assignment_ids = request.args.getlist("assignment_id")
unit_ids = request.args.getlist("unit_ids")
statuses = request.args.getlist("status")
db = app.extensions["db"]
units = []
assignments = []
assert len(
task_names) == 0, "Searching via task names not yet supported"
task_runs = [
TaskRun.get(db, task_run_id) for task_run_id in task_run_ids
]
for task_run in task_runs:
assignments += task_run.get_assignments()
assignments += [
Assignment.get(db, assignment_id)
for assignment_id in assignment_ids
]
if len(statuses) == 0:
statuses = [
AssignmentState.COMPLETED,
AssignmentState.ACCEPTED,
AssignmentState.REJECTED,
]
filtered_assignments = [
a for a in assignments if a.get_status() in statuses
]
for assignment in assignments:
units += assignment.get_units()
units += [Unit.get(db, unit_id) for unit_id in unit_ids]
all_unit_data = []
for unit in units:
unit_data = {
"assignment_id": unit.assignment_id,
"task_run_id": unit.task_run_id,
"status": unit.db_status,
"unit_id": unit.db_id,
"worker_id": unit.worker_id,
"data": None,
}
agent = unit.get_assigned_agent()
if agent is not None:
unit_data["data"] = agent.state.get_data()
unit_data["worker_id"] = agent.worker_id
all_unit_data.append(unit_data)
print(all_unit_data)
return jsonify({"success": True, "units": all_unit_data})
except Exception as e:
import traceback
traceback.print_exc()
return jsonify({"success": False, "msg": str(e)})
def main():
"""
Script to launch makeup tasks for workers that
can't be bonused via other avenues.
Creates a task for a worker, qualifying them directly,
and marks as a soft_rejected HIT for the given task name.
"""
db = LocalMephistoDB()
task_name = input(
"Please enter a task name for bookkeeping. This task name will be tied to "
"the additional spend granted through this script, and should be the same "
"as the task you originally launched that you now need to compensate for:\n>> "
)
tasks = db.find_tasks(task_name=task_name)
if len(tasks) == 0:
print("No tasks found with the given name...")
all_tasks = db.find_tasks()
all_names = set([t.task_name for t in all_tasks])
print(
f"Choose an existing task of {all_names} to use this functionality."
)
print(f"Compensation hits must be tied to an existing task")
return 0
task = tasks[0]
req_name = input(
"Please enter an MTurkRequester name to use to bonus from:\n>> ")
requesters = db.find_requesters(requester_name=req_name)
if len(requesters) == 0:
print("Could not find a requester by that name...")
return 0
requester = requesters[0]
client = requester._get_client(requester._requester_name)
print(
"You can now enter a worker id, amount, and reason for as many compensation tasks "
"as you want to launch for this.")
compensation_hits = []
amount = None
reason = None
while True:
worker_id = input(
"Enter a worker id to compensate. Leave blank to move on to launching: \n>> "
).strip()
if len(worker_id) == 0:
break
prev_amount = "" if amount is None else f" (leave blank for ${amount})"
next_amount = input(
f"Enter the amount in dollars to pay out in this compensation task{prev_amount}:\n>> $"
)
amount = float(next_amount) if len(
next_amount.strip()) != 0 else amount
assert amount is not None, "Amount can not be left blank"
prev_reason = "" if reason is None else f" (leave blank for '{reason}'"
next_reason = input(
f"Provide reason for launching this compensation task. This will be sent to the worker{prev_reason}:\n>> "
)
reason = next_reason if len(next_reason.strip()) != 0 else reason
assert reason is not None, "Reason can not be left blank"
compensation_hits.append({
"worker_id": worker_id,
"amount": amount,
"reason": reason,
})
if len(compensation_hits) == 0:
print("No compensation details provided, exiting")
return 0
print(f"You entered the following tasks:\n{compensation_hits}")
input("Input anything to confirm and continue...")
# Iterate through and launch tasks
for comp_dict in compensation_hits:
# Create the MTurk qualification for this specific worker
worker_id = comp_dict["worker_id"]
qual_name = f"compensation-for-{worker_id}-on-{task_name}"
print(f"Creating qualification for {worker_id}: {qual_name}....")
qualification = make_qualification_dict(qual_name, QUAL_EXISTS, None)
qual_map = requester.datastore.get_qualification_mapping(qual_name)
if qual_map is None:
qualification[
"QualificationTypeId"] = requester._create_new_mturk_qualification(
qual_name)
else:
qualification["QualificationTypeId"] = qual_map[
"mturk_qualification_id"]
give_worker_qualification(client, worker_id,
qualification["QualificationTypeId"])
# Create the task run for this HIT
print(f"Creating task run and data model components for this HIT")
config = build_task_config(comp_dict, requester)
init_params = OmegaConf.to_yaml(OmegaConf.structured(config))
new_run_id = db.new_task_run(
task.db_id,
requester.db_id,
json.dumps(init_params),
requester.provider_type,
"mock",
requester.is_sandbox(),
)
task_run = TaskRun.get(db, new_run_id)
# Create an assignment, unit, agent, and mark as assigned
# Assignment creation
task_args = task_run.get_task_args()
assignment_id = db.new_assignment(
task_run.task_id,
task_run.db_id,
task_run.requester_id,
task_run.task_type,
task_run.provider_type,
task_run.sandbox,
)
data = InitializationData({}, [{}])
assignment = Assignment.get(db, assignment_id)
assignment.write_assignment_data(data)
# Unit creation
unit_id = db.new_unit(
task_run.task_id,
task_run.db_id,
task_run.requester_id,
assignment_id,
COMPENSATION_UNIT_INDEX,
task_args.task_reward,
task_run.provider_type,
task_run.task_type,
task_run.sandbox,
)
compensation_unit = Unit.get(db, unit_id)
print(f"Created {task_run}, {assignment}, and {compensation_unit}...")
# Set up HIT type
hit_type_id = create_hit_type(
client,
task_run.get_task_args(),
[qualification],
auto_approve_delay=30,
skip_locale_qual=True,
)
# Create the task on MTurk, email the worker
print("Creating and deploying task on MTurk")
duration = 60 * 60 * 24
run_id = task_run.db_id
hit_link, hit_id, response = create_compensation_hit_with_hit_type(
client, comp_dict["reason"], hit_type_id)
requester.datastore.new_hit(hit_id, hit_link, duration, task_run.db_id)
print("Sending email to worker...")
result = email_worker(
client,
worker_id,
"Compensation HIT Launched",
("Hello Worker,\n We've launched a compensation hit for a task that you've worked on "
f"for us in the past. The reason supplied for this task was: {reason}. This task is "
f"only doable by you, and should reward ${comp_dict['amount']}. Thanks for being a valued "
"contributor to our tasks, and for allowing us to try and resolve the issue.\n\n"
f"Your task can be accessed at the following link: {hit_link}."),
)
if not result[0]:
print(
f"Email send failed, for reason {result[1]}\n"
f"Please send {hit_link} to {worker_id} yourself if they reached out about this issue."
)
# Mark the agent as soft_rejected, such that we've "paid" it
compensation_unit.set_db_status(AssignmentState.SOFT_REJECTED)
