def init_gold_config(
self,
task_run: "TaskRun",
args: "DictConfig",
shared_state: "GoldUnitSharedState",
) -> None:
self.use_golds = args.blueprint.get("use_golds", False)
if not self.use_golds:
return
# Runs using gold units need to keep track of the frequency and
# usage of golds
self.base_qual_name = args.blueprint.gold_qualification_base
self.golds_correct_qual_name = f"{self.base_qual_name}-correct-golds"
self.golds_failed_qual_name = f"{self.base_qual_name}-wrong-golds"
self.disqualified_qual_name = f"{self.base_qual_name}-disqualified"
self.task_count_qual_name = f"{self.base_qual_name}-completed-count"
self.get_gold_for_worker = shared_state.get_gold_for_worker
self.worker_needs_gold = shared_state.worker_needs_gold
self.worker_qualifies = shared_state.worker_qualifies
self.min_golds = args.blueprint.min_golds
self.max_incorrect_golds = args.blueprint.max_incorrect_golds
find_or_create_qualification(task_run.db, self.golds_correct_qual_name)
find_or_create_qualification(task_run.db, self.golds_failed_qual_name)
find_or_create_qualification(task_run.db, self.disqualified_qual_name)
find_or_create_qualification(task_run.db, self.task_count_qual_name)
def __init__(self, task_run: "TaskRun", args: "DictConfig",
shared_state: "SharedTaskState"):
self.args = args
self.shared_state = shared_state
self.task_run = task_run
self.running_assignments: Dict[str, RunningAssignment] = {}
self.running_units: Dict[str, RunningUnit] = {}
self.running_onboardings: Dict[str, RunningOnboarding] = {}
self.is_concurrent = False
self.block_qualification = args.blueprint.get("block_qualification",
None)
if self.block_qualification is not None:
find_or_create_qualification(task_run.db, self.block_qualification)
def init_screening_config(
self,
task_run: "TaskRun",
args: "DictConfig",
shared_state: "ScreenTaskSharedState",
) -> None:
self.use_screening_task = args.blueprint.get("use_screening_task", False)
if not self.use_screening_task:
return
# Runs that are using a qualification task should be able to assign
# a specially generated unit to unqualified workers
self.passed_qualification_name = args.blueprint.passed_qualification_name
self.failed_qualification_name = args.blueprint.block_qualification
self.screening_data_factory: Tuple[
bool, ScreenUnitDataGenerator
] = shared_state.screening_data_factory
self.screening_units_launched = 0
self.screening_unit_cap = args.blueprint.max_screening_units
find_or_create_qualification(task_run.db, self.passed_qualification_name)
find_or_create_qualification(task_run.db, self.failed_qualification_name)
def test_find_workers_by_quals(self) -> None:
"""Ensure we can find a worker by an assigned qualification"""
db = self.db
WORKER_1_NAME = "worker_1"
WORKER_2_NAME = "worker_2"
WORKER_3_NAME = "worker_3"
QUAL_NAME = "test_qualification"
worker_1 = self.get_named_test_worker(WORKER_1_NAME)
worker_2 = self.get_named_test_worker(WORKER_2_NAME)
worker_3 = self.get_named_test_worker(WORKER_3_NAME)
find_or_create_qualification(db, QUAL_NAME)
worker_1.grant_qualification(QUAL_NAME, skip_crowd=True)
worker_3.grant_qualification(QUAL_NAME, skip_crowd=True)
data_browser = DataBrowser(db)
qualified_workers = data_browser.get_workers_with_qualification(
QUAL_NAME)
qualified_ids = [w.db_id for w in qualified_workers]
self.assertEqual(
len(qualified_workers),
2,
f"Should only be two qualified workers, found {qualified_ids}",
)
self.assertIn(
worker_1.db_id,
qualified_ids,
f"Worker 1 not in qualified list, found {qualified_ids}",
)
self.assertIn(
worker_3.db_id,
qualified_ids,
f"Worker 3 not in qualified list, found {qualified_ids}",
)
self.assertNotIn(worker_2.db_id, qualified_ids,
"Worker 2 should not be in qualified list")
def init_onboarding_config(self, task_run: "TaskRun", args: "DictConfig",
shared_state: "SharedTaskState"):
assert isinstance(
shared_state, OnboardingSharedState
), f"Cannot init onboarding config with {shared_state}, need OnboardingSharedState"
self.onboarding_qualification_name: Optional[str] = args.blueprint.get(
"onboarding_qualification", None)
self.onboarding_data = shared_state.onboarding_data
self.use_onboarding = self.onboarding_qualification_name is not None
self.onboarding_qualification_id = None
if not self.use_onboarding:
return
onboarding_qualification_name = self.onboarding_qualification_name
assert onboarding_qualification_name is not None
db = task_run.db
self.onboarding_qualification_id = find_or_create_qualification(
db,
onboarding_qualification_name,
)
self.onboarding_failed_name = self.get_failed_qual(
onboarding_qualification_name)
self.onboarding_failed_id = find_or_create_qualification(
db, self.onboarding_failed_name)
def run_examine_by_worker(
db: "MephistoDB",
format_data_for_printing: Callable[[Dict[str, Any]], str],
task_name: Optional[str] = None,
block_qualification: Optional[str] = None,
approve_qualification: Optional[str] = None,
):
"""
Basic script for reviewing work, grouped by worker for convenience. First gets
the required information to run a review, then
"""
data_browser = DataBrowser(db=db)
# Get initial arguments
if task_name is None:
task_name, block_qualification, approve_qualification = prompt_for_options(
task_name, block_qualification, approve_qualification
)
tasks = db.find_tasks(task_name=task_name)
assert len(tasks) >= 1, f"No task found under name {task_name}"
print(
"You will be reviewing actual tasks with this flow. Tasks that you either Accept or Pass "
"will be paid out to the worker, while rejected tasks will not. Passed tasks will be "
"specially marked such that you can leave them out of your dataset. \n"
"You may enter the option in caps to apply it to the rest of the units for a given worker."
)
if block_qualification is not None:
created_block_qual = find_or_create_qualification(db, block_qualification)
print(
"When you pass or reject a task, the script gives you an option to disqualify the worker "
"from future tasks by assigning a qualification. If provided, this worker will no "
"longer be able to work on tasks where the set --block-qualification shares the same name "
f"you provided above: {block_qualification}\n"
)
if approve_qualification is not None:
created_approve_qual = find_or_create_qualification(db, approve_qualification)
print(
"You may use this script to establish a qualified worker pool by granting the provided "
f"approve qualification {approve_qualification} to workers you think understand the task "
"well. This will be provided as an option for workers you (A)pprove all on. "
"Future tasks can use this qual as a required qualification, as described in the "
"common qualification flows document."
)
print(
"**************\n"
"You should only reject tasks when it is clear the worker has acted in bad faith, and "
"didn't actually do the task. Prefer to pass on tasks that were misunderstandings.\n"
"**************\n"
)
units = data_browser.get_units_for_task_name(task_name)
others = [u for u in units if u.get_status() != "completed"]
units = [u for u in units if u.get_status() == "completed"]
reviews_left = len(units)
previous_work_by_worker = get_worker_stats(others)
# Determine allowed options
options = ["a", "p", "r"]
options_string = "Do you want to accept this work? (a)ccept, (r)eject, (p)ass:"
units_by_worker: Dict[str, List["Unit"]] = {}
for u in units:
w_id = u.worker_id
if w_id not in units_by_worker:
units_by_worker[w_id] = []
units_by_worker[w_id].append(u)
# Run the review
for w_id, w_units in units_by_worker.items():
worker = Worker.get(db, w_id)
worker_name = worker.worker_name
apply_all_decision = None
reason = None
for idx, unit in enumerate(w_units):
print(
f"Reviewing for worker {worker_name}, ({idx+1}/{len(w_units)}), "
f"Previous {format_worker_stats(w_id, previous_work_by_worker)} "
f"(total remaining: {reviews_left})"
)
reviews_left -= 1
print(format_data_for_printing(data_browser.get_data_from_unit(unit)))
if apply_all_decision is not None:
decision = apply_all_decision
else:
decision = input(
"Do you want to accept this work? (a)ccept, (r)eject, (p)ass: "
)
while decision.lower() not in options:
decision = input(
"Decision must be one of a, p, r. Use CAPS to apply to all remaining for worker: "
)
agent = unit.get_assigned_agent()
assert (
agent is not None
), f"Can't make decision on None agent... issue with {unit}"
if decision.lower() == "a":
agent.approve_work()
if decision == "A" and approve_qualification is not None:
should_special_qualify = input(
"Do you want to approve qualify this worker? (y)es/(n)o: "
)
if should_special_qualify.lower() in ["y", "yes"]:
worker.grant_qualification(approve_qualification, 1)
elif decision.lower() == "p":
agent.soft_reject_work()
if apply_all_decision is None and block_qualification is not None:
should_soft_block = input(
"Do you want to soft block this worker? (y)es/(n)o: "
)
if should_soft_block.lower() in ["y", "yes"]:
worker.grant_qualification(block_qualification, 1)
else: # decision = 'r'
if apply_all_decision is None:
reason = input("Why are you rejecting this work? ")
should_block = input(
"Do you want to hard block this worker? (y)es/(n)o: "
)
if should_block.lower() in ["y", "yes"]:
block_reason = input("Why permanently block this worker? ")
worker.block_worker(block_reason)
agent.reject_work(reason)
if decision.lower() != decision:
apply_all_decision = decision.lower()