def init_gold_config( self, task_run: "TaskRun", args: "DictConfig", shared_state: "GoldUnitSharedState", ) -> None: self.use_golds = args.blueprint.get("use_golds", False) if not self.use_golds: return # Runs using gold units need to keep track of the frequency and # usage of golds self.base_qual_name = args.blueprint.gold_qualification_base self.golds_correct_qual_name = f"{self.base_qual_name}-correct-golds" self.golds_failed_qual_name = f"{self.base_qual_name}-wrong-golds" self.disqualified_qual_name = f"{self.base_qual_name}-disqualified" self.task_count_qual_name = f"{self.base_qual_name}-completed-count" self.get_gold_for_worker = shared_state.get_gold_for_worker self.worker_needs_gold = shared_state.worker_needs_gold self.worker_qualifies = shared_state.worker_qualifies self.min_golds = args.blueprint.min_golds self.max_incorrect_golds = args.blueprint.max_incorrect_golds find_or_create_qualification(task_run.db, self.golds_correct_qual_name) find_or_create_qualification(task_run.db, self.golds_failed_qual_name) find_or_create_qualification(task_run.db, self.disqualified_qual_name) find_or_create_qualification(task_run.db, self.task_count_qual_name)
def __init__(self, task_run: "TaskRun", args: "DictConfig", shared_state: "SharedTaskState"): self.args = args self.shared_state = shared_state self.task_run = task_run self.running_assignments: Dict[str, RunningAssignment] = {} self.running_units: Dict[str, RunningUnit] = {} self.running_onboardings: Dict[str, RunningOnboarding] = {} self.is_concurrent = False self.block_qualification = args.blueprint.get("block_qualification", None) if self.block_qualification is not None: find_or_create_qualification(task_run.db, self.block_qualification)
def init_screening_config( self, task_run: "TaskRun", args: "DictConfig", shared_state: "ScreenTaskSharedState", ) -> None: self.use_screening_task = args.blueprint.get("use_screening_task", False) if not self.use_screening_task: return # Runs that are using a qualification task should be able to assign # a specially generated unit to unqualified workers self.passed_qualification_name = args.blueprint.passed_qualification_name self.failed_qualification_name = args.blueprint.block_qualification self.screening_data_factory: Tuple[ bool, ScreenUnitDataGenerator ] = shared_state.screening_data_factory self.screening_units_launched = 0 self.screening_unit_cap = args.blueprint.max_screening_units find_or_create_qualification(task_run.db, self.passed_qualification_name) find_or_create_qualification(task_run.db, self.failed_qualification_name)
def test_find_workers_by_quals(self) -> None: """Ensure we can find a worker by an assigned qualification""" db = self.db WORKER_1_NAME = "worker_1" WORKER_2_NAME = "worker_2" WORKER_3_NAME = "worker_3" QUAL_NAME = "test_qualification" worker_1 = self.get_named_test_worker(WORKER_1_NAME) worker_2 = self.get_named_test_worker(WORKER_2_NAME) worker_3 = self.get_named_test_worker(WORKER_3_NAME) find_or_create_qualification(db, QUAL_NAME) worker_1.grant_qualification(QUAL_NAME, skip_crowd=True) worker_3.grant_qualification(QUAL_NAME, skip_crowd=True) data_browser = DataBrowser(db) qualified_workers = data_browser.get_workers_with_qualification( QUAL_NAME) qualified_ids = [w.db_id for w in qualified_workers] self.assertEqual( len(qualified_workers), 2, f"Should only be two qualified workers, found {qualified_ids}", ) self.assertIn( worker_1.db_id, qualified_ids, f"Worker 1 not in qualified list, found {qualified_ids}", ) self.assertIn( worker_3.db_id, qualified_ids, f"Worker 3 not in qualified list, found {qualified_ids}", ) self.assertNotIn(worker_2.db_id, qualified_ids, "Worker 2 should not be in qualified list")
def init_onboarding_config(self, task_run: "TaskRun", args: "DictConfig", shared_state: "SharedTaskState"): assert isinstance( shared_state, OnboardingSharedState ), f"Cannot init onboarding config with {shared_state}, need OnboardingSharedState" self.onboarding_qualification_name: Optional[str] = args.blueprint.get( "onboarding_qualification", None) self.onboarding_data = shared_state.onboarding_data self.use_onboarding = self.onboarding_qualification_name is not None self.onboarding_qualification_id = None if not self.use_onboarding: return onboarding_qualification_name = self.onboarding_qualification_name assert onboarding_qualification_name is not None db = task_run.db self.onboarding_qualification_id = find_or_create_qualification( db, onboarding_qualification_name, ) self.onboarding_failed_name = self.get_failed_qual( onboarding_qualification_name) self.onboarding_failed_id = find_or_create_qualification( db, self.onboarding_failed_name)
def run_examine_by_worker( db: "MephistoDB", format_data_for_printing: Callable[[Dict[str, Any]], str], task_name: Optional[str] = None, block_qualification: Optional[str] = None, approve_qualification: Optional[str] = None, ): """ Basic script for reviewing work, grouped by worker for convenience. First gets the required information to run a review, then """ data_browser = DataBrowser(db=db) # Get initial arguments if task_name is None: task_name, block_qualification, approve_qualification = prompt_for_options( task_name, block_qualification, approve_qualification ) tasks = db.find_tasks(task_name=task_name) assert len(tasks) >= 1, f"No task found under name {task_name}" print( "You will be reviewing actual tasks with this flow. Tasks that you either Accept or Pass " "will be paid out to the worker, while rejected tasks will not. Passed tasks will be " "specially marked such that you can leave them out of your dataset. \n" "You may enter the option in caps to apply it to the rest of the units for a given worker." ) if block_qualification is not None: created_block_qual = find_or_create_qualification(db, block_qualification) print( "When you pass or reject a task, the script gives you an option to disqualify the worker " "from future tasks by assigning a qualification. If provided, this worker will no " "longer be able to work on tasks where the set --block-qualification shares the same name " f"you provided above: {block_qualification}\n" ) if approve_qualification is not None: created_approve_qual = find_or_create_qualification(db, approve_qualification) print( "You may use this script to establish a qualified worker pool by granting the provided " f"approve qualification {approve_qualification} to workers you think understand the task " "well. This will be provided as an option for workers you (A)pprove all on. " "Future tasks can use this qual as a required qualification, as described in the " "common qualification flows document." ) print( "**************\n" "You should only reject tasks when it is clear the worker has acted in bad faith, and " "didn't actually do the task. Prefer to pass on tasks that were misunderstandings.\n" "**************\n" ) units = data_browser.get_units_for_task_name(task_name) others = [u for u in units if u.get_status() != "completed"] units = [u for u in units if u.get_status() == "completed"] reviews_left = len(units) previous_work_by_worker = get_worker_stats(others) # Determine allowed options options = ["a", "p", "r"] options_string = "Do you want to accept this work? (a)ccept, (r)eject, (p)ass:" units_by_worker: Dict[str, List["Unit"]] = {} for u in units: w_id = u.worker_id if w_id not in units_by_worker: units_by_worker[w_id] = [] units_by_worker[w_id].append(u) # Run the review for w_id, w_units in units_by_worker.items(): worker = Worker.get(db, w_id) worker_name = worker.worker_name apply_all_decision = None reason = None for idx, unit in enumerate(w_units): print( f"Reviewing for worker {worker_name}, ({idx+1}/{len(w_units)}), " f"Previous {format_worker_stats(w_id, previous_work_by_worker)} " f"(total remaining: {reviews_left})" ) reviews_left -= 1 print(format_data_for_printing(data_browser.get_data_from_unit(unit))) if apply_all_decision is not None: decision = apply_all_decision else: decision = input( "Do you want to accept this work? (a)ccept, (r)eject, (p)ass: " ) while decision.lower() not in options: decision = input( "Decision must be one of a, p, r. Use CAPS to apply to all remaining for worker: " ) agent = unit.get_assigned_agent() assert ( agent is not None ), f"Can't make decision on None agent... issue with {unit}" if decision.lower() == "a": agent.approve_work() if decision == "A" and approve_qualification is not None: should_special_qualify = input( "Do you want to approve qualify this worker? (y)es/(n)o: " ) if should_special_qualify.lower() in ["y", "yes"]: worker.grant_qualification(approve_qualification, 1) elif decision.lower() == "p": agent.soft_reject_work() if apply_all_decision is None and block_qualification is not None: should_soft_block = input( "Do you want to soft block this worker? (y)es/(n)o: " ) if should_soft_block.lower() in ["y", "yes"]: worker.grant_qualification(block_qualification, 1) else: # decision = 'r' if apply_all_decision is None: reason = input("Why are you rejecting this work? ") should_block = input( "Do you want to hard block this worker? (y)es/(n)o: " ) if should_block.lower() in ["y", "yes"]: block_reason = input("Why permanently block this worker? ") worker.block_worker(block_reason) agent.reject_work(reason) if decision.lower() != decision: apply_all_decision = decision.lower()