def main(cfg: DictConfig) -> None: def onboarding_is_valid(onboarding_data): outputs = onboarding_data["outputs"] answer_str = outputs["answer"] # NOTE: depending on which OS Turker uses, there could be carriage returns \r or just newlines \n # this python module should handle all cases commands = answer_str.splitlines() # filter empty commands filtered_commands = [x for x in commands if x != ""] # Number check: Check that the number of commands >= 3 if len(commands) < 3: return False # Length check: Check that the average number of words in commands > 4 commands_split = [x.split(" ") for x in filtered_commands] avg_words_in_commands = sum(map(len, commands_split)) / len(commands_split) if avg_words_in_commands < 2: return False # Diversity check: Check that commands are reasonably diverse first_words = [x[0] for x in commands_split] if len(set(first_words)) == 1: return False # TODO: Grammar check: Check that there is punctuation, capitals return True shared_state = SharedStaticTaskState( onboarding_data={}, validate_onboarding=onboarding_is_valid, ) db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) world_opt = get_world_opt(cfg) onboarding_world_opt = get_onboarding_world_opt(cfg) shared_state = SharedParlAITaskState( world_opt=world_opt, onboarding_world_opt=onboarding_world_opt) check_role_training_qualification( db=db, qname=world_opt[constants.ROLE_QUALIFICATION_NAME_KEY], requester_name=cfg.mephisto.provider.requester_name, ) shared_state.task_config['minTurns'] = world_opt['min_turns'] shared_state.task_config[ 'onboardingPersona'] = constants.ONBOARDING_PERSONA shared_state.worker_can_do_unit = get_worker_eval_function( world_opt[constants.ROLE_QUALIFICATION_NAME_KEY], onboarding_world_opt['onboarding_qualification'], ) banned_words_fpath = cfg.mephisto.blueprint.banned_words_file add_banned_words_frontend_conf(shared_state, banned_words_fpath) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=300) update_persona_use_counts_file(cfg.mephisto.blueprint.persona_counts_file, world_opt['prev_persona_count'])
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) parser = ParlaiParser(True, False) opt = parser.parse_args( list(chain.from_iterable( ('--' + k, v) for k, v in cfg.teacher.items()))) agent = RepeatLabelAgent(opt) teacher = create_task(opt, agent).get_task_agent() world_opt = {"turn_timeout": cfg.turn_timeout, "teacher": teacher} custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle", None) if custom_bundle_path is not None: assert os.path.exists(custom_bundle_path), ( "Must build the custom bundle with `npm install; npm run dev` from within " f"the {TASK_DIRECTORY}/webapp directory in order to demo a custom bundle " ) world_opt["send_task_data"] = True shared_state = SharedParlAITaskState(world_opt=world_opt, onboarding_world_opt=world_opt) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def run_static_task(cfg: DictConfig, task_directory: str): """ Run static task, given configuration. """ db, cfg = load_db_and_process_config(cfg) print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}') random.seed(42) task_name = cfg.mephisto.task.get('task_name', 'turn_annotations_static') soft_block_qual_name = cfg.mephisto.blueprint.get('block_qualification', f'{task_name}_block') # Default to a task-specific name to avoid soft-block collisions soft_block_mturk_workers(cfg=cfg, db=db, soft_block_qual_name=soft_block_qual_name) build_task(task_directory) operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=cfg.monitoring_log_rate)
def run_acute_eval(self): """ Run ACUTE Eval. """ self.set_up_acute_eval() db, cfg = load_db_and_process_config(self.args) operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None) operator.wait_for_runs_then_shutdown( skip_input=True, log_rate=cfg.monitoring_log_rate )
def main(cfg: DictConfig) -> None: shared_state = SharedStaticTaskState(qualifications=[ make_qualification_dict(ALLOWLIST_QUALIFICATION, QUAL_EXISTS, None), ], ) db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def run_acute_eval(self): """ Run ACUTE Eval. """ self.set_up_acute_eval() db, cfg = load_db_and_process_config(self.args) print(f'*** RUN ID: {cfg.mephisto.task.task_name} ***') print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}') operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None: correct_config_answer = cfg.correct_answer def onboarding_is_valid(onboarding_data): inputs = onboarding_data["inputs"] outputs = onboarding_data["outputs"] return outputs.get("answer") == correct_config_answer shared_state = SharedStaticTaskState( onboarding_data={"correct_answer": correct_config_answer}, validate_onboarding=onboarding_is_valid, ) db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) operator = Operator(db) validator = validate_unit shared_state = SharedStaticTaskState(on_unit_submitted=validator, ) # Do not allow workers to take pilot task the second time shared_state.qualifications = [ make_qualification_dict( PILOT_BLOCK_QUAL_NAME, QUAL_NOT_EXIST, None, ), ] operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) teacher = get_teacher(cfg) world_opt = {"turn_timeout": cfg.turn_timeout, "teacher": teacher} custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle", None) if custom_bundle_path is not None: if not os.path.exists(custom_bundle_path): build_task(TASK_DIRECTORY) shared_state = SharedParlAITaskState(world_opt=world_opt, onboarding_world_opt=world_opt) operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) world_opt = {"num_turns": cfg.num_turns, "turn_timeout": cfg.turn_timeout} custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle", None) if custom_bundle_path is not None: assert os.path.exists(custom_bundle_path), ( "Must build the custom bundle with `npm install; npm run dev` from within " f"the {TASK_DIRECTORY}/webapp directory in order to demo a custom bundle " ) world_opt["send_task_data"] = True shared_state = SharedParlAITaskState(world_opt=world_opt, onboarding_world_opt=world_opt) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None: task_dir = cfg.task_dir def onboarding_always_valid(onboarding_data): return True shared_state = SharedStaticTaskState( static_task_data=[ {"text": "This text is good text!"}, {"text": "This text is bad text!"}, ], validate_onboarding=onboarding_always_valid, ) build_task(task_dir) db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def run_task(cfg: DictConfig, task_directory: str): """ Run task, given configuration. """ frontend_source_dir = os.path.join(task_directory, "webapp") frontend_build_dir = os.path.join(frontend_source_dir, "build") _ = frontend_build_dir # Unused at the moment db, cfg = load_db_and_process_config(cfg) print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}') random.seed(42) # Update task name when on sandbox or local to ensure data is split. task_name = cfg.mephisto.task.get('task_name', 'model_chat') architect_type = cfg.mephisto.architect._architect_type if architect_type == 'local': task_name = f"{task_name}_local" elif architect_type == 'mturk_sandbox': task_name = f"{task_name}_sandbox" cfg.mephisto.task.task_name = task_name soft_block_qual_name = cfg.mephisto.blueprint.get('block_qualification', f'{task_name}_block') # Default to a task-specific name to avoid soft-block collisions soft_block_mturk_workers(cfg=cfg, db=db, soft_block_qual_name=soft_block_qual_name) # Init shared_state = SharedModelChatTaskState(world_module=world_module) operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)