def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) world_opt = get_world_opt(cfg) onboarding_world_opt = get_onboarding_world_opt(cfg) shared_state = SharedParlAITaskState( world_opt=world_opt, onboarding_world_opt=onboarding_world_opt) check_role_training_qualification( db=db, qname=world_opt[constants.ROLE_QUALIFICATION_NAME_KEY], requester_name=cfg.mephisto.provider.requester_name, ) shared_state.task_config['minTurns'] = world_opt['min_turns'] shared_state.task_config[ 'onboardingPersona'] = constants.ONBOARDING_PERSONA shared_state.worker_can_do_unit = get_worker_eval_function( world_opt[constants.ROLE_QUALIFICATION_NAME_KEY], onboarding_world_opt['onboarding_qualification'], ) banned_words_fpath = cfg.mephisto.blueprint.banned_words_file add_banned_words_frontend_conf(shared_state, banned_words_fpath) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=300) update_persona_use_counts_file(cfg.mephisto.blueprint.persona_counts_file, world_opt['prev_persona_count'])
def main(cfg: DictConfig) -> None: def onboarding_is_valid(onboarding_data): outputs = onboarding_data["outputs"] answer_str = outputs["answer"] # NOTE: depending on which OS Turker uses, there could be carriage returns \r or just newlines \n # this python module should handle all cases commands = answer_str.splitlines() # filter empty commands filtered_commands = [x for x in commands if x != ""] # Number check: Check that the number of commands >= 3 if len(commands) < 3: return False # Length check: Check that the average number of words in commands > 4 commands_split = [x.split(" ") for x in filtered_commands] avg_words_in_commands = sum(map(len, commands_split)) / len(commands_split) if avg_words_in_commands < 2: return False # Diversity check: Check that commands are reasonably diverse first_words = [x[0] for x in commands_split] if len(set(first_words)) == 1: return False # TODO: Grammar check: Check that there is punctuation, capitals return True shared_state = SharedStaticTaskState( onboarding_data={}, validate_onboarding=onboarding_is_valid, ) db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) parser = ParlaiParser(True, False) opt = parser.parse_args( list(chain.from_iterable( ('--' + k, v) for k, v in cfg.teacher.items()))) agent = RepeatLabelAgent(opt) teacher = create_task(opt, agent).get_task_agent() world_opt = {"turn_timeout": cfg.turn_timeout, "teacher": teacher} custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle", None) if custom_bundle_path is not None: assert os.path.exists(custom_bundle_path), ( "Must build the custom bundle with `npm install; npm run dev` from within " f"the {TASK_DIRECTORY}/webapp directory in order to demo a custom bundle " ) world_opt["send_task_data"] = True shared_state = SharedParlAITaskState(world_opt=world_opt, onboarding_world_opt=world_opt) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def run_static_task(cfg: DictConfig, task_directory: str): """ Run static task, given configuration. """ db, cfg = load_db_and_process_config(cfg) print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}') random.seed(42) task_name = cfg.mephisto.task.get('task_name', 'turn_annotations_static') soft_block_qual_name = cfg.mephisto.blueprint.get('block_qualification', f'{task_name}_block') # Default to a task-specific name to avoid soft-block collisions soft_block_mturk_workers(cfg=cfg, db=db, soft_block_qual_name=soft_block_qual_name) build_task(task_directory) operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None: shared_state = SharedStaticTaskState(qualifications=[ make_qualification_dict(ALLOWLIST_QUALIFICATION, QUAL_EXISTS, None), ], ) db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def run_acute_eval(self): """ Run ACUTE Eval. """ self.set_up_acute_eval() db, cfg = load_db_and_process_config(self.args) operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None) operator.wait_for_runs_then_shutdown( skip_input=True, log_rate=cfg.monitoring_log_rate )
def run_acute_eval(self): """ Run ACUTE Eval. """ self.set_up_acute_eval() db, cfg = load_db_and_process_config(self.args) print(f'*** RUN ID: {cfg.mephisto.task.task_name} ***') print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}') operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=cfg.monitoring_log_rate)
def main(operator: Operator, cfg: DictConfig) -> None: correct_config_answer = cfg.correct_answer def onboarding_is_valid(onboarding_data): inputs = onboarding_data["inputs"] outputs = onboarding_data["outputs"] return outputs.get("answer") == correct_config_answer shared_state = SharedStaticTaskState( onboarding_data={"correct_answer": correct_config_answer}, validate_onboarding=onboarding_is_valid, ) operator.launch_task_run(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) operator = Operator(db) validator = validate_unit shared_state = SharedStaticTaskState(on_unit_submitted=validator, ) # Do not allow workers to take pilot task the second time shared_state.qualifications = [ make_qualification_dict( PILOT_BLOCK_QUAL_NAME, QUAL_NOT_EXIST, None, ), ] operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None: correct_config_answer = cfg.correct_answer def onboarding_is_valid(onboarding_data): inputs = onboarding_data["inputs"] outputs = onboarding_data["outputs"] return outputs.get("answer") == correct_config_answer shared_state = SharedStaticTaskState( onboarding_data={"correct_answer": correct_config_answer}, validate_onboarding=onboarding_is_valid, ) db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) teacher = get_teacher(cfg) world_opt = {"turn_timeout": cfg.turn_timeout, "teacher": teacher} custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle", None) if custom_bundle_path is not None: if not os.path.exists(custom_bundle_path): build_task(TASK_DIRECTORY) shared_state = SharedParlAITaskState(world_opt=world_opt, onboarding_world_opt=world_opt) operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) world_opt = {"num_turns": cfg.num_turns, "turn_timeout": cfg.turn_timeout} custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle", None) if custom_bundle_path is not None: assert os.path.exists(custom_bundle_path), ( "Must build the custom bundle with `npm install; npm run dev` from within " f"the {TASK_DIRECTORY}/webapp directory in order to demo a custom bundle " ) world_opt["send_task_data"] = True shared_state = SharedParlAITaskState(world_opt=world_opt, onboarding_world_opt=world_opt) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(operator: Operator, cfg: DictConfig) -> None: def onboarding_always_valid(onboarding_data): return True shared_state = SharedStaticTaskState( static_task_data=[ { "text": "This text is good text!" }, { "text": "This text is bad text!" }, ], validate_onboarding=onboarding_always_valid, ) task_dir = cfg.task_dir build_custom_bundle(task_dir) operator.launch_task_run(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None: task_dir = cfg.task_dir def onboarding_always_valid(onboarding_data): return True shared_state = SharedStaticTaskState( static_task_data=[ {"text": "This text is good text!"}, {"text": "This text is bad text!"}, ], validate_onboarding=onboarding_always_valid, ) build_task(task_dir) db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(operator: Operator, cfg: DictConfig) -> None: def onboarding_always_valid(onboarding_data): # NOTE you can make an onboarding task and validate it here print(onboarding_data) return True # Right now we're building locally, but should eventually # use non-local for the real thing tasks = build_tasks(cfg.num_tasks) context = build_local_context(cfg.num_tasks) def handle_with_model( _request_id: str, args: Dict[str, Any], agent_state: RemoteProcedureAgentState ) -> Dict[str, Any]: """Remote call to process external content using a 'model'""" # NOTE this body can be whatever you want print(f"The parsed args are {args}, you can do what you want with that") print(f"You can also use {agent_state.init_data}, to get task keys") assert agent_state.init_data is not None idx = agent_state.init_data["local_value_key"] print(f"And that may let you get local context, like {context[idx]}") return { "secret_local_value": context[idx], "update": f"this was request {args['arg3'] + 1}", } function_registry = { "handle_with_model": handle_with_model, } shared_state = SharedRemoteProcedureTaskState( static_task_data=tasks, validate_onboarding=onboarding_always_valid, function_registry=function_registry, ) task_dir = cfg.task_dir build_custom_bundle(task_dir) operator.launch_task_run(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def run_task(cfg: DictConfig, task_directory: str): """ Run task, given configuration. """ frontend_source_dir = os.path.join(task_directory, "webapp") frontend_build_dir = os.path.join(frontend_source_dir, "build") _ = frontend_build_dir # Unused at the moment db, cfg = load_db_and_process_config(cfg) print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}') random.seed(42) # Update task name when on sandbox or local to ensure data is split. task_name = cfg.mephisto.task.get('task_name', 'model_chat') architect_type = cfg.mephisto.architect._architect_type if architect_type == 'local': task_name = f"{task_name}_local" elif architect_type == 'mturk_sandbox': task_name = f"{task_name}_sandbox" cfg.mephisto.task.task_name = task_name soft_block_qual_name = cfg.mephisto.blueprint.get('block_qualification', f'{task_name}_block') # Default to a task-specific name to avoid soft-block collisions soft_block_mturk_workers(cfg=cfg, db=db, soft_block_qual_name=soft_block_qual_name) # Init shared_state = SharedModelChatTaskState(world_module=world_module) operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=cfg.monitoring_log_rate)
def main(operator: Operator, cfg: DictConfig) -> None: tasks: List[Dict[str, Any]] = [{}] * cfg.num_tasks mnist_model = mnist(pretrained=True) def handle_with_model( _request_id: str, args: Dict[str, Any], agent_state: RemoteProcedureAgentState) -> Dict[str, Any]: """Convert the image to be read by MNIST classifier, then classify""" img_dat = args["urlData"].split("data:image/png;base64,")[1] im = Image.open(BytesIO(base64.b64decode(img_dat))) im_gray = im.convert("L") im_resized = im_gray.resize((28, 28)) im_vals = list(im_resized.getdata()) norm_vals = [(255 - x) * 1.0 / 255.0 for x in im_vals] in_tensor = torch.tensor([norm_vals]) output = mnist_model(in_tensor) pred = output.data.max(1)[1] print("Predicted digit:", pred.item()) return { "digit_prediction": pred.item(), } function_registry = { "classify_digit": handle_with_model, } shared_state = SharedRemoteProcedureTaskState( static_task_data=tasks, function_registry=function_registry, ) task_dir = cfg.task_dir build_custom_bundle(task_dir) operator.launch_task_run(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)