def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None)
operator.wait_for_runs_then_shutdown(
skip_input=True, log_rate=cfg.monitoring_log_rate
)
def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
world_opt = {
"num_turns": cfg.num_turns,
"turn_timeout": cfg.turn_timeout,
}
custom_bundle_path = cfg.mephisto.blueprint.get('custom_source_bundle', None)
if custom_bundle_path is not None:
assert os.path.exists(custom_bundle_path), (
"Must build the custom bundle with `npm install; npm run dev` from within "
f"the {TASK_DIRECTORY}/webapp directory in order to demo a custom bundle "
)
world_opt["send_task_data"] = True
shared_state = SharedParlAITaskState(
world_opt=world_opt,
onboarding_world_opt=world_opt,
)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None:
task_dir = cfg.task_dir
def onboarding_always_valid(onboarding_data):
return True
shared_state = SharedStaticTaskState(
static_task_data=[
{
"text": "This text is good text!"
},
{
"text": "This text is bad text!"
},
],
validate_onboarding=onboarding_always_valid,
)
build_task(task_dir)
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None:
correct_config_answer = cfg.correct_answer
def onboarding_is_valid(onboarding_data):
inputs = onboarding_data["inputs"]
outputs = onboarding_data["outputs"]
return outputs.get("answer") == correct_config_answer
shared_state = SharedStaticTaskState(
onboarding_data={"correct_answer": correct_config_answer},
validate_onboarding=onboarding_is_valid,
)
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def run_task(opt):
"""
Note: launch_config opt should be something like:
parlai.crowdsourcing.tasks.turn_annotations_static.launch_config.LaunchConfig
"""
launch_config_file = opt.get('launch_config')
launch_module = import_module(launch_config_file)
launch_config = launch_module.LaunchConfig
db, arg_string = setup_mephisto(launch_config)
if 'sandbox' not in launch_config.PROVIDER:
block_workers(opt, db, launch_config.REQUESTER)
build_task()
operator = Operator(db)
operator.parse_and_launch_run_wrapper(shlex.split(arg_string),
extra_args={})
operator.wait_for_runs_then_shutdown()
def run_static_task(cfg: DictConfig, task_directory: str):
"""
Run static task, given configuration.
"""
frontend_source_dir = os.path.join(task_directory, "webapp")
frontend_build_dir = os.path.join(frontend_source_dir, "build")
db, cfg = load_db_and_process_config(cfg)
print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}')
random.seed(42)
soft_block_qual_name = cfg.mephisto.task.get('task_name',
'turn_annotations_static')
# Default to a task-specific name to avoid soft-block collisions
soft_block_mturk_workers(cfg=cfg,
db=db,
soft_block_qual_name=soft_block_qual_name)
# Build the task
return_dir = os.getcwd()
os.chdir(frontend_source_dir)
if os.path.exists(frontend_build_dir):
shutil.rmtree(frontend_build_dir)
packages_installed = subprocess.call(["npm", "install"])
if packages_installed != 0:
raise Exception("please make sure npm is installed, otherwise view "
"the above error for more info.")
webpack_complete = subprocess.call(["npm", "run", "dev"])
if webpack_complete != 0:
raise RuntimeError(
"Webpack appears to have failed to build your "
"frontend. See the above error for more information.")
os.chdir(return_dir)
operator = Operator(db)
operator.validate_and_run_config(run_config=cfg.mephisto,
shared_state=None)
operator.wait_for_runs_then_shutdown(skip_input=True,
log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
f'--task-description "\\"{task_description}\\"" '
"--task-reward 0.5 "
f"--task-tags {hit_keywords} "
f"--maximum-units-per-worker 0 " # Num of units a worker is allowed to do, 0 is infinite
f"--allowed-concurrent 1 " # Workers can only do one task at a time, or onboarding may break
)
extra_args = {
"pairings_filepath": args['pairings_filepath'],
"block_on_onboarding_fail": True,
"block_qualification": f"acute_eval_{int(time.time())}_block",
# num times to use the same conversation pair
"annotations_per_pair": args["annotations_per_pair"],
"random_seed": 42, # random seed
"subtasks_per_unit": args[
"subtasks_per_unit"
], # num comparisons to show within one unit
"num_matchup_pairs": args[
"num_matchup_pairs"
], # num pairs of conversations to be compared
# question phrasing
"s1_choice": "I would prefer to talk to <Speaker 1>",
"s2_choice": "I would prefer to talk to <Speaker 2>",
"eval_question": "Who would you prefer to talk to for a long conversation?",
"assignment_duration_in_seconds": 600,
}
operator = Operator(db)
operator.parse_and_launch_run_wrapper(shlex.split(ARG_STRING), extra_args=extra_args)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
},
]
}
# build the task
def build_task():
"""Rebuild the frontend for this task"""
return_dir = os.getcwd()
os.chdir(FRONTEND_SOURCE_DIR)
if os.path.exists(FRONTEND_BUILD_DIR):
shutil.rmtree(FRONTEND_BUILD_DIR)
packages_installed = subprocess.call(["npm", "install"])
if packages_installed != 0:
raise Exception("please make sure npm is installed, otherwise view "
"the above error for more info.")
webpack_complete = subprocess.call(["npm", "run", "dev"])
if webpack_complete != 0:
raise Exception("Webpack appears to have failed to build your "
"frontend. See the above error for more information.")
os.chdir(return_dir)
build_task()
operator = Operator(db)
operator.parse_and_launch_run_wrapper(shlex.split(ARG_STRING),
extra_args=extra_args)
operator.wait_for_runs_then_shutdown()
