def task_script( config: Optional[Type[TaskConfig]] = None, default_config_file: Optional[str] = None, config_path: str = "hydra_configs", # Override if using a different dir ) -> Callable[[TaskFunction], Any]: """ Create a decorator for the main of a Mephisto task script Must provide one of config (a TaskConfig dataclass) or default_config_file (the location of a default task config hydra yaml), the former will be preferred. May specify a config_path override if not using `hydra_configs` at the run script location. """ if config is not None: used_config = config else: assert ( default_config_file is not None), "Must provide one of config or default_config_file" used_config = build_default_task_config(default_config_file) register_script_config(name="taskconfig", module=used_config) def task_script_wrapper(script_func: TaskFunction) -> TaskFunction: @functools.wraps(script_func) def process_config_and_run_main(cfg: "DictConfig"): operator, cfg = process_config_and_get_operator(cfg) try: ret_val = script_func(operator, cfg) except Exception as e: raise e finally: if not operator.is_shutdown: operator.shutdown() return ret_val absolute_config_path = os.path.abspath( os.path.join(get_run_file_dir(), config_path)) hydra_wrapper = hydra.main(config_path=absolute_config_path, config_name="taskconfig") return cast(TaskFunction, hydra_wrapper(process_config_and_run_main)) return task_script_wrapper
@dataclass class TestScriptConfig(MTurkRunScriptConfig): defaults: List[Any] = field(default_factory=lambda: defaults) task_dir: str = TASK_DIRECTORY monitoring_log_rate: int = field( default=30, metadata={ 'help': 'Frequency in seconds of logging the monitoring of the crowdsourcing task' }, ) register_script_config(name='fast_acute_scriptconfig', module=TestScriptConfig) @hydra.main(config_path="hydra_configs", config_name="fast_acute_scriptconfig") def main(cfg: DictConfig) -> None: runner = FastAcuteExecutor(cfg) # Create self-chats runner.compile_chat_logs() # Run ACUTE-Eval runner.run_acute_eval() # Analyze the results runner.analyze_results()
defaults = [ {'mephisto/blueprint': STATIC_BLUEPRINT_TYPE}, {"mephisto/architect": "local"}, {"mephisto/provider": "mock"}, {"conf": "example"}, ] @dataclass class ScriptConfig(MTurkRunScriptConfig): defaults: List[Any] = field(default_factory=lambda: defaults) task_dir: str = TASK_DIRECTORY monitoring_log_rate: int = field( default=30, metadata={ 'help': 'Frequency in seconds of logging the monitoring of the crowdsourcing task' }, ) register_script_config(name='scriptconfig', module=ScriptConfig) @hydra.main(config_name="scriptconfig") def main(cfg: DictConfig) -> None: run_static_task(cfg=cfg, task_directory=TASK_DIRECTORY) if __name__ == "__main__": main()
task_dir: str = TASK_DIRECTORY num_turns: int = field( default=3, metadata={"help": "Number of turns before a conversation is complete"}, ) turn_timeout: int = field( default=300, metadata={ "help": "Maximum response time before kicking " "a worker out, default 300 seconds" }, ) register_script_config(name="scriptconfig", module=TestScriptConfig) @hydra.main(config_name="scriptconfig") def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) world_opt = {"num_turns": cfg.num_turns, "turn_timeout": cfg.turn_timeout} custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle", None) if custom_bundle_path is not None: assert os.path.exists(custom_bundle_path), ( "Must build the custom bundle with `npm install; npm run dev` from within " f"the {TASK_DIRECTORY}/webapp directory in order to demo a custom bundle " )
@dataclass class TestScriptConfig(MTurkRunScriptConfig): defaults: List[Any] = field(default_factory=lambda: defaults) task_dir: str = ACUTE_EVAL_TASK_DIRECTORY fast_acute_task_dir: str = FAST_ACUTE_TASK_DIRECTORY monitoring_log_rate: int = field( default=30, metadata={ 'help': 'Frequency in seconds of logging the monitoring of the crowdsourcing task' }, ) register_script_config(name='fast_acute_no_self_chat', module=TestScriptConfig) @hydra.main(config_name="fast_acute_no_self_chat") def main(cfg: DictConfig) -> None: runner = NoSelfChatFastAcuteExecutor(cfg) # Run ACUTE-Evals runner.run_acute_eval() # Analyze the results runner.analyze_results() if __name__ == '__main__':
_ = BLUEPRINT_TYPE TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) defaults = ['_self_', {"conf": "example"}] @dataclass class PerTurnEvalConfig(MTurkRunScriptConfig): defaults: List[Any] = field(default_factory=lambda: defaults) task_dir: str = TASK_DIRECTORY monitoring_log_rate: int = field( default=30, metadata={ 'help': 'Frequency in seconds of logging the monitoring of the crowdsourcing task' }, ) register_script_config(name='perturnevalconfig', module=PerTurnEvalConfig) @hydra.main(config_path="hydra_configs", config_name="perturnevalconfig") def main(cfg: DictConfig) -> None: run_task(cfg=cfg, task_directory=TASK_DIRECTORY) if __name__ == "__main__": main()