def task_script(
config: Optional[Type[TaskConfig]] = None,
default_config_file: Optional[str] = None,
config_path: str = "hydra_configs", # Override if using a different dir
) -> Callable[[TaskFunction], Any]:
"""
Create a decorator for the main of a Mephisto task script
Must provide one of config (a TaskConfig dataclass) or default_config_file
(the location of a default task config hydra yaml), the former
will be preferred.
May specify a config_path override if not using `hydra_configs` at the
run script location.
"""
if config is not None:
used_config = config
else:
assert (
default_config_file
is not None), "Must provide one of config or default_config_file"
used_config = build_default_task_config(default_config_file)
register_script_config(name="taskconfig", module=used_config)
def task_script_wrapper(script_func: TaskFunction) -> TaskFunction:
@functools.wraps(script_func)
def process_config_and_run_main(cfg: "DictConfig"):
operator, cfg = process_config_and_get_operator(cfg)
try:
ret_val = script_func(operator, cfg)
except Exception as e:
raise e
finally:
if not operator.is_shutdown:
operator.shutdown()
return ret_val
absolute_config_path = os.path.abspath(
os.path.join(get_run_file_dir(), config_path))
hydra_wrapper = hydra.main(config_path=absolute_config_path,
config_name="taskconfig")
return cast(TaskFunction, hydra_wrapper(process_config_and_run_main))
return task_script_wrapper
@dataclass
class TestScriptConfig(MTurkRunScriptConfig):
defaults: List[Any] = field(default_factory=lambda: defaults)
task_dir: str = TASK_DIRECTORY
monitoring_log_rate: int = field(
default=30,
metadata={
'help':
'Frequency in seconds of logging the monitoring of the crowdsourcing task'
},
)
register_script_config(name='fast_acute_scriptconfig', module=TestScriptConfig)
@hydra.main(config_path="hydra_configs", config_name="fast_acute_scriptconfig")
def main(cfg: DictConfig) -> None:
runner = FastAcuteExecutor(cfg)
# Create self-chats
runner.compile_chat_logs()
# Run ACUTE-Eval
runner.run_acute_eval()
# Analyze the results
runner.analyze_results()
defaults = [
{'mephisto/blueprint': STATIC_BLUEPRINT_TYPE},
{"mephisto/architect": "local"},
{"mephisto/provider": "mock"},
{"conf": "example"},
]
@dataclass
class ScriptConfig(MTurkRunScriptConfig):
defaults: List[Any] = field(default_factory=lambda: defaults)
task_dir: str = TASK_DIRECTORY
monitoring_log_rate: int = field(
default=30,
metadata={
'help': 'Frequency in seconds of logging the monitoring of the crowdsourcing task'
},
)
register_script_config(name='scriptconfig', module=ScriptConfig)
@hydra.main(config_name="scriptconfig")
def main(cfg: DictConfig) -> None:
run_static_task(cfg=cfg, task_directory=TASK_DIRECTORY)
if __name__ == "__main__":
main()
task_dir: str = TASK_DIRECTORY
num_turns: int = field(
default=3,
metadata={"help": "Number of turns before a conversation is complete"},
)
turn_timeout: int = field(
default=300,
metadata={
"help":
"Maximum response time before kicking "
"a worker out, default 300 seconds"
},
)
register_script_config(name="scriptconfig", module=TestScriptConfig)
@hydra.main(config_name="scriptconfig")
def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
world_opt = {"num_turns": cfg.num_turns, "turn_timeout": cfg.turn_timeout}
custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle",
None)
if custom_bundle_path is not None:
assert os.path.exists(custom_bundle_path), (
"Must build the custom bundle with `npm install; npm run dev` from within "
f"the {TASK_DIRECTORY}/webapp directory in order to demo a custom bundle "
)
@dataclass
class TestScriptConfig(MTurkRunScriptConfig):
defaults: List[Any] = field(default_factory=lambda: defaults)
task_dir: str = ACUTE_EVAL_TASK_DIRECTORY
fast_acute_task_dir: str = FAST_ACUTE_TASK_DIRECTORY
monitoring_log_rate: int = field(
default=30,
metadata={
'help':
'Frequency in seconds of logging the monitoring of the crowdsourcing task'
},
)
register_script_config(name='fast_acute_no_self_chat', module=TestScriptConfig)
@hydra.main(config_name="fast_acute_no_self_chat")
def main(cfg: DictConfig) -> None:
runner = NoSelfChatFastAcuteExecutor(cfg)
# Run ACUTE-Evals
runner.run_acute_eval()
# Analyze the results
runner.analyze_results()
if __name__ == '__main__':
_ = BLUEPRINT_TYPE
TASK_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
defaults = ['_self_', {"conf": "example"}]
@dataclass
class PerTurnEvalConfig(MTurkRunScriptConfig):
defaults: List[Any] = field(default_factory=lambda: defaults)
task_dir: str = TASK_DIRECTORY
monitoring_log_rate: int = field(
default=30,
metadata={
'help':
'Frequency in seconds of logging the monitoring of the crowdsourcing task'
},
)
register_script_config(name='perturnevalconfig', module=PerTurnEvalConfig)
@hydra.main(config_path="hydra_configs", config_name="perturnevalconfig")
def main(cfg: DictConfig) -> None:
run_task(cfg=cfg, task_directory=TASK_DIRECTORY)
if __name__ == "__main__":
main()
