def run_task(cfg: DictConfig, task_directory: str):
"""
Run task, given configuration.
"""
frontend_source_dir = os.path.join(task_directory, "webapp")
frontend_build_dir = os.path.join(frontend_source_dir, "build")
_ = frontend_build_dir # Unused at the moment
db, cfg = load_db_and_process_config(cfg)
print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}')
random.seed(42)
# Update task name when on sandbox or local to ensure data is split.
task_name = cfg.mephisto.task.get('task_name', 'model_chat')
architect_type = cfg.mephisto.architect._architect_type
if architect_type == 'local':
task_name = f"{task_name}_local"
elif architect_type == 'mturk_sandbox':
task_name = f"{task_name}_sandbox"
cfg.mephisto.task.task_name = task_name
soft_block_qual_name = cfg.mephisto.blueprint.get('block_qualification',
f'{task_name}_block')
# Default to a task-specific name to avoid soft-block collisions
soft_block_mturk_workers(cfg=cfg,
db=db,
soft_block_qual_name=soft_block_qual_name)
# Init
shared_state = SharedModelChatTaskState(world_module=world_module)
operator = Operator(db)
operator.validate_and_run_config(run_config=cfg.mephisto,
shared_state=shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True,
log_rate=cfg.monitoring_log_rate)
def test_base_task(self):
with testing_utils.tempdir() as tmpdir:
# Paths
expected_states_folder = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'expected_states')
expected_chat_data_path = os.path.join(expected_states_folder,
'final_chat_data.json')
expected_state_path = os.path.join(expected_states_folder,
'state.json')
model_opt_path = os.path.join(tmpdir, 'model_opts.yaml')
chat_data_folder = os.path.join(tmpdir, 'final_chat_data')
# Create a model opt file for the fixed-response model
with open(model_opt_path, 'w') as f:
model_opt_contents = f"""\
fixed_response: >
--model fixed_response
"""
f.write(model_opt_contents)
# Set up the config and database
num_convos = 10
overrides = [
f'mephisto.blueprint.conversations_needed_string=\"fixed_response:{num_convos:d}\"',
f'mephisto.blueprint.chat_data_folder={chat_data_folder}',
f'mephisto.blueprint.model_opt_path={model_opt_path}',
]
self._set_up_config(task_directory=TASK_DIRECTORY,
overrides=overrides)
# Set up the operator and server
shared_state = SharedModelChatTaskState(
world_module=world_module)
self._set_up_server(shared_state=shared_state)
# Check that the agent states are as they should be
self._get_channel_info(
).job.task_runner.task_run.get_blueprint().use_onboarding = (
False)
# Don't require onboarding for this test agent
with open(expected_state_path) as f:
expected_state = json.load(f)
self._test_agent_states(
num_agents=1,
agent_display_ids=AGENT_DISPLAY_IDS,
agent_messages=AGENT_MESSAGES,
form_messages=FORM_MESSAGES,
form_task_data=FORM_TASK_DATA,
expected_states=(expected_state, ),
agent_task_data=AGENT_TASK_DATA,
)
# Check that the contents of the chat data file are as expected
with open(expected_chat_data_path) as f:
expected_chat_data = json.load(f)
results_path = list(
glob.glob(
os.path.join(chat_data_folder,
'*/*_*_*_sandbox.json')))[0]
with open(results_path) as f:
actual_chat_data = json.load(f)
self._check_final_chat_data(actual_value=actual_chat_data,
expected_value=expected_chat_data)
def test_base_task(self):
with testing_utils.tempdir() as tmpdir:
# Paths
expected_states_folder = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'expected_states')
expected_chat_data_path = os.path.join(expected_states_folder,
'final_chat_data.json')
expected_state_path = os.path.join(expected_states_folder,
'state.json')
model_opt_path = os.path.join(tmpdir, 'model_opts.yaml')
chat_data_folder = os.path.join(tmpdir, 'final_chat_data')
# Create a model opt file for the fixed-response model
with open(model_opt_path, 'w') as f:
model_opt_contents = f"""\
fixed_response: >
--model fixed_response
"""
f.write(model_opt_contents)
# Set up the config and database
num_blender_convos = 10
args = ModelChatBlueprintArgs()
overrides = [
f'+mephisto.blueprint.{key}={val}'
for key, val in args.__dict__.items() if key in [
'max_onboard_time',
'max_resp_time',
'override_opt',
'random_seed',
'world_file',
]
] + [
'mephisto.blueprint.annotations_config_path=${task_dir}/task_config/annotations_config.json',
f'mephisto.blueprint.conversations_needed_string=\"fixed_response:{num_blender_convos:d}\"',
f'mephisto.blueprint.chat_data_folder={chat_data_folder}',
'+mephisto.blueprint.left_pane_text_path=${task_dir}/task_config/left_pane_text.html',
'+mephisto.blueprint.max_concurrent_responses=1',
f'mephisto.blueprint.model_opt_path={model_opt_path}',
f'+mephisto.blueprint.num_conversations={num_blender_convos:d}',
'+mephisto.blueprint.onboard_task_data_path=${task_dir}/task_config/onboard_task_data.json',
'+mephisto.blueprint.task_description_file=${task_dir}/task_config/task_description.html',
]
# TODO: remove all of these params once Hydra 1.1 is released with
# support for recursive defaults
self._set_up_config(
blueprint_type=BLUEPRINT_TYPE,
task_directory=TASK_DIRECTORY,
overrides=overrides,
)
# Set up the operator and server
shared_state = SharedModelChatTaskState(
world_module=world_module)
self._set_up_server(shared_state=shared_state)
# Check that the agent states are as they should be
self._get_channel_info(
).job.task_runner.task_run.get_blueprint().use_onboarding = (
False)
# Don't require onboarding for this test agent
with open(expected_state_path) as f:
expected_state = json.load(f)
self._test_agent_states(
num_agents=1,
agent_display_ids=AGENT_DISPLAY_IDS,
agent_messages=AGENT_MESSAGES,
form_messages=FORM_MESSAGES,
form_task_data=FORM_TASK_DATA,
expected_states=(expected_state, ),
agent_task_data=AGENT_TASK_DATA,
)
# Check that the contents of the chat data file are as expected
with open(expected_chat_data_path) as f:
expected_chat_data = json.load(f)
results_path = list(
glob.glob(
os.path.join(chat_data_folder,
'*_*_*_sandbox.json')))[0]
with open(results_path) as f:
actual_chat_data = json.load(f)
self._check_final_chat_data(actual_value=actual_chat_data,
expected_value=expected_chat_data)