def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None) operator.wait_for_runs_then_shutdown( skip_input=True, log_rate=cfg.monitoring_log_rate )
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) world_opt = { "num_turns": cfg.num_turns, "turn_timeout": cfg.turn_timeout, } custom_bundle_path = cfg.mephisto.blueprint.get('custom_source_bundle', None) if custom_bundle_path is not None: assert os.path.exists(custom_bundle_path), ( "Must build the custom bundle with `npm install; npm run dev` from within " f"the {TASK_DIRECTORY}/webapp directory in order to demo a custom bundle " ) world_opt["send_task_data"] = True shared_state = SharedParlAITaskState( world_opt=world_opt, onboarding_world_opt=world_opt, ) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None: task_dir = cfg.task_dir def onboarding_always_valid(onboarding_data): return True shared_state = SharedStaticTaskState( static_task_data=[ { "text": "This text is good text!" }, { "text": "This text is bad text!" }, ], validate_onboarding=onboarding_always_valid, ) build_task(task_dir) db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None: correct_config_answer = cfg.correct_answer def onboarding_is_valid(onboarding_data): inputs = onboarding_data["inputs"] outputs = onboarding_data["outputs"] return outputs.get("answer") == correct_config_answer shared_state = SharedStaticTaskState( onboarding_data={"correct_answer": correct_config_answer}, validate_onboarding=onboarding_is_valid, ) db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto, shared_state) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def run_static_task(cfg: DictConfig, task_directory: str): """ Run static task, given configuration. """ frontend_source_dir = os.path.join(task_directory, "webapp") frontend_build_dir = os.path.join(frontend_source_dir, "build") db, cfg = load_db_and_process_config(cfg) print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}') random.seed(42) soft_block_qual_name = cfg.mephisto.task.get('task_name', 'turn_annotations_static') # Default to a task-specific name to avoid soft-block collisions soft_block_mturk_workers(cfg=cfg, db=db, soft_block_qual_name=soft_block_qual_name) # Build the task return_dir = os.getcwd() os.chdir(frontend_source_dir) if os.path.exists(frontend_build_dir): shutil.rmtree(frontend_build_dir) packages_installed = subprocess.call(["npm", "install"]) if packages_installed != 0: raise Exception("please make sure npm is installed, otherwise view " "the above error for more info.") webpack_complete = subprocess.call(["npm", "run", "dev"]) if webpack_complete != 0: raise RuntimeError( "Webpack appears to have failed to build your " "frontend. See the above error for more information.") os.chdir(return_dir) operator = Operator(db) operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None: db, cfg = load_db_and_process_config(cfg) operator = Operator(db) operator.validate_and_run_config(cfg.mephisto) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
class TestOperator(unittest.TestCase): """ Unit testing for the Mephisto Supervisor """ def setUp(self): self.data_dir = tempfile.mkdtemp() database_path = os.path.join(self.data_dir, "mephisto.db") self.db = LocalMephistoDB(database_path) self.requester_name, _req_id = get_test_requester(self.db) self.operator = None def tearDown(self): if self.operator is not None: self.operator.shutdown() self.db.shutdown() shutil.rmtree(self.data_dir) self.assertTrue( len(threading.enumerate()) == 1, f"Expected only main thread at teardown, found {threading.enumerate()}", ) def wait_for_complete_assignment(self, assignment, timeout: int): start_time = time.time() while time.time() - start_time < timeout: if assignment.get_status() == AssignmentState.COMPLETED: break time.sleep(0.1) self.assertLess(time.time() - start_time, timeout, "Assignment not completed in time") def await_server_start(self, architect: "MockArchitect"): start_time = time.time() assert architect.server is not None, "Cannot wait on empty server" while time.time() - start_time < 5: if len(architect.server.subs) > 0: break time.sleep(0.1) self.assertLess(time.time() - start_time, 5, "Mock server not up in time") def test_initialize_supervisor(self): """Quick test to ensure that the operator can be initialized""" self.operator = Operator(self.db) def test_run_job_concurrent(self): """Ensure that the supervisor object can even be created""" self.operator = Operator(self.db) config = MephistoConfig( blueprint=MockBlueprintArgs( num_assignments=1, is_concurrent=True, ), provider=MockProviderArgs(requester_name=self.requester_name, ), architect=MockArchitectArgs(should_run_server=True), task=MOCK_TASK_ARGS, ) self.operator.validate_and_run_config(OmegaConf.structured(config)) tracked_runs = self.operator.get_running_task_runs() self.assertEqual(len(tracked_runs), 1, "Run not launched") task_run_id, tracked_run = list(tracked_runs.items())[0] self.assertIsNotNone(tracked_run) self.assertIsNotNone(tracked_run.task_launcher) self.assertIsNotNone(tracked_run.task_runner) self.assertIsNotNone(tracked_run.architect) self.assertIsNotNone(tracked_run.task_run) self.assertEqual(tracked_run.task_run.db_id, task_run_id) # Create two agents to step through the task architect = tracked_run.architect self.assertIsInstance(architect, MockArchitect, "Must use mock in testing") # Register a worker mock_worker_name = "MOCK_WORKER" architect.server.register_mock_worker(mock_worker_name) workers = self.db.find_workers(worker_name=mock_worker_name) worker_id = workers[0].db_id self.assertEqual(len(tracked_run.task_runner.running_assignments), 0) # Register an agent mock_agent_details = "FAKE_ASSIGNMENT" architect.server.register_mock_agent(worker_id, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 1, "Agent was not created properly") agent = agents[0] self.assertIsNotNone(agent) # Register another worker mock_worker_name = "MOCK_WORKER_2" architect.server.register_mock_worker(mock_worker_name) workers = self.db.find_workers(worker_name=mock_worker_name) worker_id = workers[0].db_id # Register an agent mock_agent_details = "FAKE_ASSIGNMENT_2" architect.server.register_mock_agent(worker_id, mock_agent_details) # Give up to 5 seconds for whole mock task to complete start_time = time.time() while time.time() - start_time < TIMEOUT_TIME: if len(self.operator.get_running_task_runs()) == 0: break time.sleep(0.1) self.assertLess(time.time() - start_time, TIMEOUT_TIME, "Task not completed in time") # Ensure the assignment is completed task_run = tracked_run.task_run assignment = task_run.get_assignments()[0] self.assertEqual(assignment.get_status(), AssignmentState.COMPLETED) def test_run_job_not_concurrent(self): """Ensure that the supervisor object can even be created""" self.operator = Operator(self.db) config = MephistoConfig( blueprint=MockBlueprintArgs( num_assignments=1, is_concurrent=False, ), provider=MockProviderArgs(requester_name=self.requester_name, ), architect=MockArchitectArgs(should_run_server=True), task=MOCK_TASK_ARGS, ) self.operator.validate_and_run_config(OmegaConf.structured(config)) tracked_runs = self.operator.get_running_task_runs() self.assertEqual(len(tracked_runs), 1, "Run not launched") task_run_id, tracked_run = list(tracked_runs.items())[0] self.assertIsNotNone(tracked_run) self.assertIsNotNone(tracked_run.task_launcher) self.assertIsNotNone(tracked_run.task_runner) self.assertIsNotNone(tracked_run.architect) self.assertIsNotNone(tracked_run.task_run) self.assertEqual(tracked_run.task_run.db_id, task_run_id) # Create two agents to step through the task architect = tracked_run.architect self.assertIsInstance(architect, MockArchitect, "Must use mock in testing") # Register a worker mock_worker_name = "MOCK_WORKER" architect.server.register_mock_worker(mock_worker_name) workers = self.db.find_workers(worker_name=mock_worker_name) worker_id = workers[0].db_id self.assertEqual(len(tracked_run.task_runner.running_assignments), 0) # Register an agent mock_agent_details = "FAKE_ASSIGNMENT" architect.server.register_mock_agent(worker_id, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 1, "Agent was not created properly") agent = agents[0] self.assertIsNotNone(agent) # Register another worker mock_worker_name = "MOCK_WORKER_2" architect.server.register_mock_worker(mock_worker_name) workers = self.db.find_workers(worker_name=mock_worker_name) worker_id = workers[0].db_id # Register an agent mock_agent_details = "FAKE_ASSIGNMENT_2" architect.server.register_mock_agent(worker_id, mock_agent_details) # Give up to 5 seconds for both tasks to complete start_time = time.time() while time.time() - start_time < TIMEOUT_TIME: if len(self.operator.get_running_task_runs()) == 0: break time.sleep(0.1) self.assertLess(time.time() - start_time, TIMEOUT_TIME, "Task not completed in time") # Ensure the assignment is completed task_run = tracked_run.task_run assignment = task_run.get_assignments()[0] self.assertEqual(assignment.get_status(), AssignmentState.COMPLETED) def test_run_jobs_with_restrictions(self): """Ensure allowed_concurrent and maximum_units_per_worker work""" self.operator = Operator(self.db) provider_args = MockProviderArgs(requester_name=self.requester_name, ) architect_args = MockArchitectArgs(should_run_server=True) config = MephistoConfig( blueprint=MockBlueprintArgs( num_assignments=3, is_concurrent=True, ), provider=provider_args, architect=architect_args, task=TaskConfigArgs( task_title="title", task_description="This is a description", task_reward="0.3", task_tags="1,2,3", maximum_units_per_worker=2, allowed_concurrent=1, task_name='max-unit-test', ), ) self.operator.validate_and_run_config(OmegaConf.structured(config)) tracked_runs = self.operator.get_running_task_runs() self.assertEqual(len(tracked_runs), 1, "Run not launched") task_run_id, tracked_run = list(tracked_runs.items())[0] self.assertIsNotNone(tracked_run) self.assertIsNotNone(tracked_run.task_launcher) self.assertIsNotNone(tracked_run.task_runner) self.assertIsNotNone(tracked_run.architect) self.assertIsNotNone(tracked_run.task_run) self.assertEqual(tracked_run.task_run.db_id, task_run_id) self.await_server_start(tracked_run.architect) # Create two agents to step through the task architect = tracked_run.architect self.assertIsInstance(architect, MockArchitect, "Must use mock in testing") # Register a worker mock_worker_name = "MOCK_WORKER" architect.server.register_mock_worker(mock_worker_name) workers = self.db.find_workers(worker_name=mock_worker_name) worker_id_1 = workers[0].db_id self.assertEqual(len(tracked_run.task_runner.running_assignments), 0) # Register an agent mock_agent_details = "FAKE_ASSIGNMENT" architect.server.register_mock_agent(worker_id_1, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 1, "Agent was not created properly") agent = agents[0] self.assertIsNotNone(agent) # Try to register a second agent, which should fail due to concurrency mock_agent_details = "FAKE_ASSIGNMENT_2" architect.server.register_mock_agent(worker_id_1, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 1, "Second agent was created") # Register another worker mock_worker_name = "MOCK_WORKER_2" architect.server.register_mock_worker(mock_worker_name) workers = self.db.find_workers(worker_name=mock_worker_name) worker_id_2 = workers[0].db_id # Register an agent mock_agent_details = "FAKE_ASSIGNMENT_2" architect.server.register_mock_agent(worker_id_2, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 2, "Second agent was not created") # wait for task to pass self.wait_for_complete_assignment( agents[1].get_unit().get_assignment(), 3) # Pass a second task as well mock_agent_details = "FAKE_ASSIGNMENT_3" architect.server.register_mock_agent(worker_id_1, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 3, "Agent was not created properly") mock_agent_details = "FAKE_ASSIGNMENT_4" architect.server.register_mock_agent(worker_id_2, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 4, "Fourth agent was not created") # wait for task to pass self.wait_for_complete_assignment( agents[3].get_unit().get_assignment(), 3) # Both workers should have saturated their tasks, and not be granted agents mock_agent_details = "FAKE_ASSIGNMENT_5" architect.server.register_mock_agent(worker_id_1, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 4, "Additional agent was created") architect.server.register_mock_agent(worker_id_2, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 4, "Additional agent was created") # new workers should be able to work on these just fine though mock_worker_name = "MOCK_WORKER_3" architect.server.register_mock_worker(mock_worker_name) workers = self.db.find_workers(worker_name=mock_worker_name) worker_id_3 = workers[0].db_id mock_worker_name = "MOCK_WORKER_4" architect.server.register_mock_worker(mock_worker_name) workers = self.db.find_workers(worker_name=mock_worker_name) worker_id_4 = workers[0].db_id # Register agents from new workers mock_agent_details = "FAKE_ASSIGNMENT_5" architect.server.register_mock_agent(worker_id_3, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 5, "Additional agent was not created") mock_agent_details = "FAKE_ASSIGNMENT_6" architect.server.register_mock_agent(worker_id_4, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 6, "Additional agent was not created") # wait for task to pass self.wait_for_complete_assignment( agents[5].get_unit().get_assignment(), 3) # Give up to 5 seconds for whole mock task to complete start_time = time.time() while time.time() - start_time < TIMEOUT_TIME: if len(self.operator.get_running_task_runs()) == 0: break time.sleep(0.1) self.assertLess(time.time() - start_time, TIMEOUT_TIME, "Task not completed in time") # Ensure all assignments are completed task_run = tracked_run.task_run assignments = task_run.get_assignments() for assignment in assignments: self.assertEqual(assignment.get_status(), AssignmentState.COMPLETED) # Create a new task config = MephistoConfig( blueprint=MockBlueprintArgs( num_assignments=1, is_concurrent=True, ), provider=MockProviderArgs(requester_name=self.requester_name, ), architect=MockArchitectArgs(should_run_server=True), task=TaskConfigArgs( task_title="title", task_description="This is a description", task_reward="0.3", task_tags="1,2,3", maximum_units_per_worker=2, allowed_concurrent=1, task_name='max-unit-test', ), ) self.operator.validate_and_run_config(OmegaConf.structured(config)) tracked_runs = self.operator.get_running_task_runs() self.assertEqual(len(tracked_runs), 1, "Run not launched") task_run_id, tracked_run = list(tracked_runs.items())[0] self.await_server_start(tracked_run.architect) architect = tracked_run.architect # Workers one and two still shouldn't be able to make agents mock_agent_details = "FAKE_ASSIGNMENT_7" architect.server.register_mock_agent(worker_id_1, mock_agent_details) agents = self.db.find_agents() self.assertEqual( len(agents), 6, "Additional agent was created for worker exceeding max units", ) mock_agent_details = "FAKE_ASSIGNMENT_7" architect.server.register_mock_agent(worker_id_2, mock_agent_details) agents = self.db.find_agents() self.assertEqual( len(agents), 6, "Additional agent was created for worker exceeding max units", ) # Three and four should though mock_agent_details = "FAKE_ASSIGNMENT_7" architect.server.register_mock_agent(worker_id_3, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 7, "Additional agent was not created") mock_agent_details = "FAKE_ASSIGNMENT_8" architect.server.register_mock_agent(worker_id_4, mock_agent_details) agents = self.db.find_agents() self.assertEqual(len(agents), 8, "Additional agent was not created") # Ensure the task run completed and that all assignments are done start_time = time.time() while time.time() - start_time < TIMEOUT_TIME: if len(self.operator.get_running_task_runs()) == 0: break time.sleep(0.1) self.assertLess(time.time() - start_time, TIMEOUT_TIME, "Task not completed in time") task_run = tracked_run.task_run assignments = task_run.get_assignments() for assignment in assignments: self.assertEqual(assignment.get_status(), AssignmentState.COMPLETED)
class TestAcuteEval(unittest.TestCase): """ Test the ACUTE-Eval crowdsourcing task. """ def setUp(self): self.operator = None def tearDown(self): if self.operator is not None: self.operator.shutdown() def test_base_task(self): # Define the configuration settings relative_task_directory = os.path.relpath( TASK_DIRECTORY, os.path.dirname(__file__) ) relative_config_path = os.path.join(relative_task_directory, 'conf') with initialize(config_path=relative_config_path): self.config = compose( config_name="example", overrides=[ f'+mephisto.blueprint._blueprint_type={AcuteEvalBlueprint.BLUEPRINT_TYPE}', f'+mephisto/architect=mock', f'+mephisto/provider=mock', f'mephisto.blueprint.block_on_onboarding_fail={False}', f'+task_dir={TASK_DIRECTORY}', f'+current_time={int(time.time())}', ], ) # TODO: when Hydra 1.1 is released with support for recursive defaults, # don't manually specify all missing blueprint args anymore, but # instead define the blueprint in the defaults list directly. # Currently, the blueprint can't be set in the defaults list without # overriding params in the YAML file, as documented at # https://github.com/facebookresearch/hydra/issues/326 and as fixed in # https://github.com/facebookresearch/hydra/pull/1044. self.data_dir = tempfile.mkdtemp() database_path = os.path.join(self.data_dir, "mephisto.db") self.db = LocalMephistoDB(database_path) self.config = augment_config_from_db(self.config, self.db) self.config.mephisto.architect.should_run_server = True self.operator = Operator(self.db) self.operator.validate_and_run_config( self.config.mephisto, shared_state=None ) channel_info = list(self.operator.supervisor.channels.values())[0] server = channel_info.job.architect.server # Register a worker mock_worker_name = "MOCK_WORKER" server.register_mock_worker(mock_worker_name) workers = self.db.find_workers(worker_name=mock_worker_name) worker_id = workers[0].db_id # Register an agent mock_agent_details = "FAKE_ASSIGNMENT" server.register_mock_agent(worker_id, mock_agent_details) agent = self.db.find_agents()[0] agent_id_1 = agent.db_id # Set initial data server.request_init_data(agent_id_1) # Make agent act server.send_agent_act( agent_id_1, {"MEPHISTO_is_submit": True, "task_data": DESIRED_OUTPUTS} ) # Check that the inputs and outputs are as expected agent = self.db.find_agents()[0] state = agent.state.get_data() self.assertEqual(DESIRED_INPUTS, state['inputs']) self.assertEqual(DESIRED_OUTPUTS, state['outputs'])
class CrowdsourcingTestMixin: """ Mixin for end-to-end tests of Mephisto-based crowdsourcing tasks. Allows for setup and teardown of the operator, as well as for config specification and agent registration. """ def setUp(self): self.operator = None def tearDown(self): if self.operator is not None: self.operator.shutdown() def _set_up_config( self, blueprint_type: str, task_directory: str, overrides: Optional[List[str]] = None, ): """ Set up the config and database. Uses the Hydra compose() API for unit testing and a temporary directory to store the test database. :param blueprint_type: string uniquely specifying Blueprint class :param task_directory: directory containing the `conf/` configuration folder. Will be injected as `${task_dir}` in YAML files. :param overrides: additional config overrides """ # Define the configuration settings relative_task_directory = os.path.relpath(task_directory, os.path.dirname(__file__)) relative_config_path = os.path.join(relative_task_directory, 'conf') if overrides is None: overrides = [] with initialize(config_path=relative_config_path): self.config = compose( config_name="example", overrides=[ f'+mephisto.blueprint._blueprint_type={blueprint_type}', f'+mephisto/architect=mock', f'+mephisto/provider=mock', f'+task_dir={task_directory}', f'+current_time={int(time.time())}', ] + overrides, ) # TODO: when Hydra 1.1 is released with support for recursive defaults, # don't manually specify all missing blueprint args anymore, but # instead define the blueprint in the defaults list directly. # Currently, the blueprint can't be set in the defaults list without # overriding params in the YAML file, as documented at # https://github.com/facebookresearch/hydra/issues/326 and as fixed in # https://github.com/facebookresearch/hydra/pull/1044. self.data_dir = tempfile.mkdtemp() database_path = os.path.join(self.data_dir, "mephisto.db") self.db = LocalMephistoDB(database_path) self.config = augment_config_from_db(self.config, self.db) self.config.mephisto.architect.should_run_server = True def _set_up_server(self, shared_state: Optional[SharedTaskState] = None): """ Set up the operator and server. """ self.operator = Operator(self.db) self.operator.validate_and_run_config(self.config.mephisto, shared_state=shared_state) channel_info = list(self.operator.supervisor.channels.values())[0] self.server = channel_info.job.architect.server def _register_mock_agents(self, num_agents: int = 1) -> List[str]: """ Register mock agents for testing, taking the place of crowdsourcing workers. Specify the number of agents to register. Return the agents' IDs after creation. """ for idx in range(num_agents): # Register the worker mock_worker_name = f"MOCK_WORKER_{idx:d}" self.server.register_mock_worker(mock_worker_name) workers = self.db.find_workers(worker_name=mock_worker_name) worker_id = workers[0].db_id # Register the agent mock_agent_details = f"FAKE_ASSIGNMENT_{idx:d}" self.server.register_mock_agent(worker_id, mock_agent_details) # Get all agents' IDs agents = self.db.find_agents() agent_ids = [agent.db_id for agent in agents] return agent_ids