def main(cfg: DictConfig) -> None:
correct_config_answer = cfg.correct_answer
def onboarding_is_valid(onboarding_data):
inputs = onboarding_data["inputs"]
outputs = onboarding_data["outputs"]
return outputs.get("answer") == correct_config_answer
shared_state = SharedStaticTaskState(
onboarding_data={"correct_answer": correct_config_answer},
validate_onboarding=onboarding_is_valid,
)
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def run_task(opt):
"""
Note: launch_config opt should be something like:
parlai.crowdsourcing.tasks.turn_annotations_static.launch_config.LaunchConfig
"""
launch_config_file = opt.get('launch_config')
launch_module = import_module(launch_config_file)
launch_config = launch_module.LaunchConfig
db, arg_string = setup_mephisto(launch_config)
if 'sandbox' not in launch_config.PROVIDER:
block_workers(opt, db, launch_config.REQUESTER)
build_task()
operator = Operator(db)
operator.parse_and_launch_run_wrapper(shlex.split(arg_string),
extra_args={})
operator.wait_for_runs_then_shutdown()
def run_static_task(cfg: DictConfig, task_directory: str):
"""
Run static task, given configuration.
"""
frontend_source_dir = os.path.join(task_directory, "webapp")
frontend_build_dir = os.path.join(frontend_source_dir, "build")
db, cfg = load_db_and_process_config(cfg)
print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}')
random.seed(42)
soft_block_qual_name = cfg.mephisto.task.get('task_name',
'turn_annotations_static')
# Default to a task-specific name to avoid soft-block collisions
soft_block_mturk_workers(cfg=cfg,
db=db,
soft_block_qual_name=soft_block_qual_name)
# Build the task
return_dir = os.getcwd()
os.chdir(frontend_source_dir)
if os.path.exists(frontend_build_dir):
shutil.rmtree(frontend_build_dir)
packages_installed = subprocess.call(["npm", "install"])
if packages_installed != 0:
raise Exception("please make sure npm is installed, otherwise view "
"the above error for more info.")
webpack_complete = subprocess.call(["npm", "run", "dev"])
if webpack_complete != 0:
raise RuntimeError(
"Webpack appears to have failed to build your "
"frontend. See the above error for more information.")
os.chdir(return_dir)
operator = Operator(db)
operator.validate_and_run_config(run_config=cfg.mephisto,
shared_state=None)
operator.wait_for_runs_then_shutdown(skip_input=True,
log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
world_opt = {"num_turns": cfg.num_turns, "turn_timeout": cfg.turn_timeout}
custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle",
None)
if custom_bundle_path is not None:
assert os.path.exists(custom_bundle_path), (
"Must build the custom bundle with `npm install; npm run dev` from within "
f"the {TASK_DIRECTORY}/webapp directory in order to demo a custom bundle "
)
world_opt["send_task_data"] = True
shared_state = SharedParlAITaskState(world_opt=world_opt,
onboarding_world_opt=world_opt)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None:
task_dir = cfg.task_dir
def onboarding_always_valid(onboarding_data):
return True
shared_state = SharedStaticTaskState(
static_task_data=[
{"text": "This text is good text!"},
{"text": "This text is bad text!"},
],
validate_onboarding=onboarding_always_valid,
)
build_task(task_dir)
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
from mephisto.core.local_database import LocalMephistoDB
import os
import atexit
import signal
app = Flask(__name__,
static_url_path="/static",
static_folder="../webapp/build/static")
app.config.from_object(Config)
app.register_blueprint(api, url_prefix="/api/v1")
# Register extensions
db = LocalMephistoDB()
operator = Operator(db)
if not hasattr(app, "extensions"):
app.extensions = {}
app.extensions["db"] = db
app.extensions["operator"] = operator
@app.route("/", defaults={"path": "index.html"})
@app.route("/<path:path>")
def index(path):
return send_file(os.path.join("..", "webapp", "build", "index.html"))
@app.after_request
def after_request(response):
response.headers.add("Access-Control-Allow-Origin", "*")
def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def test_run_job_concurrent(self):
"""Ensure that the supervisor object can even be created"""
self.operator = Operator(self.db)
config = MephistoConfig(
blueprint=MockBlueprintArgs(
num_assignments=1,
is_concurrent=True,
),
provider=MockProviderArgs(requester_name=self.requester_name, ),
architect=MockArchitectArgs(should_run_server=True),
task=MOCK_TASK_ARGS,
)
self.operator.validate_and_run_config(OmegaConf.structured(config))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.assertIsNotNone(tracked_run)
self.assertIsNotNone(tracked_run.task_launcher)
self.assertIsNotNone(tracked_run.task_runner)
self.assertIsNotNone(tracked_run.architect)
self.assertIsNotNone(tracked_run.task_run)
self.assertEqual(tracked_run.task_run.db_id, task_run_id)
# Create two agents to step through the task
architect = tracked_run.architect
self.assertIsInstance(architect, MockArchitect,
"Must use mock in testing")
# Register a worker
mock_worker_name = "MOCK_WORKER"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id = workers[0].db_id
self.assertEqual(len(tracked_run.task_runner.running_assignments), 0)
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT"
architect.server.register_mock_agent(worker_id, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Agent was not created properly")
agent = agents[0]
self.assertIsNotNone(agent)
# Register another worker
mock_worker_name = "MOCK_WORKER_2"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id = workers[0].db_id
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id, mock_agent_details)
# Give up to 5 seconds for whole mock task to complete
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
# Ensure the assignment is completed
task_run = tracked_run.task_run
assignment = task_run.get_assignments()[0]
self.assertEqual(assignment.get_status(), AssignmentState.COMPLETED)
def test_initialize_supervisor(self):
"""Quick test to ensure that the operator can be initialized"""
self.operator = Operator(self.db)
class TestOperator(unittest.TestCase):
"""
Unit testing for the Mephisto Supervisor
"""
def setUp(self):
self.data_dir = tempfile.mkdtemp()
database_path = os.path.join(self.data_dir, "mephisto.db")
self.db = LocalMephistoDB(database_path)
self.requester_name, _req_id = get_test_requester(self.db)
self.operator = None
def tearDown(self):
if self.operator is not None:
self.operator.shutdown()
self.db.shutdown()
shutil.rmtree(self.data_dir)
self.assertTrue(
len(threading.enumerate()) == 1,
f"Expected only main thread at teardown, found {threading.enumerate()}",
)
def wait_for_complete_assignment(self, assignment, timeout: int):
start_time = time.time()
while time.time() - start_time < timeout:
if assignment.get_status() == AssignmentState.COMPLETED:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, timeout,
"Assignment not completed in time")
def await_server_start(self, architect: "MockArchitect"):
start_time = time.time()
assert architect.server is not None, "Cannot wait on empty server"
while time.time() - start_time < 5:
if len(architect.server.subs) > 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, 5,
"Mock server not up in time")
def test_initialize_supervisor(self):
"""Quick test to ensure that the operator can be initialized"""
self.operator = Operator(self.db)
def test_run_job_concurrent(self):
"""Ensure that the supervisor object can even be created"""
self.operator = Operator(self.db)
config = MephistoConfig(
blueprint=MockBlueprintArgs(
num_assignments=1,
is_concurrent=True,
),
provider=MockProviderArgs(requester_name=self.requester_name, ),
architect=MockArchitectArgs(should_run_server=True),
task=MOCK_TASK_ARGS,
)
self.operator.validate_and_run_config(OmegaConf.structured(config))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.assertIsNotNone(tracked_run)
self.assertIsNotNone(tracked_run.task_launcher)
self.assertIsNotNone(tracked_run.task_runner)
self.assertIsNotNone(tracked_run.architect)
self.assertIsNotNone(tracked_run.task_run)
self.assertEqual(tracked_run.task_run.db_id, task_run_id)
# Create two agents to step through the task
architect = tracked_run.architect
self.assertIsInstance(architect, MockArchitect,
"Must use mock in testing")
# Register a worker
mock_worker_name = "MOCK_WORKER"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id = workers[0].db_id
self.assertEqual(len(tracked_run.task_runner.running_assignments), 0)
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT"
architect.server.register_mock_agent(worker_id, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Agent was not created properly")
agent = agents[0]
self.assertIsNotNone(agent)
# Register another worker
mock_worker_name = "MOCK_WORKER_2"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id = workers[0].db_id
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id, mock_agent_details)
# Give up to 5 seconds for whole mock task to complete
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
# Ensure the assignment is completed
task_run = tracked_run.task_run
assignment = task_run.get_assignments()[0]
self.assertEqual(assignment.get_status(), AssignmentState.COMPLETED)
def test_run_job_not_concurrent(self):
"""Ensure that the supervisor object can even be created"""
self.operator = Operator(self.db)
config = MephistoConfig(
blueprint=MockBlueprintArgs(
num_assignments=1,
is_concurrent=False,
),
provider=MockProviderArgs(requester_name=self.requester_name, ),
architect=MockArchitectArgs(should_run_server=True),
task=MOCK_TASK_ARGS,
)
self.operator.validate_and_run_config(OmegaConf.structured(config))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.assertIsNotNone(tracked_run)
self.assertIsNotNone(tracked_run.task_launcher)
self.assertIsNotNone(tracked_run.task_runner)
self.assertIsNotNone(tracked_run.architect)
self.assertIsNotNone(tracked_run.task_run)
self.assertEqual(tracked_run.task_run.db_id, task_run_id)
# Create two agents to step through the task
architect = tracked_run.architect
self.assertIsInstance(architect, MockArchitect,
"Must use mock in testing")
# Register a worker
mock_worker_name = "MOCK_WORKER"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id = workers[0].db_id
self.assertEqual(len(tracked_run.task_runner.running_assignments), 0)
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT"
architect.server.register_mock_agent(worker_id, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Agent was not created properly")
agent = agents[0]
self.assertIsNotNone(agent)
# Register another worker
mock_worker_name = "MOCK_WORKER_2"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id = workers[0].db_id
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id, mock_agent_details)
# Give up to 5 seconds for both tasks to complete
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
# Ensure the assignment is completed
task_run = tracked_run.task_run
assignment = task_run.get_assignments()[0]
self.assertEqual(assignment.get_status(), AssignmentState.COMPLETED)
def test_run_jobs_with_restrictions(self):
"""Ensure allowed_concurrent and maximum_units_per_worker work"""
self.operator = Operator(self.db)
provider_args = MockProviderArgs(requester_name=self.requester_name, )
architect_args = MockArchitectArgs(should_run_server=True)
config = MephistoConfig(
blueprint=MockBlueprintArgs(
num_assignments=3,
is_concurrent=True,
),
provider=provider_args,
architect=architect_args,
task=TaskConfigArgs(
task_title="title",
task_description="This is a description",
task_reward="0.3",
task_tags="1,2,3",
maximum_units_per_worker=2,
allowed_concurrent=1,
task_name='max-unit-test',
),
)
self.operator.validate_and_run_config(OmegaConf.structured(config))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.assertIsNotNone(tracked_run)
self.assertIsNotNone(tracked_run.task_launcher)
self.assertIsNotNone(tracked_run.task_runner)
self.assertIsNotNone(tracked_run.architect)
self.assertIsNotNone(tracked_run.task_run)
self.assertEqual(tracked_run.task_run.db_id, task_run_id)
self.await_server_start(tracked_run.architect)
# Create two agents to step through the task
architect = tracked_run.architect
self.assertIsInstance(architect, MockArchitect,
"Must use mock in testing")
# Register a worker
mock_worker_name = "MOCK_WORKER"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_1 = workers[0].db_id
self.assertEqual(len(tracked_run.task_runner.running_assignments), 0)
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Agent was not created properly")
agent = agents[0]
self.assertIsNotNone(agent)
# Try to register a second agent, which should fail due to concurrency
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Second agent was created")
# Register another worker
mock_worker_name = "MOCK_WORKER_2"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_2 = workers[0].db_id
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 2, "Second agent was not created")
# wait for task to pass
self.wait_for_complete_assignment(
agents[1].get_unit().get_assignment(), 3)
# Pass a second task as well
mock_agent_details = "FAKE_ASSIGNMENT_3"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 3, "Agent was not created properly")
mock_agent_details = "FAKE_ASSIGNMENT_4"
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 4, "Fourth agent was not created")
# wait for task to pass
self.wait_for_complete_assignment(
agents[3].get_unit().get_assignment(), 3)
# Both workers should have saturated their tasks, and not be granted agents
mock_agent_details = "FAKE_ASSIGNMENT_5"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 4, "Additional agent was created")
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 4, "Additional agent was created")
# new workers should be able to work on these just fine though
mock_worker_name = "MOCK_WORKER_3"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_3 = workers[0].db_id
mock_worker_name = "MOCK_WORKER_4"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_4 = workers[0].db_id
# Register agents from new workers
mock_agent_details = "FAKE_ASSIGNMENT_5"
architect.server.register_mock_agent(worker_id_3, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 5, "Additional agent was not created")
mock_agent_details = "FAKE_ASSIGNMENT_6"
architect.server.register_mock_agent(worker_id_4, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 6, "Additional agent was not created")
# wait for task to pass
self.wait_for_complete_assignment(
agents[5].get_unit().get_assignment(), 3)
# Give up to 5 seconds for whole mock task to complete
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
# Ensure all assignments are completed
task_run = tracked_run.task_run
assignments = task_run.get_assignments()
for assignment in assignments:
self.assertEqual(assignment.get_status(),
AssignmentState.COMPLETED)
# Create a new task
config = MephistoConfig(
blueprint=MockBlueprintArgs(
num_assignments=1,
is_concurrent=True,
),
provider=MockProviderArgs(requester_name=self.requester_name, ),
architect=MockArchitectArgs(should_run_server=True),
task=TaskConfigArgs(
task_title="title",
task_description="This is a description",
task_reward="0.3",
task_tags="1,2,3",
maximum_units_per_worker=2,
allowed_concurrent=1,
task_name='max-unit-test',
),
)
self.operator.validate_and_run_config(OmegaConf.structured(config))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.await_server_start(tracked_run.architect)
architect = tracked_run.architect
# Workers one and two still shouldn't be able to make agents
mock_agent_details = "FAKE_ASSIGNMENT_7"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(
len(agents),
6,
"Additional agent was created for worker exceeding max units",
)
mock_agent_details = "FAKE_ASSIGNMENT_7"
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(
len(agents),
6,
"Additional agent was created for worker exceeding max units",
)
# Three and four should though
mock_agent_details = "FAKE_ASSIGNMENT_7"
architect.server.register_mock_agent(worker_id_3, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 7, "Additional agent was not created")
mock_agent_details = "FAKE_ASSIGNMENT_8"
architect.server.register_mock_agent(worker_id_4, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 8, "Additional agent was not created")
# Ensure the task run completed and that all assignments are done
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
task_run = tracked_run.task_run
assignments = task_run.get_assignments()
for assignment in assignments:
self.assertEqual(assignment.get_status(),
AssignmentState.COMPLETED)
def test_run_jobs_with_restrictions(self):
"""Ensure allowed_concurrent and maximum_units_per_worker work"""
self.operator = Operator(self.db)
provider_args = MockProviderArgs(requester_name=self.requester_name, )
architect_args = MockArchitectArgs(should_run_server=True)
config = MephistoConfig(
blueprint=MockBlueprintArgs(
num_assignments=3,
is_concurrent=True,
),
provider=provider_args,
architect=architect_args,
task=TaskConfigArgs(
task_title="title",
task_description="This is a description",
task_reward="0.3",
task_tags="1,2,3",
maximum_units_per_worker=2,
allowed_concurrent=1,
task_name='max-unit-test',
),
)
self.operator.validate_and_run_config(OmegaConf.structured(config))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.assertIsNotNone(tracked_run)
self.assertIsNotNone(tracked_run.task_launcher)
self.assertIsNotNone(tracked_run.task_runner)
self.assertIsNotNone(tracked_run.architect)
self.assertIsNotNone(tracked_run.task_run)
self.assertEqual(tracked_run.task_run.db_id, task_run_id)
self.await_server_start(tracked_run.architect)
# Create two agents to step through the task
architect = tracked_run.architect
self.assertIsInstance(architect, MockArchitect,
"Must use mock in testing")
# Register a worker
mock_worker_name = "MOCK_WORKER"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_1 = workers[0].db_id
self.assertEqual(len(tracked_run.task_runner.running_assignments), 0)
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Agent was not created properly")
agent = agents[0]
self.assertIsNotNone(agent)
# Try to register a second agent, which should fail due to concurrency
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Second agent was created")
# Register another worker
mock_worker_name = "MOCK_WORKER_2"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_2 = workers[0].db_id
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 2, "Second agent was not created")
# wait for task to pass
self.wait_for_complete_assignment(
agents[1].get_unit().get_assignment(), 3)
# Pass a second task as well
mock_agent_details = "FAKE_ASSIGNMENT_3"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 3, "Agent was not created properly")
mock_agent_details = "FAKE_ASSIGNMENT_4"
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 4, "Fourth agent was not created")
# wait for task to pass
self.wait_for_complete_assignment(
agents[3].get_unit().get_assignment(), 3)
# Both workers should have saturated their tasks, and not be granted agents
mock_agent_details = "FAKE_ASSIGNMENT_5"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 4, "Additional agent was created")
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 4, "Additional agent was created")
# new workers should be able to work on these just fine though
mock_worker_name = "MOCK_WORKER_3"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_3 = workers[0].db_id
mock_worker_name = "MOCK_WORKER_4"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_4 = workers[0].db_id
# Register agents from new workers
mock_agent_details = "FAKE_ASSIGNMENT_5"
architect.server.register_mock_agent(worker_id_3, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 5, "Additional agent was not created")
mock_agent_details = "FAKE_ASSIGNMENT_6"
architect.server.register_mock_agent(worker_id_4, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 6, "Additional agent was not created")
# wait for task to pass
self.wait_for_complete_assignment(
agents[5].get_unit().get_assignment(), 3)
# Give up to 5 seconds for whole mock task to complete
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
# Ensure all assignments are completed
task_run = tracked_run.task_run
assignments = task_run.get_assignments()
for assignment in assignments:
self.assertEqual(assignment.get_status(),
AssignmentState.COMPLETED)
# Create a new task
config = MephistoConfig(
blueprint=MockBlueprintArgs(
num_assignments=1,
is_concurrent=True,
),
provider=MockProviderArgs(requester_name=self.requester_name, ),
architect=MockArchitectArgs(should_run_server=True),
task=TaskConfigArgs(
task_title="title",
task_description="This is a description",
task_reward="0.3",
task_tags="1,2,3",
maximum_units_per_worker=2,
allowed_concurrent=1,
task_name='max-unit-test',
),
)
self.operator.validate_and_run_config(OmegaConf.structured(config))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.await_server_start(tracked_run.architect)
architect = tracked_run.architect
# Workers one and two still shouldn't be able to make agents
mock_agent_details = "FAKE_ASSIGNMENT_7"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(
len(agents),
6,
"Additional agent was created for worker exceeding max units",
)
mock_agent_details = "FAKE_ASSIGNMENT_7"
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(
len(agents),
6,
"Additional agent was created for worker exceeding max units",
)
# Three and four should though
mock_agent_details = "FAKE_ASSIGNMENT_7"
architect.server.register_mock_agent(worker_id_3, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 7, "Additional agent was not created")
mock_agent_details = "FAKE_ASSIGNMENT_8"
architect.server.register_mock_agent(worker_id_4, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 8, "Additional agent was not created")
# Ensure the task run completed and that all assignments are done
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
task_run = tracked_run.task_run
assignments = task_run.get_assignments()
for assignment in assignments:
self.assertEqual(assignment.get_status(),
AssignmentState.COMPLETED)
f'--task-description "\\"{task_description}\\"" '
"--task-reward 0.5 "
f"--task-tags {hit_keywords} "
f"--maximum-units-per-worker 0 " # Num of units a worker is allowed to do, 0 is infinite
f"--allowed-concurrent 1 " # Workers can only do one task at a time, or onboarding may break
)
extra_args = {
"pairings_filepath": args['pairings_filepath'],
"block_on_onboarding_fail": True,
"block_qualification": f"acute_eval_{int(time.time())}_block",
# num times to use the same conversation pair
"annotations_per_pair": args["annotations_per_pair"],
"random_seed": 42, # random seed
"subtasks_per_unit": args[
"subtasks_per_unit"
], # num comparisons to show within one unit
"num_matchup_pairs": args[
"num_matchup_pairs"
], # num pairs of conversations to be compared
# question phrasing
"s1_choice": "I would prefer to talk to <Speaker 1>",
"s2_choice": "I would prefer to talk to <Speaker 2>",
"eval_question": "Who would you prefer to talk to for a long conversation?",
"assignment_duration_in_seconds": 600,
}
operator = Operator(db)
operator.parse_and_launch_run_wrapper(shlex.split(ARG_STRING), extra_args=extra_args)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
},
]
}
# build the task
def build_task():
"""Rebuild the frontend for this task"""
return_dir = os.getcwd()
os.chdir(FRONTEND_SOURCE_DIR)
if os.path.exists(FRONTEND_BUILD_DIR):
shutil.rmtree(FRONTEND_BUILD_DIR)
packages_installed = subprocess.call(["npm", "install"])
if packages_installed != 0:
raise Exception("please make sure npm is installed, otherwise view "
"the above error for more info.")
webpack_complete = subprocess.call(["npm", "run", "dev"])
if webpack_complete != 0:
raise Exception("Webpack appears to have failed to build your "
"frontend. See the above error for more information.")
os.chdir(return_dir)
build_task()
operator = Operator(db)
operator.parse_and_launch_run_wrapper(shlex.split(ARG_STRING),
extra_args=extra_args)
operator.wait_for_runs_then_shutdown()
def test_base_task(self):
# Define the configuration settings
relative_task_directory = os.path.relpath(
TASK_DIRECTORY, os.path.dirname(__file__)
)
relative_config_path = os.path.join(relative_task_directory, 'conf')
with initialize(config_path=relative_config_path):
self.config = compose(
config_name="example",
overrides=[
f'+mephisto.blueprint._blueprint_type={AcuteEvalBlueprint.BLUEPRINT_TYPE}',
f'+mephisto/architect=mock',
f'+mephisto/provider=mock',
f'mephisto.blueprint.block_on_onboarding_fail={False}',
f'+task_dir={TASK_DIRECTORY}',
f'+current_time={int(time.time())}',
],
)
# TODO: when Hydra 1.1 is released with support for recursive defaults,
# don't manually specify all missing blueprint args anymore, but
# instead define the blueprint in the defaults list directly.
# Currently, the blueprint can't be set in the defaults list without
# overriding params in the YAML file, as documented at
# https://github.com/facebookresearch/hydra/issues/326 and as fixed in
# https://github.com/facebookresearch/hydra/pull/1044.
self.data_dir = tempfile.mkdtemp()
database_path = os.path.join(self.data_dir, "mephisto.db")
self.db = LocalMephistoDB(database_path)
self.config = augment_config_from_db(self.config, self.db)
self.config.mephisto.architect.should_run_server = True
self.operator = Operator(self.db)
self.operator.validate_and_run_config(
self.config.mephisto, shared_state=None
)
channel_info = list(self.operator.supervisor.channels.values())[0]
server = channel_info.job.architect.server
# Register a worker
mock_worker_name = "MOCK_WORKER"
server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id = workers[0].db_id
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT"
server.register_mock_agent(worker_id, mock_agent_details)
agent = self.db.find_agents()[0]
agent_id_1 = agent.db_id
# Set initial data
server.request_init_data(agent_id_1)
# Make agent act
server.send_agent_act(
agent_id_1, {"MEPHISTO_is_submit": True, "task_data": DESIRED_OUTPUTS}
)
# Check that the inputs and outputs are as expected
agent = self.db.find_agents()[0]
state = agent.state.get_data()
self.assertEqual(DESIRED_INPUTS, state['inputs'])
self.assertEqual(DESIRED_OUTPUTS, state['outputs'])
class CrowdsourcingTestMixin:
"""
Mixin for end-to-end tests of Mephisto-based crowdsourcing tasks.
Allows for setup and teardown of the operator, as well as for config specification
and agent registration.
"""
def setUp(self):
self.operator = None
def tearDown(self):
if self.operator is not None:
self.operator.shutdown()
def _set_up_config(
self,
blueprint_type: str,
task_directory: str,
overrides: Optional[List[str]] = None,
):
"""
Set up the config and database.
Uses the Hydra compose() API for unit testing and a temporary directory to store
the test database.
:param blueprint_type: string uniquely specifying Blueprint class
:param task_directory: directory containing the `conf/` configuration folder.
Will be injected as `${task_dir}` in YAML files.
:param overrides: additional config overrides
"""
# Define the configuration settings
relative_task_directory = os.path.relpath(task_directory,
os.path.dirname(__file__))
relative_config_path = os.path.join(relative_task_directory, 'conf')
if overrides is None:
overrides = []
with initialize(config_path=relative_config_path):
self.config = compose(
config_name="example",
overrides=[
f'+mephisto.blueprint._blueprint_type={blueprint_type}',
f'+mephisto/architect=mock',
f'+mephisto/provider=mock',
f'+task_dir={task_directory}',
f'+current_time={int(time.time())}',
] + overrides,
)
# TODO: when Hydra 1.1 is released with support for recursive defaults,
# don't manually specify all missing blueprint args anymore, but
# instead define the blueprint in the defaults list directly.
# Currently, the blueprint can't be set in the defaults list without
# overriding params in the YAML file, as documented at
# https://github.com/facebookresearch/hydra/issues/326 and as fixed in
# https://github.com/facebookresearch/hydra/pull/1044.
self.data_dir = tempfile.mkdtemp()
database_path = os.path.join(self.data_dir, "mephisto.db")
self.db = LocalMephistoDB(database_path)
self.config = augment_config_from_db(self.config, self.db)
self.config.mephisto.architect.should_run_server = True
def _set_up_server(self, shared_state: Optional[SharedTaskState] = None):
"""
Set up the operator and server.
"""
self.operator = Operator(self.db)
self.operator.validate_and_run_config(self.config.mephisto,
shared_state=shared_state)
channel_info = list(self.operator.supervisor.channels.values())[0]
self.server = channel_info.job.architect.server
def _register_mock_agents(self, num_agents: int = 1) -> List[str]:
"""
Register mock agents for testing, taking the place of crowdsourcing workers.
Specify the number of agents to register. Return the agents' IDs after creation.
"""
for idx in range(num_agents):
# Register the worker
mock_worker_name = f"MOCK_WORKER_{idx:d}"
self.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id = workers[0].db_id
# Register the agent
mock_agent_details = f"FAKE_ASSIGNMENT_{idx:d}"
self.server.register_mock_agent(worker_id, mock_agent_details)
# Get all agents' IDs
agents = self.db.find_agents()
agent_ids = [agent.db_id for agent in agents]
return agent_ids
def test_run_job_concurrent(self):
"""Ensure that the supervisor object can even be created"""
self.operator = Operator(self.db)
ARG_STRING = ("--blueprint-type mock "
"--architect-type mock "
f"--requester-name {self.requester_name} "
"--num-assignments 1 "
"--task-title title "
"--task-description description "
"--task-reward 0.3 "
"--task-tags 1,2,3 "
"--should-run-server true "
"--is-concurrent true ")
self.operator.parse_and_launch_run(shlex.split(ARG_STRING))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.assertIsNotNone(tracked_run)
self.assertIsNotNone(tracked_run.task_launcher)
self.assertIsNotNone(tracked_run.task_runner)
self.assertIsNotNone(tracked_run.architect)
self.assertIsNotNone(tracked_run.task_run)
self.assertEqual(tracked_run.task_run.db_id, task_run_id)
# Create two agents to step through the task
architect = tracked_run.architect
self.assertIsInstance(architect, MockArchitect,
"Must use mock in testing")
# Register a worker
mock_worker_name = "MOCK_WORKER"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id = workers[0].db_id
self.assertEqual(len(tracked_run.task_runner.running_assignments), 0)
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT"
architect.server.register_mock_agent(worker_id, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Agent was not created properly")
agent = agents[0]
self.assertIsNotNone(agent)
# Register another worker
mock_worker_name = "MOCK_WORKER_2"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id = workers[0].db_id
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id, mock_agent_details)
# Give up to 5 seconds for whole mock task to complete
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
# Ensure the assignment is completed
task_run = tracked_run.task_run
assignment = task_run.get_assignments()[0]
self.assertEqual(assignment.get_status(), AssignmentState.COMPLETED)
def test_run_jobs_with_restrictions(self):
"""Ensure allowed_concurrent and maximum_units_per_worker work"""
self.operator = Operator(self.db)
ARG_STRING = ("--blueprint-type mock "
"--architect-type mock "
f"--requester-name {self.requester_name} "
"--num-assignments 3 "
"--task-title title "
"--task-description description "
"--task-reward 0.3 "
"--task-tags 1,2,3 "
"--should-run-server true "
"--is-concurrent true "
"--task-name max-unit-test "
"--allowed-concurrent 1 "
"--maximum-units-per-worker 2 ")
self.operator.parse_and_launch_run(shlex.split(ARG_STRING))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.assertIsNotNone(tracked_run)
self.assertIsNotNone(tracked_run.task_launcher)
self.assertIsNotNone(tracked_run.task_runner)
self.assertIsNotNone(tracked_run.architect)
self.assertIsNotNone(tracked_run.task_run)
self.assertEqual(tracked_run.task_run.db_id, task_run_id)
self.await_server_start(tracked_run.architect)
# Create two agents to step through the task
architect = tracked_run.architect
self.assertIsInstance(architect, MockArchitect,
"Must use mock in testing")
# Register a worker
mock_worker_name = "MOCK_WORKER"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_1 = workers[0].db_id
self.assertEqual(len(tracked_run.task_runner.running_assignments), 0)
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Agent was not created properly")
agent = agents[0]
self.assertIsNotNone(agent)
# Try to register a second agent, which should fail due to concurrency
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Second agent was created")
# Register another worker
mock_worker_name = "MOCK_WORKER_2"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_2 = workers[0].db_id
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 2, "Second agent was not created")
# wait for task to pass
self.wait_for_complete_assignment(
agents[1].get_unit().get_assignment(), 3)
# Pass a second task as well
mock_agent_details = "FAKE_ASSIGNMENT_3"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 3, "Agent was not created properly")
mock_agent_details = "FAKE_ASSIGNMENT_4"
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 4, "Fourth agent was not created")
# wait for task to pass
self.wait_for_complete_assignment(
agents[3].get_unit().get_assignment(), 3)
# Both workers should have saturated their tasks, and not be granted agents
mock_agent_details = "FAKE_ASSIGNMENT_5"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 4, "Additional agent was created")
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 4, "Additional agent was created")
# new workers should be able to work on these just fine though
mock_worker_name = "MOCK_WORKER_3"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_3 = workers[0].db_id
mock_worker_name = "MOCK_WORKER_4"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_4 = workers[0].db_id
# Register agents from new workers
mock_agent_details = "FAKE_ASSIGNMENT_5"
architect.server.register_mock_agent(worker_id_3, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 5, "Additional agent was not created")
mock_agent_details = "FAKE_ASSIGNMENT_6"
architect.server.register_mock_agent(worker_id_4, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 6, "Additional agent was not created")
# wait for task to pass
self.wait_for_complete_assignment(
agents[5].get_unit().get_assignment(), 3)
# Give up to 5 seconds for whole mock task to complete
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
# Ensure all assignments are completed
task_run = tracked_run.task_run
assignments = task_run.get_assignments()
for assignment in assignments:
self.assertEqual(assignment.get_status(),
AssignmentState.COMPLETED)
# Create a new task
ARG_STRING = ("--blueprint-type mock "
"--architect-type mock "
f"--requester-name {self.requester_name} "
"--num-assignments 1 "
"--task-title title "
"--task-description description "
"--task-reward 0.3 "
"--task-tags 1,2,3 "
"--should-run-server true "
"--is-concurrent true "
"--task-name max-unit-test "
"--allowed-concurrent 1 "
"--maximum-units-per-worker 2 ")
self.operator.parse_and_launch_run(shlex.split(ARG_STRING))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.await_server_start(tracked_run.architect)
architect = tracked_run.architect
# Workers one and two still shouldn't be able to make agents
mock_agent_details = "FAKE_ASSIGNMENT_7"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(
len(agents),
6,
"Additional agent was created for worker exceeding max units",
)
mock_agent_details = "FAKE_ASSIGNMENT_7"
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(
len(agents),
6,
"Additional agent was created for worker exceeding max units",
)
# Three and four should though
mock_agent_details = "FAKE_ASSIGNMENT_7"
architect.server.register_mock_agent(worker_id_3, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 7, "Additional agent was not created")
mock_agent_details = "FAKE_ASSIGNMENT_8"
architect.server.register_mock_agent(worker_id_4, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 8, "Additional agent was not created")
# Ensure the task run completed and that all assignments are done
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
task_run = tracked_run.task_run
assignments = task_run.get_assignments()
for assignment in assignments:
self.assertEqual(assignment.get_status(),
AssignmentState.COMPLETED)
