def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(run_config=cfg.mephisto,
shared_state=None)
operator.wait_for_runs_then_shutdown(skip_input=True,
log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None:
def onboarding_is_valid(onboarding_data):
outputs = onboarding_data["outputs"]
answer_str = outputs["answer"]
# NOTE: depending on which OS Turker uses, there could be carriage returns \r or just newlines \n
# this python module should handle all cases
commands = answer_str.splitlines()
# filter empty commands
filtered_commands = [x for x in commands if x != ""]
# Number check: Check that the number of commands >= 3
if len(commands) < 3:
return False
# Length check: Check that the average number of words in commands > 4
commands_split = [x.split(" ") for x in filtered_commands]
avg_words_in_commands = sum(map(len,
commands_split)) / len(commands_split)
if avg_words_in_commands < 2:
return False
# Diversity check: Check that commands are reasonably diverse
first_words = [x[0] for x in commands_split]
if len(set(first_words)) == 1:
return False
# TODO: Grammar check: Check that there is punctuation, capitals
return True
shared_state = SharedStaticTaskState(
onboarding_data={},
validate_onboarding=onboarding_is_valid,
)
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
world_opt = get_world_opt(cfg)
onboarding_world_opt = get_onboarding_world_opt(cfg)
shared_state = SharedParlAITaskState(
world_opt=world_opt, onboarding_world_opt=onboarding_world_opt)
check_role_training_qualification(
db=db,
qname=world_opt[constants.ROLE_QUALIFICATION_NAME_KEY],
requester_name=cfg.mephisto.provider.requester_name,
)
shared_state.task_config['minTurns'] = world_opt['min_turns']
shared_state.task_config[
'onboardingPersona'] = constants.ONBOARDING_PERSONA
shared_state.worker_can_do_unit = get_worker_eval_function(
world_opt[constants.ROLE_QUALIFICATION_NAME_KEY],
onboarding_world_opt['onboarding_qualification'],
)
banned_words_fpath = cfg.mephisto.blueprint.banned_words_file
add_banned_words_frontend_conf(shared_state, banned_words_fpath)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=300)
update_persona_use_counts_file(cfg.mephisto.blueprint.persona_counts_file,
world_opt['prev_persona_count'])
def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
parser = ParlaiParser(True, False)
opt = parser.parse_args(
list(chain.from_iterable(
('--' + k, v) for k, v in cfg.teacher.items())))
agent = RepeatLabelAgent(opt)
teacher = create_task(opt, agent).get_task_agent()
world_opt = {"turn_timeout": cfg.turn_timeout, "teacher": teacher}
custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle",
None)
if custom_bundle_path is not None:
assert os.path.exists(custom_bundle_path), (
"Must build the custom bundle with `npm install; npm run dev` from within "
f"the {TASK_DIRECTORY}/webapp directory in order to demo a custom bundle "
)
world_opt["send_task_data"] = True
shared_state = SharedParlAITaskState(world_opt=world_opt,
onboarding_world_opt=world_opt)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def run_static_task(cfg: DictConfig, task_directory: str):
"""
Run static task, given configuration.
"""
db, cfg = load_db_and_process_config(cfg)
print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}')
random.seed(42)
task_name = cfg.mephisto.task.get('task_name', 'turn_annotations_static')
soft_block_qual_name = cfg.mephisto.blueprint.get('block_qualification',
f'{task_name}_block')
# Default to a task-specific name to avoid soft-block collisions
soft_block_mturk_workers(cfg=cfg,
db=db,
soft_block_qual_name=soft_block_qual_name)
build_task(task_directory)
operator = Operator(db)
operator.validate_and_run_config(run_config=cfg.mephisto,
shared_state=None)
operator.wait_for_runs_then_shutdown(skip_input=True,
log_rate=cfg.monitoring_log_rate)
def _set_up_server(self, shared_state: Optional[SharedTaskState] = None):
"""
Set up the operator and server.
"""
self.operator = Operator(self.db)
self.operator.validate_and_run_config(self.config.mephisto,
shared_state=shared_state)
self.server = self._get_channel_info().job.architect.server
def _set_up_server(self, shared_state: Optional[SharedTaskState] = None):
"""
Set up the operator and server.
"""
self.operator = Operator(self.db)
self.operator.validate_and_run_config(self.config.mephisto,
shared_state=shared_state)
channel_info = list(self.operator.supervisor.channels.values())[0]
self.server = channel_info.job.architect.server
def main(cfg: DictConfig) -> None:
shared_state = SharedStaticTaskState(qualifications=[
make_qualification_dict(ALLOWLIST_QUALIFICATION, QUAL_EXISTS, None),
], )
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def run_acute_eval(self):
"""
Run ACUTE Eval.
"""
self.set_up_acute_eval()
db, cfg = load_db_and_process_config(self.args)
operator = Operator(db)
operator.validate_and_run_config(run_config=cfg.mephisto, shared_state=None)
operator.wait_for_runs_then_shutdown(
skip_input=True, log_rate=cfg.monitoring_log_rate
)
def process_config_and_get_operator(cfg: DictConfig,
print_config=False
) -> Tuple["Operator", DictConfig]:
"""
Using a Hydra DictConfig built from a TaskConfig,
return an operator for that task as well as a validated config.
Takes in an option to print out the configuration before returning
"""
db, valid_config = load_db_and_process_config(cfg,
print_config=print_config)
return Operator(db), valid_config
def run_acute_eval(self):
"""
Run ACUTE Eval.
"""
self.set_up_acute_eval()
db, cfg = load_db_and_process_config(self.args)
print(f'*** RUN ID: {cfg.mephisto.task.task_name} ***')
print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}')
operator = Operator(db)
operator.validate_and_run_config(run_config=cfg.mephisto,
shared_state=None)
operator.wait_for_runs_then_shutdown(skip_input=True,
log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None:
correct_config_answer = cfg.correct_answer
def onboarding_is_valid(onboarding_data):
inputs = onboarding_data["inputs"]
outputs = onboarding_data["outputs"]
return outputs.get("answer") == correct_config_answer
shared_state = SharedStaticTaskState(
onboarding_data={"correct_answer": correct_config_answer},
validate_onboarding=onboarding_is_valid,
)
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
validator = validate_unit
shared_state = SharedStaticTaskState(on_unit_submitted=validator, )
# Do not allow workers to take pilot task the second time
shared_state.qualifications = [
make_qualification_dict(
PILOT_BLOCK_QUAL_NAME,
QUAL_NOT_EXIST,
None,
),
]
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main():
app = Flask(
__name__, static_url_path="/static", static_folder="webapp/build/static"
)
app.config.from_object(Config)
app.register_blueprint(api, url_prefix="/api/v1")
# Register extensions
db = LocalMephistoDB()
operator = Operator(db)
if not hasattr(app, "extensions"):
app.extensions = {}
app.extensions["db"] = db
app.extensions["operator"] = operator
@app.route("/", defaults={"path": "index.html"})
@app.route("/<path:path>")
def index(path):
return send_file(os.path.join("webapp", "build", "index.html"))
@app.after_request
def after_request(response):
response.headers.add("Access-Control-Allow-Origin", "*")
response.headers.add(
"Access-Control-Allow-Headers", "Content-Type,Authorization"
)
response.headers.add(
"Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS"
)
response.headers.add("Cache-Control", "no-store")
return response
term_handler = signal.getsignal(signal.SIGINT)
def cleanup_resources(*args, **kwargs):
operator.shutdown()
db.shutdown()
term_handler(*args, **kwargs)
atexit.register(cleanup_resources)
signal.signal(signal.SIGINT, cleanup_resources)
def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
teacher = get_teacher(cfg)
world_opt = {"turn_timeout": cfg.turn_timeout, "teacher": teacher}
custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle",
None)
if custom_bundle_path is not None:
if not os.path.exists(custom_bundle_path):
build_task(TASK_DIRECTORY)
shared_state = SharedParlAITaskState(world_opt=world_opt,
onboarding_world_opt=world_opt)
operator = Operator(db)
operator.validate_and_run_config(run_config=cfg.mephisto,
shared_state=shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True,
log_rate=cfg.monitoring_log_rate)
def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
world_opt = {"num_turns": cfg.num_turns, "turn_timeout": cfg.turn_timeout}
custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle",
None)
if custom_bundle_path is not None:
assert os.path.exists(custom_bundle_path), (
"Must build the custom bundle with `npm install; npm run dev` from within "
f"the {TASK_DIRECTORY}/webapp directory in order to demo a custom bundle "
)
world_opt["send_task_data"] = True
shared_state = SharedParlAITaskState(world_opt=world_opt,
onboarding_world_opt=world_opt)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None:
task_dir = cfg.task_dir
def onboarding_always_valid(onboarding_data):
return True
shared_state = SharedStaticTaskState(
static_task_data=[
{"text": "This text is good text!"},
{"text": "This text is bad text!"},
],
validate_onboarding=onboarding_always_valid,
)
build_task(task_dir)
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def run_task(cfg: DictConfig, task_directory: str):
"""
Run task, given configuration.
"""
frontend_source_dir = os.path.join(task_directory, "webapp")
frontend_build_dir = os.path.join(frontend_source_dir, "build")
_ = frontend_build_dir # Unused at the moment
db, cfg = load_db_and_process_config(cfg)
print(f'\nHydra config:\n{OmegaConf.to_yaml(cfg)}')
random.seed(42)
# Update task name when on sandbox or local to ensure data is split.
task_name = cfg.mephisto.task.get('task_name', 'model_chat')
architect_type = cfg.mephisto.architect._architect_type
if architect_type == 'local':
task_name = f"{task_name}_local"
elif architect_type == 'mturk_sandbox':
task_name = f"{task_name}_sandbox"
cfg.mephisto.task.task_name = task_name
soft_block_qual_name = cfg.mephisto.blueprint.get('block_qualification',
f'{task_name}_block')
# Default to a task-specific name to avoid soft-block collisions
soft_block_mturk_workers(cfg=cfg,
db=db,
soft_block_qual_name=soft_block_qual_name)
# Init
shared_state = SharedModelChatTaskState(world_module=world_module)
operator = Operator(db)
operator.validate_and_run_config(run_config=cfg.mephisto,
shared_state=shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True,
log_rate=cfg.monitoring_log_rate)
from mephisto.abstractions.databases.local_database import LocalMephistoDB
import os
import atexit
import signal
app = Flask(__name__,
static_url_path="/static",
static_folder="webapp/build/static")
app.config.from_object(Config)
app.register_blueprint(api, url_prefix="/api/v1")
# Register extensions
db = LocalMephistoDB()
operator = Operator(db)
if not hasattr(app, "extensions"):
app.extensions = {}
app.extensions["db"] = db
app.extensions["operator"] = operator
@app.route("/", defaults={"path": "index.html"})
@app.route("/<path:path>")
def index(path):
return send_file(os.path.join("webapp", "build", "index.html"))
@app.after_request
def after_request(response):
response.headers.add("Access-Control-Allow-Origin", "*")
def test_run_jobs_with_restrictions(self):
"""Ensure allowed_concurrent and maximum_units_per_worker work"""
self.operator = Operator(self.db)
provider_args = MockProviderArgs(requester_name=self.requester_name)
architect_args = MockArchitectArgs(should_run_server=True)
config = MephistoConfig(
blueprint=MockBlueprintArgs(num_assignments=3, is_concurrent=True),
provider=provider_args,
architect=architect_args,
task=TaskConfigArgs(
task_title="title",
task_description="This is a description",
task_reward="0.3",
task_tags="1,2,3",
maximum_units_per_worker=2,
allowed_concurrent=1,
task_name="max-unit-test",
),
)
self.operator.validate_and_run_config(OmegaConf.structured(config))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.assertIsNotNone(tracked_run)
self.assertIsNotNone(tracked_run.task_launcher)
self.assertIsNotNone(tracked_run.task_runner)
self.assertIsNotNone(tracked_run.architect)
self.assertIsNotNone(tracked_run.task_run)
self.assertEqual(tracked_run.task_run.db_id, task_run_id)
self.await_server_start(tracked_run.architect)
# Create two agents to step through the task
architect = tracked_run.architect
self.assertIsInstance(architect, MockArchitect,
"Must use mock in testing")
# Register a worker
mock_worker_name = "MOCK_WORKER"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_1 = workers[0].db_id
self.assertEqual(len(tracked_run.task_runner.running_assignments), 0)
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Agent was not created properly")
agent = agents[0]
self.assertIsNotNone(agent)
# Try to register a second agent, which should fail due to concurrency
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Second agent was created")
# Register another worker
mock_worker_name = "MOCK_WORKER_2"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_2 = workers[0].db_id
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 2, "Second agent was not created")
# wait for task to pass
self.wait_for_complete_assignment(
agents[1].get_unit().get_assignment(), 3)
# Pass a second task as well
mock_agent_details = "FAKE_ASSIGNMENT_3"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 3, "Agent was not created properly")
mock_agent_details = "FAKE_ASSIGNMENT_4"
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 4, "Fourth agent was not created")
# wait for task to pass
self.wait_for_complete_assignment(
agents[3].get_unit().get_assignment(), 3)
# Both workers should have saturated their tasks, and not be granted agents
mock_agent_details = "FAKE_ASSIGNMENT_5"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 4, "Additional agent was created")
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 4, "Additional agent was created")
# new workers should be able to work on these just fine though
mock_worker_name = "MOCK_WORKER_3"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_3 = workers[0].db_id
mock_worker_name = "MOCK_WORKER_4"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id_4 = workers[0].db_id
# Register agents from new workers
mock_agent_details = "FAKE_ASSIGNMENT_5"
architect.server.register_mock_agent(worker_id_3, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 5, "Additional agent was not created")
mock_agent_details = "FAKE_ASSIGNMENT_6"
architect.server.register_mock_agent(worker_id_4, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 6, "Additional agent was not created")
# wait for task to pass
self.wait_for_complete_assignment(
agents[5].get_unit().get_assignment(), 3)
# Give up to 5 seconds for whole mock task to complete
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
# Ensure all assignments are completed
task_run = tracked_run.task_run
assignments = task_run.get_assignments()
for assignment in assignments:
self.assertEqual(assignment.get_status(),
AssignmentState.COMPLETED)
# Create a new task
config = MephistoConfig(
blueprint=MockBlueprintArgs(num_assignments=1, is_concurrent=True),
provider=MockProviderArgs(requester_name=self.requester_name),
architect=MockArchitectArgs(should_run_server=True),
task=TaskConfigArgs(
task_title="title",
task_description="This is a description",
task_reward="0.3",
task_tags="1,2,3",
maximum_units_per_worker=2,
allowed_concurrent=1,
task_name="max-unit-test",
),
)
self.operator.validate_and_run_config(OmegaConf.structured(config))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.await_server_start(tracked_run.architect)
architect = tracked_run.architect
# Workers one and two still shouldn't be able to make agents
mock_agent_details = "FAKE_ASSIGNMENT_7"
architect.server.register_mock_agent(worker_id_1, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(
len(agents),
6,
"Additional agent was created for worker exceeding max units",
)
mock_agent_details = "FAKE_ASSIGNMENT_7"
architect.server.register_mock_agent(worker_id_2, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(
len(agents),
6,
"Additional agent was created for worker exceeding max units",
)
# Three and four should though
mock_agent_details = "FAKE_ASSIGNMENT_7"
architect.server.register_mock_agent(worker_id_3, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 7, "Additional agent was not created")
mock_agent_details = "FAKE_ASSIGNMENT_8"
architect.server.register_mock_agent(worker_id_4, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 8, "Additional agent was not created")
# Ensure the task run completed and that all assignments are done
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
task_run = tracked_run.task_run
assignments = task_run.get_assignments()
for assignment in assignments:
self.assertEqual(assignment.get_status(),
AssignmentState.COMPLETED)
def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def test_initialize_supervisor(self):
"""Quick test to ensure that the operator can be initialized"""
self.operator = Operator(self.db)
def test_run_job_concurrent(self):
"""Ensure that the supervisor object can even be created"""
self.operator = Operator(self.db)
config = MephistoConfig(
blueprint=MockBlueprintArgs(num_assignments=1, is_concurrent=True),
provider=MockProviderArgs(requester_name=self.requester_name),
architect=MockArchitectArgs(should_run_server=True),
task=MOCK_TASK_ARGS,
)
self.operator.validate_and_run_config(OmegaConf.structured(config))
tracked_runs = self.operator.get_running_task_runs()
self.assertEqual(len(tracked_runs), 1, "Run not launched")
task_run_id, tracked_run = list(tracked_runs.items())[0]
self.assertIsNotNone(tracked_run)
self.assertIsNotNone(tracked_run.task_launcher)
self.assertIsNotNone(tracked_run.task_runner)
self.assertIsNotNone(tracked_run.architect)
self.assertIsNotNone(tracked_run.task_run)
self.assertEqual(tracked_run.task_run.db_id, task_run_id)
# Create two agents to step through the task
architect = tracked_run.architect
self.assertIsInstance(architect, MockArchitect,
"Must use mock in testing")
# Register a worker
mock_worker_name = "MOCK_WORKER"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id = workers[0].db_id
self.assertEqual(len(tracked_run.task_runner.running_assignments), 0)
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT"
architect.server.register_mock_agent(worker_id, mock_agent_details)
agents = self.db.find_agents()
self.assertEqual(len(agents), 1, "Agent was not created properly")
agent = agents[0]
self.assertIsNotNone(agent)
# Register another worker
mock_worker_name = "MOCK_WORKER_2"
architect.server.register_mock_worker(mock_worker_name)
workers = self.db.find_workers(worker_name=mock_worker_name)
worker_id = workers[0].db_id
# Register an agent
mock_agent_details = "FAKE_ASSIGNMENT_2"
architect.server.register_mock_agent(worker_id, mock_agent_details)
# Give up to 5 seconds for whole mock task to complete
start_time = time.time()
while time.time() - start_time < TIMEOUT_TIME:
if len(self.operator.get_running_task_runs()) == 0:
break
time.sleep(0.1)
self.assertLess(time.time() - start_time, TIMEOUT_TIME,
"Task not completed in time")
# Ensure the assignment is completed
task_run = tracked_run.task_run
assignment = task_run.get_assignments()[0]
self.assertEqual(assignment.get_status(), AssignmentState.COMPLETED)
