def main(cfg: DictConfig) -> None:
def onboarding_is_valid(onboarding_data):
outputs = onboarding_data["outputs"]
answer_str = outputs["answer"]
# NOTE: depending on which OS Turker uses, there could be carriage returns \r or just newlines \n
# this python module should handle all cases
commands = answer_str.splitlines()
# filter empty commands
filtered_commands = [x for x in commands if x != ""]
# Number check: Check that the number of commands >= 3
if len(commands) < 3:
return False
# Length check: Check that the average number of words in commands > 4
commands_split = [x.split(" ") for x in filtered_commands]
avg_words_in_commands = sum(map(len,
commands_split)) / len(commands_split)
if avg_words_in_commands < 2:
return False
# Diversity check: Check that commands are reasonably diverse
first_words = [x[0] for x in commands_split]
if len(set(first_words)) == 1:
return False
# TODO: Grammar check: Check that there is punctuation, capitals
return True
shared_state = SharedStaticTaskState(qualifications=[
make_qualification_dict(ALLOWLIST_QUALIFICATION, QUAL_EXISTS, None),
], )
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
# operator.validate_and_run_config(cfg.mephisto)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None:
shared_state = SharedStaticTaskState(qualifications=[
make_qualification_dict(ALLOWLIST_QUALIFICATION, QUAL_EXISTS, None),
], )
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def main(cfg: DictConfig) -> None:
db, cfg = load_db_and_process_config(cfg)
operator = Operator(db)
validator = validate_unit
shared_state = SharedStaticTaskState(on_unit_submitted=validator, )
# Do not allow workers to take pilot task the second time
shared_state.qualifications = [
make_qualification_dict(
PILOT_BLOCK_QUAL_NAME,
QUAL_NOT_EXIST,
None,
),
]
operator.validate_and_run_config(cfg.mephisto, shared_state)
operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30)
def validate_and_run_config_or_die(
self,
run_config: DictConfig,
shared_state: Optional[SharedTaskState] = None) -> str:
"""
Parse the given arguments and launch a job.
"""
if shared_state is None:
shared_state = SharedTaskState()
# First try to find the requester:
requester_name = run_config.provider.requester_name
requesters = self.db.find_requesters(requester_name=requester_name)
if len(requesters) == 0:
if run_config.provider.requester_name == "MOCK_REQUESTER":
requesters = [get_mock_requester(self.db)]
else:
raise EntryDoesNotExistException(
f"No requester found with name {requester_name}")
requester = requesters[0]
requester_id = requester.db_id
provider_type = requester.provider_type
assert provider_type == run_config.provider._provider_type, (
f"Found requester for name {requester_name} is not "
f"of the specified type {run_config.provider._provider_type}, "
f"but is instead {provider_type}.")
# Next get the abstraction classes, and run validation
# before anything is actually created in the database
blueprint_type = run_config.blueprint._blueprint_type
architect_type = run_config.architect._architect_type
BlueprintClass = get_blueprint_from_type(blueprint_type)
ArchitectClass = get_architect_from_type(architect_type)
CrowdProviderClass = get_crowd_provider_from_type(provider_type)
BlueprintClass.assert_task_args(run_config, shared_state)
ArchitectClass.assert_task_args(run_config, shared_state)
CrowdProviderClass.assert_task_args(run_config, shared_state)
# Find an existing task or create a new one
task_name = run_config.task.get("task_name", None)
if task_name is None:
task_name = blueprint_type
logger.warning(
f"Task is using the default blueprint name {task_name} as a name, "
"as no task_name is provided")
tasks = self.db.find_tasks(task_name=task_name)
task_id = None
if len(tasks) == 0:
task_id = self.db.new_task(task_name, blueprint_type)
else:
task_id = tasks[0].db_id
logger.info(f"Creating a task run under task name: {task_name}")
# Create a new task run
new_run_id = self.db.new_task_run(
task_id,
requester_id,
json.dumps(OmegaConf.to_container(run_config, resolve=True)),
provider_type,
blueprint_type,
requester.is_sandbox(),
)
task_run = TaskRun(self.db, new_run_id)
try:
# Register the blueprint with args to the task run,
# ensure cached
blueprint = task_run.get_blueprint(args=run_config,
shared_state=shared_state)
# If anything fails after here, we have to cleanup the architect
build_dir = os.path.join(task_run.get_run_dir(), "build")
os.makedirs(build_dir, exist_ok=True)
architect = ArchitectClass(self.db, run_config, shared_state,
task_run, build_dir)
# Setup and deploy the server
built_dir = architect.prepare()
task_url = architect.deploy()
# TODO(#102) maybe the cleanup (destruction of the server configuration?) should only
# happen after everything has already been reviewed, this way it's possible to
# retrieve the exact build directory to review a task for real
architect.cleanup()
# Create the backend runner
task_runner = BlueprintClass.TaskRunnerClass(
task_run, run_config, shared_state)
# Small hack for auto appending block qualification
existing_qualifications = shared_state.qualifications
if run_config.blueprint.get("block_qualification",
None) is not None:
existing_qualifications.append(
make_qualification_dict(
run_config.blueprint.block_qualification,
QUAL_NOT_EXIST, None))
if run_config.blueprint.get("onboarding_qualification",
None) is not None:
existing_qualifications.append(
make_qualification_dict(
OnboardingRequired.get_failed_qual(
run_config.blueprint.onboarding_qualification),
QUAL_NOT_EXIST,
None,
))
shared_state.qualifications = existing_qualifications
# Register the task with the provider
provider = CrowdProviderClass(self.db)
provider.setup_resources_for_task_run(task_run, run_config,
shared_state, task_url)
initialization_data_array = blueprint.get_initialization_data()
# Link the job together
job = self.supervisor.register_job(architect, task_runner,
provider,
existing_qualifications)
if self.supervisor.sending_thread is None:
self.supervisor.launch_sending_thread()
except (KeyboardInterrupt, Exception) as e:
logger.error(
"Encountered error while launching run, shutting down",
exc_info=True)
try:
architect.shutdown()
except (KeyboardInterrupt, Exception) as architect_exception:
logger.exception(
f"Could not shut down architect: {architect_exception}",
exc_info=True,
)
raise e
launcher = TaskLauncher(self.db, task_run, initialization_data_array)
launcher.create_assignments()
launcher.launch_units(task_url)
self._task_runs_tracked[task_run.db_id] = TrackedRun(
task_run=task_run,
task_launcher=launcher,
task_runner=task_runner,
architect=architect,
job=job,
)
task_run.update_completion_progress(status=False)
return task_run.db_id
def parse_and_launch_run(
self,
arg_list: Optional[List[str]] = None,
extra_args: Optional[Dict[str, Any]] = None,
) -> str:
"""
Parse the given arguments and launch a job.
"""
if extra_args is None:
extra_args = {}
# Extract the abstractions being used
parser = self._get_baseline_argparser()
type_args, task_args_string = parser.parse_known_args(arg_list)
requesters = self.db.find_requesters(requester_name=type_args.requester_name)
if len(requesters) == 0:
raise EntryDoesNotExistException(
f"No requester found with name {type_args.requester_name}"
)
requester = requesters[0]
requester_id = requester.db_id
provider_type = requester.provider_type
# Parse the arguments for the abstractions to ensure
# everything required is set
BlueprintClass = get_blueprint_from_type(type_args.blueprint_type)
ArchitectClass = get_architect_from_type(type_args.architect_type)
CrowdProviderClass = get_crowd_provider_from_type(provider_type)
task_args, _unknown = self._parse_args_from_classes(
BlueprintClass, ArchitectClass, CrowdProviderClass, task_args_string
)
task_args.update(extra_args)
# Load the classes to force argument validation before anything
# is actually created in the database
# TODO(#94) perhaps parse the arguments for these things one at a time?
BlueprintClass.assert_task_args(task_args)
ArchitectClass.assert_task_args(task_args)
CrowdProviderClass.assert_task_args(task_args)
# Find an existing task or create a new one
task_name = task_args.get("task_name")
if task_name is None:
task_name = type_args.blueprint_type
logger.warning(
f"Task is using the default blueprint name {task_name} as a name, as no task_name is provided"
)
tasks = self.db.find_tasks(task_name=task_name)
task_id = None
if len(tasks) == 0:
task_id = self.db.new_task(task_name, type_args.blueprint_type)
else:
task_id = tasks[0].db_id
logger.info(f"Creating a task run under task name: {task_name}")
# Create a new task run
new_run_id = self.db.new_task_run(
task_id,
requester_id,
" ".join([shlex.quote(x) for x in task_args_string]),
provider_type,
type_args.blueprint_type,
requester.is_sandbox(),
)
task_run = TaskRun(self.db, new_run_id)
try:
# If anything fails after here, we have to cleanup the architect
build_dir = os.path.join(task_run.get_run_dir(), "build")
os.makedirs(build_dir, exist_ok=True)
architect = ArchitectClass(self.db, task_args, task_run, build_dir)
# Register the blueprint with args to the task run,
# ensure cached
blueprint = BlueprintClass(task_run, task_args)
task_run.get_blueprint(opts=task_args)
# Setup and deploy the server
built_dir = architect.prepare()
task_url = architect.deploy()
# TODO(#102) maybe the cleanup (destruction of the server configuration?) should only
# happen after everything has already been reviewed, this way it's possible to
# retrieve the exact build directory to review a task for real
architect.cleanup()
# Create the backend runner
task_runner = BlueprintClass.TaskRunnerClass(task_run, task_args)
# Small hack for auto appending block qualification
existing_qualifications = task_args.get("qualifications", [])
if task_args.get("block_qualification") is not None:
existing_qualifications.append(
make_qualification_dict(
task_args["block_qualification"], QUAL_NOT_EXIST, None
)
)
if task_args.get("onboarding_qualification") is not None:
existing_qualifications.append(
make_qualification_dict(
OnboardingRequired.get_failed_qual(
task_args["onboarding_qualification"]
),
QUAL_NOT_EXIST,
None,
)
)
task_args["qualifications"] = existing_qualifications
# Register the task with the provider
provider = CrowdProviderClass(self.db)
provider.setup_resources_for_task_run(task_run, task_args, task_url)
initialization_data_array = blueprint.get_initialization_data()
# Link the job together
job = self.supervisor.register_job(
architect, task_runner, provider, existing_qualifications
)
if self.supervisor.sending_thread is None:
self.supervisor.launch_sending_thread()
except (KeyboardInterrupt, Exception) as e:
logger.error(
"Encountered error while launching run, shutting down", exc_info=True
)
try:
architect.shutdown()
except (KeyboardInterrupt, Exception) as architect_exception:
logger.exception(
f"Could not shut down architect: {architect_exception}",
exc_info=True,
)
raise e
launcher = TaskLauncher(self.db, task_run, initialization_data_array)
launcher.create_assignments()
launcher.launch_units(task_url)
self._task_runs_tracked[task_run.db_id] = TrackedRun(
task_run=task_run,
task_launcher=launcher,
task_runner=task_runner,
architect=architect,
job=job,
)
return task_run.db_id