def __new__(cls, agent: Union["Agent", "OnboardingAgent"]) -> "AgentState": """Return the correct agent state for the given agent""" if cls == AgentState: from mephisto.data_model.agent import Agent from mephisto.core.registry import get_blueprint_from_type # We are trying to construct an AgentState, find what type to use and # create that instead if isinstance(agent, Agent): correct_class = get_blueprint_from_type( agent.task_type).AgentStateClass else: correct_class = get_blueprint_from_type( agent.task_type).OnboardingAgentStateClass return super().__new__(correct_class) else: # We are constructing another instance directly return super().__new__(cls)
def __new__(cls, task_run: "TaskRun", args: "DictConfig") -> "TaskBuilder": """Get the correct TaskBuilder for this task run""" from mephisto.core.registry import get_blueprint_from_type if cls == TaskBuilder: # We are trying to construct an TaskBuilder, find what type to use and # create that instead correct_class = get_blueprint_from_type(task_run.task_type).TaskBuilderClass return super().__new__(correct_class) else: # We are constructing another instance directly return super().__new__(cls)
def get_blueprint(self, opts: Optional[Dict[str, Any]] = None) -> "Blueprint": """Return the runner associated with this task run""" if self.__blueprint is None: cache = False task_args = self.get_task_config().args if opts is not None: task_args.update(opts) cache = True BlueprintClass = get_blueprint_from_type(self.task_type) if not cache: return BlueprintClass(self, task_args) self.__blueprint = BlueprintClass(self, task_args) return self.__blueprint
def __new__(cls, task_run: "TaskRun", opts: Dict[str, Any]) -> "TaskRunner": """Get the correct TaskRunner for this task run""" if cls == TaskRunner: from mephisto.core.registry import get_blueprint_from_type # We are trying to construct an AgentState, find what type to use and # create that instead correct_class = get_blueprint_from_type( task_run.task_type).TaskRunnerClass return super().__new__(correct_class) else: # We are constructing another instance directly return super().__new__(cls)
def __init__(self, task_run: "TaskRun"): self.db = task_run.db # Try to find existing parsed args arg_path = os.path.join(task_run.get_run_dir(), CONFIG_FILE_PATH) if os.path.exists(arg_path): with open(arg_path, "r") as config_file: args = json.load(config_file) else: # parse new arguments BlueprintClass = get_blueprint_from_type(task_run.task_type) CrowdProviderClass = get_crowd_provider_from_type( task_run.provider_type) param_string = task_run.param_string parser = argparse.ArgumentParser() blueprint_group = parser.add_argument_group("blueprint") BlueprintClass.add_args_to_group(blueprint_group) provider_group = parser.add_argument_group("crowd_provider") CrowdProviderClass.add_args_to_group(provider_group) task_group = parser.add_argument_group("task_config") TaskConfig.add_args_to_group(task_group) try: arg_namespace, _unknown = parser.parse_known_args( shlex.split(param_string)) except SystemExit: raise Exception(f"Argparse broke on {param_string} - must fix") args = vars(arg_namespace) with open(arg_path, "w+") as config_file: json.dump(args, config_file) # Parse out specific arguments for the task_config self.args: Dict[str, Any] = args self.task_title: str = args["task_title"] self.task_description: str = args["task_description"] self.task_reward: float = args["task_reward"] self.task_tags: List[str] = [ s.strip() for s in args["task_tags"].split(",") ] self.assignment_duration_in_seconds: int = args[ "assignment_duration_in_seconds"] self.allowed_concurrent: int = args["allowed_concurrent"] self.maximum_units_per_worker: int = args["maximum_units_per_worker"]
def get_blueprint( self, args: Optional["DictConfig"] = None, shared_state: Optional["SharedTaskState"] = None) -> "Blueprint": """Return the runner associated with this task run""" from mephisto.core.registry import get_blueprint_from_type from mephisto.data_model.blueprint import SharedTaskState if self.__blueprint is None: cache = False if args is None: args = self.args else: cache = True if shared_state is None: shared_state = SharedTaskState() BlueprintClass = get_blueprint_from_type(self.task_type) if not cache: return BlueprintClass(self, args, shared_state) self.__blueprint = BlueprintClass(self, args, shared_state) return self.__blueprint
def parse_and_launch_run( self, arg_list: Optional[List[str]] = None, extra_args: Optional[Dict[str, Any]] = None, ) -> str: """ Parse the given arguments and launch a job. """ if extra_args is None: extra_args = {} # Extract the abstractions being used parser = self._get_baseline_argparser() type_args, task_args_string = parser.parse_known_args(arg_list) requesters = self.db.find_requesters(requester_name=type_args.requester_name) if len(requesters) == 0: raise EntryDoesNotExistException( f"No requester found with name {type_args.requester_name}" ) requester = requesters[0] requester_id = requester.db_id provider_type = requester.provider_type # Parse the arguments for the abstractions to ensure # everything required is set BlueprintClass = get_blueprint_from_type(type_args.blueprint_type) ArchitectClass = get_architect_from_type(type_args.architect_type) CrowdProviderClass = get_crowd_provider_from_type(provider_type) task_args, _unknown = self._parse_args_from_classes( BlueprintClass, ArchitectClass, CrowdProviderClass, task_args_string ) task_args.update(extra_args) # Load the classes to force argument validation before anything # is actually created in the database # TODO(#94) perhaps parse the arguments for these things one at a time? BlueprintClass.assert_task_args(task_args) ArchitectClass.assert_task_args(task_args) CrowdProviderClass.assert_task_args(task_args) # Find an existing task or create a new one task_name = task_args.get("task_name") if task_name is None: task_name = type_args.blueprint_type logger.warning( f"Task is using the default blueprint name {task_name} as a name, as no task_name is provided" ) tasks = self.db.find_tasks(task_name=task_name) task_id = None if len(tasks) == 0: task_id = self.db.new_task(task_name, type_args.blueprint_type) else: task_id = tasks[0].db_id logger.info(f"Creating a task run under task name: {task_name}") # Create a new task run new_run_id = self.db.new_task_run( task_id, requester_id, " ".join([shlex.quote(x) for x in task_args_string]), provider_type, type_args.blueprint_type, requester.is_sandbox(), ) task_run = TaskRun(self.db, new_run_id) try: # If anything fails after here, we have to cleanup the architect build_dir = os.path.join(task_run.get_run_dir(), "build") os.makedirs(build_dir, exist_ok=True) architect = ArchitectClass(self.db, task_args, task_run, build_dir) # Register the blueprint with args to the task run, # ensure cached blueprint = BlueprintClass(task_run, task_args) task_run.get_blueprint(opts=task_args) # Setup and deploy the server built_dir = architect.prepare() task_url = architect.deploy() # TODO(#102) maybe the cleanup (destruction of the server configuration?) should only # happen after everything has already been reviewed, this way it's possible to # retrieve the exact build directory to review a task for real architect.cleanup() # Create the backend runner task_runner = BlueprintClass.TaskRunnerClass(task_run, task_args) # Small hack for auto appending block qualification existing_qualifications = task_args.get("qualifications", []) if task_args.get("block_qualification") is not None: existing_qualifications.append( make_qualification_dict( task_args["block_qualification"], QUAL_NOT_EXIST, None ) ) if task_args.get("onboarding_qualification") is not None: existing_qualifications.append( make_qualification_dict( OnboardingRequired.get_failed_qual( task_args["onboarding_qualification"] ), QUAL_NOT_EXIST, None, ) ) task_args["qualifications"] = existing_qualifications # Register the task with the provider provider = CrowdProviderClass(self.db) provider.setup_resources_for_task_run(task_run, task_args, task_url) initialization_data_array = blueprint.get_initialization_data() # Link the job together job = self.supervisor.register_job( architect, task_runner, provider, existing_qualifications ) if self.supervisor.sending_thread is None: self.supervisor.launch_sending_thread() except (KeyboardInterrupt, Exception) as e: logger.error( "Encountered error while launching run, shutting down", exc_info=True ) try: architect.shutdown() except (KeyboardInterrupt, Exception) as architect_exception: logger.exception( f"Could not shut down architect: {architect_exception}", exc_info=True, ) raise e launcher = TaskLauncher(self.db, task_run, initialization_data_array) launcher.create_assignments() launcher.launch_units(task_url) self._task_runs_tracked[task_run.db_id] = TrackedRun( task_run=task_run, task_launcher=launcher, task_runner=task_runner, architect=architect, job=job, ) return task_run.db_id
def validate_and_run_config_or_die( self, run_config: DictConfig, shared_state: Optional[SharedTaskState] = None, ) -> str: """ Parse the given arguments and launch a job. """ if shared_state is None: shared_state = SharedTaskState() # First try to find the requester: requester_name = run_config.provider.requester_name requesters = self.db.find_requesters(requester_name=requester_name) if len(requesters) == 0: if run_config.provider.requester_name == "MOCK_REQUESTER": requesters = [get_mock_requester(self.db)] else: raise EntryDoesNotExistException( f"No requester found with name {requester_name}") requester = requesters[0] requester_id = requester.db_id provider_type = requester.provider_type assert provider_type == run_config.provider._provider_type, ( f"Found requester for name {requester_name} is not " f"of the specified type {run_config.provider._provider_type}, " f"but is instead {provider_type}.") # Next get the abstraction classes, and run validation # before anything is actually created in the database blueprint_type = run_config.blueprint._blueprint_type architect_type = run_config.architect._architect_type BlueprintClass = get_blueprint_from_type(blueprint_type) ArchitectClass = get_architect_from_type(architect_type) CrowdProviderClass = get_crowd_provider_from_type(provider_type) BlueprintClass.assert_task_args(run_config, shared_state) ArchitectClass.assert_task_args(run_config, shared_state) CrowdProviderClass.assert_task_args(run_config, shared_state) # Find an existing task or create a new one task_name = run_config.task.get("task_name", None) if task_name is None: task_name = blueprint_type logger.warning( f"Task is using the default blueprint name {task_name} as a name, " "as no task_name is provided") tasks = self.db.find_tasks(task_name=task_name) task_id = None if len(tasks) == 0: task_id = self.db.new_task(task_name, blueprint_type) else: task_id = tasks[0].db_id logger.info(f"Creating a task run under task name: {task_name}") # Create a new task run new_run_id = self.db.new_task_run( task_id, requester_id, json.dumps(OmegaConf.to_container(run_config, resolve=True)), provider_type, blueprint_type, requester.is_sandbox(), ) task_run = TaskRun(self.db, new_run_id) try: # If anything fails after here, we have to cleanup the architect build_dir = os.path.join(task_run.get_run_dir(), "build") os.makedirs(build_dir, exist_ok=True) architect = ArchitectClass(self.db, run_config, shared_state, task_run, build_dir) # Register the blueprint with args to the task run, # ensure cached blueprint = BlueprintClass(task_run, run_config, shared_state) task_run.get_blueprint(args=run_config, shared_state=shared_state) # Setup and deploy the server built_dir = architect.prepare() task_url = architect.deploy() # TODO(#102) maybe the cleanup (destruction of the server configuration?) should only # happen after everything has already been reviewed, this way it's possible to # retrieve the exact build directory to review a task for real architect.cleanup() # Create the backend runner task_runner = BlueprintClass.TaskRunnerClass( task_run, run_config, shared_state) # Small hack for auto appending block qualification existing_qualifications = shared_state.qualifications if run_config.blueprint.get("block_qualification", None) is not None: existing_qualifications.append( make_qualification_dict( run_config.blueprint.block_qualification, QUAL_NOT_EXIST, None)) if run_config.blueprint.get("onboarding_qualification", None) is not None: existing_qualifications.append( make_qualification_dict( OnboardingRequired.get_failed_qual( run_config.blueprint.onboarding_qualification, ), QUAL_NOT_EXIST, None, )) shared_state.qualifications = existing_qualifications # Register the task with the provider provider = CrowdProviderClass(self.db) provider.setup_resources_for_task_run(task_run, run_config, task_url) initialization_data_array = blueprint.get_initialization_data() # Link the job together job = self.supervisor.register_job(architect, task_runner, provider, existing_qualifications) if self.supervisor.sending_thread is None: self.supervisor.launch_sending_thread() except (KeyboardInterrupt, Exception) as e: logger.error( "Encountered error while launching run, shutting down", exc_info=True) try: architect.shutdown() except (KeyboardInterrupt, Exception) as architect_exception: logger.exception( f"Could not shut down architect: {architect_exception}", exc_info=True, ) raise e launcher = TaskLauncher(self.db, task_run, initialization_data_array) launcher.create_assignments() launcher.launch_units(task_url) self._task_runs_tracked[task_run.db_id] = TrackedRun( task_run=task_run, task_launcher=launcher, task_runner=task_runner, architect=architect, job=job, ) task_run.update_completion_progress(status=False) return task_run.db_id
def get_help_arguments(args): if len(args) == 0: click.echo( "Usage: mephisto wut <abstraction>[=<type>] [...specific args to check]" ) return from mephisto.core.registry import ( get_blueprint_from_type, get_crowd_provider_from_type, get_architect_from_type, get_valid_blueprint_types, get_valid_provider_types, get_valid_architect_types, ) from mephisto.core.argparse_parser import get_extra_argument_dicts VALID_ABSTRACTIONS = [ "blueprint", "architect", "requester", "provider", "task" ] abstraction_equal_split = args[0].split("=", 1) abstraction = abstraction_equal_split[0] if abstraction not in VALID_ABSTRACTIONS: click.echo( f"Given abstraction {abstraction} not in valid abstractions {VALID_ABSTRACTIONS}" ) return if abstraction == "task": from mephisto.data_model.task_config import TaskConfig target_class = TaskConfig else: if len(abstraction_equal_split) == 1: # querying about the general abstraction if abstraction == "blueprint": click.echo( f"The blueprint determines the task content. Valid blueprints are {get_valid_blueprint_types()}" ) return elif abstraction == "architect": click.echo( f"The architect determines the server where a task is hosted. Valid architects are {get_valid_architect_types()}" ) return elif abstraction == "requester": click.echo( f"The requester is an account for a crowd provider. Valid requester types are {get_valid_provider_types()}. \n" "Use `mephisto requesters` to see registered requesters, and `mephisto register <requester type>` to register." ) return elif abstraction == "provider": click.echo( f"The crowd provider determines the source of the crowd workers. Valid provider are {get_valid_provider_types()}" ) return # There's a specific abstraction to check abstract_value = abstraction_equal_split[1] target_class = None valid = None if abstraction == "blueprint": try: target_class = get_blueprint_from_type(abstract_value) except: valid = get_valid_blueprint_types() elif abstraction == "architect": try: target_class = get_architect_from_type(abstract_value) except: valid = get_valid_architect_types() elif abstraction == "provider": try: target_class = get_crowd_provider_from_type(abstract_value) except: valid = get_valid_provider_types() elif abstraction == "requester": try: target_class = get_crowd_provider_from_type( abstract_value).RequesterClass except: valid = get_valid_provider_types() if valid is not None: click.echo( f"The valid types for {abstraction} are {valid}. '{abstract_value}' not found." ) return from tabulate import tabulate arg_dict = get_extra_argument_dicts(target_class)[0] click.echo(arg_dict["desc"]) checking_args = arg_dict["args"] if len(args) > 1: checking_args = { k: v for k, v in checking_args.items() if k in args[1:] } click.echo(tabulate(checking_args.values(), headers="keys"))
def get_blueprint_arguments(blueprint_type): if blueprint_type == "none": return jsonify({"success": True, "options": {}}) BlueprintClass = get_blueprint_from_type(blueprint_type) params = get_extra_argument_dicts(BlueprintClass) return jsonify({"success": True, "options": params})