def test_invalid_scheme(self): failed = False try: validate("invalid_schema", {"name": "foobar"}) except ValidationSchemaMissing as e: failed = True assert failed
def update(self, snapshot_id, config=None, stats=None, message=None, label=None, visible=None): """Update the snapshot metadata""" if not snapshot_id: raise RequiredArgumentMissing( __("error", "controller.snapshot.delete.arg", "snapshot_id")) update_snapshot_input_dict = {'id': snapshot_id} validate( "update_snapshot", { "config": config, "stats": stats, "message": message, "label": label, "visible": visible }) if config is not None: update_snapshot_input_dict['config'] = config if stats is not None: update_snapshot_input_dict['stats'] = stats if message is not None: update_snapshot_input_dict['message'] = message if label is not None: update_snapshot_input_dict['label'] = label if visible is not None: update_snapshot_input_dict['visible'] = visible return self.dal.snapshot.update(update_snapshot_input_dict)
def update(self, task_id, workspace=None, command=None, command_list=None, interactive=False): """Update the task metadata""" if not task_id: raise RequiredArgumentMissing( __("error", "controller.task.delete.arg", "id")) if command_list: command = " ".join(command_list) elif command: command_list = shlex.split(command) validate( "update_task", { "workspace": workspace, "command": command, "command_list": command_list, "interactive": interactive }) update_task_input_dict = {'id': task_id} if workspace is not None: update_task_input_dict['workspace'] = workspace if command is not None: update_task_input_dict['command'] = command if command_list is not None: update_task_input_dict['command_list'] = command_list if interactive: update_task_input_dict['interactive'] = interactive return self.dal.task.update(update_task_input_dict)
def test_validate_fail(self): failed = False try: validate("create_project", {"name": 3}) except ValidationFailed as e: failed = True assert e.errors assert isinstance(e.errors, dict) assert failed
def create(self, incoming_dictionary): # Look for existing session first and return if it exists validate("create_session", incoming_dictionary) results = self.dal.session.query({ "model_id": self.model.id, "name": incoming_dictionary['name'] }) if len(results) == 1: return results[0] session_dict = { "model_id": self.model.id, "name": incoming_dictionary["name"] } # Create new session and return return self.dal.session.create(session_dict)
def init(self, name, description): """ Initialize the project This function will initialize the project or reinitialize it the project is already initialized. Parameters ---------- name : str description : str Returns ------- bool """ is_new_model = False old_model = self.model if not self.model: is_new_model = True try: # Always validate inputs to the init function validate("create_project", { "name": name, "description": description }) # Initialize File Driver if needed if not self.file_driver.is_initialized: self.file_driver.init() # Initialize the dal if not self.dal.is_initialized: self.dal.init() # Initialize Code Driver if needed if not self.code_driver.is_initialized: self.code_driver.init() # Initialize Environment Driver if needed if not self.environment_driver.is_initialized: self.environment_driver.init() # Initialize the config JSON store self.config_store = JSONStore( os.path.join(self.home, Config().datmo_directory_name, ".config")) # Create model if new else update if is_new_model: _ = self.dal.model.create( Model({ "name": name, "description": description })) else: self._model = self.dal.model.update({ "id": self.model.id, "name": name, "description": description }) # Connect Environment Driver if needed # (not required but will warn if not present) try: if not self.environment_driver.is_connected: self.environment_driver.connect() except EnvironmentConnectFailed: self.logger.warning( __("warn", "controller.general.environment.failed")) # Build the initial default Environment (NOT NECESSARY) # self.environment_driver.build_image(tag="datmo-" + \ # self.model.name) return True except Exception: # if any error occurred with new model, ensure no initialize occurs and raise previous error # if any error occurred with existing model, ensure no updates were made, raise previous error if is_new_model: self.cleanup() else: self._model = self.dal.model.update({ "id": old_model.id, "name": old_model.name, "description": old_model.description }) raise
def create(self, dictionary, save_hardware_file=True): """Create an environment Parameters ---------- dictionary : dict optional values to populate required environment entity args paths : list, optional list of absolute or relative filepaths and/or dirpaths to collect with destination names (e.g. "/path/to/file>hello", "/path/to/file2", "/path/to/dir>newdir") (default if none provided is to pull from project environment folder and project root. If none found create default definition) name : str, optional name of the environment (default is None) description : str, optional description of the environment (default is None) save_hardware_file : bool boolean to save hardware file along with other files (default is True to save the file and create distinct hashes based on software and hardware) Returns ------- Environment returns an object representing the environment created Raises ------ EnvironmentDoesNotExist if there is no environment found after given parameters and defaults are checked PathDoesNotExist if any source paths provided do not exist """ # Validate Inputs create_dict = {"model_id": self.model.id} create_dict["driver_type"] = self.environment_driver.type validate("create_environment", dictionary) # Create temp environment folder _temp_env_dir = get_datmo_temp_path(self.home) # Step 1: Populate a path list from the user inputs in a format compatible # with the input of the File Collection create function paths = [] # a. add in user given paths as is if they exist if "paths" in dictionary and dictionary['paths']: paths.extend(dictionary['paths']) # b. if there exists projet environment directory AND no paths exist, add in absolute paths if not paths and os.path.isdir(self.file_driver.environment_directory): paths.extend([ os.path.join(self.file_driver.environment_directory, filepath) for filepath in list_all_filepaths( self.file_driver.environment_directory) ]) # c. add in default environment definition filepath as specified by the environment driver # if path exists and NO OTHER PATHS exist src_environment_filename = self.environment_driver.get_default_definition_filename( ) src_environment_filepath = os.path.join(self.home, src_environment_filename) _, environment_filename = os.path.split(src_environment_filepath) create_dict['definition_filename'] = environment_filename if not paths and os.path.exists(src_environment_filepath): paths.append(src_environment_filepath) # Step 2: Check existing paths and create files as needed to populate the # full environment within the temporary directory paths = self._setup_compatible_environment( create_dict, paths, _temp_env_dir, save_hardware_file=save_hardware_file) # Step 3: Pass in all paths for the environment to the file collection create # If PathDoesNotExist is found for any source paths, then error if not paths: raise EnvironmentDoesNotExist() try: file_collection_obj = self.file_collection.create(paths) except PathDoesNotExist as e: raise PathDoesNotExist( __("error", "controller.environment.create.filepath.dne", str(e))) # Step 4: Add file collection information to create dict and check unique hash create_dict['file_collection_id'] = file_collection_obj.id create_dict['unique_hash'] = file_collection_obj.filehash # Check if unique hash is unique or not. # If not, DO NOT CREATE Environment and return existing Environment object results = self.dal.environment.query( {"unique_hash": file_collection_obj.filehash}) if results: return results[0] # Step 5: Delete the temporary directory shutil.rmtree(_temp_env_dir) # Step 6: Add optional arguments to the Environment entity for optional_arg in ["name", "description"]: if optional_arg in dictionary: create_dict[optional_arg] = dictionary[optional_arg] # Step 7: Create environment and return return self.dal.environment.create(Environment(create_dict))
def run(self, task_id, snapshot_dict=None, task_dict=None): """Run a task with parameters. If dictionary specified, create a new task with new run parameters. Snapshot objects are created before and after the task to keep track of the state. During the run, you can access task outputs using environment variable DATMO_TASK_DIR or `/task` which points to location for the task files. Create config.json, stats.json and any weights or any file such as graphs and visualizations within that directory for quick access Parameters ---------- task_id : str id for the task you would like to run snapshot_dict : dict set of parameters to create a snapshot (see SnapshotController for details. default is None, which means dictionary with `visible` False will be added to hide auto-generated snapshot) NOTE: `visible` False will always be False regardless of whether the user provides another value for `visible`. task_dict : dict set of parameters to characterize the task run (default is None, which translate to {}, see datmo.core.entity.task.Task for more details on inputs) Returns ------- Task the Task object which completed its run with updated parameters Raises ------ TaskRunError If there is any error in creating files for the task or downstream errors """ # Ensure visible=False is present in the snapshot dictionary if not snapshot_dict: snapshot_dict = {"visible": False} else: snapshot_dict['visible'] = False if not task_dict: task_dict = {} # Obtain Task to run task_obj = self.dal.task.get_by_id(task_id) # Ensure that at least 1 of command, command_list, or interactive is present in task_dict important_task_args = ["command", "command_list", "interactive"] if not task_dict.get('command', task_obj.command) and \ not task_dict.get('command_list', task_obj.command_list) and \ not task_dict.get('interactive', task_obj.interactive): raise RequiredArgumentMissing( __("error", "controller.task.run.arg", " or ".join(important_task_args))) if task_obj.status is None: task_obj.status = "RUNNING" else: raise TaskRunError( __("error", "cli.run.run.already_running", task_obj.id)) # Create Task directory for user during run task_dirpath = os.path.join(".datmo", "tasks", task_obj.id) try: _ = self.file_driver.create(task_dirpath, directory=True) except Exception: raise TaskRunError( __("error", "controller.task.run", task_dirpath)) # Create the before snapshot prior to execution before_snapshot_dict = snapshot_dict.copy() before_snapshot_dict[ 'message'] = "autogenerated snapshot created before task %s is run" % task_obj.id before_snapshot_obj = self.snapshot.create(before_snapshot_dict) # Update the task with pre-execution parameters, prefer list first then look for string command # List command will overwrite a string command if given if task_dict.get('command_list', task_obj.command_list): task_dict['command'] = " ".join( task_dict.get('command_list', task_obj.command_list)) else: if task_dict.get('command', task_obj.command): task_dict['command_list'] = shlex.split( task_dict.get('command', task_obj.command)) elif not task_dict.get('interactive', task_obj.interactive): # If it's not interactive then there is not expected task raise TaskNoCommandGiven() validate("create_task", task_dict) task_obj = self.dal.task.update({ "id": task_obj.id, "before_snapshot_id": task_dict.get('before_snapshot_id', before_snapshot_obj.id), "command": task_dict.get('command', task_obj.command), "command_list": task_dict.get('command_list', task_obj.command_list), "gpu": task_dict.get('gpu', False), "mem_limit": task_dict.get('mem_limit', None), "workspace": task_dict.get('workspace', None), "interactive": task_dict.get('interactive', task_obj.interactive), "detach": task_dict.get('detach', task_obj.detach), "ports": task_dict.get('ports', task_obj.ports), "task_dirpath": task_dict.get('task_dirpath', task_dirpath), "log_filepath": task_dict.get('log_filepath', os.path.join(task_dirpath, "task.log")), "start_time": task_dict.get('start_time', datetime.utcnow()), "status": task_obj.status }) # Copy over files from the before_snapshot file collection to task dir file_collection_obj = \ self.dal.file_collection.get_by_id(before_snapshot_obj.file_collection_id) self.file_driver.copytree( os.path.join(self.home, file_collection_obj.path), os.path.join(self.home, task_obj.task_dirpath)) return_code, run_id, logs = 0, None, None try: # Set the parameters set in the task if task_obj.detach and task_obj.interactive: raise TaskInteractiveDetachError( __("error", "controller.task.run.args.detach.interactive")) environment_run_options = { "command": task_obj.command_list, "ports": [] if task_obj.ports is None else task_obj.ports, "name": "datmo-task-" + self.model.id + "-" + task_obj.id, "volumes": { os.path.join(self.home, task_obj.task_dirpath): { 'bind': '/task/', 'mode': 'rw' }, self.home: { 'bind': '/home/', 'mode': 'rw' } }, "mem_limit": task_obj.mem_limit, "workspace": task_obj.workspace, "gpu": task_obj.gpu, "detach": task_obj.detach, "stdin_open": task_obj.interactive, "tty": task_obj.interactive, "api": False } # Run environment via the helper function return_code, run_id, logs = \ self._run_helper(before_snapshot_obj.environment_id, environment_run_options, os.path.join(self.home, task_obj.log_filepath)) except Exception as e: return_code = 1 logs += "Error running task: %" % e.message finally: # Create the after snapshot after execution is completed with new paths after_snapshot_dict = snapshot_dict.copy() after_snapshot_dict[ 'message'] = "autogenerated snapshot created after task %s is run" % task_obj.id # Add in absolute paths from running task directory absolute_task_dir_path = os.path.join(self.home, task_obj.task_dirpath) absolute_paths = [] for item in os.listdir(absolute_task_dir_path): path = os.path.join(absolute_task_dir_path, item) if os.path.isfile(path) or os.path.isdir(path): absolute_paths.append(path) after_snapshot_dict.update({ "paths": absolute_paths, "environment_id": before_snapshot_obj.environment_id, }) after_snapshot_obj = self.snapshot.create(after_snapshot_dict) # (optional) Remove temporary task directory path # Update the task with post-execution parameters end_time = datetime.utcnow() duration = (end_time - task_obj.start_time).total_seconds() update_task_dict = { "id": task_obj.id, "after_snapshot_id": after_snapshot_obj.id, "logs": logs, "status": "SUCCESS" if return_code == 0 else "FAILED", # "results": task_obj.results, # TODO: update during run "end_time": end_time, "duration": duration } if logs is not None: update_task_dict["results"] = self._parse_logs_for_results( logs) if run_id is not None: update_task_dict["run_id"] = run_id return self.dal.task.update(update_task_dict)
def create(self, dictionary): """Create snapshot object Parameters ---------- dictionary : dict for each of the 5 key components, this function will search for one of the variables below starting from the top. Default functionality is described below for each component as well for reference if none of the variables are given. code : code_id : str, optional code reference associated with the snapshot; if not provided will look to inputs below for code creation commit_id : str, optional commit id provided by the user if already available Default ------- commits will be taken and code created via the CodeController and are added to the snapshot at the time of snapshot creation environment : environment_id : str, optional id for environment used to create snapshot environment_paths : list, optional list of absolute or relative filepaths and/or dirpaths to collect with destination names (e.g. "/path/to/file>hello", "/path/to/file2", "/path/to/dir>newdir") Default ------- default environment files will be searched and environment will be created with the EnvironmentController and added to the snapshot at the time of snapshot creation file_collection : file_collection_id : str, optional file collection associated with the snapshot paths : list, optional list of absolute or relative filepaths and/or dirpaths to collect with destination names (e.g. "/path/to/file:hello", "/path/to/file2", "/path/to/dir:newdir") Default ------- paths will be considered empty ([]), and the FileCollectionController will create a blank FileCollection that is empty. config : config : dict, optional key, value pairs of configurations config_filepath : str, optional absolute filepath to configuration parameters file config_filename : str, optional name of file with configuration parameters Default ------- config will be considered empty ({}) and saved to the snapshot stats : stats : dict, optional key, value pairs of metrics and statistics stats_filepath : str, optional absolute filepath to stats parameters file stats_filename : str, optional name of file with metrics and statistics. Default ------- stats will be considered empty ({}) and saved to the snapshot for the remaining optional arguments it will search for them in the input dictionary message : str long description of snapshot session_id : str, optional session id within which snapshot is created, will overwrite default if given task_id : str, optional task id associated with snapshot label : str, optional short description of snapshot visible : bool, optional True if visible to user via list command else False Returns ------- datmo.core.entity.snapshot.Snapshot snapshot object with all relevant parameters Raises ------ RequiredArgumentMissing if required arguments are not given by the user FileIOError if files are not present or there is an error in File IO """ # Validate Inputs create_dict = { "model_id": self.model.id, "session_id": self.current_session.id, } validate("create_snapshot", dictionary) # Message must be present if "message" in dictionary: create_dict['message'] = dictionary['message'] else: raise RequiredArgumentMissing( __("error", "controller.snapshot.create.arg", "message")) # Code setup self._code_setup(dictionary, create_dict) # Environment setup self._env_setup(dictionary, create_dict) # File setup self._file_setup(dictionary, create_dict) # Config setup self._config_setup(dictionary, create_dict) # Stats setup self._stats_setup(dictionary, create_dict) # If snapshot object with required args already exists, return it # DO NOT create a new snapshot with the same required arguments results = self.dal.snapshot.query({ "model_id": create_dict["model_id"], "code_id": create_dict['code_id'], "environment_id": create_dict['environment_id'], "file_collection_id": create_dict['file_collection_id'], "config": create_dict['config'], "stats": create_dict['stats'] }) if results: return results[0] # Optional args for Snapshot entity optional_args = ["task_id", "label", "visible"] for optional_arg in optional_args: if optional_arg in dictionary: create_dict[optional_arg] = dictionary[optional_arg] # Create snapshot and return return self.dal.snapshot.create(Snapshot(create_dict))
def create_from_task(self, message, task_id, label=None, config=None, stats=None): """Create snapshot from a completed task. # TODO: enable create from task DURING a run Parameters ---------- message : str long description of snapshot task_id : str task object to use to create snapshot label: str, optional short description of snapshot config : dict, optional key, value pairs of configurations stats : dict, optional key, value pairs of metrics and statistics Returns ------- datmo.core.entity.snapshot.Snapshot snapshot object with all relevant parameters Raises ------ TaskNotComplete if task specified has not been completed """ validate( "create_snapshot_from_task", { "message": message, "task_id": task_id, "label": label, "config": config, "stats": stats }) task_obj = self.dal.task.get_by_id(task_id) if not task_obj.status and not task_obj.after_snapshot_id: raise TaskNotComplete( __("error", "controller.snapshot.create_from_task", str(task_obj.id))) after_snapshot_obj = self.dal.snapshot.get_by_id( task_obj.after_snapshot_id) snapshot_update_dict = { "id": task_obj.after_snapshot_id, "message": message, "visible": True } if label: snapshot_update_dict["label"] = label if config: snapshot_update_dict["config"] = config if stats: snapshot_update_dict["stats"] = stats else: # Append to any existing stats already present snapshot_update_dict["stats"] = {} if after_snapshot_obj.stats is not None: snapshot_update_dict["stats"].update(after_snapshot_obj.stats) if task_obj.results is not None: snapshot_update_dict["stats"].update(task_obj.results) if snapshot_update_dict["stats"] == {}: snapshot_update_dict["stats"] = None return self.dal.snapshot.update(snapshot_update_dict)
def init(self, name, description): """ Initialize the project This function will initialize the project or reinitialize it the project is already initialized. Parameters ---------- name : str description : str Returns ------- bool Raises ------ SessionDoesNotExist """ # Create the Model, is it new or update? is_new_model = False old_model = self.model if not self.model: is_new_model = True try: # Always validate inputs to the init function validate("create_project", { "name": name, "description": description }) # Create model if new else update if is_new_model: _ = self.dal.model.create( Model({ "name": name, "description": description })) else: self._model = self.dal.model.update({ "id": self.model.id, "name": name, "description": description }) # Initialize Code Driver if needed if not self.code_driver.is_initialized: self.code_driver.init() # Initialize File Driver if needed if not self.file_driver.is_initialized: self.file_driver.init() # Initialize Environment Driver if needed # (not required but will warn if not present) try: if not self.environment_driver.is_initialized: self.environment_driver.init() except EnvironmentInitFailed: self.logger.warning( __("warn", "controller.general.environment.failed")) # Build the initial default Environment (NOT NECESSARY) # self.environment_driver.build_image(tag="datmo-" + \ # self.model.name) # Create and set current session if is_new_model: # Create new default session _ = self.dal.session.create( Session({ "name": "default", "model_id": self.model.id, "current": True })) else: if not self.current_session: default_session_objs = self.dal.session.query({ "name": "default", "model_id": self.model.id }) if not default_session_objs: # Creating a default session since none exists _ = self.dal.session.create( Session({ "name": "default", "model_id": self.model.id, "current": True })) else: # Update default session to be current default_session_obj = default_session_objs[0] self.dal.session.update({ "id": default_session_obj.id, "current": True }) return True except Exception: # if any error occurred with new model, ensure no initialize occurs and raise previous error # if any error occurred with existing model, ensure no updates were made, raise previous error if is_new_model: self.cleanup() else: self._model = self.dal.model.update({ "id": old_model.id, "name": old_model.name, "description": old_model.description }) raise
def test_validate_success(self): assert validate("create_project", { "name": "foobar", "description": "barbaz" })
def init(self, name, description): # Create the Model, is it new or update? is_new_model = False if not self.model: is_new_model = True # If model is new validate inputs if is_new_model: validate("create_project", { "name": name, "description": description }) # Create model if new else update if is_new_model: _ = self.dal.model.create( Model({ "name": name, "description": description })) else: self._model = self.dal.model.update({ "id": self.model.id, "name": name, "description": description }) # Initialize Code Manager if needed if not self.code_driver.is_initialized: self.code_driver.init() # Initialize File Manager if needed if not self.file_driver.is_initialized: self.file_driver.init() # Initialize Environment Manager if needed if not self.environment_driver.is_initialized: self.environment_driver.init() # Build the initial default Environment (NOT NECESSARY) # self.environment_driver.build_image(tag="datmo-" + \ # self.model.name) # Add in Project template files if specified # TODO: Add in project template files # Create and set current session if is_new_model: # Create new default session _ = self.dal.session.create( Session({ "name": "default", "model_id": self.model.id, "current": True })) else: if not self.current_session: default_session_obj = self.dal.session.query({ "name": "default", "model_id": self.model.id }) if not default_session_obj: raise SessionDoesNotExistException( __("error", "controller.project.init")) # Update default session to be current self.dal.session.update({ "id": default_session_obj.id, "current": True }) return True