def _find_in_filecollection(self, file_to_find, file_collection_id): """ Attempts to find a file within the file collection Returns ------- dict output dictionary of the JSON file """ file_collection_obj = self.file_collection.dal.file_collection.\ get_by_id(file_collection_id) file_collection_path = \ self.file_collection.file_driver.get_collection_path( file_collection_obj.filehash) # find all of the possible paths it could exist possible_paths = [os.path.join(self.home, file_to_find)] + \ [os.path.join(self.home, item[0], file_to_find) for item in os.walk(file_collection_path)] existing_possible_paths = [ possible_path for possible_path in possible_paths if os.path.isfile(possible_path) ] if not existing_possible_paths: # TODO: Add some info / warning that no file was found # create some default stats return {} else: # If any such path exists, transform file to stats dict json_file = JSONStore(existing_possible_paths[0]) return json_file.to_dict()
def _config_setup(self, incoming_dictionary, create_dict): """ Fills in snapshot config by having one of the following: 1. config = JSON object 2. config_filepath = some location where a json file exists 3. config_filename = just the file name Parameters ---------- incoming_dictionary : dict dictionary for the create function defined above create_dict : dict dictionary for creating the Snapshot entity Raises ------ FileIOError """ if "config" in incoming_dictionary: create_dict['config'] = incoming_dictionary['config'] elif "config_filepath" in incoming_dictionary: if not os.path.isfile(incoming_dictionary['config_filepath']): raise FileIOError( __("error", "controller.snapshot.create.file_config")) # If path exists transform file to config dict config_json_driver = JSONStore( incoming_dictionary['config_filepath']) create_dict['config'] = config_json_driver.to_dict() elif "config_filename" in incoming_dictionary: config_filename = incoming_dictionary['config_filename'] create_dict['config'] = self._find_in_filecollection( config_filename, create_dict['file_collection_id']) else: config_filename = "config.json" create_dict['config'] = self._find_in_filecollection( config_filename, create_dict['file_collection_id'])
def remote_credentials(self): """ Returns credentials if present Returns ------- MASTER_SERVER_IP : str return if present else None DATMO_API_KEY : str return if present else None END_POINT : str return if present else None """ # 1) Load from the environment if datmo config not already saved globally MASTER_SERVER_IP = os.environ.get('MASTER_SERVER_IP', None) DATMO_API_KEY = os.environ.get('DATMO_API_KEY', None) # 2) loading the datmo config if present datmo_config_filepath = os.path.join(os.path.expanduser("~"), ".datmo", "config") if os.path.isfile(datmo_config_filepath): datmo_config = JSONStore(datmo_config_filepath) config_dict = datmo_config.to_dict() if MASTER_SERVER_IP is None: MASTER_SERVER_IP = config_dict.get('MASTER_SERVER_IP', None) if DATMO_API_KEY is None: DATMO_API_KEY = config_dict.get('DATMO_API_KEY', None) if MASTER_SERVER_IP: END_POINT = 'http://' + MASTER_SERVER_IP + ':2083/api/v1' else: END_POINT = None return MASTER_SERVER_IP, DATMO_API_KEY, END_POINT
def test_load_new_json_file(self): new_json_filepath = os.path.join(self.temp_dir, "test.json") with open(new_json_filepath, "wb") as f: f.write(to_bytes("")) # Test returns empty dict for empty file storage = JSONStore(new_json_filepath) assert storage.to_dict() == {} # Test FileIOError if not JSON decodeable with open(new_json_filepath, "wb") as f: f.write(to_bytes("this is not json decodeable")) failed = False try: storage = JSONStore(new_json_filepath) storage.to_dict() except FileIOError: failed = True assert failed
class ProjectController(BaseController): """ProjectController inherits from BaseController and manages business logic related to the project. One model is associated with each project currently. Methods ------- init(name, description) Initialize the project repository as a new model or update the existing project cleanup() Remove all datmo references from the current repository. NOTE: THIS WILL DELETE ALL DATMO WORK status() Give the user a picture of the status of the project, snapshots, and tasks """ def __init__(self): super(ProjectController, self).__init__() def init(self, name, description): """ Initialize the project This function will initialize the project or reinitialize it the project is already initialized. Parameters ---------- name : str description : str Returns ------- bool """ is_new_model = False old_model = self.model if not self.model: is_new_model = True try: # Always validate inputs to the init function validate("create_project", { "name": name, "description": description }) # Initialize File Driver if needed if not self.file_driver.is_initialized: self.file_driver.init() # Initialize the dal if not self.dal.is_initialized: self.dal.init() # Initialize Code Driver if needed if not self.code_driver.is_initialized: self.code_driver.init() # Initialize Environment Driver if needed if not self.environment_driver.is_initialized: self.environment_driver.init() # Initialize the config JSON store self.config_store = JSONStore( os.path.join(self.home, Config().datmo_directory_name, ".config")) # Create model if new else update if is_new_model: _ = self.dal.model.create( Model({ "name": name, "description": description })) else: self._model = self.dal.model.update({ "id": self.model.id, "name": name, "description": description }) # Connect Environment Driver if needed # (not required but will warn if not present) try: if not self.environment_driver.is_connected: self.environment_driver.connect() except EnvironmentConnectFailed: self.logger.warning( __("warn", "controller.general.environment.failed")) # Build the initial default Environment (NOT NECESSARY) # self.environment_driver.build_image(tag="datmo-" + \ # self.model.name) return True except Exception: # if any error occurred with new model, ensure no initialize occurs and raise previous error # if any error occurred with existing model, ensure no updates were made, raise previous error if is_new_model: self.cleanup() else: self._model = self.dal.model.update({ "id": old_model.id, "name": old_model.name, "description": old_model.description }) raise def cleanup(self): """Cleans the project structure completely Notes ----- This function will not error out but will gracefully exit, since it is used in cases where init fails as a check against mid-initialized projects Returns ------- bool """ if not self.is_initialized: self.logger.warning( __("warn", "controller.project.cleanup.not_init")) # Remove Datmo environment_driver references, give warning if error try: # Obtain image id before cleaning up if exists images = self.environment_driver.list_images(name="datmo-" + \ self.model.name) image_id = images[0].id if images else None except Exception: self.logger.warning( __("warn", "controller.project.cleanup.environment")) # Remove Datmo code_driver references, give warning if error try: if self.code_driver.is_initialized: for ref in self.code_driver.list_refs(): self.code_driver.delete_ref(ref) except Exception: self.logger.warning(__("warn", "controller.project.cleanup.code")) try: # Remove Hidden Datmo file structure, give warning if error self.file_driver.delete_hidden_datmo_file_structure() except (FileIOError, PathDoesNotExist): self.logger.warning(__("warn", "controller.project.cleanup.files")) try: if image_id: # Remove image created during init self.environment_driver.remove_image( image_id_or_name=image_id, force=True) # Remove any dangling images (optional) # Stop and remove all running environments with image_id self.environment_driver.stop_remove_containers_by_term( image_id, force=True) except Exception: self.logger.warning( __("warn", "controller.project.cleanup.environment")) return True def status(self): """Return the project status information if initialized Returns ------- status_dict : dict dictionary with project metadata and config current_snapshot : datmo.core.entity.snapshot.Snapshot snapshot object of the current state of the repo if present else None latest_snapshot_user_generated : datmo.core.entity.snapshot.Snapshot snapshot object of the latest snapshot generated by the user if present else None latest_snapshot_auto_generated : datmo.core.entity.snapshot.Snapshot snapshot object of the latest snapshot generated automatically by datmo if present else None unstaged_code : bool True if code has unstaged changes unstaged_environment : bool True if environment has unstaged changes unstaged_files : bool True if files have unstaged changes """ if not self.is_initialized: raise ProjectNotInitialized( __("error", "controller.project.status")) # TODO: Add in note when environment is not setup or intialized # Add in project metadata status_dict = self.model.to_dictionary().copy() # Find all project settings status_dict["config"] = self.config_store.to_dict() # Find the latest snapshot generated by the user descending_snapshots = self.dal.snapshot.query( { "visible": True }, sort_key="created_at", sort_order="descending") latest_snapshot_user_generated = descending_snapshots[ 0] if descending_snapshots else None # Show the latest snapshot generated automatically by datmo descending_snapshots = self.dal.snapshot.query( { "visible": False }, sort_key="created_at", sort_order="descending") latest_snapshot_auto_generated = descending_snapshots[ 0] if descending_snapshots else None # TODO: add in latest run self.code_controller = CodeController() try: unstaged_code = self.code_controller.check_unstaged_changes() except UnstagedChanges: unstaged_code = True self.environment_controller = EnvironmentController() try: unstaged_environment = self.environment_controller.check_unstaged_changes( ) except UnstagedChanges: unstaged_environment = True self.file_collection_controller = FileCollectionController() try: unstaged_files = self.file_collection_controller.check_unstaged_changes( ) except UnstagedChanges: unstaged_files = True # If exists, obtain the current snapshot, if unstaged changes, will be None self.snapshot_controller = SnapshotController() try: current_snapshot = self.snapshot_controller.current_snapshot() except UnstagedChanges: current_snapshot = None return status_dict, current_snapshot, latest_snapshot_user_generated, latest_snapshot_auto_generated, \ unstaged_code, unstaged_environment, unstaged_files
def __save_dictionary(self, dictionary, path): json_obj = JSONStore(path) data = json_obj.to_dict() data.update(dictionary) json_obj.to_file(data) return data