def test_get_datmo_temp_path(self): datmo_temp_path = get_datmo_temp_path(self.temp_dir) exists = False if os.path.isdir(datmo_temp_path): exists = True assert exists # Test if subsequent temp dirs are different datmo_temp_path_1 = get_datmo_temp_path(self.temp_dir) assert datmo_temp_path != datmo_temp_path_1 datmo_temp_path_2 = get_datmo_temp_path(self.temp_dir) assert datmo_temp_path != datmo_temp_path_2 assert datmo_temp_path_1 != datmo_temp_path_2
def test_get_dirhash(self): temp_dir_1 = get_datmo_temp_path(self.temp_dir) filepath = os.path.join(temp_dir_1, "test.txt") with open(filepath, "wb") as f: f.write(to_bytes("hello\n")) result = self.local_file_driver.get_dirhash(temp_dir_1) assert result == "57ae7aad8abe2f317e460c92d3ed1178" temp_dir_2 = get_datmo_temp_path(self.temp_dir) filepath_2 = os.path.join(temp_dir_2, "test.txt") with open(filepath_2, "wb") as f: f.write(to_bytes("hello\n")) result_2 = self.local_file_driver.get_dirhash(temp_dir_2) assert result == result_2
def test_calculate_hash_paths_simple(self): self.local_file_driver.init() # Create test directories to move self.local_file_driver.create("dirpath1", directory=True) self.local_file_driver.create("dirpath2", directory=True) self.local_file_driver.create("filepath1") self.local_file_driver.create("filepath2") dirpath1 = os.path.join(self.local_file_driver.root, "dirpath1") dirpath2 = os.path.join(self.local_file_driver.root, "dirpath2") filepath1 = os.path.join(self.local_file_driver.root, "filepath1") filepath2 = os.path.join(self.local_file_driver.root, "filepath2") # check with just 1 blank filepath paths = [filepath1] temp_dir = get_datmo_temp_path(self.local_file_driver.root) result = self.local_file_driver.calculate_hash_paths(paths, temp_dir) assert result == "74be16979710d4c4e7c6647856088456" shutil.rmtree(temp_dir) # check with 1 empty directory and 1 blank filepath (empty directories do NOT change hash) paths = [filepath1, dirpath1] temp_dir = get_datmo_temp_path(self.local_file_driver.root) result = self.local_file_driver.calculate_hash_paths(paths, temp_dir) assert result == "74be16979710d4c4e7c6647856088456" shutil.rmtree(temp_dir) # check with 2 empty directories and 1 blank filepath (empty directories do NOT change hash) paths = [filepath1, dirpath1, dirpath2] temp_dir = get_datmo_temp_path(self.local_file_driver.root) result = self.local_file_driver.calculate_hash_paths(paths, temp_dir) assert result == "74be16979710d4c4e7c6647856088456" shutil.rmtree(temp_dir) # check 2 blank filepaths (should be different) paths = [filepath1, filepath2] temp_dir = get_datmo_temp_path(self.local_file_driver.root) result = self.local_file_driver.calculate_hash_paths(paths, temp_dir) assert result == "020eb29b524d7ba672d9d48bc72db455" shutil.rmtree(temp_dir) # check 1 blank filepath with a different name (same because name not factored into hash) paths = [filepath2] temp_dir = get_datmo_temp_path(self.local_file_driver.root) result = self.local_file_driver.calculate_hash_paths(paths, temp_dir) assert result == "74be16979710d4c4e7c6647856088456" shutil.rmtree(temp_dir)
def _calculate_project_environment_hash(self, save_hardware_file=True): """Return the environment hash from contents in project environment directory. If environment_directory not present then will assume it is empty Parameters ---------- save_hardware_file : bool include the hardware info file within the hash Returns ------- str unique hash of the project environment directory """ # Populate paths from the project environment directory paths = [] if os.path.isdir(self.environment_driver.environment_directory_path): paths.extend([ os.path.join( self.environment_driver.environment_directory_path, filepath) for filepath in list_all_filepaths( self.environment_driver.environment_directory_path) ]) # Create a temp dir to save any additional files necessary _temp_dir = get_datmo_temp_path(self.home) # Setup compatible environment and create add paths paths = self._setup_compatible_environment( { "definition_filename": self.environment_driver.get_default_definition_filename() }, paths, _temp_dir, save_hardware_file=save_hardware_file) # Create new temp directory _temp_dir_2 = get_datmo_temp_path(self.home) # Hash the paths of the environment with a different temp dir dirhash = self.file_driver.calculate_hash_paths(paths, _temp_dir_2) # Remove both temporary directories shutil.rmtree(_temp_dir) shutil.rmtree(_temp_dir_2) return dirhash
def _calculate_project_files_hash(self): """Return the file hash of the file collections filepaths for project files directory Returns ------- str unique hash of the project files directory """ # Populate paths from the project files directory paths = [] if os.path.isdir(self.file_driver.files_directory): paths.extend([ os.path.join(self.file_driver.files_directory, filepath) for filepath in list_all_filepaths( self.file_driver.files_directory) ]) # Create a temp dir to use for calculating the hash _temp_dir = get_datmo_temp_path(self.home) # Hash the paths of the files dirhash = self.file_driver.calculate_hash_paths(paths, _temp_dir) # Remove temporary directory shutil.rmtree(_temp_dir) return dirhash
def create_collection(self, paths): if not self.is_initialized: raise FileStructureError( __("error", "controller.file.driver.local.create_collection.structure")) self.ensure_collections_dir() temp_collection_path = get_datmo_temp_path(self.root) filehash = self.calculate_hash_paths(paths, temp_collection_path) # Move contents to folder with filehash as name and remove temp_collection_path collection_path = os.path.join(self.datmo_directory, "collections", filehash) if os.path.isdir(collection_path): return filehash # raise FileStructureError("exception.file.create_collection", { # "exception": "File collection with id already exists." # }) os.makedirs(collection_path) self.copytree(temp_collection_path, collection_path) # Change permissions to read only for collection_path. File collection is immutable mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH for root, dirs, files in os.walk(collection_path, topdown=False): for dir in [os.path.join(root, d) for d in dirs]: os.chmod(dir, mode) for file in [os.path.join(root, f) for f in files]: os.chmod(file, mode) return filehash
def build(self, environment_id, workspace=None): """Build environment from definition file Parameters ---------- environment_id : str environment object id to build workspace : str workspace to be used Returns ------- bool returns True if success Raises ------ EnvironmentDoesNotExist if the specified Environment does not exist. """ self.environment_driver.init() if not self.exists(environment_id): raise EnvironmentDoesNotExist( __("error", "controller.environment.build", environment_id)) environment_obj = self.dal.environment.get_by_id(environment_id) file_collection_obj = self.dal.file_collection.\ get_by_id(environment_obj.file_collection_id) # TODO: Check hardware info here if different from creation time # Add in files for that environment id environment_definition_path = os.path.join(self.home, file_collection_obj.path) # Copy to temp folder and remove files that are datmo specific _temp_env_dir = get_datmo_temp_path(self.home) self.file_driver.copytree(environment_definition_path, _temp_env_dir) # get definition filepath for the temp folder environment_definition_filepath = os.path.join( _temp_env_dir, environment_obj.definition_filename) try: # Build the Environment with the driver self.spinner.start() result = self.environment_driver.build( environment_id, path=environment_definition_filepath, workspace=workspace) finally: self.spinner.stop() # Remove both temporary directories shutil.rmtree(_temp_env_dir) return result
def test_calculate_hash_paths_single_line(self): self.local_file_driver.init() # Create test directories to move self.local_file_driver.create("filepath1") filepath1 = os.path.join(self.local_file_driver.root, "filepath1") paths = [filepath1] # Add contents to the file in python and verify hash temp_dir = get_datmo_temp_path(self.local_file_driver.root) with open(filepath1, "wb") as f: f.write(to_bytes("hello\n")) result = self.local_file_driver.calculate_hash_paths(paths, temp_dir) shutil.rmtree(temp_dir) assert result == "57ae7aad8abe2f317e460c92d3ed1178"
def test_calculate_hash_paths_multiple_lines(self): self.local_file_driver.init() # Create test directories to move self.local_file_driver.create("filepath1") filepath1 = os.path.join(self.local_file_driver.root, "filepath1") paths = [filepath1] # Add contents to the file in python and verify hash temp_dir = get_datmo_temp_path(self.local_file_driver.root) with open(filepath1, "wb") as f: f.write(to_bytes("FROM something:something\n")) f.write(to_bytes("test multiple lines\n")) result = self.local_file_driver.calculate_hash_paths(paths, temp_dir) shutil.rmtree(temp_dir) assert result == "a14de65c0fc13bc50cb246cc518195af"
def create(self, dictionary, save_hardware_file=True): """Create an environment Parameters ---------- dictionary : dict optional values to populate required environment entity args paths : list, optional list of absolute or relative filepaths and/or dirpaths to collect with destination names (e.g. "/path/to/file>hello", "/path/to/file2", "/path/to/dir>newdir") (default if none provided is to pull from project environment folder and project root. If none found create default definition) name : str, optional name of the environment (default is None) description : str, optional description of the environment (default is None) save_hardware_file : bool boolean to save hardware file along with other files (default is True to save the file and create distinct hashes based on software and hardware) Returns ------- Environment returns an object representing the environment created Raises ------ EnvironmentDoesNotExist if there is no environment found after given parameters and defaults are checked PathDoesNotExist if any source paths provided do not exist """ # Validate Inputs create_dict = {"model_id": self.model.id} create_dict["driver_type"] = self.environment_driver.type validate("create_environment", dictionary) # Create temp environment folder _temp_env_dir = get_datmo_temp_path(self.home) # Step 1: Populate a path list from the user inputs in a format compatible # with the input of the File Collection create function paths = [] # a. add in user given paths as is if they exist if "paths" in dictionary and dictionary['paths']: paths.extend(dictionary['paths']) # b. if there exists projet environment directory AND no paths exist, add in absolute paths if not paths and os.path.isdir(self.file_driver.environment_directory): paths.extend([ os.path.join(self.file_driver.environment_directory, filepath) for filepath in list_all_filepaths( self.file_driver.environment_directory) ]) # c. add in default environment definition filepath as specified by the environment driver # if path exists and NO OTHER PATHS exist src_environment_filename = self.environment_driver.get_default_definition_filename( ) src_environment_filepath = os.path.join(self.home, src_environment_filename) _, environment_filename = os.path.split(src_environment_filepath) create_dict['definition_filename'] = environment_filename if not paths and os.path.exists(src_environment_filepath): paths.append(src_environment_filepath) # Step 2: Check existing paths and create files as needed to populate the # full environment within the temporary directory paths = self._setup_compatible_environment( create_dict, paths, _temp_env_dir, save_hardware_file=save_hardware_file) # Step 3: Pass in all paths for the environment to the file collection create # If PathDoesNotExist is found for any source paths, then error if not paths: raise EnvironmentDoesNotExist() try: file_collection_obj = self.file_collection.create(paths) except PathDoesNotExist as e: raise PathDoesNotExist( __("error", "controller.environment.create.filepath.dne", str(e))) # Step 4: Add file collection information to create dict and check unique hash create_dict['file_collection_id'] = file_collection_obj.id create_dict['unique_hash'] = file_collection_obj.filehash # Check if unique hash is unique or not. # If not, DO NOT CREATE Environment and return existing Environment object results = self.dal.environment.query( {"unique_hash": file_collection_obj.filehash}) if results: return results[0] # Step 5: Delete the temporary directory shutil.rmtree(_temp_env_dir) # Step 6: Add optional arguments to the Environment entity for optional_arg in ["name", "description"]: if optional_arg in dictionary: create_dict[optional_arg] = dictionary[optional_arg] # Step 7: Create environment and return return self.dal.environment.create(Environment(create_dict))
def checkout(self, environment_id): """Checkout to specific environment id Parameters ---------- environment_id : str environment id to checkout to Returns ------- bool True if success Raises ------ EnvironmentNotInitialized error if not initialized (must initialize first) PathDoesNotExist if environment id does not exist UnstagedChanges error if not there exists unstaged changes in environment """ if not self.is_initialized: raise EnvironmentNotInitialized() if not self.exists(environment_id): raise EnvironmentDoesNotExist( __("error", "controller.environment.checkout_env", environment_id)) # Check if unstaged changes exist if self._has_unstaged_changes(): raise UnstagedChanges() # Check if environment has is same as current results = self.dal.environment.query({"id": environment_id}) environment_obj = results[0] environment_hash = environment_obj.unique_hash if self._calculate_project_environment_hash() == environment_hash: return True # Remove all content from project environment directory for file in os.listdir(self.file_driver.environment_directory): file_path = os.path.join(self.file_driver.environment_directory, file) try: if os.path.isfile(file_path): os.remove(file_path) elif os.path.isdir(file_path): shutil.rmtree(file_path) except Exception as e: print(e) # Add in files for that environment id file_collection_obj = self.dal.file_collection.\ get_by_id(environment_obj.file_collection_id) environment_definition_path = os.path.join(self.home, file_collection_obj.path) # Copy to temp folder and remove files that are datmo specific _temp_env_dir = get_datmo_temp_path(self.home) self.file_driver.copytree(environment_definition_path, _temp_env_dir) for filename in self.environment_driver.get_datmo_definition_filenames( ): os.remove(os.path.join(_temp_env_dir, filename)) # Copy from temp folder to project environment directory self.file_driver.copytree(_temp_env_dir, self.file_driver.environment_directory) shutil.rmtree(_temp_env_dir) return True