def test_get_datmo_temp_path(self):
datmo_temp_path = get_datmo_temp_path(self.temp_dir)
exists = False
if os.path.isdir(datmo_temp_path):
exists = True
assert exists
# Test if subsequent temp dirs are different
datmo_temp_path_1 = get_datmo_temp_path(self.temp_dir)
assert datmo_temp_path != datmo_temp_path_1
datmo_temp_path_2 = get_datmo_temp_path(self.temp_dir)
assert datmo_temp_path != datmo_temp_path_2
assert datmo_temp_path_1 != datmo_temp_path_2
def test_get_dirhash(self):
temp_dir_1 = get_datmo_temp_path(self.temp_dir)
filepath = os.path.join(temp_dir_1, "test.txt")
with open(filepath, "wb") as f:
f.write(to_bytes("hello\n"))
result = self.local_file_driver.get_dirhash(temp_dir_1)
assert result == "57ae7aad8abe2f317e460c92d3ed1178"
temp_dir_2 = get_datmo_temp_path(self.temp_dir)
filepath_2 = os.path.join(temp_dir_2, "test.txt")
with open(filepath_2, "wb") as f:
f.write(to_bytes("hello\n"))
result_2 = self.local_file_driver.get_dirhash(temp_dir_2)
assert result == result_2
def test_calculate_hash_paths_simple(self):
self.local_file_driver.init()
# Create test directories to move
self.local_file_driver.create("dirpath1", directory=True)
self.local_file_driver.create("dirpath2", directory=True)
self.local_file_driver.create("filepath1")
self.local_file_driver.create("filepath2")
dirpath1 = os.path.join(self.local_file_driver.root, "dirpath1")
dirpath2 = os.path.join(self.local_file_driver.root, "dirpath2")
filepath1 = os.path.join(self.local_file_driver.root, "filepath1")
filepath2 = os.path.join(self.local_file_driver.root, "filepath2")
# check with just 1 blank filepath
paths = [filepath1]
temp_dir = get_datmo_temp_path(self.local_file_driver.root)
result = self.local_file_driver.calculate_hash_paths(paths, temp_dir)
assert result == "74be16979710d4c4e7c6647856088456"
shutil.rmtree(temp_dir)
# check with 1 empty directory and 1 blank filepath (empty directories do NOT change hash)
paths = [filepath1, dirpath1]
temp_dir = get_datmo_temp_path(self.local_file_driver.root)
result = self.local_file_driver.calculate_hash_paths(paths, temp_dir)
assert result == "74be16979710d4c4e7c6647856088456"
shutil.rmtree(temp_dir)
# check with 2 empty directories and 1 blank filepath (empty directories do NOT change hash)
paths = [filepath1, dirpath1, dirpath2]
temp_dir = get_datmo_temp_path(self.local_file_driver.root)
result = self.local_file_driver.calculate_hash_paths(paths, temp_dir)
assert result == "74be16979710d4c4e7c6647856088456"
shutil.rmtree(temp_dir)
# check 2 blank filepaths (should be different)
paths = [filepath1, filepath2]
temp_dir = get_datmo_temp_path(self.local_file_driver.root)
result = self.local_file_driver.calculate_hash_paths(paths, temp_dir)
assert result == "020eb29b524d7ba672d9d48bc72db455"
shutil.rmtree(temp_dir)
# check 1 blank filepath with a different name (same because name not factored into hash)
paths = [filepath2]
temp_dir = get_datmo_temp_path(self.local_file_driver.root)
result = self.local_file_driver.calculate_hash_paths(paths, temp_dir)
assert result == "74be16979710d4c4e7c6647856088456"
shutil.rmtree(temp_dir)
def _calculate_project_environment_hash(self, save_hardware_file=True):
"""Return the environment hash from contents in project environment directory.
If environment_directory not present then will assume it is empty
Parameters
----------
save_hardware_file : bool
include the hardware info file within the hash
Returns
-------
str
unique hash of the project environment directory
"""
# Populate paths from the project environment directory
paths = []
if os.path.isdir(self.environment_driver.environment_directory_path):
paths.extend([
os.path.join(
self.environment_driver.environment_directory_path,
filepath) for filepath in list_all_filepaths(
self.environment_driver.environment_directory_path)
])
# Create a temp dir to save any additional files necessary
_temp_dir = get_datmo_temp_path(self.home)
# Setup compatible environment and create add paths
paths = self._setup_compatible_environment(
{
"definition_filename":
self.environment_driver.get_default_definition_filename()
},
paths,
_temp_dir,
save_hardware_file=save_hardware_file)
# Create new temp directory
_temp_dir_2 = get_datmo_temp_path(self.home)
# Hash the paths of the environment with a different temp dir
dirhash = self.file_driver.calculate_hash_paths(paths, _temp_dir_2)
# Remove both temporary directories
shutil.rmtree(_temp_dir)
shutil.rmtree(_temp_dir_2)
return dirhash
def _calculate_project_files_hash(self):
"""Return the file hash of the file collections filepaths for project files directory
Returns
-------
str
unique hash of the project files directory
"""
# Populate paths from the project files directory
paths = []
if os.path.isdir(self.file_driver.files_directory):
paths.extend([
os.path.join(self.file_driver.files_directory,
filepath) for filepath in list_all_filepaths(
self.file_driver.files_directory)
])
# Create a temp dir to use for calculating the hash
_temp_dir = get_datmo_temp_path(self.home)
# Hash the paths of the files
dirhash = self.file_driver.calculate_hash_paths(paths, _temp_dir)
# Remove temporary directory
shutil.rmtree(_temp_dir)
return dirhash
def create_collection(self, paths):
if not self.is_initialized:
raise FileStructureError(
__("error",
"controller.file.driver.local.create_collection.structure"))
self.ensure_collections_dir()
temp_collection_path = get_datmo_temp_path(self.root)
filehash = self.calculate_hash_paths(paths, temp_collection_path)
# Move contents to folder with filehash as name and remove temp_collection_path
collection_path = os.path.join(self.datmo_directory, "collections",
filehash)
if os.path.isdir(collection_path):
return filehash
# raise FileStructureError("exception.file.create_collection", {
# "exception": "File collection with id already exists."
# })
os.makedirs(collection_path)
self.copytree(temp_collection_path, collection_path)
# Change permissions to read only for collection_path. File collection is immutable
mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
for root, dirs, files in os.walk(collection_path, topdown=False):
for dir in [os.path.join(root, d) for d in dirs]:
os.chmod(dir, mode)
for file in [os.path.join(root, f) for f in files]:
os.chmod(file, mode)
return filehash
def build(self, environment_id, workspace=None):
"""Build environment from definition file
Parameters
----------
environment_id : str
environment object id to build
workspace : str
workspace to be used
Returns
-------
bool
returns True if success
Raises
------
EnvironmentDoesNotExist
if the specified Environment does not exist.
"""
self.environment_driver.init()
if not self.exists(environment_id):
raise EnvironmentDoesNotExist(
__("error", "controller.environment.build", environment_id))
environment_obj = self.dal.environment.get_by_id(environment_id)
file_collection_obj = self.dal.file_collection.\
get_by_id(environment_obj.file_collection_id)
# TODO: Check hardware info here if different from creation time
# Add in files for that environment id
environment_definition_path = os.path.join(self.home,
file_collection_obj.path)
# Copy to temp folder and remove files that are datmo specific
_temp_env_dir = get_datmo_temp_path(self.home)
self.file_driver.copytree(environment_definition_path, _temp_env_dir)
# get definition filepath for the temp folder
environment_definition_filepath = os.path.join(
_temp_env_dir, environment_obj.definition_filename)
try:
# Build the Environment with the driver
self.spinner.start()
result = self.environment_driver.build(
environment_id,
path=environment_definition_filepath,
workspace=workspace)
finally:
self.spinner.stop()
# Remove both temporary directories
shutil.rmtree(_temp_env_dir)
return result
def test_calculate_hash_paths_single_line(self):
self.local_file_driver.init()
# Create test directories to move
self.local_file_driver.create("filepath1")
filepath1 = os.path.join(self.local_file_driver.root, "filepath1")
paths = [filepath1]
# Add contents to the file in python and verify hash
temp_dir = get_datmo_temp_path(self.local_file_driver.root)
with open(filepath1, "wb") as f:
f.write(to_bytes("hello\n"))
result = self.local_file_driver.calculate_hash_paths(paths, temp_dir)
shutil.rmtree(temp_dir)
assert result == "57ae7aad8abe2f317e460c92d3ed1178"
def test_calculate_hash_paths_multiple_lines(self):
self.local_file_driver.init()
# Create test directories to move
self.local_file_driver.create("filepath1")
filepath1 = os.path.join(self.local_file_driver.root, "filepath1")
paths = [filepath1]
# Add contents to the file in python and verify hash
temp_dir = get_datmo_temp_path(self.local_file_driver.root)
with open(filepath1, "wb") as f:
f.write(to_bytes("FROM something:something\n"))
f.write(to_bytes("test multiple lines\n"))
result = self.local_file_driver.calculate_hash_paths(paths, temp_dir)
shutil.rmtree(temp_dir)
assert result == "a14de65c0fc13bc50cb246cc518195af"
def create(self, dictionary, save_hardware_file=True):
"""Create an environment
Parameters
----------
dictionary : dict
optional values to populate required environment entity args
paths : list, optional
list of absolute or relative filepaths and/or dirpaths to collect with destination names
(e.g. "/path/to/file>hello", "/path/to/file2", "/path/to/dir>newdir")
(default if none provided is to pull from project environment folder and project root. If none found create default definition)
name : str, optional
name of the environment
(default is None)
description : str, optional
description of the environment
(default is None)
save_hardware_file : bool
boolean to save hardware file along with other files
(default is True to save the file and create distinct hashes based on software and hardware)
Returns
-------
Environment
returns an object representing the environment created
Raises
------
EnvironmentDoesNotExist
if there is no environment found after given parameters and defaults are checked
PathDoesNotExist
if any source paths provided do not exist
"""
# Validate Inputs
create_dict = {"model_id": self.model.id}
create_dict["driver_type"] = self.environment_driver.type
validate("create_environment", dictionary)
# Create temp environment folder
_temp_env_dir = get_datmo_temp_path(self.home)
# Step 1: Populate a path list from the user inputs in a format compatible
# with the input of the File Collection create function
paths = []
# a. add in user given paths as is if they exist
if "paths" in dictionary and dictionary['paths']:
paths.extend(dictionary['paths'])
# b. if there exists projet environment directory AND no paths exist, add in absolute paths
if not paths and os.path.isdir(self.file_driver.environment_directory):
paths.extend([
os.path.join(self.file_driver.environment_directory, filepath)
for filepath in list_all_filepaths(
self.file_driver.environment_directory)
])
# c. add in default environment definition filepath as specified by the environment driver
# if path exists and NO OTHER PATHS exist
src_environment_filename = self.environment_driver.get_default_definition_filename(
)
src_environment_filepath = os.path.join(self.home,
src_environment_filename)
_, environment_filename = os.path.split(src_environment_filepath)
create_dict['definition_filename'] = environment_filename
if not paths and os.path.exists(src_environment_filepath):
paths.append(src_environment_filepath)
# Step 2: Check existing paths and create files as needed to populate the
# full environment within the temporary directory
paths = self._setup_compatible_environment(
create_dict,
paths,
_temp_env_dir,
save_hardware_file=save_hardware_file)
# Step 3: Pass in all paths for the environment to the file collection create
# If PathDoesNotExist is found for any source paths, then error
if not paths:
raise EnvironmentDoesNotExist()
try:
file_collection_obj = self.file_collection.create(paths)
except PathDoesNotExist as e:
raise PathDoesNotExist(
__("error", "controller.environment.create.filepath.dne",
str(e)))
# Step 4: Add file collection information to create dict and check unique hash
create_dict['file_collection_id'] = file_collection_obj.id
create_dict['unique_hash'] = file_collection_obj.filehash
# Check if unique hash is unique or not.
# If not, DO NOT CREATE Environment and return existing Environment object
results = self.dal.environment.query(
{"unique_hash": file_collection_obj.filehash})
if results: return results[0]
# Step 5: Delete the temporary directory
shutil.rmtree(_temp_env_dir)
# Step 6: Add optional arguments to the Environment entity
for optional_arg in ["name", "description"]:
if optional_arg in dictionary:
create_dict[optional_arg] = dictionary[optional_arg]
# Step 7: Create environment and return
return self.dal.environment.create(Environment(create_dict))
def checkout(self, environment_id):
"""Checkout to specific environment id
Parameters
----------
environment_id : str
environment id to checkout to
Returns
-------
bool
True if success
Raises
------
EnvironmentNotInitialized
error if not initialized (must initialize first)
PathDoesNotExist
if environment id does not exist
UnstagedChanges
error if not there exists unstaged changes in environment
"""
if not self.is_initialized:
raise EnvironmentNotInitialized()
if not self.exists(environment_id):
raise EnvironmentDoesNotExist(
__("error", "controller.environment.checkout_env",
environment_id))
# Check if unstaged changes exist
if self._has_unstaged_changes():
raise UnstagedChanges()
# Check if environment has is same as current
results = self.dal.environment.query({"id": environment_id})
environment_obj = results[0]
environment_hash = environment_obj.unique_hash
if self._calculate_project_environment_hash() == environment_hash:
return True
# Remove all content from project environment directory
for file in os.listdir(self.file_driver.environment_directory):
file_path = os.path.join(self.file_driver.environment_directory,
file)
try:
if os.path.isfile(file_path):
os.remove(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception as e:
print(e)
# Add in files for that environment id
file_collection_obj = self.dal.file_collection.\
get_by_id(environment_obj.file_collection_id)
environment_definition_path = os.path.join(self.home,
file_collection_obj.path)
# Copy to temp folder and remove files that are datmo specific
_temp_env_dir = get_datmo_temp_path(self.home)
self.file_driver.copytree(environment_definition_path, _temp_env_dir)
for filename in self.environment_driver.get_datmo_definition_filenames(
):
os.remove(os.path.join(_temp_env_dir, filename))
# Copy from temp folder to project environment directory
self.file_driver.copytree(_temp_env_dir,
self.file_driver.environment_directory)
shutil.rmtree(_temp_env_dir)
return True
