def test_create_environment_by_dictionary(self):
environment = self.dal.environment.create(
Environment(self.environment_input_dict))
assert environment.id
assert environment.driver_type == self.environment_input_dict[
'driver_type']
assert environment.file_collection_id == self.environment_input_dict[
'file_collection_id']
assert environment.definition_filename == self.environment_input_dict[
'definition_filename']
assert environment.hardware_info == self.environment_input_dict[
'hardware_info']
assert environment.unique_hash == self.environment_input_dict[
'unique_hash']
assert environment.created_at
assert environment.updated_at
environment_2 = self.dal.environment.create(
Environment(self.environment_input_dict))
assert environment_2.id != environment.id
test_environment_input_dict = self.environment_input_dict.copy()
test_environment_input_dict['id'] = "environment_id"
environment_3 = self.dal.environment.create(
Environment(test_environment_input_dict))
assert environment_3.id == test_environment_input_dict['id']
def test_query_environments_multiple(self):
environment_1 = self.dal.environment.create(
Environment(self.environment_input_dict))
environment_2 = self.dal.environment.create(
Environment(self.environment_input_dict))
environment_3 = self.dal.environment.create(
Environment(self.environment_input_dict))
results = self.dal.environment.query({},
sort_key="created_at",
sort_order="ascending")
assert len(results) == 3
assert results[0].created_at == environment_1.created_at
assert results[1].created_at == environment_2.created_at
results = self.dal.environment.query({},
sort_key="created_at",
sort_order="descending")
assert len(results) == 3
assert results[0].created_at == environment_3.created_at
assert results[1].created_at == environment_2.created_at
# Wrong order being passed in
failed = False
try:
_ = self.dal.environment.query({},
sort_key='created_at',
sort_order='wrong_order')
except InvalidArgumentType:
failed = True
assert failed
# Wrong key and order being passed in
failed = False
try:
_ = self.dal.environment.query({},
sort_key='wrong_key',
sort_order='wrong_order')
except InvalidArgumentType:
failed = True
assert failed
# wrong key and right order being passed in
expected_items = self.dal.environment.query({},
sort_key='created_at',
sort_order='ascending')
items = self.dal.environment.query({},
sort_key='wrong_key',
sort_order='ascending')
expected_ids = [item.id for item in expected_items]
ids = [item.id for item in items]
assert set(expected_ids) == set(ids)
def test_query_environments_range_query(self):
_ = self.dal.environment.create(
Environment(self.environment_input_dict))
_ = self.dal.environment.create(
Environment(self.environment_input_dict))
_ = self.dal.environment.create(
Environment(self.environment_input_dict))
environments = self.dal.environment.query({},
sort_key="created_at",
sort_order="descending")
result = self.dal.environment.query({
"created_at": {
"$lt":
environments[1].created_at.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
}
})
assert len(environments) == 3
assert len(result) == 1
def test_init_with_id(self):
self.input_dict['id'] = "test"
environment_entity = Environment(self.input_dict)
for k, v in self.input_dict.items():
assert getattr(environment_entity, k) == v
assert environment_entity.name == None
assert environment_entity.description == None
assert environment_entity.created_at
assert environment_entity.updated_at
def test_init_no_id(self):
environment_entity = Environment(self.input_dict)
for k, v in self.input_dict.items():
assert getattr(environment_entity, k) == v
assert environment_entity.id == None
assert environment_entity.description == None
assert environment_entity.created_at
assert environment_entity.updated_at
assert environment_entity.language
def test_delete_environment(self):
environment = self.dal.environment.create(Environment(self.environment_input_dict))
self.dal.environment.delete(environment.id)
deleted = False
try:
self.dal.environment.get_by_id(environment.id)
except EntityNotFound:
deleted = True
assert deleted
def test_get_by_id_environment_new_driver_instance(self):
environment = self.dal.environment.create(Environment(self.environment_input_dict))
# create new dal with new driver instance (success)
new_driver_instance = BlitzDBDALDriver("file", self.temp_dir)
new_dal_instance = LocalDAL(new_driver_instance)
new_environment_1 = new_dal_instance.environment.get_by_id(environment.id)
assert new_environment_1.id == environment.id
# create new dal instance with same driver (success)
new_dal_instance = LocalDAL(self.datadriver)
new_environment_2 = new_dal_instance.environment.get_by_id(environment.id)
assert new_environment_2.id == environment.id
def test_update_environment(self):
environment = self.dal.environment.create(Environment(self.environment_input_dict))
# Update required and optional parameters
updated_environment_input_dict = self.environment_input_dict.copy()
updated_environment_input_dict['id'] = environment.id
updated_environment_input_dict['driver_type'] = "new_driver"
updated_environment_input_dict['created_at'] = datetime.utcnow()
updated_environment = self.dal.environment.update(updated_environment_input_dict)
assert environment.id == updated_environment.id
assert environment.updated_at < updated_environment.updated_at
assert updated_environment.driver_type == updated_environment_input_dict['driver_type']
assert updated_environment.created_at == updated_environment_input_dict['created_at']
def create(self, dictionary, save_hardware_file=True):
"""Create an environment
Parameters
----------
dictionary : dict
optional values to populate required environment entity args
paths : list, optional
list of absolute or relative filepaths and/or dirpaths to collect with destination names
(e.g. "/path/to/file>hello", "/path/to/file2", "/path/to/dir>newdir")
(default if none provided is to pull from project environment folder and project root. If none found create default definition)
name : str, optional
name of the environment
(default is None)
description : str, optional
description of the environment
(default is None)
save_hardware_file : bool
boolean to save hardware file along with other files
(default is True to save the file and create distinct hashes based on software and hardware)
Returns
-------
Environment
returns an object representing the environment created
Raises
------
EnvironmentDoesNotExist
if there is no environment found after given parameters and defaults are checked
PathDoesNotExist
if any source paths provided do not exist
"""
# Validate Inputs
create_dict = {"model_id": self.model.id}
create_dict["driver_type"] = self.environment_driver.type
validate("create_environment", dictionary)
# Create temp environment folder
_temp_env_dir = get_datmo_temp_path(self.home)
# Step 1: Populate a path list from the user inputs in a format compatible
# with the input of the File Collection create function
paths = []
# a. add in user given paths as is if they exist
if "paths" in dictionary and dictionary['paths']:
paths.extend(dictionary['paths'])
# b. if there exists projet environment directory AND no paths exist, add in absolute paths
if not paths and os.path.isdir(self.file_driver.environment_directory):
paths.extend([
os.path.join(self.file_driver.environment_directory, filepath)
for filepath in list_all_filepaths(
self.file_driver.environment_directory)
])
# c. add in default environment definition filepath as specified by the environment driver
# if path exists and NO OTHER PATHS exist
src_environment_filename = self.environment_driver.get_default_definition_filename(
)
src_environment_filepath = os.path.join(self.home,
src_environment_filename)
_, environment_filename = os.path.split(src_environment_filepath)
create_dict['definition_filename'] = environment_filename
if not paths and os.path.exists(src_environment_filepath):
paths.append(src_environment_filepath)
# Step 2: Check existing paths and create files as needed to populate the
# full environment within the temporary directory
paths = self._setup_compatible_environment(
create_dict,
paths,
_temp_env_dir,
save_hardware_file=save_hardware_file)
# Step 3: Pass in all paths for the environment to the file collection create
# If PathDoesNotExist is found for any source paths, then error
if not paths:
raise EnvironmentDoesNotExist()
try:
file_collection_obj = self.file_collection.create(paths)
except PathDoesNotExist as e:
raise PathDoesNotExist(
__("error", "controller.environment.create.filepath.dne",
str(e)))
# Step 4: Add file collection information to create dict and check unique hash
create_dict['file_collection_id'] = file_collection_obj.id
create_dict['unique_hash'] = file_collection_obj.filehash
# Check if unique hash is unique or not.
# If not, DO NOT CREATE Environment and return existing Environment object
results = self.dal.environment.query(
{"unique_hash": file_collection_obj.filehash})
if results: return results[0]
# Step 5: Delete the temporary directory
shutil.rmtree(_temp_env_dir)
# Step 6: Add optional arguments to the Environment entity
for optional_arg in ["name", "description"]:
if optional_arg in dictionary:
create_dict[optional_arg] = dictionary[optional_arg]
# Step 7: Create environment and return
return self.dal.environment.create(Environment(create_dict))
def create(self, dictionary):
"""Create an environment
Parameters
----------
dictionary : dict
optional values to populate required environment entity args
definition_filepath : str, optional
absolute filepath to the environment definition file
(default is to use driver default filepath)
hardware_info : dict, optional
information about the environment hardware
(default is to extract hardware from platform currently running)
language : str, optional
programming language used
(default is None, which allows Driver to determine default)
optional values to populate optional environment entity args
description : str, optional
description of the environment
(default is blank)
Returns
-------
Environment
returns an object representing the environment created
Raises
------
RequiredArgumentMissing
if any arguments above are not provided.
"""
# Validate Inputs
create_dict = {
"model_id": self.model.id,
}
create_dict["driver_type"] = self.environment_driver.type
create_dict["language"] = dictionary.get("language", None)
if "definition_filepath" in dictionary and dictionary[
'definition_filepath']:
original_definition_filepath = dictionary['definition_filepath']
# Split up the given path and save definition filename
definition_path, definition_filename = \
os.path.split(original_definition_filepath)
create_dict['definition_filename'] = definition_filename
# Create datmo environment definition in the same dir as definition filepath
datmo_definition_filepath = \
os.path.join(definition_path, "datmo" + definition_filename)
_, _, _, requirements_filepath = self.environment_driver.create(
path=dictionary['definition_filepath'],
output_path=datmo_definition_filepath)
else:
# If path is not given, then only use the language to create a default environment
# Use the default create to find environment definition
_, original_definition_filepath, datmo_definition_filepath, requirements_filepath = \
self.environment_driver.create(language=create_dict['language'])
# Split up the default path obtained to save the definition name
definition_path, definition_filename = \
os.path.split(original_definition_filepath)
create_dict['definition_filename'] = definition_filename
hardware_info_filepath = self._store_hardware_info(
dictionary, create_dict, definition_path)
# Add all environment files to collection:
# definition path, datmo_definition_path, hardware_info
filepaths = [
original_definition_filepath, datmo_definition_filepath,
hardware_info_filepath
]
if requirements_filepath:
filepaths.append(requirements_filepath)
file_collection_obj = self.file_collection.create(filepaths)
create_dict['file_collection_id'] = file_collection_obj.id
# Delete temporary files created once transfered into file collection
if requirements_filepath:
os.remove(requirements_filepath)
os.remove(original_definition_filepath)
os.remove(datmo_definition_filepath)
os.remove(hardware_info_filepath)
create_dict['unique_hash'] = file_collection_obj.filehash
# Check if unique hash is unique or not.
# If not, DO NOT CREATE Environment and return existing Environment object
results = self.dal.environment.query(
{"unique_hash": file_collection_obj.filehash})
if results: return results[0]
# Optional args for Environment entity
for optional_arg in ["description"]:
if optional_arg in dictionary:
create_dict[optional_arg] = dictionary[optional_arg]
# Create environment and return
return self.dal.environment.create(Environment(create_dict))
def test_get_by_shortened_id_environment(self):
environment = self.dal.environment.create(
Environment(self.environment_input_dict))
result = self.dal.environment.get_by_shortened_id(environment.id[:10])
assert environment.id == result.id
def test_query_environments_basic(self):
environment = self.dal.environment.create(
Environment(self.environment_input_dict))
assert len(self.dal.environment.query({"id": environment.id})) == 1
def test_to_dictionary(self):
environment_entity = Environment(self.input_dict)
output_dict = environment_entity.to_dictionary()
for k, v in output_dict.items():
assert v == getattr(environment_entity, k)
def test_eq(self):
environment_entity_1 = Environment(self.input_dict)
environment_entity_2 = Environment(self.input_dict)
assert environment_entity_1 == environment_entity_2