def create_requirements_file(self, execpath="pipreqs"):
"""Create python requirements txt file for the project
Parameters
----------
execpath : str, optional
execpath for the pipreqs command to form requirements.txt file
(default is "pipreqs")
Returns
-------
str
absolute filepath for requirements file
Raises
------
EnvironmentDoesNotExist
no python requirements found for environment
EnvironmentRequirementsCreateException
error in running pipreqs command to extract python requirements
"""
try:
subprocess.check_output([execpath, self.filepath, "--force"],
cwd=self.filepath).strip()
requirements_filepath = os.path.join(self.filepath,
"requirements.txt")
except Exception as e:
raise EnvironmentRequirementsCreateException(
__("error", "controller.environment.requirements.create",
str(e)))
if open(requirements_filepath, "r").read() == "\n":
raise EnvironmentDoesNotExist()
return requirements_filepath
def delete(self, environment_id):
"""Delete all traces of an environment
Parameters
----------
environment_id : str
environment object id to remove
Returns
-------
bool
True if success
Raises
------
EnvironmentDoesNotExist
if the specified Environment does not exist.
"""
self.environment_driver.init()
if not self.exists(environment_id):
raise EnvironmentDoesNotExist(
__("error", "controller.environment.delete", environment_id))
# Remove file collection
environment_obj = self.dal.environment.get_by_id(environment_id)
file_collection_deleted = self.file_collection.delete(
environment_obj.file_collection_id)
# Remove artifacts associated with the environment_driver
environment_artifacts_removed = self.environment_driver.remove(
environment_id, force=True)
# Delete environment_driver object
delete_success = self.dal.environment.delete(environment_obj.id)
return file_collection_deleted and environment_artifacts_removed and \
delete_success
def create(self, path=None, output_path=None, language=None):
if not path:
path = os.path.join(self.filepath, "Dockerfile")
if not output_path:
directory, filename = os.path.split(path)
output_path = os.path.join(directory, "datmo" + filename)
if not language:
language = "python3"
requirements_filepath = None
if not os.path.isfile(path):
if language == "python3":
# Create requirements txt file for python
requirements_filepath = self.create_requirements_file()
# Create Dockerfile for ubuntu
path = self.create_default_dockerfile(
requirements_filepath=requirements_filepath,
language=language)
else:
raise EnvironmentDoesNotExist(
__("error",
"controller.environment.driver.docker.create.dne",
path))
if os.path.isfile(output_path):
raise FileAlreadyExistsException(
__("error",
"controller.environment.driver.docker.create.exists",
output_path))
success = self.form_datmo_definition_file(
input_definition_path=path, output_definition_path=output_path)
return success, path, output_path, requirements_filepath
def update(self, environment_id, name=None, description=None):
"""Update the environment metadata"""
if not self.exists(environment_id):
raise EnvironmentDoesNotExist()
update_environment_input_dict = {"id": environment_id}
if name:
update_environment_input_dict['name'] = name
if description:
update_environment_input_dict['description'] = description
return self.dal.environment.update(update_environment_input_dict)
def setup(self, options, definition_path):
environment_type = options.get("environment_type", None)
environment_framework = options.get("environment_framework", None)
environment_language = options.get("environment_language", None)
environment_types = self.get_environment_types()
if environment_type not in environment_types:
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.setup.dne",
environment_type))
available_environment_info = self.get_supported_frameworks(
environment_type)
available_environments = [
item[0] for item in available_environment_info
]
if environment_framework not in available_environments:
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.setup.dne",
environment_framework))
available_environment_languages = self.get_supported_languages(
environment_type, environment_framework)
if available_environment_languages and environment_language not in available_environment_languages:
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.setup.dne",
environment_language))
# Validate the given definition path exists
if not os.path.isdir(definition_path):
raise PathDoesNotExist()
# To setup the environment definition file
definition_filepath = os.path.join(definition_path, "Dockerfile")
with open(definition_filepath, "wb") as f:
if environment_language:
f.write(
to_bytes("FROM datmo/%s:%s-%s%s%s" %
(environment_framework, environment_type,
environment_language, os.linesep, os.linesep)))
else:
f.write(
to_bytes("FROM datmo/%s:%s%s%s" %
(environment_framework, environment_type,
os.linesep, os.linesep)))
return True
def build(self, environment_id, workspace=None):
"""Build environment from definition file
Parameters
----------
environment_id : str
environment object id to build
workspace : str
workspace to be used
Returns
-------
bool
returns True if success
Raises
------
EnvironmentDoesNotExist
if the specified Environment does not exist.
"""
self.environment_driver.init()
if not self.exists(environment_id):
raise EnvironmentDoesNotExist(
__("error", "controller.environment.build", environment_id))
environment_obj = self.dal.environment.get_by_id(environment_id)
file_collection_obj = self.dal.file_collection.\
get_by_id(environment_obj.file_collection_id)
# TODO: Check hardware info here if different from creation time
# Add in files for that environment id
environment_definition_path = os.path.join(self.home,
file_collection_obj.path)
# Copy to temp folder and remove files that are datmo specific
_temp_env_dir = get_datmo_temp_path(self.home)
self.file_driver.copytree(environment_definition_path, _temp_env_dir)
# get definition filepath for the temp folder
environment_definition_filepath = os.path.join(
_temp_env_dir, environment_obj.definition_filename)
try:
# Build the Environment with the driver
self.spinner.start()
result = self.environment_driver.build(
environment_id,
path=environment_definition_filepath,
workspace=workspace)
finally:
self.spinner.stop()
# Remove both temporary directories
shutil.rmtree(_temp_env_dir)
return result
def setup(self, options, definition_path):
name = options.get("name", None)
available_environments = self.get_supported_environments()
# Validate that the name exists
if not name or name not in [n for n, _ in available_environments]:
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.setup.dne",
name))
# Validate the given definition path exists
if not os.path.isdir(definition_path):
raise PathDoesNotExist()
# To setup the environment definition file
definition_filepath = os.path.join(definition_path, "Dockerfile")
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM datmo/%s\n\n" % name))
return True
def create(self, path=None, output_path=None):
if not path:
path = os.path.join(self.filepath, "Dockerfile")
if not output_path:
directory, filename = os.path.split(path)
output_path = os.path.join(directory, "datmo" + filename)
if not os.path.isfile(path):
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.create.dne",
path))
if os.path.isfile(output_path):
raise FileAlreadyExistsError(
__("error",
"controller.environment.driver.docker.create.exists",
output_path))
success = self.create_datmo_definition(
input_definition_path=path, output_definition_path=output_path)
return success, path, output_path
def build(self, environment_id):
"""Build environment from definition file
Parameters
----------
environment_id : str
environment object id to build
Returns
-------
bool
returns True if success
Raises
------
EnvironmentDoesNotExist
if the specified Environment does not exist.
"""
self.environment_driver.init()
if not self.exists(environment_id):
raise EnvironmentDoesNotExist(
__("error", "controller.environment.build", environment_id))
environment_obj = self.dal.environment.get_by_id(environment_id)
file_collection_obj = self.dal.file_collection.\
get_by_id(environment_obj.file_collection_id)
# TODO: Check hardware info here if different from creation time
# Build the Environment with the driver
datmo_definition_filepath = os.path.join(
self.home, file_collection_obj.path,
"datmo" + environment_obj.definition_filename)
try:
self.spinner.start()
result = self.environment_driver.build(
environment_id, path=datmo_definition_filepath)
finally:
self.spinner.stop()
return result
def create(self, dictionary, save_hardware_file=True):
"""Create an environment
Parameters
----------
dictionary : dict
optional values to populate required environment entity args
paths : list, optional
list of absolute or relative filepaths and/or dirpaths to collect with destination names
(e.g. "/path/to/file>hello", "/path/to/file2", "/path/to/dir>newdir")
(default if none provided is to pull from project environment folder and project root. If none found create default definition)
name : str, optional
name of the environment
(default is None)
description : str, optional
description of the environment
(default is None)
save_hardware_file : bool
boolean to save hardware file along with other files
(default is True to save the file and create distinct hashes based on software and hardware)
Returns
-------
Environment
returns an object representing the environment created
Raises
------
EnvironmentDoesNotExist
if there is no environment found after given parameters and defaults are checked
PathDoesNotExist
if any source paths provided do not exist
"""
# Validate Inputs
create_dict = {"model_id": self.model.id}
create_dict["driver_type"] = self.environment_driver.type
validate("create_environment", dictionary)
# Create temp environment folder
_temp_env_dir = get_datmo_temp_path(self.home)
# Step 1: Populate a path list from the user inputs in a format compatible
# with the input of the File Collection create function
paths = []
# a. add in user given paths as is if they exist
if "paths" in dictionary and dictionary['paths']:
paths.extend(dictionary['paths'])
# b. if there exists projet environment directory AND no paths exist, add in absolute paths
if not paths and os.path.isdir(self.file_driver.environment_directory):
paths.extend([
os.path.join(self.file_driver.environment_directory, filepath)
for filepath in list_all_filepaths(
self.file_driver.environment_directory)
])
# c. add in default environment definition filepath as specified by the environment driver
# if path exists and NO OTHER PATHS exist
src_environment_filename = self.environment_driver.get_default_definition_filename(
)
src_environment_filepath = os.path.join(self.home,
src_environment_filename)
_, environment_filename = os.path.split(src_environment_filepath)
create_dict['definition_filename'] = environment_filename
if not paths and os.path.exists(src_environment_filepath):
paths.append(src_environment_filepath)
# Step 2: Check existing paths and create files as needed to populate the
# full environment within the temporary directory
paths = self._setup_compatible_environment(
create_dict,
paths,
_temp_env_dir,
save_hardware_file=save_hardware_file)
# Step 3: Pass in all paths for the environment to the file collection create
# If PathDoesNotExist is found for any source paths, then error
if not paths:
raise EnvironmentDoesNotExist()
try:
file_collection_obj = self.file_collection.create(paths)
except PathDoesNotExist as e:
raise PathDoesNotExist(
__("error", "controller.environment.create.filepath.dne",
str(e)))
# Step 4: Add file collection information to create dict and check unique hash
create_dict['file_collection_id'] = file_collection_obj.id
create_dict['unique_hash'] = file_collection_obj.filehash
# Check if unique hash is unique or not.
# If not, DO NOT CREATE Environment and return existing Environment object
results = self.dal.environment.query(
{"unique_hash": file_collection_obj.filehash})
if results: return results[0]
# Step 5: Delete the temporary directory
shutil.rmtree(_temp_env_dir)
# Step 6: Add optional arguments to the Environment entity
for optional_arg in ["name", "description"]:
if optional_arg in dictionary:
create_dict[optional_arg] = dictionary[optional_arg]
# Step 7: Create environment and return
return self.dal.environment.create(Environment(create_dict))
def checkout(self, environment_id):
"""Checkout to specific environment id
Parameters
----------
environment_id : str
environment id to checkout to
Returns
-------
bool
True if success
Raises
------
EnvironmentNotInitialized
error if not initialized (must initialize first)
PathDoesNotExist
if environment id does not exist
UnstagedChanges
error if not there exists unstaged changes in environment
"""
if not self.is_initialized:
raise EnvironmentNotInitialized()
if not self.exists(environment_id):
raise EnvironmentDoesNotExist(
__("error", "controller.environment.checkout_env",
environment_id))
# Check if unstaged changes exist
if self._has_unstaged_changes():
raise UnstagedChanges()
# Check if environment has is same as current
results = self.dal.environment.query({"id": environment_id})
environment_obj = results[0]
environment_hash = environment_obj.unique_hash
if self._calculate_project_environment_hash() == environment_hash:
return True
# Remove all content from project environment directory
for file in os.listdir(self.file_driver.environment_directory):
file_path = os.path.join(self.file_driver.environment_directory,
file)
try:
if os.path.isfile(file_path):
os.remove(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception as e:
print(e)
# Add in files for that environment id
file_collection_obj = self.dal.file_collection.\
get_by_id(environment_obj.file_collection_id)
environment_definition_path = os.path.join(self.home,
file_collection_obj.path)
# Copy to temp folder and remove files that are datmo specific
_temp_env_dir = get_datmo_temp_path(self.home)
self.file_driver.copytree(environment_definition_path, _temp_env_dir)
for filename in self.environment_driver.get_datmo_definition_filenames(
):
os.remove(os.path.join(_temp_env_dir, filename))
# Copy from temp folder to project environment directory
self.file_driver.copytree(_temp_env_dir,
self.file_driver.environment_directory)
shutil.rmtree(_temp_env_dir)
return True