def transfer_collection(self, filehash, dst_dirpath):
if not self.exists_collection(filehash):
raise PathDoesNotExist(
__("error", "controller.file.driver.local.transfer_collection",
filehash))
if not os.path.isdir(dst_dirpath):
raise PathDoesNotExist(
__("error",
"controller.file.driver.local.transfer_collection.dst",
dst_dirpath))
collection_path = os.path.join(self.filepath, ".datmo", "collections",
filehash)
return self.copytree(collection_path, dst_dirpath)
def copyfile(filepath, dst_dirpath):
if not os.path.isfile(filepath):
raise PathDoesNotExist(
__("error", "controller.file.driver.local.copyfile.core",
filepath))
if not os.path.isdir(dst_dirpath):
raise PathDoesNotExist(
__("error", "controller.file.driver.local.copyfile.dst",
dst_dirpath))
dst_filepath = LocalFileDriver.get_safe_dst_filepath(
filepath, dst_dirpath)
shutil.copy2(filepath, dst_filepath)
return True
def delete(self, environment_id):
"""Delete all traces of an environment
Parameters
----------
environment_id : str
environment object id to remove
Returns
-------
bool
True if success
Raises
------
PathDoesNotExist
if the specified Environment does not exist.
"""
environment_obj = self.dal.environment.get_by_id(environment_id)
if not environment_obj:
raise PathDoesNotExist(
__("error", "controller.environment.delete", environment_id))
# Remove file collection
file_collection_deleted = self.file_collection.delete(
environment_obj.file_collection_id)
# Remove artifacts associated with the environment_driver
environment_artifacts_removed = self.environment_driver.remove(
environment_id, force=True)
# Delete environment_driver object
delete_success = self.dal.environment.delete(environment_obj.id)
return file_collection_deleted and environment_artifacts_removed and \
delete_success
def delete(self, file_collection_id):
"""Delete all traces of FileCollection object
Parameters
----------
file_collection_id : str
file collection id to remove
Returns
-------
bool
returns True if success else False
Raises
------
PathDoesNotExist
if the specified FileCollection does not exist.
"""
file_collection_obj = self.dal.file_collection.get_by_id(
file_collection_id)
if not file_collection_obj:
raise PathDoesNotExist(
__("error", "controller.file_collection.delete",
file_collection_id))
# Remove file collection files
delete_file_collection_success = self.file_driver.delete_collection(
file_collection_obj.filehash)
# Delete FileCollection
delete_file_collection_obj_success = self.dal.file_collection.delete(
file_collection_obj.id)
return delete_file_collection_success and delete_file_collection_obj_success
def calculate_hash_paths(self, paths, directory):
try:
files, dirs, _, _ = parse_paths(self.root, paths, directory)
except PathDoesNotExist as e:
raise PathDoesNotExist(
__("error",
"controller.file.driver.local.create_collection.filepath",
str(e)))
# Populate collection from left to right in lists
for file_tuple in files:
src_abs_filepath, dest_abs_filepath = file_tuple
if os.path.exists(dest_abs_filepath):
raise FileAlreadyExistsError(
__("error",
"controller.file.driver.create_collection.file_exists",
dest_abs_filepath))
# File is copied over to the new destination path
shutil.copy2(src_abs_filepath, dest_abs_filepath)
for dir_tuple in dirs:
src_abs_dirpath, dest_abs_dirpath = dir_tuple
if os.path.exists(dest_abs_dirpath):
raise DirAlreadyExistsError(
__("error",
"controller.file.driver.create_collection.dir_exists",
dest_abs_dirpath))
os.makedirs(dest_abs_dirpath)
# All contents of directory is copied over to the new directory path
self.copytree(src_abs_dirpath, dest_abs_dirpath)
# Hash the files to find filehash
return self.get_dirhash(directory)
def delete(self, code_id):
"""Delete all traces of Code object
Parameters
----------
code_id : str
code object id to remove
Returns
-------
bool
returns True if success else False
Raises
------
PathDoesNotExist
if the specified Code does not exist.
"""
code_obj = self.dal.code.get_by_id(code_id)
if not code_obj:
raise PathDoesNotExist(
__("error", "controller.code.delete", code_id))
# Remove code reference
delete_code_success = self.code_driver.delete_ref(code_obj.commit_id)
# Delete code object
delete_code_obj_success = self.dal.code.delete(code_obj.id)
return delete_code_success and delete_code_obj_success
def __init__(self,
root,
datmo_directory_name,
docker_execpath="docker",
docker_socket=None):
super(DockerEnvironmentDriver, self).__init__()
if not docker_socket:
if platform.system() != "Windows":
docker_socket = "unix:///var/run/docker.sock"
self.root = root
# Check if filepath exists
if not os.path.exists(self.root):
raise PathDoesNotExist(
__("error",
"controller.environment.driver.docker.__init__.dne", root))
self._datmo_directory_name = datmo_directory_name
self._datmo_directory_path = os.path.join(self.root,
self._datmo_directory_name)
self.environment_directory_name = "environment"
self.environment_directory_path = os.path.join(
self._datmo_directory_path, self.environment_directory_name)
self.docker_execpath = docker_execpath
self.docker_socket = docker_socket
if self.docker_socket:
self.client = DockerClient(base_url=self.docker_socket)
self.prefix = [self.docker_execpath, "-H", self.docker_socket]
else:
self.client = DockerClient()
self.prefix = [self.docker_execpath]
self._is_connected = False
self._is_initialized = self.is_initialized
self.type = "docker"
with open(docker_config_filepath) as f:
self.docker_config = json.load(f)
def build(self, environment_id):
"""Build environment from definition file
Parameters
----------
environment_id : str
environment object id to build
Returns
-------
bool
returns True if success
Raises
------
PathDoesNotExist
if the specified Environment does not exist.
"""
environment_obj = self.dal.environment.get_by_id(environment_id)
if not environment_obj:
raise PathDoesNotExist(
__("error", "controller.environment.build", environment_id))
file_collection_obj = self.dal.file_collection.\
get_by_id(environment_obj.file_collection_id)
# TODO: Check hardware info here if different from creation time
# Build the Environment with the driver
datmo_definition_filepath = os.path.join(
self.home, file_collection_obj.path,
"datmo" + environment_obj.definition_filename)
result = self.environment_driver.build(environment_id,
path=datmo_definition_filepath)
return result
def __init__(self, filepath):
super(LocalFileDriver, self).__init__()
self.filepath = filepath
# Check if filepath exists
if not os.path.exists(self.filepath):
raise PathDoesNotExist(
__("error", "controller.file.driver.local.__init__", filepath))
self._is_initialized = self.is_initialized
self.type = "local"
def delete(self, relative_path, directory=False):
if not os.path.exists(os.path.join(self.filepath, relative_path)):
raise PathDoesNotExist(
__("error", "controller.file.driver.local.delete",
os.path.join(self.filepath, relative_path)))
if directory:
shutil.rmtree(relative_path)
else:
os.remove(os.path.join(self.filepath, relative_path))
return True
def copytree(src_dirpath, dst_dirpath, symlinks=False, ignore=None):
if not os.path.isdir(src_dirpath):
raise PathDoesNotExist(
__("error", "controller.file.driver.local.copytree.core",
src_dirpath))
if not os.path.isdir(dst_dirpath):
raise PathDoesNotExist(
__("error", "controller.file.driver.local.copytree.dst",
dst_dirpath))
for item in os.listdir(src_dirpath):
src_filepath = os.path.join(src_dirpath, item)
dst_filepath = os.path.join(dst_dirpath, item)
if os.path.isdir(src_filepath):
if os.path.exists(dst_filepath):
shutil.rmtree(dst_filepath)
shutil.copytree(src_filepath, dst_filepath, symlinks, ignore)
else:
if os.path.exists(dst_filepath):
os.remove(dst_filepath)
shutil.copy2(src_filepath, dst_filepath)
return True
def __init__(self, filepath, execpath, remote_url=None):
super(GitCodeDriver, self).__init__()
self.filepath = filepath
# Check if filepath exists
if not os.path.exists(self.filepath):
raise PathDoesNotExist(
__("error", "controller.code.driver.git.__init__.dne",
filepath))
self.execpath = execpath
# Check the execpath and the version
try:
p = subprocess.Popen(
[self.execpath, "version"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=self.filepath)
out, err = p.communicate()
out, err = out.decode(), err.decode()
if err:
raise GitExecutionError(
__("error", "controller.code.driver.git.__init__.giterror",
err))
version = str(out.split()[2].split(".windows")[0])
if not semver.match(version, ">=1.9.7"):
raise GitExecutionError(
__("error",
"controller.code.driver.git.__init__.gitversion",
out.split()[2]))
except Exception as e:
raise GitExecutionError(
__("error", "controller.code.driver.git.__init__.giterror",
str(e)))
# TODO: handle multiple remote urls
# self.git_host_driver = GitHostDriver()
self.remote_url = remote_url
self._is_initialized = self.is_initialized
if self._is_initialized:
# If initialized ensure .datmo is not in working tree else error
if self.exists_datmo_files_in_worktree():
raise DatmoFolderInWorkTree(
__("error", "controller.code.driver.git.__init__.datmo"))
# If initialized ensure .datmo is ignored (in .git/info/exclude)
self.ensure_datmo_files_ignored()
# If initialized update remote information
# if remote_url:
# self.remote("set-url", "origin", remote_url)
# self._remote_url = self.remote_url
# self._remote_access = False
self.type = "git"
def get_filehash(filepath):
if not os.path.isfile(filepath):
raise PathDoesNotExist(
__("error", "util.misc_functions.get_filehash", filepath))
BUFF_SIZE = 65536
sha1 = hashlib.md5()
with open(filepath, "rb") as f:
while True:
data = f.read(BUFF_SIZE)
if not data:
break
sha1.update(data)
return sha1.hexdigest()
def parse_paths(default_src_prefix, paths, dest_prefix):
"""Parse user given paths. Checks only source paths and destination are valid
Parameters
----------
default_src_prefix : str
default directory prefix to append if source path is not an absolute path
paths : list
list of absolute or relative filepaths and/or dirpaths to collect with destination names
(e.g. "/path/to/file>hello", "/path/to/file2", "/path/to/dir>newdir")
dest_prefix : str
destination directory prefix to append to the destination filename
Returns
-------
files : list
list of file tuples of the form (absolute_source_path, absolute_dest_filepath)
directories : list
list of directory tuples of the form (absolute_source_path, absolute_dest_filepath)
files_rel : list
list of file tuples of the form (absolute_source_path, relative_dest_path)
directories_rel : list
list of directory tuples of the form (absolute_source_path, relative_dest_path)
Raises
------
InvalidDestinationName
destination specified in paths is not valid
PathDoesNotExist
if the path does not exist
"""
files, files_rel, directories, directories_rel = [], [], [], []
for path in paths:
src_path, dest_name = parse_path(path)
# For dest_name, append the dest_prefix
dest_abs_path = os.path.join(dest_prefix, dest_name)
# For src_path if not absolute, append the default src_prefix
if not os.path.isabs(path):
src_abs_path = os.path.join(default_src_prefix, src_path)
else:
src_abs_path = src_path
# Check if source is file or directory and save accordingly
if os.path.isfile(src_abs_path):
files.append((src_abs_path, dest_abs_path))
files_rel.append((src_abs_path, dest_name))
elif os.path.isdir(src_abs_path):
directories.append((src_abs_path, dest_abs_path))
directories_rel.append((src_abs_path, dest_name))
else:
raise PathDoesNotExist(src_abs_path)
return files, directories, files_rel, directories_rel
def __init__(self, root, datmo_directory_name):
super(FileCodeDriver, self).__init__()
self.root = root
# Check if filepath exists
if not os.path.exists(self.root):
raise PathDoesNotExist(
__("error", "controller.code.driver.git.__init__.dne", root))
self._datmo_directory_name = datmo_directory_name
self._datmo_directory_path = os.path.join(self.root,
self._datmo_directory_name)
self._code_filepath = os.path.join(self._datmo_directory_path, "code")
self._datmo_ignore_filepath = os.path.join(self.root, ".datmoignore")
self._is_initialized = self.is_initialized
self.type = "file"
def get_files(self, task_id, mode="r"):
"""Get list of file objects for task id. It will look in the following areas in the following order
1) look in the after snapshot for file collection
2) look in the running task file collection
3) look in the before snapshot for file collection
Parameters
----------
task_id : str
id for the task you would like to get file objects for
mode : str
file open mode
(default is "r" to open file for read)
Returns
-------
list
list of python file objects
Raises
------
PathDoesNotExist
no file objects exist for the task
"""
task_obj = self.dal.task.get_by_id(task_id)
if task_obj.after_snapshot_id:
# perform number 1) and return file list
after_snapshot_obj = \
self.dal.snapshot.get_by_id(task_obj.after_snapshot_id)
file_collection_obj = \
self.dal.file_collection.get_by_id(after_snapshot_obj.file_collection_id)
return self.file_driver.\
get_collection_files(file_collection_obj.filehash, mode=mode)
elif task_obj.task_dirpath:
# perform number 2) and return file list
return self.file_driver.get(task_obj.task_dirpath,
mode=mode,
directory=True)
elif task_obj.before_snapshot_id:
# perform number 3) and return file list
before_snapshot_obj = \
self.dal.snapshot.get_by_id(task_obj.before_snapshot_id)
file_collection_obj = \
self.dal.file_collection.get_by_id(before_snapshot_obj.file_collection_id)
return self.file_driver. \
get_collection_files(file_collection_obj.filehash, mode=mode)
else:
# Error because the task does not have any files associated with it
raise PathDoesNotExist()
def get_safe_dst_filepath(filepath, dst_dirpath):
if not os.path.isfile(filepath):
raise PathDoesNotExist(
__("error",
"controller.file.driver.local.get_safe_dst_filepath.core",
filepath))
if not os.path.isdir(dst_dirpath):
raise PathDoesNotExist(
__("error",
"controller.file.driver.local.get_safe_dst_filepath.dst",
dst_dirpath))
_, filename = os.path.split(filepath)
dst_filepath = os.path.join(dst_dirpath, filename)
number_of_items = glob.glob(dst_filepath)
if number_of_items:
filepath_without_ext = os.path.splitext(dst_filepath)[0]
extension = os.path.splitext(dst_filepath)[1]
number_of_items = len(glob.glob(filepath_without_ext + '*'))
new_filepath = filepath_without_ext + "_" + str(number_of_items -
1)
new_filepath_with_ext = new_filepath + extension
return new_filepath_with_ext
else:
return dst_filepath
def setup(self, options, definition_path):
name = options.get("name", None)
available_environments = self.get_supported_environments()
# Validate that the name exists
if not name or name not in [n for n, _ in available_environments]:
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.setup.dne",
name))
# Validate the given definition path exists
if not os.path.isdir(definition_path):
raise PathDoesNotExist()
# To setup the environment definition file
definition_filepath = os.path.join(definition_path, "Dockerfile")
with open(definition_filepath, "wb") as f:
f.write(to_bytes("FROM datmo/%s\n\n" % name))
return True
def checkout(self, file_collection_id):
"""Checkout to specific file collection id
Returns
-------
bool
True if success
Raises
------
FileNotInitialized
error if not initialized (must initialize first)
UnstagedChanges
error if not there exists unstaged changes in files
"""
if not self.is_initialized:
raise FileNotInitialized()
if not self.exists(file_collection_id=file_collection_id):
raise PathDoesNotExist(
__("error", "controller.file_collection.checkout_file"))
# Check if unstaged changes exist
self.check_unstaged_changes()
# Check if environment has is same as current
results = self.dal.file_collection.query({"id": file_collection_id})
file_collection_obj = results[0]
file_hash = file_collection_obj.filehash
if self._calculate_project_files_hash() == file_hash:
return True
# Remove all content from `datmo_file` folder
# TODO Use datmo environment path as a class attribute
for file in os.listdir(self.file_driver.files_directory):
file_path = os.path.join(self.file_driver.files_directory, file)
try:
if os.path.isfile(file_path):
os.remove(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception as e:
print(e)
# Add in files for that file collection id
file_collection_path = os.path.join(self.home,
file_collection_obj.path)
self.file_driver.copytree(file_collection_path,
self.file_driver.files_directory)
return True
def __init__(self, root):
super(LocalFileDriver, self).__init__()
self.root = root
# Check if filepath exists
if not os.path.exists(self.root):
raise PathDoesNotExist(
__("error", "controller.file.driver.local.__init__", root))
self.datmo_directory_name = ".datmo"
self.datmo_directory = os.path.join(self.root,
self.datmo_directory_name)
self.environment_directory_name = "datmo_environment"
self.environment_directory = os.path.join(
self.root, self.environment_directory_name)
self.files_directory_name = "datmo_files"
self.files_directory = os.path.join(self.root,
self.files_directory_name)
self._is_initialized = self.is_initialized
self.type = "local"
def setup(self, options, definition_path):
environment_type = options.get("environment_type", None)
environment_framework = options.get("environment_framework", None)
environment_language = options.get("environment_language", None)
environment_types = self.get_environment_types()
if environment_type not in environment_types:
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.setup.dne",
environment_type))
available_environment_info = self.get_supported_frameworks(
environment_type)
available_environments = [
item[0] for item in available_environment_info
]
if environment_framework not in available_environments:
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.setup.dne",
environment_framework))
available_environment_languages = self.get_supported_languages(
environment_type, environment_framework)
if available_environment_languages and environment_language not in available_environment_languages:
raise EnvironmentDoesNotExist(
__("error", "controller.environment.driver.docker.setup.dne",
environment_language))
# Validate the given definition path exists
if not os.path.isdir(definition_path):
raise PathDoesNotExist()
# To setup the environment definition file
definition_filepath = os.path.join(definition_path, "Dockerfile")
with open(definition_filepath, "wb") as f:
if environment_language:
f.write(
to_bytes("FROM datmo/%s:%s-%s%s%s" %
(environment_framework, environment_type,
environment_language, os.linesep, os.linesep)))
else:
f.write(
to_bytes("FROM datmo/%s:%s%s%s" %
(environment_framework, environment_type,
os.linesep, os.linesep)))
return True
def get(self, relative_path, mode="r", directory=False):
filepath = os.path.join(self.root, relative_path)
if not os.path.exists(filepath):
raise PathDoesNotExist(
__("error", "controller.file.driver.local.get", filepath))
if directory:
absolute_filepaths = []
for dirname, _, filenames in os.walk(filepath):
# print path to all filenames.
for filename in filenames:
absolute_filepaths.append(os.path.join(dirname, filename))
# Return a list of file objects
return [
open(absolute_filepath, mode)
for absolute_filepath in absolute_filepaths
]
else:
filepath = os.path.join(self.root, relative_path)
return open(filepath, mode)
def __init__(self, root, datmo_directory_name):
super(LocalFileDriver, self).__init__()
self.root = root
# Check if root exists
if not os.path.exists(self.root):
raise PathDoesNotExist(
__("error", "controller.file.driver.local.__init__", root))
self.datmo_directory_name = datmo_directory_name
self.datmo_directory = os.path.join(self.root,
self.datmo_directory_name)
self.collections_directory_name = "collections"
self.collections_directory = os.path.join(
self.root, self.datmo_directory_name,
self.collections_directory_name)
self.files_directory_name = "files"
self.files_directory = os.path.join(self.root,
self.datmo_directory_name,
self.files_directory_name)
self._is_initialized = self.is_initialized
self.type = "local"
def __init__(self,
filepath="",
docker_execpath="docker",
docker_socket=None):
if not docker_socket:
if platform.system() != "Windows":
docker_socket = "unix:///var/run/docker.sock"
super(DockerEnvironmentDriver, self).__init__()
self.filepath = filepath
# Check if filepath exists
if not os.path.exists(self.filepath):
raise PathDoesNotExist(
__("error",
"controller.environment.driver.docker.__init__.dne",
filepath))
# TODO: separate methods for instantiation into init function below
# Initiate Docker execution
self.docker_execpath = docker_execpath
try:
self.docker_socket = docker_socket
if self.docker_socket:
self.client = DockerClient(base_url=self.docker_socket)
self.cpu_prefix = [
self.docker_execpath, "-H", self.docker_socket
]
else:
self.client = DockerClient()
self.cpu_prefix = [self.docker_execpath]
self.info = self.client.info()
except Exception:
raise EnvironmentInitFailed(
__("error", "controller.environment.driver.docker.__init__",
platform.system()))
self.is_connected = True if self.info["Images"] != None else False
self._is_initialized = self.is_initialized
self.type = "docker"
def __init__(self, filepath):
super(FileCodeDriver, self).__init__()
self.filepath = filepath
# Check if filepath exists
if not os.path.exists(self.filepath):
raise PathDoesNotExist(
__("error", "controller.code.driver.git.__init__.dne",
filepath))
self._datmo_directory_name = ".datmo"
self._datmo_directory_path = os.path.join(self.filepath,
self._datmo_directory_name)
self._environment_directory_name = "datmo_environment"
self._environment_directory_path = os.path.join(
self.filepath, self._environment_directory_name)
self._files_directory_name = "datmo_files"
self._files_directory_path = os.path.join(self.filepath,
self._files_directory_name)
self._code_filepath = os.path.join(self._datmo_directory_path, "code")
self._datmo_ignore_filepath = os.path.join(self.filepath,
".datmoignore")
self._is_initialized = self.is_initialized
self.type = "file"
def create(self, dictionary, save_hardware_file=True):
"""Create an environment
Parameters
----------
dictionary : dict
optional values to populate required environment entity args
paths : list, optional
list of absolute or relative filepaths and/or dirpaths to collect with destination names
(e.g. "/path/to/file>hello", "/path/to/file2", "/path/to/dir>newdir")
(default if none provided is to pull from project environment folder and project root. If none found create default definition)
name : str, optional
name of the environment
(default is None)
description : str, optional
description of the environment
(default is None)
save_hardware_file : bool
boolean to save hardware file along with other files
(default is True to save the file and create distinct hashes based on software and hardware)
Returns
-------
Environment
returns an object representing the environment created
Raises
------
EnvironmentDoesNotExist
if there is no environment found after given parameters and defaults are checked
PathDoesNotExist
if any source paths provided do not exist
"""
# Validate Inputs
create_dict = {"model_id": self.model.id}
create_dict["driver_type"] = self.environment_driver.type
validate("create_environment", dictionary)
# Create temp environment folder
_temp_env_dir = get_datmo_temp_path(self.home)
# Step 1: Populate a path list from the user inputs in a format compatible
# with the input of the File Collection create function
paths = []
# a. add in user given paths as is if they exist
if "paths" in dictionary and dictionary['paths']:
paths.extend(dictionary['paths'])
# b. if there exists projet environment directory AND no paths exist, add in absolute paths
if not paths and os.path.isdir(self.file_driver.environment_directory):
paths.extend([
os.path.join(self.file_driver.environment_directory, filepath)
for filepath in list_all_filepaths(
self.file_driver.environment_directory)
])
# c. add in default environment definition filepath as specified by the environment driver
# if path exists and NO OTHER PATHS exist
src_environment_filename = self.environment_driver.get_default_definition_filename(
)
src_environment_filepath = os.path.join(self.home,
src_environment_filename)
_, environment_filename = os.path.split(src_environment_filepath)
create_dict['definition_filename'] = environment_filename
if not paths and os.path.exists(src_environment_filepath):
paths.append(src_environment_filepath)
# Step 2: Check existing paths and create files as needed to populate the
# full environment within the temporary directory
paths = self._setup_compatible_environment(
create_dict,
paths,
_temp_env_dir,
save_hardware_file=save_hardware_file)
# Step 3: Pass in all paths for the environment to the file collection create
# If PathDoesNotExist is found for any source paths, then error
if not paths:
raise EnvironmentDoesNotExist()
try:
file_collection_obj = self.file_collection.create(paths)
except PathDoesNotExist as e:
raise PathDoesNotExist(
__("error", "controller.environment.create.filepath.dne",
str(e)))
# Step 4: Add file collection information to create dict and check unique hash
create_dict['file_collection_id'] = file_collection_obj.id
create_dict['unique_hash'] = file_collection_obj.filehash
# Check if unique hash is unique or not.
# If not, DO NOT CREATE Environment and return existing Environment object
results = self.dal.environment.query(
{"unique_hash": file_collection_obj.filehash})
if results: return results[0]
# Step 5: Delete the temporary directory
shutil.rmtree(_temp_env_dir)
# Step 6: Add optional arguments to the Environment entity
for optional_arg in ["name", "description"]:
if optional_arg in dictionary:
create_dict[optional_arg] = dictionary[optional_arg]
# Step 7: Create environment and return
return self.dal.environment.create(Environment(create_dict))
def create_collection(self, filepaths):
if not self.is_initialized:
raise FileStructureException(
__("error",
"controller.file.driver.local.create_collection.structure"))
# Ensure all filepaths are valid before proceeding
for filepath in filepaths:
if not os.path.isdir(filepath) and \
not os.path.isfile(filepath):
raise PathDoesNotExist(
__(
"error",
"controller.file.driver.local.create_collection.filepath",
filepath))
# Create temp hash and folder to move all contents from filepaths
temp_hash = hashlib.sha1(str(uuid.uuid4()). \
encode("UTF=8")).hexdigest()[:20]
self.ensure_collections_dir()
temp_collection_path = os.path.join(self.filepath, ".datmo",
"collections", temp_hash)
os.makedirs(temp_collection_path)
# Populate collection
for filepath in filepaths:
_, dirname = os.path.split(filepath)
if os.path.isdir(filepath):
dst_dirpath = os.path.join(temp_collection_path, dirname)
self.create(dst_dirpath, directory=True)
# All contents of directory are copied into the dst_dirpath
self.copytree(filepath, dst_dirpath)
elif os.path.isfile(filepath):
# File is copied into the collection_path
self.copyfile(filepath, temp_collection_path)
# Hash the files to find filehash
filehash = checksumdir.dirhash(temp_collection_path)
# Move contents to folder with filehash as name and remove temp_collection_path
collection_path = os.path.join(self.filepath, ".datmo", "collections",
filehash)
if os.path.isdir(collection_path):
shutil.rmtree(temp_collection_path)
return filehash
# raise FileStructureException("exception.file.create_collection", {
# "exception": "File collection with id already exists."
# })
os.makedirs(collection_path)
self.copytree(temp_collection_path, collection_path)
shutil.rmtree(temp_collection_path)
# Change permissions to read only for collection_path. File collection is immutable
mode = 0o755
for root, dirs, files in os.walk(collection_path, topdown=False):
for dir in [os.path.join(root, d) for d in dirs]:
os.chmod(dir, mode)
for file in [os.path.join(root, f) for f in files]:
os.chmod(file, mode)
return filehash