def calculate_hash_paths(self, paths, directory):
try:
files, dirs, _, _ = parse_paths(self.root, paths, directory)
except PathDoesNotExist as e:
raise PathDoesNotExist(
__("error",
"controller.file.driver.local.create_collection.filepath",
str(e)))
# Populate collection from left to right in lists
for file_tuple in files:
src_abs_filepath, dest_abs_filepath = file_tuple
if os.path.exists(dest_abs_filepath):
raise FileAlreadyExistsError(
__("error",
"controller.file.driver.create_collection.file_exists",
dest_abs_filepath))
# File is copied over to the new destination path
shutil.copy2(src_abs_filepath, dest_abs_filepath)
for dir_tuple in dirs:
src_abs_dirpath, dest_abs_dirpath = dir_tuple
if os.path.exists(dest_abs_dirpath):
raise DirAlreadyExistsError(
__("error",
"controller.file.driver.create_collection.dir_exists",
dest_abs_dirpath))
os.makedirs(dest_abs_dirpath)
# All contents of directory is copied over to the new directory path
self.copytree(src_abs_dirpath, dest_abs_dirpath)
# Hash the files to find filehash
return self.get_dirhash(directory)
def test_parse_paths(self):
# Create a file and a directory to test on
filepath = os.path.join(self.temp_dir, "test.txt")
dirpath = os.path.join(self.temp_dir, "test_dir")
dirfilepath = os.path.join(dirpath, "test.txt")
with open(filepath, "wb") as f:
f.write(to_bytes("test" + "\n"))
os.makedirs(dirpath)
with open(dirfilepath, "wb") as f:
f.write(to_bytes("test" + "\n"))
# Define user paths
default_source_prefix = self.temp_dir
dest_prefix = os.path.join(self.temp_dir, "some_dest_dir")
# Test default source path and default dest path
paths = ["test.txt", "test_dir"]
result = parse_paths(default_source_prefix, paths, dest_prefix)
assert result[0] == [(os.path.join(default_source_prefix, "test.txt"),
os.path.join(dest_prefix, "test.txt"))]
assert result[1] == [(os.path.join(default_source_prefix, "test_dir"),
os.path.join(dest_prefix, "test_dir"))]
# Test absolute source path and no dest path
paths = [filepath, dirpath]
result = parse_paths(default_source_prefix, paths, dest_prefix)
assert result[0] == [(filepath, os.path.join(dest_prefix, "test.txt"))]
assert result[1] == [(dirpath, os.path.join(dest_prefix, "test_dir"))]
# Test default source path and given dest path
paths = ["test.txt>new_name.txt", "test_dir>new_dirname"]
result = parse_paths(default_source_prefix, paths, dest_prefix)
assert result[0] == [(os.path.join(default_source_prefix, "test.txt"),
os.path.join(dest_prefix, "new_name.txt"))]
assert result[1] == [(os.path.join(default_source_prefix, "test_dir"),
os.path.join(dest_prefix, "new_dirname"))]
# Test failure if path does not exist
paths = ["sldfkj.txt>new_name.txt", "sldkfj>new_dirname"]
failed = False
try:
parse_paths(default_source_prefix, paths, dest_prefix)
except PathDoesNotExist:
failed = True
assert failed
# Test success absolute path and destination
paths = [filepath + ">new_name.txt", dirpath + ">new_dirname"]
result = parse_paths(default_source_prefix, paths, dest_prefix)
assert result[0] == [(filepath,
os.path.join(dest_prefix, "new_name.txt"))]
assert result[1] == [(dirpath, os.path.join(dest_prefix,
"new_dirname"))]
def create_collection(self, paths):
if not self.is_initialized:
raise FileStructureError(
__("error",
"controller.file.driver.local.create_collection.structure"))
self.ensure_collections_dir()
temp_collection_path = get_datmo_temp_path(self.root)
_, _, files_rel, dirs_rel = parse_paths(self.root, paths,
temp_collection_path)
filehash = self.calculate_hash_paths(paths, temp_collection_path)
# Move contents to folder with filehash as name and remove temp_collection_path
collection_path = os.path.join(self.datmo_directory, "collections",
filehash)
if os.path.isdir(collection_path):
return filehash, files_rel, dirs_rel
# raise FileStructureError("exception.file.create_collection", {
# "exception": "File collection with id already exists."
# })
os.makedirs(collection_path)
self.copytree(temp_collection_path, collection_path)
# Change permissions to read only for collection_path. File collection is immutable
mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
for root, dirs, files in os.walk(collection_path, topdown=False):
for dir in [os.path.join(root, d) for d in dirs]:
os.chmod(dir, mode)
for file in [os.path.join(root, f) for f in files]:
os.chmod(file, mode)
# Removing temp collection path
shutil.rmtree(temp_collection_path)
return filehash, files_rel, dirs_rel
def task_run_helper(self,
task_dict,
snapshot_dict,
error_identifier,
data_paths=None):
"""
Run task with given parameters and provide error identifier
Parameters
----------
task_dict : dict
input task dictionary for task run controller
snapshot_dict : dict
input snapshot dictionary for task run controller
error_identifier : str
identifier to print error
data_paths : list
list of data paths being passed for task run
Returns
-------
Task or False
the Task object which completed its run with updated parameters.
returns False if an error occurs
"""
self.task_controller = TaskController()
task_obj = self.task_controller.create()
updated_task_obj = task_obj
# Pass in the task
status = "NOT STARTED"
try:
if data_paths:
try:
_, _, task_dict['data_file_path_map'], task_dict['data_directory_path_map'] = \
parse_paths(self.task_controller.home, data_paths, '/data')
except PathDoesNotExist as e:
status = "NOT STARTED"
workspace = task_dict.get('workspace', None)
command = task_dict.get('command', None)
command_list = task_dict.get('command_list', None)
interactive = task_dict.get('interactive', False)
self.task_controller.update(task_obj.id,
workspace=workspace,
command=command,
command_list=command_list,
interactive=interactive)
self.cli_helper.echo(
__("error", "cli.run.parse.paths", str(e)))
return False
updated_task_obj = self.task_controller.run(
task_obj.id, snapshot_dict=snapshot_dict, task_dict=task_dict)
status = "SUCCESS"
self.cli_helper.echo(__("info", "cli.run.run.stop"))
except Exception as e:
status = "FAILED"
self.logger.error("%s %s" % (e, task_dict))
self.cli_helper.echo("%s" % e)
self.cli_helper.echo(__("error", error_identifier, task_obj.id))
return False
finally:
self.task_controller.stop(task_id=updated_task_obj.id,
status=status)
self.cli_helper.echo(
__("info", "cli.run.run.complete", updated_task_obj.id))
return updated_task_obj