def test_has_changed_fast_from_loaded(self, mock_dataset_with_manifest):
ds, manifest, working_dir = mock_dataset_with_manifest
sh = SmartHash(ds.root_dir, manifest.cache_mgr.cache_root,
manifest.dataset_revision)
cache_dir = manifest.cache_mgr.cache_root
revision = manifest.dataset_revision
assert sh.fast_hash_data == {}
filename = "test1.txt"
helper_append_file(cache_dir, revision, filename, "pupper")
hash_result = sh.fast_hash([filename])
hash_result = hash_result[0]
fname, fsize, mtime = hash_result.split("||")
assert fname == "test1.txt"
assert fsize == '6'
assert sh.fast_hash_data is not None
assert os.path.exists(os.path.join(cache_dir, revision,
".smarthash")) is True
assert sh.is_cached(filename) is True
assert sh.has_changed_fast(filename) is False
sh2 = SmartHash(ds.root_dir, cache_dir, revision)
assert sh2.fast_hash_data is not None
assert sh2.is_cached(filename) is True
assert sh2.has_changed_fast(filename) is False
assert sh2.fast_hash_data[filename] == hash_result
def test_has_changed_fast(self, mock_dataset_with_manifest):
ds, manifest, working_dir = mock_dataset_with_manifest
sh = SmartHash(ds.root_dir, manifest.cache_mgr.cache_root,
manifest.dataset_revision)
cache_dir = manifest.cache_mgr.cache_root
revision = manifest.dataset_revision
assert sh.fast_hash_data == {}
assert os.path.exists(os.path.join(cache_dir, revision,
".smarthash")) is False
filename = "test1.txt"
helper_append_file(cache_dir, revision, filename, "pupper")
assert sh.is_cached(filename) is False
hash_result = sh.fast_hash([filename])
hash_result = hash_result[0]
fname, fsize, mtime = hash_result.split("||")
assert fname == "test1.txt"
assert fsize == '6'
assert sh.fast_hash_data is not None
assert os.path.exists(os.path.join(cache_dir, revision,
".smarthash")) is True
assert sh.is_cached(filename) is True
assert sh.has_changed_fast(filename) is False
time.sleep(1.1)
assert sh.has_changed_fast(filename) is False
# Change file
helper_append_file(cache_dir, revision, filename, "jgfdjfdgsjfdgsj")
assert sh.has_changed_fast(filename) is True
assert sh.has_changed_fast(filename) is True
sh.fast_hash([filename])
assert sh.has_changed_fast(filename) is False
# Touch file, so only change mtime
time.sleep(1.1)
Path(sh.get_abs_path(filename)).touch()
assert sh.has_changed_fast(filename) is True
sh.fast_hash([filename])
assert sh.has_changed_fast(filename) is False
async def test_hash_same_as_nonchunked_multiple(
self, event_loop, mock_dataset_with_manifest):
ds, manifest, working_dir = mock_dataset_with_manifest
sh = SmartHash(ds.root_dir, manifest.cache_mgr.cache_root,
manifest.dataset_revision)
cache_dir = manifest.cache_mgr.cache_root
revision = manifest.dataset_revision
filename1 = "test1.txt"
helper_append_file(cache_dir, revision, filename1,
"asdfdsfgkdfshuhwedfgft345wfd" * 100000)
assert sh.is_cached(filename1) is False
filename2 = "test2.txt"
helper_append_file(cache_dir, revision, filename2,
"gfggfgfgfgwee" * 100000)
assert sh.is_cached(filename2) is False
assert os.path.exists(os.path.join(cache_dir, revision,
".smarthash")) is False
assert sh.fast_hash_data == {}
h = blake2b()
with open(sh.get_abs_path(filename1), 'rb') as fh:
h.update(fh.read())
hash1 = h.hexdigest()
h = blake2b()
with open(sh.get_abs_path(filename2), 'rb') as fh:
h.update(fh.read())
hash2 = h.hexdigest()
hash_result = await sh.hash([filename1, filename2])
assert hash1 == hash_result[0]
assert hash2 == hash_result[1]
hash_result = await sh.hash([filename2, filename1])
assert hash2 == hash_result[0]
assert hash1 == hash_result[1]
async def test_hash(self, event_loop, mock_dataset_with_manifest):
ds, manifest, working_dir = mock_dataset_with_manifest
sh = SmartHash(ds.root_dir, manifest.cache_mgr.cache_root,
manifest.dataset_revision)
cache_dir = manifest.cache_mgr.cache_root
revision = manifest.dataset_revision
assert sh.fast_hash_data == {}
filename = "test1.txt"
helper_append_file(cache_dir, revision, filename, "pupper")
assert sh.fast_hash_data == {}
assert sh.is_cached(filename) is False
assert os.path.exists(os.path.join(cache_dir, revision,
".smarthash")) is False
hash_result = await sh.hash([filename])
hash_result = hash_result[0]
assert len(hash_result) == 128
class Manifest(object):
"""Class to handle file file manifest"""
def __init__(self,
dataset: Dataset,
logged_in_username: Optional[str] = None) -> None:
self.dataset = dataset
cache_mgr_class = get_cache_manager_class(self.dataset.client_config)
self.cache_mgr: CacheManager = cache_mgr_class(self.dataset,
logged_in_username)
self.hasher = SmartHash(dataset.root_dir, self.cache_mgr.cache_root,
self.dataset.git.repo.head.commit.hexsha)
self.ignore_file = os.path.join(dataset.root_dir, ".gigantumignore")
self.manifest = self._load_manifest()
# TODO: Support ignoring files
# self.ignored = self._load_ignored()
@property
def dataset_revision(self) -> str:
"""Property to get the current revision hash of the dataset
Returns:
str
"""
return self.dataset.git.repo.head.commit.hexsha
def _load_manifest(self) -> OrderedDict:
"""Method to load the manifest file
Returns:
dict
"""
manifest_file = os.path.join(self.dataset.root_dir, 'manifest',
'manifest0')
if os.path.exists(manifest_file):
with open(manifest_file, 'rb') as mf:
return pickle.load(mf)
else:
return OrderedDict()
def _save_manifest(self) -> None:
"""Method to load the manifest file
Returns:
dict
"""
with open(os.path.join(self.dataset.root_dir, 'manifest', 'manifest0'),
'wb') as mf:
pickle.dump(self.manifest, mf, pickle.HIGHEST_PROTOCOL)
def get_abs_path(self, relative_path: str) -> str:
"""Method to generate the absolute path to a file in the cache at the current revision
Args:
relative_path:
Returns:
"""
return self.hasher.get_abs_path(relative_path)
def queue_to_push(self, obj: str, rel_path: str, revision: str) -> None:
"""Method to queue and object for push to remote storage backend
Objects to push are stored in a file named with the revision at which the files were written. This is different
from the revision that contains the files (after written and untracked, changes are committed and then an
activity record is created with another commit)
Args:
obj: object path
revision: revision of the dataset the object exists in
rel_path: Objects relative file path in the dataset
Returns:
"""
if not os.path.exists(obj):
raise ValueError(
"Object does not exist. Failed to add to push queue.")
push_dir = os.path.join(self.cache_mgr.cache_root, 'objects', '.push')
if not os.path.exists(push_dir):
os.makedirs(push_dir)
with open(os.path.join(push_dir, revision), 'at') as fh:
fh.write(f"{rel_path},{obj}\n")
def get_change_type(self, path) -> FileChangeType:
"""
Args:
path:
Returns:
"""
if self.hasher.is_cached(path):
if self.hasher.has_changed_fast(path):
result = FileChangeType.MODIFIED
else:
result = FileChangeType.NOCHANGE
else:
if path in self.manifest.keys():
# No fast hash, but exists in manifest. User just edited a file that hasn't been pulled
result = FileChangeType.MODIFIED
else:
# No fast hash, not in manifest.
result = FileChangeType.CREATED
return result
def status(self) -> StatusResult:
"""
Returns:
"""
# TODO: think about how to send batches to get_change_type
status: Dict[str, List] = {
"created": [],
"modified": [],
"deleted": []
}
all_files = list()
revision_directory = os.path.join(self.cache_mgr.cache_root,
self.dataset_revision)
for root, dirs, files in os.walk(revision_directory):
_, folder = root.split(revision_directory)
if len(folder) > 0:
if folder[0] == os.path.sep:
folder = folder[1:]
for d in dirs:
# TODO: Check for ignored
rel_path = os.path.join(
folder, d
) + os.path.sep # All folders are represented with a trailing slash
all_files.append(rel_path)
change = self.get_change_type(rel_path)
if change == FileChangeType.NOCHANGE:
continue
elif change == FileChangeType.MODIFIED:
# Don't record directory modifications
pass
elif change == FileChangeType.CREATED:
status['created'].append(rel_path)
else:
raise ValueError(f"Invalid Change type: {change}")
for file in files:
# TODO: Check for ignored
if file in ['.smarthash', '.DS_STORE', '.DS_Store']:
continue
rel_path = os.path.join(folder, file)
all_files.append(rel_path)
change = self.get_change_type(rel_path)
if change == FileChangeType.NOCHANGE:
continue
elif change == FileChangeType.MODIFIED:
status['modified'].append(rel_path)
elif change == FileChangeType.CREATED:
status['created'].append(rel_path)
else:
raise ValueError(f"Invalid Change type: {change}")
# De-dup and sort
status['created'] = list(set(status['created']))
status['modified'] = list(set(status['modified']))
status['modified'] = natsorted(status['modified'])
status['created'] = natsorted(status['created'])
all_files = list(set(all_files))
return StatusResult(created=status.get('created'),
modified=status.get('modified'),
deleted=self.hasher.get_deleted_files(all_files))
@staticmethod
def _get_object_subdirs(object_id) -> Tuple[str, str]:
"""
Args:
object_id:
Returns:
"""
return object_id[0:8], object_id[8:16]
def dataset_to_object_path(self, dataset_path: str) -> str:
"""
Args:
dataset_path:
Returns:
"""
data: Optional[dict] = self.manifest.get(dataset_path)
if not data:
raise ValueError(f"{dataset_path} not found in Dataset manifest.")
hash_str: str = data['h']
level1, level2 = self._get_object_subdirs(hash_str)
return os.path.join(self.cache_mgr.cache_root, 'objects', level1,
level2, hash_str)
@staticmethod
def _blocking_move_and_link(source, destination):
if os.path.isfile(destination):
# Object already exists, no need to store again
os.remove(source)
else:
# Move file to new object
shutil.move(source, destination)
# Link object back
os.link(destination, source)
async def _move_to_object_cache(self, relative_path, hash_str):
"""
Args:
relative_path:
hash_str:
Returns:
"""
source = os.path.join(self.cache_mgr.cache_root, self.dataset_revision,
relative_path)
if os.path.isfile(source):
level1, level2 = self._get_object_subdirs(hash_str)
os.makedirs(os.path.join(self.cache_mgr.cache_root, 'objects',
level1),
exist_ok=True)
os.makedirs(os.path.join(self.cache_mgr.cache_root, 'objects',
level1, level2),
exist_ok=True)
destination = os.path.join(self.cache_mgr.cache_root, 'objects',
level1, level2, hash_str)
# Move file to new object
loop = get_event_loop()
await loop.run_in_executor(None, self._blocking_move_and_link,
source, destination)
# Queue new object for push
self.queue_to_push(destination, relative_path,
self.dataset_revision)
else:
destination = source
return destination
def update(self, status: StatusResult = None) -> StatusResult:
"""
Args:
status:
Returns:
"""
if not status:
status = self.status()
update_files = copy.deepcopy(status.created)
update_files.extend(status.modified)
if update_files:
# Hash Files
loop = get_event_loop()
hash_task = asyncio.ensure_future(self.hasher.hash(update_files))
loop.run_until_complete(asyncio.gather(hash_task))
# Move files into object cache and link back to the revision directory
hash_result = hash_task.result()
tasks = [
asyncio.ensure_future(self._move_to_object_cache(f, h))
for f, h in zip(update_files, hash_result)
]
loop.run_until_complete(asyncio.gather(*tasks))
# Update fast hash after objects have been moved/relinked
fast_hash_result = self.hasher.fast_hash(update_files, save=True)
# Update manifest file
for f, h, fh in zip(update_files, hash_result, fast_hash_result):
if not fh:
raise ValueError(
f"Failed to update manifest for {f}. File not found.")
_, file_bytes, mtime = fh.split("||")
self.manifest[f] = {'h': h, 'm': mtime, 'b': file_bytes}
if status.deleted:
self.hasher.delete_fast_hashes(status.deleted)
for f in status.deleted:
del self.manifest[f]
self._save_manifest()
return status
def _file_info(self, key, item) -> Dict[str, Any]:
"""
Args:
key:
item:
Returns:
"""
# TODO: Support favorites
abs_path = os.path.join(self.cache_mgr.cache_root,
self.dataset_revision, key)
return {
'key': key,
'size': item.get('b'),
'is_favorite': False,
'is_local': os.path.exists(abs_path),
'is_dir': os.path.isdir(abs_path),
'modified_at': float(item.get('m'))
}
def gen_file_info(self, key) -> Dict[str, Any]:
"""
Args:
key:
item:
Returns:
"""
# TODO: Support favorites
abs_path = self.get_abs_path(key)
stat = os.stat(abs_path)
is_dir = os.path.isdir(abs_path)
return {
'key': key,
'size': str(stat.st_size) if not is_dir else '0',
'is_favorite': False,
'is_local': os.path.exists(abs_path),
'is_dir': is_dir,
'modified_at': stat.st_mtime
}
def get(self, dataset_path: str) -> dict:
"""Method to get the file info for a single file from the manifest
Args:
dataset_path: Relative path to the object within the dataset
Returns:
"""
item = self.manifest.get(dataset_path)
return self._file_info(dataset_path, item)
def list(self,
first: int = None,
after_index: int = 0) -> List[Dict[str, Any]]:
"""
Args:
first:
after_index:
Returns:
"""
if first:
if first <= 0:
raise ValueError("`first` must be greater than 0")
if after_index:
if after_index < 0:
raise ValueError(
"`after_index` must be greater or equal than 0")
result = list()
if first is not None:
first = min(first + after_index, len(self.manifest))
for key, item in list(self.manifest.items())[after_index:first]:
result.append(self._file_info(key, item))
return result
def delete(self, path_list: List[str]) -> None:
"""Method to delete a list of files/folders from the dataset
Args:
path_list: List of relative paths in the dataset
Returns:
"""
revision_directory = os.path.join(self.cache_mgr.cache_root,
self.dataset_revision)
for path in path_list:
target_path = os.path.join(revision_directory, path)
if os.path.isdir(target_path):
shutil.rmtree(target_path)
else:
os.remove(target_path)
self.sweep_all_changes()
def move(self, src_path: str, dest_path: str) -> List[Dict[str, Any]]:
"""Method to move/rename a file or directory in a dataset
Args:
src_path: The relative path in the dataset to the source file/folder
dest_path: The relative path in the dataset to the destination file/folder
Returns:
"""
revision_directory = os.path.join(self.cache_mgr.cache_root,
self.dataset_revision)
src_rel_path = self.dataset.make_path_relative(
src_path.replace('..', ''))
dest_rel_path = self.dataset.make_path_relative(
dest_path.replace('..', ''))
src_abs_path = os.path.join(revision_directory, src_rel_path)
dest_abs_path = os.path.join(revision_directory, dest_rel_path)
src_type = 'directory' if os.path.isdir(src_abs_path) else 'file'
if not os.path.exists(src_abs_path):
raise ValueError(
f"No src file or folder exists at `{src_abs_path}`")
# Move
result_path = shutil.move(src_abs_path, dest_abs_path)
msg = f"Moved {src_type} `{src_rel_path}` to `{dest_rel_path}`"
previous_revision_directory = os.path.join(self.cache_mgr.cache_root,
self.dataset_revision)
self.sweep_all_changes(extra_msg=msg)
# Update paths due to relinking
revision_directory = os.path.join(self.cache_mgr.cache_root,
self.dataset_revision)
final_rel_path = self.dataset.make_path_relative(
result_path.replace(previous_revision_directory, ''))
dest_abs_path = os.path.join(revision_directory, final_rel_path)
if os.path.isfile(dest_abs_path):
manifest_data = self.manifest.get(final_rel_path)
return [self._file_info(final_rel_path, manifest_data)]
elif os.path.isdir(dest_abs_path):
moved_files = list()
moved_files.append(self.gen_file_info(final_rel_path))
for root, dirs, files in os.walk(dest_abs_path):
dirs.sort()
rt = root.replace(revision_directory, '')
rt = self.dataset.make_path_relative(rt)
for d in dirs:
if d[-1] != os.path.sep:
d = d + '/'
moved_files.append(self.gen_file_info(os.path.join(rt, d)))
for f in filter(lambda n: n != '.gitkeep', sorted(files)):
rel_path = os.path.join(rt, f)
manifest_data = self.manifest.get(rel_path)
moved_files.append(self._file_info(rel_path,
manifest_data))
else:
raise ValueError(
"Destination path does not exist after move operation")
logger.info(msg)
return moved_files
def create_directory(self, path: str) -> Dict[str, Any]:
"""Method to create an empty directory in a dataset
Args:
path: Relative path to the directory
Returns:
dict
"""
relative_path = self.dataset.make_path_relative(path)
new_directory_path = os.path.join(self.cache_mgr.cache_root,
self.dataset_revision, relative_path)
previous_revision = self.dataset_revision
if os.path.exists(new_directory_path):
raise ValueError(f"Directory already exists: `{relative_path}`")
else:
logger.info(
f"Creating new empty directory in `{new_directory_path}`")
if os.path.isdir(Path(new_directory_path).parent) is False:
raise ValueError(
f"Parent directory does not exist. Failed to create `{new_directory_path}` "
)
# create dir
os.makedirs(new_directory_path)
self.update()
if relative_path not in self.manifest:
raise ValueError("Failed to add directory to manifest")
# Create detail record
adr = ActivityDetailRecord(ActivityDetailType.DATASET,
show=False,
importance=0,
action=ActivityAction.CREATE)
msg = f"Created new empty directory `{relative_path}`"
adr.add_value('text/markdown', msg)
commit = self.dataset.git.commit(msg)
# Create activity record
ar = ActivityRecord(ActivityType.DATASET,
message=msg,
linked_commit=commit.hexsha,
show=True,
importance=255,
tags=['directory-create'])
ar.add_detail_object(adr)
# Store
ars = ActivityStore(self.dataset)
ars.create_activity_record(ar)
# Relink after the commit
self.link_revision()
if os.path.isdir(
os.path.join(self.cache_mgr.cache_root,
previous_revision)):
shutil.rmtree(
os.path.join(self.cache_mgr.cache_root, previous_revision))
return self.gen_file_info(relative_path)
def link_revision(self) -> None:
"""Method to link all the objects in the cache to the current revision directory, so that all files are
accessible with the correct file names.
Note: This update the current revision in the hashing class
Returns:
None
"""
current_revision = self.dataset_revision
self.hasher.current_revision = current_revision
revision_directory = os.path.join(self.cache_mgr.cache_root,
current_revision)
if not os.path.exists(revision_directory):
os.makedirs(revision_directory)
for f in self.manifest:
hash_str = self.manifest[f].get('h')
level1, level2 = self._get_object_subdirs(hash_str)
target = os.path.join(revision_directory, f)
if target[-1] == os.path.sep:
# Create directory from manifest
if not os.path.exists(target):
os.makedirs(target)
else:
# Link file
source = os.path.join(self.cache_mgr.cache_root, 'objects',
level1, level2, hash_str)
target_dir = os.path.dirname(target)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# Link if not already linked
if not os.path.exists(target):
try:
if os.path.exists(source):
# Only try to link if the source object has been materialized
os.link(source, target)
except Exception as err:
logger.exception(err)
continue
# Completely re-compute the fast hash index
self.hasher.fast_hash_data = dict()
self.hasher.fast_hash(list(self.manifest.keys()))
def sweep_all_changes(self,
upload: bool = False,
extra_msg: str = None) -> None:
"""
Args:
upload:
extra_msg:
Returns:
"""
def _item_type(key):
if key[-1] == os.path.sep:
return 'directory'
else:
return 'file'
previous_revision = self.dataset_revision
# Update manifest
status = self.update()
if len(status.deleted) > 0 or len(status.created) > 0 or len(
status.modified) > 0:
# commit changed manifest file
self.dataset.git.add_all()
self.dataset.git.commit("Commit changes to manifest file.")
ar = ActivityRecord(
ActivityType.DATASET,
message="msg is set below after detail record processing...",
show=True,
importance=255,
linked_commit=self.dataset.git.commit_hash,
tags=[])
if upload:
ar.tags.append('upload')
for cnt, f in enumerate(status.created):
adr = ActivityDetailRecord(ActivityDetailType.DATASET,
show=False,
importance=max(255 - cnt, 0),
action=ActivityAction.CREATE)
msg = f"Created new {_item_type(f)} `{f}`"
adr.add_value('text/markdown', msg)
ar.add_detail_object(adr)
for cnt, f in enumerate(status.modified):
adr = ActivityDetailRecord(ActivityDetailType.DATASET,
show=False,
importance=max(255 - cnt, 0),
action=ActivityAction.EDIT)
msg = f"Modified {_item_type(f)} `{f}`"
adr.add_value('text/markdown', msg)
ar.add_detail_object(adr)
for cnt, f in enumerate(status.deleted):
adr = ActivityDetailRecord(ActivityDetailType.DATASET,
show=False,
importance=max(255 - cnt, 0),
action=ActivityAction.DELETE)
msg = f"Deleted {_item_type(f)} `{f}`"
adr.add_value('text/markdown', msg)
ar.add_detail_object(adr)
nmsg = f"{len(status.created)} new file(s). " if len(
status.created) > 0 else ""
mmsg = f"{len(status.modified)} modified file(s). " if len(
status.modified) > 0 else ""
dmsg = f"{len(status.deleted)} deleted file(s). " if len(
status.deleted) > 0 else ""
ar.message = f"{extra_msg if extra_msg else ''}" \
f"{'Uploaded ' if upload else ''}" \
f"{nmsg}{mmsg}{dmsg}"
ars = ActivityStore(self.dataset)
ars.create_activity_record(ar)
# Re-link new revision, unlink old revision
self.link_revision()
if os.path.isdir(
os.path.join(self.cache_mgr.cache_root, previous_revision)):
shutil.rmtree(
os.path.join(self.cache_mgr.cache_root, previous_revision))
class Manifest(object):
"""Class to handle file file manifest"""
def __init__(self, dataset: 'Dataset', logged_in_username: Optional[str] = None) -> None:
self.dataset = dataset
self.logged_in_username = logged_in_username
cache_mgr_class = get_cache_manager_class(self.dataset.client_config)
self.cache_mgr: CacheManager = cache_mgr_class(self.dataset, logged_in_username)
self.hasher = SmartHash(dataset.root_dir, self.cache_mgr.cache_root,
self.dataset.git.repo.head.commit.hexsha)
self._manifest_io = ManifestFileCache(dataset, logged_in_username)
# TODO: Support ignoring files
# self.ignore_file = os.path.join(dataset.root_dir, ".gigantumignore")
# self.ignored = self._load_ignored()
self._legacy_manifest_file = os.path.join(self.dataset.root_dir, 'manifest', 'manifest0')
@property
def dataset_revision(self) -> str:
"""Property to get the current revision hash of the dataset
Returns:
str
"""
return self.dataset.git.repo.head.commit.hexsha
@property
def current_revision_dir(self) -> str:
"""Method to return the directory containing files for the current dataset revision. If the dir doesn't exist,
relink it (updates to a dataset will remove a revision dir, but linked datasets may still need old revisions)
Returns:
str
"""
crd = self.cache_mgr.current_revision_dir
if not os.path.exists(crd):
self.link_revision()
return crd
@property
def manifest(self) -> OrderedDict:
"""Property to get the current manifest as the union of all manifest files, with caching supported
Returns:
OrderedDict
"""
return self._manifest_io.get_manifest()
@staticmethod
def _get_object_subdirs(object_id) -> Tuple[str, str]:
"""Get the subdirectories when accessing an object ID
Args:
object_id:
Returns:
"""
return object_id[0:8], object_id[8:16]
def get_num_hashing_cpus(self) -> int:
"""
Returns:
"""
config_val = self.dataset.client_config.config['datasets']['hash_cpu_limit']
if config_val == 'auto':
num_cpus = os.cpu_count()
if not num_cpus:
num_cpus = 1
return num_cpus
else:
return int(config_val)
def dataset_to_object_path(self, dataset_path: str) -> str:
"""Helper method to compute the absolute object path from the relative dataset path
Args:
dataset_path: relative dataset path
Returns:
str
"""
data: Optional[dict] = self.manifest.get(dataset_path)
if not data:
raise ValueError(f"{dataset_path} not found in Dataset manifest.")
hash_str: str = data['h']
level1, level2 = self._get_object_subdirs(hash_str)
return os.path.join(self.cache_mgr.cache_root, 'objects', level1, level2, hash_str)
def get_abs_path(self, relative_path: str) -> str:
"""Method to generate the absolute path to a file in the cache at the current revision
Args:
relative_path:
Returns:
"""
return self.hasher.get_abs_path(relative_path)
def queue_to_push(self, obj: str, rel_path: str, revision: str) -> None:
"""Method to queue and object for push to remote storage backend
Objects to push are stored in a file named with the revision at which the files were written. This is different
from the revision that contains the files (after written and untracked, changes are committed and then an
activity record is created with another commit)
Args:
obj: object path
revision: revision of the dataset the object exists in
rel_path: Objects relative file path in the dataset
Returns:
"""
if not os.path.exists(obj):
raise ValueError("Object does not exist. Failed to add to push queue.")
push_dir = os.path.join(self.cache_mgr.cache_root, 'objects', '.push')
if not os.path.exists(push_dir):
os.makedirs(push_dir)
with open(os.path.join(push_dir, revision), 'at') as fh:
fh.write(f"{rel_path},{obj}\n")
def get_change_type(self, path) -> FileChangeType:
"""Helper method to get the type of change from the manifest/fast hash
Args:
path:
Returns:
"""
if self.hasher.is_cached(path):
if self.hasher.has_changed_fast(path):
result = FileChangeType.MODIFIED
else:
result = FileChangeType.NOCHANGE
else:
if path in self.manifest:
# No fast hash, but exists in manifest. User just edited a file that hasn't been pulled
result = FileChangeType.MODIFIED
else:
# No fast hash, not in manifest.
result = FileChangeType.CREATED
return result
def status(self) -> StatusResult:
"""Method to compute the changes (create, modified, delete) of a dataset, comparing local state to the
manifest and fast hash
Returns:
StatusResult
"""
# TODO: think about how to send batches to get_change_type
status: Dict[str, List] = {"created": [], "modified": [], "deleted": []}
all_files = list()
revision_directory = os.path.join(self.cache_mgr.cache_root, self.dataset_revision)
for root, dirs, files in os.walk(revision_directory):
_, folder = root.split(revision_directory)
if len(folder) > 0:
if folder[0] == os.path.sep:
folder = folder[1:]
for d in dirs:
# TODO: Check for ignored
rel_path = os.path.join(folder, d) + os.path.sep # All folders are represented with a trailing slash
all_files.append(rel_path)
change = self.get_change_type(rel_path)
if change == FileChangeType.NOCHANGE:
continue
elif change == FileChangeType.MODIFIED:
# Don't record directory modifications
pass
elif change == FileChangeType.CREATED:
status['created'].append(rel_path)
else:
raise ValueError(f"Invalid Change type: {change}")
for file in files:
# TODO: Check for ignored
if file in ['.smarthash', '.DS_STORE', '.DS_Store']:
continue
rel_path = os.path.join(folder, file)
all_files.append(rel_path)
change = self.get_change_type(rel_path)
if change == FileChangeType.NOCHANGE:
continue
elif change == FileChangeType.MODIFIED:
status['modified'].append(rel_path)
elif change == FileChangeType.CREATED:
status['created'].append(rel_path)
else:
raise ValueError(f"Invalid Change type: {change}")
# De-dup and sort
status['created'] = list(set(status['created']))
status['modified'] = list(set(status['modified']))
status['modified'] = natsorted(status['modified'])
status['created'] = natsorted(status['created'])
all_files = list(set(all_files))
return StatusResult(created=status.get('created'), modified=status.get('modified'),
deleted=self.hasher.get_deleted_files(all_files))
@staticmethod
def _blocking_move_and_link(source, destination):
"""Blocking method to move a file and hard link it
Args:
source: source path
destination: destination path
Returns:
"""
if os.path.isfile(destination):
# Object already exists, no need to store again
os.remove(source)
else:
# Move file to new object
shutil.move(source, destination)
# Link object back
try:
os.link(destination, source)
except PermissionError:
os.symlink(destination, source)
async def _move_to_object_cache(self, relative_path, hash_str):
"""Method to move a file to the object cache
Args:
relative_path: relative path to the file
hash_str: content hash of the file
Returns:
"""
source = os.path.join(self.cache_mgr.cache_root, self.dataset_revision, relative_path)
if os.path.isfile(source):
level1, level2 = self._get_object_subdirs(hash_str)
os.makedirs(os.path.join(self.cache_mgr.cache_root, 'objects', level1), exist_ok=True)
os.makedirs(os.path.join(self.cache_mgr.cache_root, 'objects', level1, level2), exist_ok=True)
destination = os.path.join(self.cache_mgr.cache_root, 'objects', level1, level2, hash_str)
# Move file to new object
loop = get_event_loop()
await loop.run_in_executor(None, self._blocking_move_and_link, source, destination)
# Queue new object for push
self.queue_to_push(destination, relative_path, self.dataset_revision)
else:
destination = source
return destination
def hash_files(self, update_files: List[str]) -> Tuple[List[Optional[str]], List[Optional[str]]]:
"""Method to run the update process on the manifest based on change status (optionally computing changes if
status is not set)
Args:
update_files: The current change status of the dataset, of omitted, it will be computed
Returns:
StatusResult
"""
# Hash Files
loop = get_event_loop()
hash_task = asyncio.ensure_future(self.hasher.hash(update_files))
loop.run_until_complete(asyncio.gather(hash_task))
# Move files into object cache and link back to the revision directory
hash_result = hash_task.result()
tasks = [asyncio.ensure_future(self._move_to_object_cache(f, h)) for f, h in zip(update_files, hash_result)]
loop.run_until_complete(asyncio.gather(*tasks))
# Update fast hash after objects have been moved/relinked
fast_hash_result = self.hasher.fast_hash(update_files, save=True)
return hash_result, fast_hash_result
def update(self, status: StatusResult = None) -> StatusResult:
"""Method to run the update process on the manifest based on change status (optionally computing changes if
status is not set)
Args:
status: The current change status of the dataset, of omitted, it will be computed
Returns:
StatusResult
"""
if not status:
status = self.status()
update_files = copy.deepcopy(status.created)
update_files.extend(status.modified)
if update_files:
hash_result, fast_hash_result = self.hash_files(update_files)
# Update manifest file
for f, h, fh in zip(update_files, hash_result, fast_hash_result):
if not fh or not h:
raise ValueError(f"Failed to update manifest for {f}. File not found.")
_, file_bytes, mtime = fh.split("||")
self._manifest_io.add_or_update(f, h, mtime, file_bytes)
if status.deleted:
self.hasher.delete_fast_hashes(status.deleted)
for relative_path in status.deleted:
self._manifest_io.remove(relative_path)
self._manifest_io.persist()
return status
def _file_info(self, key, item) -> Dict[str, Any]:
"""Method to populate file info (e.g. size, mtime, etc.) using data from the manifest
Args:
key: relative path to the file
item: data from the manifest
Returns:
dict
"""
abs_path = os.path.join(self.cache_mgr.cache_root, self.dataset_revision, key)
return {'key': key,
'size': item.get('b'),
'is_local': os.path.exists(abs_path),
'is_dir': True if abs_path[-1] == "/" else False,
'modified_at': float(item.get('m'))}
def gen_file_info(self, key) -> Dict[str, Any]:
"""Method to generate file info (e.g. size, mtime, etc.)
Args:
key: relative path to the file
Returns:
dict
"""
abs_path = self.get_abs_path(key)
stat = os.stat(abs_path)
is_dir = True if S_ISDIR(stat.st_mode) else False
return {'key': key,
'size': str(stat.st_size) if not is_dir else '0',
'is_local': True,
'is_dir': is_dir,
'modified_at': stat.st_mtime}
def get(self, dataset_path: str) -> dict:
"""Method to get the file info for a single file from the manifest
Args:
dataset_path: Relative path to the object within the dataset
Returns:
"""
item = self.manifest.get(dataset_path)
return self._file_info(dataset_path, item)
def list(self, first: int = None, after_index: int = 0) -> Tuple[List[Dict[str, Any]], List[int]]:
"""
Args:
first:
after_index:
Returns:
"""
if first:
if first <= 0:
raise ValueError("`first` must be greater than 0")
if after_index:
if after_index < 0:
raise ValueError("`after_index` must be greater or equal than 0")
result = list()
indexes = list()
if after_index != 0:
after_index = after_index + 1
if first is not None:
end = min(first + after_index, len(self.manifest))
else:
end = len(self.manifest)
data = list(self.manifest.items())
for idx in range(after_index, end):
result.append(self._file_info(data[idx][0], data[idx][1]))
indexes.append(idx)
return result, indexes
def delete(self, path_list: List[str]) -> None:
"""Method to delete a list of files/folders from the dataset
Args:
path_list: List of relative paths in the dataset
Returns:
"""
revision_directory = os.path.join(self.cache_mgr.cache_root, self.dataset_revision)
for path in path_list:
target_path = os.path.join(revision_directory, path)
if os.path.isdir(target_path):
shutil.rmtree(target_path)
else:
os.remove(target_path)
self.sweep_all_changes()
def move(self, src_path: str, dest_path: str) -> List[Dict[str, Any]]:
"""Method to move/rename a file or directory in a dataset
Args:
src_path: The relative path in the dataset to the source file/folder
dest_path: The relative path in the dataset to the destination file/folder
Returns:
"""
revision_directory = os.path.join(self.cache_mgr.cache_root, self.dataset_revision)
src_rel_path = self.dataset.make_path_relative(src_path.replace('..', ''))
dest_rel_path = self.dataset.make_path_relative(dest_path.replace('..', ''))
src_abs_path = os.path.join(revision_directory, src_rel_path)
dest_abs_path = os.path.join(revision_directory, dest_rel_path)
src_type = 'directory' if os.path.isdir(src_abs_path) else 'file'
if not os.path.exists(src_abs_path):
raise ValueError(f"No src file or folder exists at `{src_abs_path}`")
# Move
result_path = shutil.move(src_abs_path, dest_abs_path)
msg = f"Moved {src_type} `{src_rel_path}` to `{dest_rel_path}`"
previous_revision_directory = os.path.join(self.cache_mgr.cache_root, self.dataset_revision)
self.sweep_all_changes(extra_msg=msg)
# Update paths due to relinking
revision_directory = os.path.join(self.cache_mgr.cache_root, self.dataset_revision)
final_rel_path = self.dataset.make_path_relative(result_path.replace(previous_revision_directory, ''))
dest_abs_path = os.path.join(revision_directory, final_rel_path)
if os.path.isfile(dest_abs_path):
manifest_data = self.manifest.get(final_rel_path)
return [self._file_info(final_rel_path, manifest_data)]
elif os.path.isdir(dest_abs_path):
moved_files = list()
moved_files.append(self.gen_file_info(final_rel_path))
for root, dirs, files in os.walk(dest_abs_path):
dirs.sort()
rt = root.replace(revision_directory, '')
rt = self.dataset.make_path_relative(rt)
for d in dirs:
if d[-1] != os.path.sep:
d = d + '/'
moved_files.append(self.gen_file_info(os.path.join(rt, d)))
for f in filter(lambda n: n != '.gitkeep', sorted(files)):
rel_path = os.path.join(rt, f)
manifest_data = self.manifest.get(rel_path)
moved_files.append(self._file_info(rel_path, manifest_data))
else:
raise ValueError("Destination path does not exist after move operation")
return moved_files
def create_directory(self, path: str) -> Dict[str, Any]:
"""Method to create an empty directory in a dataset
Args:
path: Relative path to the directory
Returns:
dict
"""
relative_path = self.dataset.make_path_relative(path)
new_directory_path = os.path.join(self.cache_mgr.cache_root, self.dataset_revision, relative_path)
previous_revision = self.dataset_revision
if os.path.exists(new_directory_path):
raise ValueError(f"Directory already exists: `{relative_path}`")
else:
logger.info(f"Creating new empty directory in `{new_directory_path}`")
if os.path.isdir(Path(new_directory_path).parent) is False:
raise ValueError(f"Parent directory does not exist. Failed to create `{new_directory_path}` ")
# create dir
os.makedirs(new_directory_path)
self.update()
if relative_path not in self.manifest:
raise ValueError("Failed to add directory to manifest")
# Create detail record
adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=0,
action=ActivityAction.CREATE)
msg = f"Created new empty directory `{relative_path}`"
adr.add_value('text/markdown', msg)
commit = self.dataset.git.commit(msg)
# Create activity record
ar = ActivityRecord(ActivityType.DATASET,
message=msg,
linked_commit=commit.hexsha,
show=True,
importance=255,
tags=['directory-create'])
ar.add_detail_object(adr)
# Store
ars = ActivityStore(self.dataset)
ars.create_activity_record(ar)
# Relink after the commit
self.link_revision()
if os.path.isdir(os.path.join(self.cache_mgr.cache_root, previous_revision)):
shutil.rmtree(os.path.join(self.cache_mgr.cache_root, previous_revision))
return self.gen_file_info(relative_path)
def link_revision(self) -> None:
"""Method to link all the objects in the cache to the current revision directory, so that all files are
accessible with the correct file names.
Note: This update the current revision in the hashing class
Returns:
None
"""
current_revision = self.dataset_revision
self.hasher.current_revision = current_revision
revision_directory = os.path.join(self.cache_mgr.cache_root, current_revision)
if not os.path.exists(revision_directory):
os.makedirs(revision_directory)
for f in self.manifest:
hash_str = self.manifest[f].get('h')
level1, level2 = self._get_object_subdirs(hash_str)
target = os.path.join(revision_directory, f)
if target[-1] == os.path.sep:
# Create directory from manifest
if not os.path.exists(target):
os.makedirs(target)
else:
# Link file
source = os.path.join(self.cache_mgr.cache_root, 'objects', level1, level2, hash_str)
target_dir = os.path.dirname(target)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# Link if not already linked
if not os.path.exists(target):
try:
if os.path.exists(source):
# Only try to link if the source object has been materialized
os.link(source, target)
except Exception as err:
logger.exception(err)
continue
# Completely re-compute the fast hash index
self.hasher.fast_hash_data = dict()
self.hasher.fast_hash(list(self.manifest.keys()))
def create_update_activity_record(self, status: StatusResult, upload: bool = False, extra_msg: str = None) -> None:
"""
Args:
status(StatusResult): a StatusResult object after updating the manifest
upload(bool): flag indicating if this is a record for an upload
extra_msg(str): any extra string to add to the activity record
Returns:
None
"""
def _item_type(key):
if key[-1] == os.path.sep:
return 'directory'
else:
return 'file'
if len(status.deleted) > 0 or len(status.created) > 0 or len(status.modified) > 0:
# commit changed manifest file
self.dataset.git.add_all()
self.dataset.git.commit("Commit changes to manifest file.")
ar = ActivityRecord(ActivityType.DATASET,
message="msg is set below after detail record processing...",
show=True,
importance=255,
linked_commit=self.dataset.git.commit_hash,
tags=[])
for cnt, f in enumerate(status.created):
adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=max(255 - cnt, 0),
action=ActivityAction.CREATE)
msg = f"Created new {_item_type(f)} `{f}`"
adr.add_value('text/markdown', msg)
ar.add_detail_object(adr)
for cnt, f in enumerate(status.modified):
adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=max(255 - cnt, 0),
action=ActivityAction.EDIT)
msg = f"Modified {_item_type(f)} `{f}`"
adr.add_value('text/markdown', msg)
ar.add_detail_object(adr)
for cnt, f in enumerate(status.deleted):
adr = ActivityDetailRecord(ActivityDetailType.DATASET, show=False, importance=max(255 - cnt, 0),
action=ActivityAction.DELETE)
msg = f"Deleted {_item_type(f)} `{f}`"
adr.add_value('text/markdown', msg)
ar.add_detail_object(adr)
num_files_created = sum([_item_type(x) == "file" for x in status.created])
num_files_modified = sum([_item_type(x) == "file" for x in status.modified])
num_files_deleted = sum([_item_type(x) == "file" for x in status.deleted])
upload_str = "Uploaded" if upload else ''
nmsg = f"{upload_str} {num_files_created} new file(s). " if num_files_created > 0 else ""
mmsg = f"{upload_str} {num_files_modified} modified file(s). " if num_files_modified > 0 else ""
dmsg = f"{num_files_deleted} deleted file(s). " if num_files_deleted > 0 else ""
if not nmsg and not mmsg and not dmsg:
# You didn't edit any files, only an empty directory
num_dirs_created = sum([_item_type(x) == "directory" for x in status.created])
num_dirs_modified = sum([_item_type(x) == "directory" for x in status.modified])
num_dirs_deleted = sum([_item_type(x) == "directory" for x in status.deleted])
nmsg = f"{num_dirs_created} new folder(s). " if num_dirs_created > 0 else ""
mmsg = f"{num_dirs_modified} modified folder(s). " if num_dirs_modified > 0 else ""
dmsg = f"{num_dirs_deleted} deleted folder(s). " if num_dirs_deleted > 0 else ""
ar.message = f"{extra_msg if extra_msg else ''}" \
f"{nmsg}{mmsg}{dmsg}"
ars = ActivityStore(self.dataset)
ars.create_activity_record(ar)
def sweep_all_changes(self, upload: bool = False, extra_msg: str = None,
status: Optional[StatusResult] = None) -> None:
"""
Args:
upload(bool): flag indicating if this is a record for an upload
extra_msg(str): any extra string to add to the activity record
status(StatusResult): a StatusResult object after updating the manifest
Returns:
"""
previous_revision = self.dataset_revision
# If `status` is set, assume update() has been run already
if not status:
status = self.update()
# Update manifest
self.create_update_activity_record(status, upload=upload, extra_msg=extra_msg)
# Re-link new revision
self.link_revision()
if os.path.isdir(os.path.join(self.cache_mgr.cache_root, previous_revision)):
shutil.rmtree(os.path.join(self.cache_mgr.cache_root, previous_revision))
def force_reload(self) -> None:
"""Method to force reloading manifest data from the filesystem
This is useful when an update to the manifest occurs, but within a checkout context. This can happen with
linked local datasets for example.
Returns:
None
"""
self._manifest_io.evict()
_ = self.manifest