def test_add(self):
mfpath = os.path.join(self.tmp_dir, 'manifest.yaml')
mf = Manifest(mfpath)
mf.add('zdj7WgHSKJkoJST5GWGgS53ARqV7oqMGYVvWzEWku3MBfnQ9u',
'data/think-hires.jpg')
self.assertTrue(
mf.exists('zdj7WgHSKJkoJST5GWGgS53ARqV7oqMGYVvWzEWku3MBfnQ9u'))
def test_search(self):
mfpath = os.path.join(self.tmp_dir, 'manifest.yaml')
mf = Manifest(mfpath)
mf.add('zdj7WgHSKJkoJST5GWGgS53ARqV7oqMGYVvWzEWku3MBfnQ9u',
'data/think-hires.jpg')
mf.add('zdj7WgHSKJkoJST5GWGgS53ARqV7oqMGYVvWzEWku3MBfnQ9u',
'data/think-hires2.jpg')
self.assertEqual(mf.search('data/think-hires.jpg'),
'zdj7WgHSKJkoJST5GWGgS53ARqV7oqMGYVvWzEWku3MBfnQ9u')
self.assertEqual(mf.search('data/think-hires2.jpg'),
'zdj7WgHSKJkoJST5GWGgS53ARqV7oqMGYVvWzEWku3MBfnQ9u')
def __commit_manifest(self, full_metadata_path, index_path, changed_files, mutability):
# Append index/files/MANIFEST.yaml to .ml-git/dataset/metadata/ <categories>/MANIFEST.yaml
idx_path = os.path.join(index_path, 'metadata', self._spec, 'MANIFEST.yaml')
if os.path.exists(idx_path) is False:
log.error('No manifest file found in [%s]' % idx_path, class_name=METADATA_CLASS_NAME)
return False
full_path = os.path.join(full_metadata_path, 'MANIFEST.yaml')
mobj = Manifest(full_path)
if mutability == Mutability.MUTABLE.value or mutability == Mutability.FLEXIBLE.value:
for key, file in changed_files:
mobj.rm(key, file)
mobj.merge(idx_path)
mobj.save()
del (mobj)
os.unlink(idx_path)
return True
def test_manifest_diff(self):
mfpath = os.path.join(self.tmp_dir, 'manifest.yaml')
mf_1 = Manifest(mfpath)
mf_1.add('zdj7WemKEtQMVL81UU6PSuYaoxvBQ6CiUMq1fMvoXBhPUsCK2',
'data/image.jpg')
mf_2 = Manifest(mfpath)
mf_2.add('zdj7WemKEtQMVL81UU6PSuYaoxvBQ6CiUMq1fMvoXBhPUsCK2',
'data/image.jpg')
mf_2.add('zdj7WgHSKJkoJST5GWGgS53ARqV7oqMGYVvWzEWku3MBfnQ9u',
'data/think-hires.jpg')
mf_diff, _ = mf_1.get_diff(mf_2)
self.assertEqual(
mf_diff, {
'zdj7WgHSKJkoJST5GWGgS53ARqV7oqMGYVvWzEWku3MBfnQ9u':
{'data/think-hires.jpg'}
})
def _get_index(self, idxpath):
metadatapath = os.path.join(idxpath, 'metadata', self._spec)
ensure_path_exists(metadatapath)
mfpath = os.path.join(metadatapath, 'MANIFEST.yaml')
return Manifest(mfpath)
def _get_index(self, idxpath):
metadatapath = os.path.join(idxpath, 'metadata', self._spec)
ensure_path_exists(metadatapath)
fidxpath = os.path.join(metadatapath, INDEX_FILE)
return Manifest(fidxpath)
def get_metadata_manifest(self, path):
if os.path.isfile(path):
return Manifest(path)
return None
def reset(self, spec, reset_type, head):
log.info(output_messages['INFO_INITIALIZING_RESET'] %
(reset_type, head),
class_name=REPOSITORY_CLASS_NAME)
if (reset_type == '--soft'
or reset_type == '--mixed') and head == HEAD:
return
try:
repo_type = self.__repo_type
metadata_path = get_metadata_path(self.__config, repo_type)
index_path = get_index_path(self.__config, repo_type)
refs_path = get_refs_path(self.__config, repo_type)
object_path = get_objects_path(self.__config, repo_type)
met = Metadata(spec, metadata_path, self.__config, repo_type)
ref = Refs(refs_path, spec, repo_type)
idx = MultihashIndex(spec, index_path, object_path)
fidx = FullIndex(spec, index_path)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
# get tag before reset
tag = met.get_current_tag()
categories_path = get_path_with_categories(str(tag))
# current manifest file before reset
manifest_path = os.path.join(metadata_path, categories_path, spec,
MANIFEST_FILE)
_manifest = Manifest(manifest_path).load()
if head == HEAD_1: # HEAD~1
try:
# reset the repo
met.reset()
except Exception:
return
# get tag after reset
tag_after_reset = met.get_current_tag()
sha = met.sha_from_tag(tag_after_reset)
# update ml-git ref HEAD
ref.update_head(str(tag_after_reset), sha)
# # get path to reset workspace in case of --hard
path, file = None, None
try:
path, file = search_spec_file(self.__repo_type, spec,
categories_path)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
if reset_type == '--hard' and path is None:
return
# get manifest from metadata after reset
_manifest_changed = Manifest(manifest_path)
hash_files, file_names = _manifest_changed.get_diff(_manifest)
idx_mf = idx.get_index().load()
if reset_type == '--soft':
# add in index/metadata/<entity-name>/MANIFEST
idx.update_index_manifest(idx_mf)
idx.update_index_manifest(hash_files)
fidx.update_index_status(file_names, Status.a.name)
else: # --hard or --mixed
# remove hash from index/hashsh/store.log
file_names.update(*idx_mf.values())
objs = MultihashFS(index_path)
for key_hash in hash_files:
objs.remove_hash(key_hash)
idx.remove_manifest()
fidx.remove_from_index_yaml(file_names)
fidx.remove_uncommitted()
if reset_type == '--hard': # reset workspace
remove_from_workspace(file_names, path, spec)