def log(self, spec, stat=False, fullstat=False):
try:
repo_type = self.__repo_type
metadata_path = get_metadata_path(self.__config, repo_type)
metadata = Metadata(spec, metadata_path, self.__config, repo_type)
index_path = get_index_path(self.__config, repo_type)
log_info = metadata.get_log_info(spec, fullstat)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
fidx = FullIndex(spec, index_path)
if stat or fullstat:
workspace_size = fidx.get_total_size()
amount_message = 'Total of files: %s' % fidx.get_total_count()
size_message = 'Workspace size: %s' % humanize.naturalsize(
workspace_size)
workspace_info = '------------------------------------------------- \n{}\t{}' \
.format(amount_message, size_message)
log_info = '{}\n{}'.format(log_info, workspace_info)
log.info(log_info, class_name=REPOSITORY_CLASS_NAME)
def remove_deleted_files(self, idx, index_path, m, manifest_path, path,
spec, deleted_files):
fidx = FullIndex(spec, index_path)
manifest = m.get_metadata_manifest(manifest_path)
fidx.remove_deleted_files(deleted_files)
idx.remove_deleted_files_index_manifest(deleted_files)
m.remove_deleted_files_meta_manifest(manifest, deleted_files)
def test_get_update_links_wspace(self):
wspath = os.path.join(self.tmp_dir, 'wspace')
hfspath = os.path.join(self.tmp_dir, 'objectsfs')
ohfs = MultihashFS(hfspath)
key = ohfs.put(HDATA_IMG_1)
fidx = FullIndex(self.tmp_dir, self.tmp_dir)
cachepath = os.path.join(self.tmp_dir, 'cachefs')
cache = Cache(cachepath, '', '')
testbucketname = os.getenv('MLGIT_TEST_BUCKET', 'ml-git-datasets')
c = get_sample_config_spec(testbucketname, testprofile, testregion)
r = LocalRepository(c, hfspath)
r._update_cache(cache, key)
mfiles = {}
files = {DATA_IMG_1}
r._update_links_wspace(cache, fidx, files, key, wspath, mfiles, Status.u.name, 'strict')
wspace_file = os.path.join(wspath, DATA_IMG_1)
set_write_read(wspace_file)
self.assertTrue(os.path.exists(wspace_file))
self.assertEqual(self.md5sum(HDATA_IMG_1), self.md5sum(wspace_file))
st = os.stat(wspace_file)
fi = fidx.get_index()
for k, v in fi.items():
self.assertEqual(k, os.path.join('data', 'imghires.jpg'))
self.assertEqual(v['hash'], 'zdj7WjdojNAZN53Wf29rPssZamfbC6MVerzcGwd9tNciMpsQh')
self.assertEqual(v['status'], 'u')
self.assertEqual(v['ctime'], st.st_ctime)
self.assertEqual(v['mtime'], st.st_mtime)
self.assertTrue(st.st_nlink == 2)
self.assertEqual(mfiles, {DATA_IMG_1: 'zdj7WjdojNAZN53Wf29rPssZamfbC6MVerzcGwd9tNciMpsQh'})
def test_get_update_links_wspace_with_duplicates(self):
wspath = os.path.join(self.tmp_dir, 'wspace')
hfspath = os.path.join(self.tmp_dir, 'objectsfs')
ohfs = MultihashFS(hfspath)
key = ohfs.put(HDATA_IMG_1)
fidx = FullIndex(self.tmp_dir, self.tmp_dir)
cachepath = os.path.join(self.tmp_dir, 'cachefs')
cache = Cache(cachepath, '', '')
testbucketname = os.getenv('MLGIT_TEST_BUCKET', 'ml-git-datasets')
c = get_sample_config_spec(testbucketname, testprofile, testregion)
r = LocalRepository(c, hfspath)
r._update_cache(cache, key)
mfiles = {}
files = {DATA_IMG_1, DATA_IMG_2}
r._update_links_wspace(cache, fidx, files, key, wspath, mfiles, Status.u.name, 'strict')
wspace_file = os.path.join(wspath, DATA_IMG_1)
self.assertTrue(os.path.exists(wspace_file))
self.assertEqual(self.md5sum(HDATA_IMG_1), self.md5sum(wspace_file))
wspace_file = os.path.join(wspath, DATA_IMG_2)
self.assertTrue(os.path.exists(wspace_file))
self.assertEqual(self.md5sum(HDATA_IMG_1), self.md5sum(wspace_file))
st = os.stat(wspace_file)
self.assertTrue(st.st_nlink == 3)
self.assertEqual(mfiles, {DATA_IMG_1: 'zdj7WjdojNAZN53Wf29rPssZamfbC6MVerzcGwd9tNciMpsQh',
DATA_IMG_2: 'zdj7WjdojNAZN53Wf29rPssZamfbC6MVerzcGwd9tNciMpsQh'})
wspath = os.path.join(self.tmp_dir, 'wspace')
ensure_path_exists(wspath)
to_be_removed = os.path.join(wspath, 'to_be_removed')
with open(to_be_removed, 'w') as f:
f.write('DEAD\n')
hfspath = os.path.join(self.tmp_dir, 'objectsfs')
ohfs = MultihashFS(hfspath)
key = ohfs.put(HDATA_IMG_1)
fidx = FullIndex(self.tmp_dir, self.tmp_dir)
cachepath = os.path.join(self.tmp_dir, 'cachefs')
cache = Cache(cachepath, '', '')
c = yaml_load('hdata/config.yaml')
r = LocalRepository(c, hfspath)
r._update_cache(cache, key)
mfiles = {}
files = {DATA_IMG_1, DATA_IMG_2}
r._update_links_wspace(cache, fidx, files, key, wspath, mfiles, Status.u.name, 'strict')
r._remove_unused_links_wspace(wspath, mfiles)
self.assertFalse(os.path.exists(to_be_removed))
def commit_objects(self, index_path, ws_path):
added_files = []
deleted_files = []
idx = MultihashFS(self._objects_path)
fidx = FullIndex(self.__spec, index_path)
findex = fidx.get_index()
log_path = os.path.join(self._logpath, STORAGE_LOG)
with open(log_path, 'a') as log_file:
for k, v in findex.items():
if not os.path.exists(os.path.join(ws_path, k)):
deleted_files.append(k)
elif v['status'] == Status.a.name:
idx.fetch_scid(v['hash'], log_file)
v['status'] = Status.u.name
if 'previous_hash' in v:
added_files.append((v['previous_hash'], k))
fidx.get_manifest_index().save()
return added_files, deleted_files
def test_push(self):
indexpath = os.path.join(self.tmp_dir, 'index-test')
mdpath = os.path.join(self.tmp_dir, 'metadata-test')
objectpath = os.path.join(self.tmp_dir, 'objects-test')
specpath = os.path.join(mdpath, 'vision-computing/images/dataset-ex')
ensure_path_exists(indexpath)
ensure_path_exists(specpath)
shutil.copy('hdata/dataset-ex.spec', specpath + '/dataset-ex.spec')
manifestpath = os.path.join(specpath, 'MANIFEST.yaml')
yaml_save(files_mock, manifestpath)
idx = MultihashIndex(specpath, indexpath, objectpath)
idx.add('data-test-push-1/', manifestpath)
fidx = FullIndex(specpath, indexpath)
self.assertTrue(os.path.exists(indexpath))
c = yaml_load('hdata/config.yaml')
o = Objects(specpath, objectpath)
o.commit_index(indexpath, self.tmp_dir)
self.assertTrue(os.path.exists(objectpath))
r = LocalRepository(c, objectpath)
self.assertTrue(r.push(objectpath, specpath + '/dataset-ex.spec') == 0)
self.assertTrue(len(fidx.get_index()) == 1)
def reset(self, spec, reset_type, head):
log.info(output_messages['INFO_INITIALIZING_RESET'] %
(reset_type, head),
class_name=REPOSITORY_CLASS_NAME)
if (reset_type == '--soft'
or reset_type == '--mixed') and head == HEAD:
return
try:
repo_type = self.__repo_type
metadata_path = get_metadata_path(self.__config, repo_type)
index_path = get_index_path(self.__config, repo_type)
refs_path = get_refs_path(self.__config, repo_type)
object_path = get_objects_path(self.__config, repo_type)
met = Metadata(spec, metadata_path, self.__config, repo_type)
ref = Refs(refs_path, spec, repo_type)
idx = MultihashIndex(spec, index_path, object_path)
fidx = FullIndex(spec, index_path)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
return
# get tag before reset
tag = met.get_current_tag()
categories_path = get_path_with_categories(str(tag))
# current manifest file before reset
manifest_path = os.path.join(metadata_path, categories_path, spec,
MANIFEST_FILE)
_manifest = Manifest(manifest_path).load()
if head == HEAD_1: # HEAD~1
try:
# reset the repo
met.reset()
except Exception:
return
# get tag after reset
tag_after_reset = met.get_current_tag()
sha = met.sha_from_tag(tag_after_reset)
# update ml-git ref HEAD
ref.update_head(str(tag_after_reset), sha)
# # get path to reset workspace in case of --hard
path, file = None, None
try:
path, file = search_spec_file(self.__repo_type, spec,
categories_path)
except Exception as e:
log.error(e, class_name=REPOSITORY_CLASS_NAME)
if reset_type == '--hard' and path is None:
return
# get manifest from metadata after reset
_manifest_changed = Manifest(manifest_path)
hash_files, file_names = _manifest_changed.get_diff(_manifest)
idx_mf = idx.get_index().load()
if reset_type == '--soft':
# add in index/metadata/<entity-name>/MANIFEST
idx.update_index_manifest(idx_mf)
idx.update_index_manifest(hash_files)
fidx.update_index_status(file_names, Status.a.name)
else: # --hard or --mixed
# remove hash from index/hashsh/store.log
file_names.update(*idx_mf.values())
objs = MultihashFS(index_path)
for key_hash in hash_files:
objs.remove_hash(key_hash)
idx.remove_manifest()
fidx.remove_from_index_yaml(file_names)
fidx.remove_uncommitted()
if reset_type == '--hard': # reset workspace
remove_from_workspace(file_names, path, spec)