def test_get_update_links_wspace(self): wspath = os.path.join(self.tmp_dir, 'wspace') hfspath = os.path.join(self.tmp_dir, 'objectsfs') ohfs = MultihashFS(hfspath) key = ohfs.put(HDATA_IMG_1) fidx = FullIndex(self.tmp_dir, self.tmp_dir) cachepath = os.path.join(self.tmp_dir, 'cachefs') cache = Cache(cachepath, '', '') testbucketname = os.getenv('MLGIT_TEST_BUCKET', 'ml-git-datasets') c = get_sample_config_spec(testbucketname, testprofile, testregion) r = LocalRepository(c, hfspath) r._update_cache(cache, key) mfiles = {} files = {DATA_IMG_1} r._update_links_wspace(cache, fidx, files, key, wspath, mfiles, Status.u.name, 'strict') wspace_file = os.path.join(wspath, DATA_IMG_1) set_write_read(wspace_file) self.assertTrue(os.path.exists(wspace_file)) self.assertEqual(self.md5sum(HDATA_IMG_1), self.md5sum(wspace_file)) st = os.stat(wspace_file) fi = fidx.get_index() for k, v in fi.items(): self.assertEqual(k, os.path.join('data', 'imghires.jpg')) self.assertEqual(v['hash'], 'zdj7WjdojNAZN53Wf29rPssZamfbC6MVerzcGwd9tNciMpsQh') self.assertEqual(v['status'], 'u') self.assertEqual(v['ctime'], st.st_ctime) self.assertEqual(v['mtime'], st.st_mtime) self.assertTrue(st.st_nlink == 2) self.assertEqual(mfiles, {DATA_IMG_1: 'zdj7WjdojNAZN53Wf29rPssZamfbC6MVerzcGwd9tNciMpsQh'})
def commit_objects(self, index_path, ws_path): added_files = [] deleted_files = [] idx = MultihashFS(self._objects_path) fidx = FullIndex(self.__spec, index_path) findex = fidx.get_index() log_path = os.path.join(self._logpath, STORAGE_LOG) with open(log_path, 'a') as log_file: for k, v in findex.items(): if not os.path.exists(os.path.join(ws_path, k)): deleted_files.append(k) elif v['status'] == Status.a.name: idx.fetch_scid(v['hash'], log_file) v['status'] = Status.u.name if 'previous_hash' in v: added_files.append((v['previous_hash'], k)) fidx.get_manifest_index().save() return added_files, deleted_files
def test_push(self): indexpath = os.path.join(self.tmp_dir, 'index-test') mdpath = os.path.join(self.tmp_dir, 'metadata-test') objectpath = os.path.join(self.tmp_dir, 'objects-test') specpath = os.path.join(mdpath, 'vision-computing/images/dataset-ex') ensure_path_exists(indexpath) ensure_path_exists(specpath) shutil.copy('hdata/dataset-ex.spec', specpath + '/dataset-ex.spec') manifestpath = os.path.join(specpath, 'MANIFEST.yaml') yaml_save(files_mock, manifestpath) idx = MultihashIndex(specpath, indexpath, objectpath) idx.add('data-test-push-1/', manifestpath) fidx = FullIndex(specpath, indexpath) self.assertTrue(os.path.exists(indexpath)) c = yaml_load('hdata/config.yaml') o = Objects(specpath, objectpath) o.commit_index(indexpath, self.tmp_dir) self.assertTrue(os.path.exists(objectpath)) r = LocalRepository(c, objectpath) self.assertTrue(r.push(objectpath, specpath + '/dataset-ex.spec') == 0) self.assertTrue(len(fidx.get_index()) == 1)