def test_meta_storage(): os.makedirs("./data/test/test_meta_storage/internal_tensor", exist_ok=True) fs: fsspec.AbstractFileSystem = fsspec.filesystem("file") meta_map = fs.get_mapper("./data/test/test_meta_storage") meta_map["meta.json"] = bytes(json.dumps(dict()), "utf-8") fs_map = fs.get_mapper("./data/test/test_meta_storage/internal_tensor") MetaStorage("internal_tensor", fs_map, meta_map)
def _generate_storage_tensors(self): for t in self._flat_tensors: t_dtype, t_path = t path = posixpath.join(self._path, t_path[1:]) self._fs.makedirs(posixpath.join(path, "--dynamic--")) yield t_path, DynamicTensor( fs_map=MetaStorage( t_path, get_storage_map( self._fs, path, self._cache, self.lock_cache, storage_cache=self._storage_cache, ), self._fs_map, self, ), mode=self._mode, shape=self._shape + t_dtype.shape, max_shape=self._shape + t_dtype.max_shape, dtype=_get_dynamic_tensor_dtype(t_dtype), chunks=t_dtype.chunks, compressor=_get_compressor(t_dtype.compressor), )
def _open_storage_tensors(self): for t in self._flat_tensors: t_dtype, t_path = t path = posixpath.join(self._path, t_path[1:]) yield t_path, DynamicTensor( fs_map=MetaStorage( t_path, get_storage_map( self._fs, path, self._cache, self.lock_cache, storage_cache=self._storage_cache, ), self._fs_map, ), mode=self._mode, # FIXME We don't need argument below here shape=self._shape + t_dtype.shape, )