def test_meta_storage():
os.makedirs("./data/test/test_meta_storage/internal_tensor", exist_ok=True)
fs: fsspec.AbstractFileSystem = fsspec.filesystem("file")
meta_map = fs.get_mapper("./data/test/test_meta_storage")
meta_map["meta.json"] = bytes(json.dumps(dict()), "utf-8")
fs_map = fs.get_mapper("./data/test/test_meta_storage/internal_tensor")
MetaStorage("internal_tensor", fs_map, meta_map)
def _generate_storage_tensors(self):
for t in self._flat_tensors:
t_dtype, t_path = t
path = posixpath.join(self._path, t_path[1:])
self._fs.makedirs(posixpath.join(path, "--dynamic--"))
yield t_path, DynamicTensor(
fs_map=MetaStorage(
t_path,
get_storage_map(
self._fs,
path,
self._cache,
self.lock_cache,
storage_cache=self._storage_cache,
),
self._fs_map,
self,
),
mode=self._mode,
shape=self._shape + t_dtype.shape,
max_shape=self._shape + t_dtype.max_shape,
dtype=_get_dynamic_tensor_dtype(t_dtype),
chunks=t_dtype.chunks,
compressor=_get_compressor(t_dtype.compressor),
)
def _open_storage_tensors(self):
for t in self._flat_tensors:
t_dtype, t_path = t
path = posixpath.join(self._path, t_path[1:])
yield t_path, DynamicTensor(
fs_map=MetaStorage(
t_path,
get_storage_map(
self._fs,
path,
self._cache,
self.lock_cache,
storage_cache=self._storage_cache,
),
self._fs_map,
),
mode=self._mode,
# FIXME We don't need argument below here
shape=self._shape + t_dtype.shape,
)
