def _generate_storage_tensors(self):
for t in self._flat_tensors:
t_dtype, t_path = t
path = posixpath.join(self._path, t_path[1:])
self._fs.makedirs(posixpath.join(path, "--dynamic--"))
yield t_path, DynamicTensor(
fs_map=MetaStorage(
t_path,
get_storage_map(
self._fs,
path,
self._cache,
self.lock_cache,
storage_cache=self._storage_cache,
),
self._fs_map,
self,
),
mode=self._mode,
shape=self._shape + t_dtype.shape,
max_shape=self._shape + t_dtype.max_shape,
dtype=_get_dynamic_tensor_dtype(t_dtype),
chunks=t_dtype.chunks,
compressor=_get_compressor(t_dtype.compressor),
)
def test_get_compression():
assert _get_compressor("lz4") == numcodecs.LZ4(numcodecs.lz4.DEFAULT_ACCELERATION)
assert _get_compressor(None) is None
assert _get_compressor("default") == "default"
assert _get_compressor("zstd") == numcodecs.Zstd(numcodecs.zstd.DEFAULT_CLEVEL)
assert _get_compressor("png") == PngCodec(solo_channel=True)
with pytest.raises(ValueError):
_get_compressor("abcd")