def register_codecs(codecs=None, force=False, verbose=True):
"""Register codecs in this module with numcodecs."""
for name, cls in globals().items():
if not hasattr(cls, 'codec_id') or name == 'Codec':
continue
if codecs is not None and cls.codec_id not in codecs:
continue
try:
try:
get_codec({'id': cls.codec_id})
except TypeError:
# registered, but failed
pass
except ValueError:
# not registered yet
pass
else:
if not force:
if verbose:
log_warning(
f'numcodec {cls.codec_id!r} already registered')
continue
if verbose:
log_warning(f'replacing registered numcodec {cls.codec_id!r}')
register_codec(cls)
def __init__(self,
dtype,
chunk_shape,
compressor_config=None,
compressor=None):
self.dtype = np.dtype(dtype)
self.chunk_shape = tuple(chunk_shape)
# is the dtype a little endian format?
self._little_endian = (self.dtype.byteorder == '<'
or (self.dtype.byteorder == '='
and sys.byteorder == 'little'))
if compressor: # pragma: no cover
if compressor_config is not None:
raise ValueError(
"Only one of compressor_config or compressor should be given."
)
compressor_config = compressor.get_config()
if (compressor_config is None and compressor is None
or compressor_config['id'] == 'raw'):
self.compressor_config = None
self._compressor = None
else:
self._compressor = get_codec(compressor_config)
self.compressor_config = self._compressor.get_config()
def check_backwards_compatibility(codec_id,
arrays,
codecs,
precision=None,
prefix=None):
# setup directory to hold data fixture
if prefix:
fixture_dir = os.path.join('fixture', codec_id, prefix)
else:
fixture_dir = os.path.join('fixture', codec_id)
if not os.path.exists(fixture_dir): # pragma: no cover
os.makedirs(fixture_dir)
# save fixture data
for i, arr in enumerate(arrays):
arr_fn = os.path.join(fixture_dir, 'array.{:02d}.npy'.format(i))
if not os.path.exists(arr_fn): # pragma: no cover
np.save(arr_fn, arr)
# load fixture data
for arr_fn in glob(os.path.join(fixture_dir, 'array.*.npy')):
# setup
i = int(arr_fn.split('.')[-2])
arr = np.load(arr_fn, allow_pickle=True)
arr_bytes = arr.tobytes(order='A')
if arr.flags.f_contiguous:
order = 'F'
else:
order = 'C'
for j, codec in enumerate(codecs):
# setup a directory to hold encoded data
codec_dir = os.path.join(fixture_dir, 'codec.{:02d}'.format(j))
if not os.path.exists(codec_dir): # pragma: no cover
os.makedirs(codec_dir)
# file with codec configuration information
codec_fn = os.path.join(codec_dir, 'config.json')
# one time save config
if not os.path.exists(codec_fn): # pragma: no cover
with open(codec_fn, mode='w') as cf:
_json.dump(codec.get_config(),
cf,
sort_keys=True,
indent=4)
# load config and compare with expectation
with open(codec_fn, mode='r') as cf:
config = _json.load(cf)
assert codec == get_codec(config)
enc_fn = os.path.join(codec_dir, 'encoded.{:02d}.dat'.format(i))
# one time encode and save array
if not os.path.exists(enc_fn): # pragma: no cover
enc = codec.encode(arr)
enc = ensure_bytes(enc)
with open(enc_fn, mode='wb') as ef:
ef.write(enc)
# load and decode data
with open(enc_fn, mode='rb') as ef:
enc = ef.read()
dec = codec.decode(enc)
dec_arr = ensure_ndarray(dec).reshape(-1, order='A')
dec_arr = dec_arr.view(dtype=arr.dtype).reshape(arr.shape,
order=order)
if precision and precision[j] is not None:
assert_array_almost_equal(arr,
dec_arr,
decimal=precision[j])
elif arr.dtype == 'object':
assert_array_items_equal(arr, dec_arr)
else:
assert_array_equal(arr, dec_arr)
assert arr_bytes == ensure_bytes(dec)
def check_config(codec):
config = codec.get_config()
# round-trip through JSON to check serialization
config = _json.loads(_json.dumps(config))
assert codec == get_codec(config)
def test_registry_errors():
with pytest.raises(ValueError):
get_codec({'id': 'foo'})
def test_get_codec_argument():
# Check that get_codec doesn't modify its argument.
arg = {"id": "json"}
before = dict(arg)
get_codec(arg)
assert before == arg
def test_alias():
config = dict(id='gzip', level=1)
codec = get_codec(config)
assert Zlib(1) == codec
