def register_codecs(codecs=None, force=False, verbose=True): """Register codecs in this module with numcodecs.""" for name, cls in globals().items(): if not hasattr(cls, 'codec_id') or name == 'Codec': continue if codecs is not None and cls.codec_id not in codecs: continue try: try: get_codec({'id': cls.codec_id}) except TypeError: # registered, but failed pass except ValueError: # not registered yet pass else: if not force: if verbose: log_warning( f'numcodec {cls.codec_id!r} already registered') continue if verbose: log_warning(f'replacing registered numcodec {cls.codec_id!r}') register_codec(cls)
def __init__(self, dtype, chunk_shape, compressor_config=None, compressor=None): self.dtype = np.dtype(dtype) self.chunk_shape = tuple(chunk_shape) # is the dtype a little endian format? self._little_endian = (self.dtype.byteorder == '<' or (self.dtype.byteorder == '=' and sys.byteorder == 'little')) if compressor: # pragma: no cover if compressor_config is not None: raise ValueError( "Only one of compressor_config or compressor should be given." ) compressor_config = compressor.get_config() if (compressor_config is None and compressor is None or compressor_config['id'] == 'raw'): self.compressor_config = None self._compressor = None else: self._compressor = get_codec(compressor_config) self.compressor_config = self._compressor.get_config()
def check_backwards_compatibility(codec_id, arrays, codecs, precision=None, prefix=None): # setup directory to hold data fixture if prefix: fixture_dir = os.path.join('fixture', codec_id, prefix) else: fixture_dir = os.path.join('fixture', codec_id) if not os.path.exists(fixture_dir): # pragma: no cover os.makedirs(fixture_dir) # save fixture data for i, arr in enumerate(arrays): arr_fn = os.path.join(fixture_dir, 'array.{:02d}.npy'.format(i)) if not os.path.exists(arr_fn): # pragma: no cover np.save(arr_fn, arr) # load fixture data for arr_fn in glob(os.path.join(fixture_dir, 'array.*.npy')): # setup i = int(arr_fn.split('.')[-2]) arr = np.load(arr_fn, allow_pickle=True) arr_bytes = arr.tobytes(order='A') if arr.flags.f_contiguous: order = 'F' else: order = 'C' for j, codec in enumerate(codecs): # setup a directory to hold encoded data codec_dir = os.path.join(fixture_dir, 'codec.{:02d}'.format(j)) if not os.path.exists(codec_dir): # pragma: no cover os.makedirs(codec_dir) # file with codec configuration information codec_fn = os.path.join(codec_dir, 'config.json') # one time save config if not os.path.exists(codec_fn): # pragma: no cover with open(codec_fn, mode='w') as cf: _json.dump(codec.get_config(), cf, sort_keys=True, indent=4) # load config and compare with expectation with open(codec_fn, mode='r') as cf: config = _json.load(cf) assert codec == get_codec(config) enc_fn = os.path.join(codec_dir, 'encoded.{:02d}.dat'.format(i)) # one time encode and save array if not os.path.exists(enc_fn): # pragma: no cover enc = codec.encode(arr) enc = ensure_bytes(enc) with open(enc_fn, mode='wb') as ef: ef.write(enc) # load and decode data with open(enc_fn, mode='rb') as ef: enc = ef.read() dec = codec.decode(enc) dec_arr = ensure_ndarray(dec).reshape(-1, order='A') dec_arr = dec_arr.view(dtype=arr.dtype).reshape(arr.shape, order=order) if precision and precision[j] is not None: assert_array_almost_equal(arr, dec_arr, decimal=precision[j]) elif arr.dtype == 'object': assert_array_items_equal(arr, dec_arr) else: assert_array_equal(arr, dec_arr) assert arr_bytes == ensure_bytes(dec)
def check_config(codec): config = codec.get_config() # round-trip through JSON to check serialization config = _json.loads(_json.dumps(config)) assert codec == get_codec(config)
def test_registry_errors(): with pytest.raises(ValueError): get_codec({'id': 'foo'})
def test_get_codec_argument(): # Check that get_codec doesn't modify its argument. arg = {"id": "json"} before = dict(arg) get_codec(arg) assert before == arg
def test_alias(): config = dict(id='gzip', level=1) codec = get_codec(config) assert Zlib(1) == codec