def compare(res):
if precision is None:
eq(arr_bytes, buffer_tobytes(res))
else:
res = ndarray_from_buffer(res, dtype=arr.dtype)
res = res.reshape(arr.shape, order=order)
assert_array_almost_equal(arr, res, decimal=precision)
def decode(self, buf, out=None):
arr = ndarray_from_buffer(buf, dtype='u1')
expect = arr[:4].view('<u4')[0]
checksum = self.checksum(arr[4:]) & 0xffffffff
if expect != checksum:
raise RuntimeError('checksum failed')
return buffer_copy(arr[4:], out)
def encode(self, buf):
arr = ndarray_from_buffer(buf, dtype='u1')
checksum = self.checksum(arr) & 0xffffffff
enc = np.empty(arr.nbytes + 4, dtype='u1')
enc[:4].view('<u4')[0] = checksum
enc[4:] = arr
return enc
def encode(self, buf):
# view input as ndarray
arr = ndarray_from_buffer(buf, bool)
# determine size of packed data
n = arr.size
n_bytes_packed = (n // 8)
n_bits_leftover = n % 8
if n_bits_leftover > 0:
n_bytes_packed += 1
# setup output
enc = np.empty(n_bytes_packed + 1, dtype='u1')
# store how many bits were padded
if n_bits_leftover:
n_bits_padded = 8 - n_bits_leftover
else:
n_bits_padded = 0
enc[0] = n_bits_padded
# apply encoding
enc[1:] = np.packbits(arr)
return enc
def encode(self, buf):
# view input data as 1D array
arr = ndarray_from_buffer(buf, self.decode_dtype)
# convert and copy
enc = arr.astype(self.encode_dtype)
return enc
def decode(self, buf, out=None):
# view encoded data as 1D array
enc = ndarray_from_buffer(buf, self.encode_dtype)
# convert and copy
dec = enc.astype(self.decode_dtype)
# handle output
out = buffer_copy(dec, out)
return out
def decode(self, buf, out=None):
# interpret buffer as 1D array
enc = ndarray_from_buffer(buf, self.astype)
# decode scale offset
dec = (enc / self.scale) + self.offset
# convert dtype
dec = dec.astype(self.dtype, copy=False)
# handle output
return buffer_copy(dec, out)
def encode(self, buf):
# view input as ndarray
arr = ndarray_from_buffer(buf, self.dtype)
# setup output array
enc = np.zeros_like(arr, dtype=self.astype)
# apply encoding, reserving 0 for values not specified in labels
for i, l in enumerate(self.labels):
enc[arr == l] = i + 1
return enc
def encode(self, buf):
# interpret buffer as 1D array
arr = ndarray_from_buffer(buf, self.dtype)
# compute scale offset
enc = (arr - self.offset) * self.scale
# round to nearest integer
enc = np.around(enc)
# convert dtype
enc = enc.astype(self.astype, copy=False)
return enc
def encode(self, buf):
# view input data as 1D array
arr = ndarray_from_buffer(buf, self.dtype)
# setup encoded output
enc = np.empty_like(arr, dtype=self.astype)
# set first element
enc[0] = arr[0]
# compute differences
enc[1:] = np.diff(arr)
return enc
def encode(self, buf):
# interpret buffer as 1D array
arr = ndarray_from_buffer(buf, self.dtype)
# apply scaling
precision = 10.**-self.digits
exp = math.log(precision, 10)
if exp < 0:
exp = int(math.floor(exp))
else:
exp = int(math.ceil(exp))
bits = math.ceil(math.log(10.**-exp, 2))
scale = 2.**bits
enc = np.around(scale * arr) / scale
# cast dtype
enc = enc.astype(self.astype, copy=False)
return enc
def decode(self, buf, out=None):
# view encoded data as ndarray
enc = ndarray_from_buffer(buf, 'u1')
# find out how many bits were padded
n_bits_padded = int(enc[0])
# apply decoding
dec = np.unpackbits(enc[1:])
# remove padded bits
if n_bits_padded:
dec = dec[:-n_bits_padded]
# view as boolean array
dec = dec.view(bool)
# handle destination
return buffer_copy(dec, out)
def compare_arrays(arr, res, precision=None):
# ensure numpy array
if not isinstance(res, np.ndarray):
res = ndarray_from_buffer(res, dtype=arr.dtype)
elif res.dtype != arr.dtype:
res = res.view(arr.dtype)
# convert to correct shape
if arr.flags.f_contiguous:
order = 'F'
else:
order = 'C'
res = res.reshape(arr.shape, order=order)
# exact compare
if precision is None:
assert_array_equal(arr, res)
# fuzzy compare
else:
assert_array_almost_equal(arr, res, decimal=precision)
def decode(self, buf, out=None):
# view encoded data as 1D array
enc = ndarray_from_buffer(buf, self.astype)
# setup decoded output
if isinstance(out, np.ndarray):
# optimization, can decode directly to out
dec = out.reshape(-1, order='A')
copy_needed = False
else:
dec = np.empty_like(enc, dtype=self.dtype)
copy_needed = True
# decode differences
np.cumsum(enc, out=dec)
# handle output
if copy_needed:
out = buffer_copy(dec, out)
return out
def decode(self, buf, out=None):
# view encoded data as ndarray
enc = ndarray_from_buffer(buf, self.astype)
# setup output
if isinstance(out, np.ndarray):
# optimization, decode directly to output
dec = out.reshape(-1, order='A')
copy_needed = False
else:
dec = np.zeros_like(enc, dtype=self.dtype)
copy_needed = True
# apply decoding
for i, l in enumerate(self.labels):
dec[enc == (i + 1)] = l
# handle output
if copy_needed:
dec = buffer_copy(dec, out)
return dec
def decode(self, buf, out=None):
# filter is lossy, decoding is no-op
dec = ndarray_from_buffer(buf, self.astype)
dec = dec.astype(self.dtype, copy=False)
return buffer_copy(dec, out)
def check_backwards_compatibility(codec_id, arrays, codecs, precision=None, prefix=None):
# setup directory to hold data fixture
if prefix:
fixture_dir = os.path.join('fixture', codec_id, prefix)
else:
fixture_dir = os.path.join('fixture', codec_id)
if not os.path.exists(fixture_dir): # pragma: no cover
os.makedirs(fixture_dir)
# save fixture data
for i, arr in enumerate(arrays):
arr_fn = os.path.join(fixture_dir, 'array.{:02d}.npy'.format(i))
if not os.path.exists(arr_fn): # pragma: no cover
np.save(arr_fn, arr)
# load fixture data
for arr_fn in glob(os.path.join(fixture_dir, 'array.*.npy')):
# setup
i = int(arr_fn.split('.')[-2])
arr = np.load(arr_fn)
arr_bytes = buffer_tobytes(arr)
if arr.flags.f_contiguous:
order = 'F'
else:
order = 'C'
for j, codec in enumerate(codecs):
# setup a directory to hold encoded data
codec_dir = os.path.join(fixture_dir, 'codec.{:02d}'.format(j))
if not os.path.exists(codec_dir): # pragma: no cover
os.makedirs(codec_dir)
# file with codec configuration information
codec_fn = os.path.join(codec_dir, 'config.json')
# one time save config
if not os.path.exists(codec_fn): # pragma: no cover
with open(codec_fn, mode='w') as cf:
json.dump(codec.get_config(), cf, sort_keys=True, indent=4)
# load config and compare with expectation
with open(codec_fn, mode='r') as cf:
config = json.load(cf)
eq(codec, get_codec(config))
enc_fn = os.path.join(codec_dir, 'encoded.{:02d}.dat'.format(i))
# one time encode and save array
if not os.path.exists(enc_fn): # pragma: no cover
enc = codec.encode(arr)
with open(enc_fn, mode='wb') as ef:
ef.write(enc)
# load and decode data
with open(enc_fn, mode='rb') as ef:
enc = ef.read()
dec = codec.decode(enc)
dec_arr = ndarray_from_buffer(dec, dtype=arr.dtype)
dec_arr = dec_arr.reshape(arr.shape, order=order)
if precision and precision[j] is not None:
assert_array_almost_equal(arr, dec_arr, decimal=precision[j])
elif arr.dtype == 'object':
assert_array_items_equal(arr, dec_arr)
else:
assert_array_equal(arr, dec_arr)
eq(arr_bytes, buffer_tobytes(dec))
