def decode(self, buf, out=None):
# normalise input
enc = ensure_ndarray(buf).view('u1')
# flatten to simplify implementation
enc = enc.reshape(-1, order='A')
# find out how many bits were padded
n_bits_padded = int(enc[0])
# apply decoding
dec = np.unpackbits(enc[1:])
# remove padded bits
if n_bits_padded:
dec = dec[:-n_bits_padded]
# view as boolean array
dec = dec.view(bool)
# given a flattened version of what was originally an nxmx2 array,
# reshape to group adjacent bits in second dimension and
# convert back to int based on each little-endian bit pair
dec = np.packbits(dec.reshape((-1, 2)), bitorder='little', axis=1)
dec = dec.squeeze(axis=1)
# handle destination
return ndarray_copy(dec, out)
def encode(self, buf):
# normalise input
arr = ensure_ndarray(buf)
# broadcast to 3rd dimension having 2 elements, least significant
# bit then second least, in that order; results in nxmx2 array
# containing individual bits as bools
# see: https://stackoverflow.com/questions/22227595/convert-integer-to-binary-array-with-suitable-padding
arr = arr[:, :, None] & (1 << np.arange(2)) > 0
return super().encode(arr)
def check_encode_decode_array(arr, codec):
enc = codec.encode(arr)
dec = codec.decode(enc)
assert_array_items_equal(arr, dec)
out = np.empty_like(arr)
codec.decode(enc, out=out)
assert_array_items_equal(arr, out)
enc = codec.encode(arr)
dec = codec.decode(ensure_ndarray(enc))
assert_array_items_equal(arr, dec)
def encode(self, buf: np.ndarray) -> np.ndarray:
"""Encodes a 2 dimensional array (variant x sample) of allele counts"""
# normalise input
arr = ensure_ndarray(buf)
# broadcast to 3rd dimension having 2 elements, least significant
# bit then second least, in that order; results in nxmx2 array
# containing individual bits as bools
# see: https://stackoverflow.com/questions/22227595/convert-integer-to-binary-array-with-suitable-padding
arr = arr[:, :, None] & self.__bit_mask
bool_arr = arr > 0
# NOTE: -1 codes as [True, True] which equals to 3 in uint8 bit encoding
return super().encode(bool_arr)
def _decode_chunk(self, encoded_data):
# decompress
if self.compressor is not None:
chunk = self.compressor.decode(encoded_data)
else:
chunk = encoded_data
# view as numpy array with correct dtype
chunk = ensure_ndarray(chunk)
chunk = chunk.view(self.dtype)
# ensure correct chunk shape
chunk = chunk.reshape(-1, order='A')
chunk = chunk.reshape(self.chunk_shape, order='C')
return chunk
def encode(self, buf):
"""The method to encode a raw image into jpeg format.
Parameters
----------
buf : ndarray
The raw image to be encoded into jpeg format
Returns
-------
ndarray
The image in jpeg format
"""
bufa = ensure_ndarray(buf)
assert 2 <= bufa.ndim <= 3
return jpeg_encode(bufa, level=self.quality)
def compare_arrays(arr, res, precision=None):
# ensure numpy array with matching dtype
res = ensure_ndarray(res).view(arr.dtype)
# convert to correct shape
if arr.flags.f_contiguous:
order = "F"
else:
order = "C"
res = res.reshape(arr.shape, order=order)
# exact compare
if precision is None:
assert_array_equal(arr, res)
# fuzzy compare
else:
assert_array_almost_equal(arr, res, decimal=precision)
def encode(self, buf):
arr = ensure_ndarray(buf).reshape(-1, order='A')
if arr.dtype != self.dtype:
raise ValueError(
f'cannot use DOD encoder with array of type {arr.dtype}', )
out = np.empty(len(arr) - 2 + 12, dtype=self.astype)
header = out[:12].view(self.dtype)
header[0] = len(arr)
if len(arr) == 0:
return out
header[1] = arr[0]
if len(arr) == 1:
return out
header[2] = arr[1] - arr[0]
delta = np.diff(arr)
out[12:] = np.diff(delta)
return out
def encode_chunk(chunk, filters=None, compressor=None):
"""helper function largely copied from zarr.Array"""
# apply filters
if filters:
for f in filters:
chunk = f.encode(chunk)
# check object encoding
if ensure_ndarray(chunk).dtype == object:
raise RuntimeError('cannot write object array without object codec')
# compress
if compressor:
cdata = compressor.encode(chunk)
else:
cdata = chunk
return cdata
def decode(self, buf, out=None):
enc = ensure_ndarray(buf).view(self.astype)
header = enc[:12].view(self.dtype)
size = header[0]
if out is None:
out = np.empty(size, dtype=self.dtype)
else:
out = out.view(self.dtype)
if len(out) != size:
raise ValueError('out size does not match data size')
if size == 0:
return out
out[0] = header[1]
if size == 1:
return out
out[1] = header[2]
out[2:] = enc[12:]
np.cumsum(out[1:], out=out[1:])
return np.cumsum(out, out=out)
def encode(self, buf):
bufa = ensure_ndarray(buf)
axis_reduction = self.axis_reduction
if bufa.ndim < 2:
raise ValueError(
f'Invalid dimensionality of input array.\n Input must have dimensionality of at least 2; got {buf.ndim}'
)
if len(self.input_shape) != len(bufa.shape):
raise ValueError(
f'Invalid input size.\n Input must have dimensionality matching the input_shape parameter of this compressor, i.e. {self.input_shape}, which has a dimensionality of {len(self.input_shape)}.\n Got input with shape {bufa.shape} instead, which has a dimensionality of {len(bufa.shape)}.'
)
if not all(chnk >= shpe
for chnk, shpe in zip(self.input_shape, bufa.shape)):
raise ValueError(
f'Invalid input size. Input must be less than or equal to the input_shape parameter of this compressor, i.e. {self.input_shape}. Got input with shape {bufa.shape} instead'
)
new_shape = [
np.prod([bufa.shape[dim] for dim in axis], dtype='int')
for axis in axis_reduction
]
tiled = bufa.reshape(new_shape)
return jpeg_encode(tiled, level=self.quality)
def encode(self, buf):
buf = ensure_ndarray(buf)
buf = cv.cvtColor(buf, cv.COLOR_RGB2YUV)
return buf
def buffer_size(v):
return ensure_ndarray(v).nbytes
def check_backwards_compatibility(codec_id,
arrays,
codecs,
precision=None,
prefix=None):
# setup directory to hold data fixture
if prefix:
fixture_dir = os.path.join('fixture', codec_id, prefix)
else:
fixture_dir = os.path.join('fixture', codec_id)
if not os.path.exists(fixture_dir): # pragma: no cover
os.makedirs(fixture_dir)
# save fixture data
for i, arr in enumerate(arrays):
arr_fn = os.path.join(fixture_dir, 'array.{:02d}.npy'.format(i))
if not os.path.exists(arr_fn): # pragma: no cover
np.save(arr_fn, arr)
# load fixture data
for arr_fn in glob(os.path.join(fixture_dir, 'array.*.npy')):
# setup
i = int(arr_fn.split('.')[-2])
arr = np.load(arr_fn, allow_pickle=True)
arr_bytes = arr.tobytes(order='A')
if arr.flags.f_contiguous:
order = 'F'
else:
order = 'C'
for j, codec in enumerate(codecs):
# setup a directory to hold encoded data
codec_dir = os.path.join(fixture_dir, 'codec.{:02d}'.format(j))
if not os.path.exists(codec_dir): # pragma: no cover
os.makedirs(codec_dir)
# file with codec configuration information
codec_fn = os.path.join(codec_dir, 'config.json')
# one time save config
if not os.path.exists(codec_fn): # pragma: no cover
with open(codec_fn, mode='w') as cf:
_json.dump(codec.get_config(),
cf,
sort_keys=True,
indent=4)
# load config and compare with expectation
with open(codec_fn, mode='r') as cf:
config = _json.load(cf)
assert codec == get_codec(config)
enc_fn = os.path.join(codec_dir, 'encoded.{:02d}.dat'.format(i))
# one time encode and save array
if not os.path.exists(enc_fn): # pragma: no cover
enc = codec.encode(arr)
enc = ensure_bytes(enc)
with open(enc_fn, mode='wb') as ef:
ef.write(enc)
# load and decode data
with open(enc_fn, mode='rb') as ef:
enc = ef.read()
dec = codec.decode(enc)
dec_arr = ensure_ndarray(dec).reshape(-1, order='A')
dec_arr = dec_arr.view(dtype=arr.dtype).reshape(arr.shape,
order=order)
if precision and precision[j] is not None:
assert_array_almost_equal(arr,
dec_arr,
decimal=precision[j])
elif arr.dtype == 'object':
assert_array_items_equal(arr, dec_arr)
else:
assert_array_equal(arr, dec_arr)
assert arr_bytes == ensure_bytes(dec)
def decode(self, buf):
buf = ensure_ndarray(buf)
buf = buf.reshape(450, 300)
buf = cv.cvtColor(buf, cv.COLOR_YUV2RGB_I420)
buf = buf[0:-1, 0:-1]
return buf
def encode(self, buf):
if self.level is not None:
return jpeg2k_encode(ensure_ndarray(buf), level=self.level)
else:
return jpeg2k_encode(ensure_ndarray(buf))
def encode(self, buf):
buf = ensure_ndarray(buf)
buf = np.pad(buf, ((0, 1), (0, 1), (0, 0)),
'edge') # YUV422 and YUV420 require even sizes
buf = cv.cvtColor(buf, cv.COLOR_RGB2YUV_I420)
return buf
def decode(self, buf):
buf = ensure_ndarray(buf)
buf = buf.reshape(300, 600)
buf = yuv422_to_rgb(buf)
buf = buf[0:-1, 0:-1]
return buf
def encode(self, buf):
buf = ensure_ndarray(buf)
buf = np.pad(buf, ((0, 1), (0, 1), (0, 0)),
'edge') # YUV422 and YUV420 require even sizes
buf = rgb_to_yuv422(buf)
return buf
def decode(self, buf):
buf = ensure_ndarray(buf)
buf = cv.cvtColor(buf, cv.COLOR_YUV2RGB)
return buf
def decode(self, buf):
return jpeg2k_decode(ensure_ndarray(buf))
