def compare(res): if precision is None: eq(arr_bytes, buffer_tobytes(res)) else: res = ndarray_from_buffer(res, dtype=arr.dtype) res = res.reshape(arr.shape, order=order) assert_array_almost_equal(arr, res, decimal=precision)
def decode(self, buf, out=None): arr = ndarray_from_buffer(buf, dtype='u1') expect = arr[:4].view('<u4')[0] checksum = self.checksum(arr[4:]) & 0xffffffff if expect != checksum: raise RuntimeError('checksum failed') return buffer_copy(arr[4:], out)
def encode(self, buf): arr = ndarray_from_buffer(buf, dtype='u1') checksum = self.checksum(arr) & 0xffffffff enc = np.empty(arr.nbytes + 4, dtype='u1') enc[:4].view('<u4')[0] = checksum enc[4:] = arr return enc
def encode(self, buf): # view input as ndarray arr = ndarray_from_buffer(buf, bool) # determine size of packed data n = arr.size n_bytes_packed = (n // 8) n_bits_leftover = n % 8 if n_bits_leftover > 0: n_bytes_packed += 1 # setup output enc = np.empty(n_bytes_packed + 1, dtype='u1') # store how many bits were padded if n_bits_leftover: n_bits_padded = 8 - n_bits_leftover else: n_bits_padded = 0 enc[0] = n_bits_padded # apply encoding enc[1:] = np.packbits(arr) return enc
def encode(self, buf): # view input data as 1D array arr = ndarray_from_buffer(buf, self.decode_dtype) # convert and copy enc = arr.astype(self.encode_dtype) return enc
def decode(self, buf, out=None): # view encoded data as 1D array enc = ndarray_from_buffer(buf, self.encode_dtype) # convert and copy dec = enc.astype(self.decode_dtype) # handle output out = buffer_copy(dec, out) return out
def decode(self, buf, out=None): # interpret buffer as 1D array enc = ndarray_from_buffer(buf, self.astype) # decode scale offset dec = (enc / self.scale) + self.offset # convert dtype dec = dec.astype(self.dtype, copy=False) # handle output return buffer_copy(dec, out)
def encode(self, buf): # view input as ndarray arr = ndarray_from_buffer(buf, self.dtype) # setup output array enc = np.zeros_like(arr, dtype=self.astype) # apply encoding, reserving 0 for values not specified in labels for i, l in enumerate(self.labels): enc[arr == l] = i + 1 return enc
def encode(self, buf): # interpret buffer as 1D array arr = ndarray_from_buffer(buf, self.dtype) # compute scale offset enc = (arr - self.offset) * self.scale # round to nearest integer enc = np.around(enc) # convert dtype enc = enc.astype(self.astype, copy=False) return enc
def encode(self, buf): # view input data as 1D array arr = ndarray_from_buffer(buf, self.dtype) # setup encoded output enc = np.empty_like(arr, dtype=self.astype) # set first element enc[0] = arr[0] # compute differences enc[1:] = np.diff(arr) return enc
def encode(self, buf): # interpret buffer as 1D array arr = ndarray_from_buffer(buf, self.dtype) # apply scaling precision = 10.**-self.digits exp = math.log(precision, 10) if exp < 0: exp = int(math.floor(exp)) else: exp = int(math.ceil(exp)) bits = math.ceil(math.log(10.**-exp, 2)) scale = 2.**bits enc = np.around(scale * arr) / scale # cast dtype enc = enc.astype(self.astype, copy=False) return enc
def decode(self, buf, out=None): # view encoded data as ndarray enc = ndarray_from_buffer(buf, 'u1') # find out how many bits were padded n_bits_padded = int(enc[0]) # apply decoding dec = np.unpackbits(enc[1:]) # remove padded bits if n_bits_padded: dec = dec[:-n_bits_padded] # view as boolean array dec = dec.view(bool) # handle destination return buffer_copy(dec, out)
def compare_arrays(arr, res, precision=None): # ensure numpy array if not isinstance(res, np.ndarray): res = ndarray_from_buffer(res, dtype=arr.dtype) elif res.dtype != arr.dtype: res = res.view(arr.dtype) # convert to correct shape if arr.flags.f_contiguous: order = 'F' else: order = 'C' res = res.reshape(arr.shape, order=order) # exact compare if precision is None: assert_array_equal(arr, res) # fuzzy compare else: assert_array_almost_equal(arr, res, decimal=precision)
def decode(self, buf, out=None): # view encoded data as 1D array enc = ndarray_from_buffer(buf, self.astype) # setup decoded output if isinstance(out, np.ndarray): # optimization, can decode directly to out dec = out.reshape(-1, order='A') copy_needed = False else: dec = np.empty_like(enc, dtype=self.dtype) copy_needed = True # decode differences np.cumsum(enc, out=dec) # handle output if copy_needed: out = buffer_copy(dec, out) return out
def decode(self, buf, out=None): # view encoded data as ndarray enc = ndarray_from_buffer(buf, self.astype) # setup output if isinstance(out, np.ndarray): # optimization, decode directly to output dec = out.reshape(-1, order='A') copy_needed = False else: dec = np.zeros_like(enc, dtype=self.dtype) copy_needed = True # apply decoding for i, l in enumerate(self.labels): dec[enc == (i + 1)] = l # handle output if copy_needed: dec = buffer_copy(dec, out) return dec
def decode(self, buf, out=None): # filter is lossy, decoding is no-op dec = ndarray_from_buffer(buf, self.astype) dec = dec.astype(self.dtype, copy=False) return buffer_copy(dec, out)
def check_backwards_compatibility(codec_id, arrays, codecs, precision=None, prefix=None): # setup directory to hold data fixture if prefix: fixture_dir = os.path.join('fixture', codec_id, prefix) else: fixture_dir = os.path.join('fixture', codec_id) if not os.path.exists(fixture_dir): # pragma: no cover os.makedirs(fixture_dir) # save fixture data for i, arr in enumerate(arrays): arr_fn = os.path.join(fixture_dir, 'array.{:02d}.npy'.format(i)) if not os.path.exists(arr_fn): # pragma: no cover np.save(arr_fn, arr) # load fixture data for arr_fn in glob(os.path.join(fixture_dir, 'array.*.npy')): # setup i = int(arr_fn.split('.')[-2]) arr = np.load(arr_fn) arr_bytes = buffer_tobytes(arr) if arr.flags.f_contiguous: order = 'F' else: order = 'C' for j, codec in enumerate(codecs): # setup a directory to hold encoded data codec_dir = os.path.join(fixture_dir, 'codec.{:02d}'.format(j)) if not os.path.exists(codec_dir): # pragma: no cover os.makedirs(codec_dir) # file with codec configuration information codec_fn = os.path.join(codec_dir, 'config.json') # one time save config if not os.path.exists(codec_fn): # pragma: no cover with open(codec_fn, mode='w') as cf: json.dump(codec.get_config(), cf, sort_keys=True, indent=4) # load config and compare with expectation with open(codec_fn, mode='r') as cf: config = json.load(cf) eq(codec, get_codec(config)) enc_fn = os.path.join(codec_dir, 'encoded.{:02d}.dat'.format(i)) # one time encode and save array if not os.path.exists(enc_fn): # pragma: no cover enc = codec.encode(arr) with open(enc_fn, mode='wb') as ef: ef.write(enc) # load and decode data with open(enc_fn, mode='rb') as ef: enc = ef.read() dec = codec.decode(enc) dec_arr = ndarray_from_buffer(dec, dtype=arr.dtype) dec_arr = dec_arr.reshape(arr.shape, order=order) if precision and precision[j] is not None: assert_array_almost_equal(arr, dec_arr, decimal=precision[j]) elif arr.dtype == 'object': assert_array_items_equal(arr, dec_arr) else: assert_array_equal(arr, dec_arr) eq(arr_bytes, buffer_tobytes(dec))