def test_basic_conformation(): # corresponds to the base64 encoded compression of # np.array([[[1]]], dtype=np.float32).tobytes() # Was compressed by fpzip C program. one_fpz = b'ZnB5KYcO8R7gAP8AAAD/AAAA/wAAAP8A8zvT3FAAAAA=\n' one_fpz = base64.decodebytes( one_fpz) # decodebytes is the modern name, but py27 doesn't have it one_array = np.array([[[1]]], dtype=np.float32) compressed = fpzip.compress(one_array) assert len(one_fpz) == len(compressed) assert np.all(fpzip.decompress(compressed) == fpzip.decompress(one_fpz)) # encoded np.array([[[1,2,3], [4,5,6]]], dtype=np.float32) # with open("six.raw", 'wb') as f: # f.write(six_array.tobytes('C')) # # ./fpzip -i six.raw -3 3 2 1 -o six.fpz # >> outbytes=44 ratio=0.55 # # with open('six.fpz', 'rb') as f: # encoded = f.read() # six_fpz = base64.encodestring(encoded) six_fpz = b'ZnB5KYcO8R7gAv0AAAH+AAAA/wAAAP8A8zvT3GsIJgDRE0yNUZgAHeZbgAA=\n' # 3 2 1 six_fpz = base64.decodebytes(six_fpz) six_array = np.array([[[1, 2, 3], [4, 5, 6]]], dtype=np.float32) compressed = fpzip.compress(six_array) assert len(six_fpz) == len(compressed) assert np.all( fpzip.decompress(six_fpz) == fpzip.decompress(compressed)[0, :, :, :])
def encode(img_chunk: np.ndarray, encoding: str, block_size: Optional[Sequence[int]] = None) -> bytes: if encoding == "raw": return encode_raw(img_chunk) elif encoding == "kempressed": return encode_kempressed(img_chunk) elif encoding == "fpzip": img_chunk = np.asfortranarray(img_chunk) return fpzip.compress(img_chunk, order='F') elif encoding == "compressed_segmentation": return encode_compressed_segmentation(img_chunk, block_size=block_size) elif encoding == "compresso": return compresso.compress(img_chunk[:, :, :, 0]) elif encoding == "jpeg": return encode_jpeg(img_chunk) elif encoding == "png": return encode_png(img_chunk) elif encoding == "npz": return encode_npz(img_chunk) elif encoding == "npz_uint8": chunk = img_chunk * 255 chunk = chunk.astype(np.uint8) return encode_npz(chunk) else: raise NotImplementedError(encoding)
def test_noncontiguous_memory(): x = np.random.random_sample(size=(15, 15, 15, 1)).astype(np.float32) x_broken = x[2::2, ::3, ::2] for order in ('C', 'F'): y = fpzip.compress(x_broken, order=order) z = fpzip.decompress(y, order=order) assert np.all(x_broken == z)
def test_doubles(): size = (128, 127, 126, 2) for dims in range(4): print(dims) x = np.random.random_sample(size=size[:dims]).astype(np.float64) x = np.ascontiguousarray(x) y = fpzip.compress(x, order='C') z = fpzip.decompress(y, order='C') assert np.all(x == z) x = np.asfortranarray(x) y = fpzip.compress(x, order='F') z = fpzip.decompress(y, order='F') z = np.squeeze(z) assert np.all(x == z) x = np.random.random_sample(size=[0] * dims).astype(np.float64) y = fpzip.compress(x) z = fpzip.decompress(y) assert np.all(x == z)
def encode(img_chunk, encoding, block_size=None): if encoding == "raw": return encode_raw(img_chunk) elif encoding == "kempressed": return encode_kempressed(img_chunk) elif encoding == "fpzip": return fpzip.compress(img_chunk) elif encoding == "compressed_segmentation": return encode_compressed_segmentation(img_chunk, block_size=block_size) elif encoding == "jpeg": return encode_jpeg(img_chunk) elif encoding == "npz": return encode_npz(img_chunk) elif encoding == "npz_uint8": chunk = img_chunk * 255 chunk = chunk.astype(np.uint8) return encode_npz(chunk) else: raise NotImplementedError(encoding)
def test_floats(): size = (128, 127, 126, 2) for dims in range(4): print(dims) x = np.random.random_sample(size=size[:dims]).astype(np.float64) x = np.ascontiguousarray(x) y = fpzip.compress(x) z = fpzip.decompress(y, order='C') assert np.all(x == z) x = np.asfortranarray(x) y = fpzip.compress(x, order='F') z = fpzip.decompress(y, order='F') z = np.squeeze(z) assert np.all(x == z) x = np.random.random_sample(size=(128, 128, 128)).astype(np.float32) x = np.ascontiguousarray(x) y = fpzip.compress(x) z = fpzip.decompress(y) z = np.squeeze(z, axis=0) # is3d logic assert np.all(x == z) x = np.asfortranarray(x) y = fpzip.compress(x, order='F') z = fpzip.decompress(y, order='F') z = np.squeeze(z, axis=3) assert np.all(x == z) x = np.random.random_sample(size=(0, 0, 0, 0)).astype(np.float32) x = np.ascontiguousarray(x) y = fpzip.compress(x, order='C') z = fpzip.decompress(y, order='C') assert np.all(x == z) x = np.asfortranarray(x) y = fpzip.compress(x, order='F') z = fpzip.decompress(y, order='F') assert np.all(x == z)
def fpzip_compress(typed_column): """ compresses floats with fpzip codec. """ # converts all data to floats (can take in integers, but will convert). numpy_array = np.array(typed_column, dtype=np.float32, order='C') try: compressed_bitstring = fpzip.compress(numpy_array, precision=0, order='C') except fpzip.FpzipWriteError: print('try a bigger block size') # replace problem value #for n in range(len(numpy_array)): # if np.isnan(numpy_array[n]): # numpy_array[n] = numpy_array[n] #numpy_array = np.multiply(numpy_array,1) #try: # compressed_bitstring = fpzip.compress(numpy_array, precision=0, order='C') #except fpzip.FpzipWriteError: # print(numpy_array) return compressed_bitstring
def encode_kempressed(subvol): data = 2.0 + np.swapaxes(subvol, 2, 3) return fpzip.compress(data, order='F')
output_files = [ os.path.join(args.output_dir, x + '.bin') for x in filenames ] output_infs = [ os.path.join(args.output_dir, x + '.inf.bin') for x in filenames ] pcQueue_512, pcQueue_1024, pcQueue_2048 = data_provider.load_data( args.input_dir, args.source_extension) centroid_list = [] furthest_distance_list = [] meta_matrix, nor_pc_512, nor_pc_1024, nor_pc_2048 = data_provider.gen_meta( centroid_list, furthest_distance_list, pcQueue_512, pcQueue_1024, pcQueue_2048) compressed_bytes = fpzip.compress(meta_matrix.astype(np.float32), precision=0, order='C') with open(output_infs[0], 'wb') as f: f.write(compressed_bytes) f.close() def patch_compress(points, model, checkpoint): estimator = tf.estimator.Estimator(model_fn=model.model_fn, model_dir=checkpoint, params={ 'checkpoint_dir': checkpoint, 'batch_size': args.batch_size, 'knn': args.knn, }) result = estimator.predict( input_fn=lambda: model.input_fn(points,