def test_basic_conformation():
# corresponds to the base64 encoded compression of
# np.array([[[1]]], dtype=np.float32).tobytes()
# Was compressed by fpzip C program.
one_fpz = b'ZnB5KYcO8R7gAP8AAAD/AAAA/wAAAP8A8zvT3FAAAAA=\n'
one_fpz = base64.decodebytes(
one_fpz) # decodebytes is the modern name, but py27 doesn't have it
one_array = np.array([[[1]]], dtype=np.float32)
compressed = fpzip.compress(one_array)
assert len(one_fpz) == len(compressed)
assert np.all(fpzip.decompress(compressed) == fpzip.decompress(one_fpz))
# encoded np.array([[[1,2,3], [4,5,6]]], dtype=np.float32)
# with open("six.raw", 'wb') as f:
# f.write(six_array.tobytes('C'))
#
# ./fpzip -i six.raw -3 3 2 1 -o six.fpz
# >> outbytes=44 ratio=0.55
#
# with open('six.fpz', 'rb') as f:
# encoded = f.read()
# six_fpz = base64.encodestring(encoded)
six_fpz = b'ZnB5KYcO8R7gAv0AAAH+AAAA/wAAAP8A8zvT3GsIJgDRE0yNUZgAHeZbgAA=\n' # 3 2 1
six_fpz = base64.decodebytes(six_fpz)
six_array = np.array([[[1, 2, 3], [4, 5, 6]]], dtype=np.float32)
compressed = fpzip.compress(six_array)
assert len(six_fpz) == len(compressed)
assert np.all(
fpzip.decompress(six_fpz) == fpzip.decompress(compressed)[0, :, :, :])
def encode(img_chunk: np.ndarray,
encoding: str,
block_size: Optional[Sequence[int]] = None) -> bytes:
if encoding == "raw":
return encode_raw(img_chunk)
elif encoding == "kempressed":
return encode_kempressed(img_chunk)
elif encoding == "fpzip":
img_chunk = np.asfortranarray(img_chunk)
return fpzip.compress(img_chunk, order='F')
elif encoding == "compressed_segmentation":
return encode_compressed_segmentation(img_chunk, block_size=block_size)
elif encoding == "compresso":
return compresso.compress(img_chunk[:, :, :, 0])
elif encoding == "jpeg":
return encode_jpeg(img_chunk)
elif encoding == "png":
return encode_png(img_chunk)
elif encoding == "npz":
return encode_npz(img_chunk)
elif encoding == "npz_uint8":
chunk = img_chunk * 255
chunk = chunk.astype(np.uint8)
return encode_npz(chunk)
else:
raise NotImplementedError(encoding)
def test_noncontiguous_memory():
x = np.random.random_sample(size=(15, 15, 15, 1)).astype(np.float32)
x_broken = x[2::2, ::3, ::2]
for order in ('C', 'F'):
y = fpzip.compress(x_broken, order=order)
z = fpzip.decompress(y, order=order)
assert np.all(x_broken == z)
def test_doubles():
size = (128, 127, 126, 2)
for dims in range(4):
print(dims)
x = np.random.random_sample(size=size[:dims]).astype(np.float64)
x = np.ascontiguousarray(x)
y = fpzip.compress(x, order='C')
z = fpzip.decompress(y, order='C')
assert np.all(x == z)
x = np.asfortranarray(x)
y = fpzip.compress(x, order='F')
z = fpzip.decompress(y, order='F')
z = np.squeeze(z)
assert np.all(x == z)
x = np.random.random_sample(size=[0] * dims).astype(np.float64)
y = fpzip.compress(x)
z = fpzip.decompress(y)
assert np.all(x == z)
def encode(img_chunk, encoding, block_size=None):
if encoding == "raw":
return encode_raw(img_chunk)
elif encoding == "kempressed":
return encode_kempressed(img_chunk)
elif encoding == "fpzip":
return fpzip.compress(img_chunk)
elif encoding == "compressed_segmentation":
return encode_compressed_segmentation(img_chunk, block_size=block_size)
elif encoding == "jpeg":
return encode_jpeg(img_chunk)
elif encoding == "npz":
return encode_npz(img_chunk)
elif encoding == "npz_uint8":
chunk = img_chunk * 255
chunk = chunk.astype(np.uint8)
return encode_npz(chunk)
else:
raise NotImplementedError(encoding)
def test_floats():
size = (128, 127, 126, 2)
for dims in range(4):
print(dims)
x = np.random.random_sample(size=size[:dims]).astype(np.float64)
x = np.ascontiguousarray(x)
y = fpzip.compress(x)
z = fpzip.decompress(y, order='C')
assert np.all(x == z)
x = np.asfortranarray(x)
y = fpzip.compress(x, order='F')
z = fpzip.decompress(y, order='F')
z = np.squeeze(z)
assert np.all(x == z)
x = np.random.random_sample(size=(128, 128, 128)).astype(np.float32)
x = np.ascontiguousarray(x)
y = fpzip.compress(x)
z = fpzip.decompress(y)
z = np.squeeze(z, axis=0) # is3d logic
assert np.all(x == z)
x = np.asfortranarray(x)
y = fpzip.compress(x, order='F')
z = fpzip.decompress(y, order='F')
z = np.squeeze(z, axis=3)
assert np.all(x == z)
x = np.random.random_sample(size=(0, 0, 0, 0)).astype(np.float32)
x = np.ascontiguousarray(x)
y = fpzip.compress(x, order='C')
z = fpzip.decompress(y, order='C')
assert np.all(x == z)
x = np.asfortranarray(x)
y = fpzip.compress(x, order='F')
z = fpzip.decompress(y, order='F')
assert np.all(x == z)
def fpzip_compress(typed_column):
"""
compresses floats with fpzip codec.
"""
# converts all data to floats (can take in integers, but will convert).
numpy_array = np.array(typed_column, dtype=np.float32, order='C')
try:
compressed_bitstring = fpzip.compress(numpy_array,
precision=0,
order='C')
except fpzip.FpzipWriteError:
print('try a bigger block size')
# replace problem value
#for n in range(len(numpy_array)):
# if np.isnan(numpy_array[n]):
# numpy_array[n] = numpy_array[n]
#numpy_array = np.multiply(numpy_array,1)
#try:
# compressed_bitstring = fpzip.compress(numpy_array, precision=0, order='C')
#except fpzip.FpzipWriteError:
# print(numpy_array)
return compressed_bitstring
def encode_kempressed(subvol):
data = 2.0 + np.swapaxes(subvol, 2, 3)
return fpzip.compress(data, order='F')
output_files = [
os.path.join(args.output_dir, x + '.bin') for x in filenames
]
output_infs = [
os.path.join(args.output_dir, x + '.inf.bin') for x in filenames
]
pcQueue_512, pcQueue_1024, pcQueue_2048 = data_provider.load_data(
args.input_dir, args.source_extension)
centroid_list = []
furthest_distance_list = []
meta_matrix, nor_pc_512, nor_pc_1024, nor_pc_2048 = data_provider.gen_meta(
centroid_list, furthest_distance_list, pcQueue_512, pcQueue_1024,
pcQueue_2048)
compressed_bytes = fpzip.compress(meta_matrix.astype(np.float32),
precision=0,
order='C')
with open(output_infs[0], 'wb') as f:
f.write(compressed_bytes)
f.close()
def patch_compress(points, model, checkpoint):
estimator = tf.estimator.Estimator(model_fn=model.model_fn,
model_dir=checkpoint,
params={
'checkpoint_dir': checkpoint,
'batch_size': args.batch_size,
'knn': args.knn,
})
result = estimator.predict(
input_fn=lambda: model.input_fn(points,
