def compress(data, quality=0.025, adaptive=False):
# Compress using automatic parameter selection
# Different from Tucker because of different parameter selection
st_hosvd_rel_error, factor_matrix_parameter = calculate_parameters(quality)
compressed1 = st_hosvd.compress_orthogonality(
st_hosvd.compress_tucker(data,
st_hosvd_rel_error,
reshape=True,
method="tensor_trains"))
if adaptive:
# Initialization
current_compression = st_hosvd.compress_quantize(
deepcopy(compressed1),
factor_matrix_parameter=factor_matrix_parameter)
current_error = st_hosvd.rel_error(
data,
st_hosvd.decompress_tucker(
st_hosvd.decompress_orthogonality(
st_hosvd.decompress_quantize(current_compression))))
current_size = st_hosvd.get_compress_quantize_size(current_compression)
# Iterate as long as error stays below quality
while True:
alt_compression = st_hosvd.compress_quantize(
deepcopy(compressed1),
factor_matrix_parameter=factor_matrix_parameter - 1)
alt_error = st_hosvd.rel_error(
data,
st_hosvd.decompress_tucker(
st_hosvd.decompress_orthogonality(
st_hosvd.decompress_quantize(alt_compression))))
alt_size = st_hosvd.get_compress_quantize_size(alt_compression)
if alt_error < quality and alt_size < current_size:
# Continue
factor_matrix_parameter -= 1
current_compression = alt_compression
current_error = alt_error
current_size = alt_size
else:
break
return current_compression
else:
# Non-adaptive, use initial factor matrix parameter value
return st_hosvd.compress_quantize(
compressed1, factor_matrix_parameter=factor_matrix_parameter)
def test_time():
# Tests performance on random data, ignoring compression ratio
data = np.random.rand(101, 101, 10001)
compressed = st_hosvd.compress_tucker(data,
0,
rank=(22, 21, 19),
print_progress=True)
decompressed = st_hosvd.decompress_tucker(compressed)
print_difference(data, decompressed)
st_hosvd.print_compression_rate_tucker(data, compressed)
plot_comparison(data, decompressed)
def compress_mauna_kea():
data = load_mauna_kea()
compressed = st_hosvd.compress_tucker(data,
0.025,
print_progress=True,
use_pure_gramian=True)
print(
"Relative error:",
custom_norm(st_hosvd.decompress_tucker(compressed).__isub__(data)) /
custom_norm(data))
st_hosvd.print_compression_rate_tucker(data, compressed)
def test_compression_ratio_tucker():
print("=" * 20 + " Phase 1 " + "=" * 20)
data = load_pavia_centre()
compressed1 = st_hosvd.compress_tucker(data,
0.025,
reshape=False,
method="tucker")
decompressed = st_hosvd.decompress_tucker(compressed1)
st_hosvd.print_compression_rate_tucker(data, compressed1)
print_difference(data, decompressed)
"""
def compare_times():
data = load_cuprite()
original_size = data.dtype.itemsize * data.size
st_hosvd.print_compression_rate_tucker(
data, st_hosvd.compress_tucker(data, 0.025))
start = time()
compressed = st_hosvd.compress_quantize2(
st_hosvd.compress_tucker(data, 0.025))
print("Time for compression:", time() - start)
start = time()
decompressed = st_hosvd.decompress_tucker(
st_hosvd.decompress_quantize2(compressed))
print("Time for decompression:", time() - start)
st_hosvd.print_compression_rate_quantize2(data, compressed)
print_difference(data, decompressed)
start = time()
compressed = other_compression.compress_jpeg(data, 50)
print("Time for compression:", time() - start)
start = time()
decompressed = other_compression.decompress_jpeg(compressed)
print("Time for decompression:", time() - start)
print("Compressed size:",
other_compression.get_compress_jpeg_size(compressed))
print("Compression ratio:",
other_compression.get_compress_jpeg_size(compressed) / original_size)
print_difference(data, decompressed)
start = time()
compressed = other_compression.compress_video(data, 28)
print("Time for compression:", time() - start)
start = time()
decompressed = other_compression.decompress_video(compressed)
print("Time for decompression:", time() - start)
print("Compressed size:",
other_compression.get_compress_video_size(compressed))
print(
"Compression ratio:",
other_compression.get_compress_video_size(compressed) / original_size)
print_difference(data, decompressed)
def test_compress_variable_tucker():
# Test series of Tucker compressions
data = load_cuprite()
original_size = data.dtype.itemsize * data.size
rel_errors = []
compression_ratios = []
for rel_error in (0.01, 0.015, 0.02, 0.025, 0.03, 0.04, 0.05, 0.06, 0.07,
0.08, 0.1, 0.12, 0.14, 0.16, 0.18, 0.2):
print(rel_error)
compressed = st_hosvd.compress_quantize2(
st_hosvd.compress_tucker(data, rel_error, print_progress=False))
decompressed = st_hosvd.decompress_tucker(
st_hosvd.decompress_quantize2(compressed))
rel_errors.append(
np.linalg.norm(data - decompressed) / np.linalg.norm(data))
compression_ratios.append(
st_hosvd.get_compress_quantize2_size(compressed) / original_size)
print("rel_errors =", rel_errors)
print("compression_ratios =", compression_ratios)
def sweep_parameters(dataset_name="Indian_Pines", tensor_trains=False):
# Sweeps parameter space by brute-forcing combinations of parameters from sets of legal values
# Constants
reset = False
method = "tensor_trains" if tensor_trains else "tucker"
measurements_path = "../measurements/parameters_measurements_%s_%s.json" % (
method, dataset_name)
measurements_temp_path = measurements_path + ".tmp"
if dataset_name == "Cuprite":
data = load_cuprite_cropped() if tensor_trains else load_cuprite()
elif dataset_name == "Indian_Pines":
data = load_indian_pines_cropped(
) if tensor_trains else load_indian_pines()
else:
raise Exception("Invalid dataset name!")
# Reset measurements if necessary
if reset or not os.path.isfile(measurements_path):
measurements = {}
with open(measurements_path, "w") as f:
json.dump(measurements, f)
else:
with open(measurements_path, "r") as f:
measurements = json.load(f)
# Iterate through parameter space
# Parameter iterables are defined in order of increasing error (decreasing compression factor)
# Cuprite
st_hosvd_relative_errors = [
val / 10000 for val in range(50, 501, 25)
] if method == "tucker" else [val / 10000 for val in range(50, 501, 2)]
core_tensor_parameters = list(range(16, 0,
-1)) if method == "tucker" else [
16,
]
factor_matrix_parameters = list(range(16, 0, -1))
max_relative_error = 0.05 # Don't bother performing experiments that will surely give us a relative error beyond this bound
for i, st_hosvd_relative_error in enumerate(st_hosvd_relative_errors):
# Perform first compression step already
compressed1 = st_hosvd.compress_orthogonality(
st_hosvd.compress_tucker(data,
st_hosvd_relative_error,
reshape=tensor_trains,
method=method))
for j, core_tensor_parameter in enumerate(core_tensor_parameters):
for k, factor_matrix_parameter in enumerate(
factor_matrix_parameters):
# Test one parameter combination
# Only perform experiment if measurement hasn't been made yet
key = str((st_hosvd_relative_error, core_tensor_parameter,
factor_matrix_parameter))
if not key in measurements:
# Only consider measuring if it won't exceed the error bound
indices = [i, j, k]
break_flag = False
for index in range(3):
if (indices[index] > 0):
indices[index] -= 1
lower_key = str(
(st_hosvd_relative_errors[indices[0]],
core_tensor_parameters[indices[1]],
factor_matrix_parameters[indices[2]]))
indices[index] += 1
if lower_key in measurements and (
measurements[lower_key]
== "Error too large"
or measurements[lower_key][0] >=
max_relative_error):
# Set this measurement to None, indicating that the error is already too large
measurements[
key] = "Error too large" # As string since we're storing this as JSON
with open(measurements_temp_path, "w") as f:
json.dump(measurements, f)
os.rename(measurements_temp_path,
measurements_path)
break_flag = True
if break_flag:
continue
# Perform actual measurements
print("Measuring key %s" % key)
compressed2 = st_hosvd.compress_quantize(
compressed1,
copy=True,
endian="big",
encoding_method="adaptive",
allow_approximate_huffman=False,
use_zlib=True,
core_tensor_method="layered-constant-step",
core_tensor_parameter=core_tensor_parameter,
core_tensor_unquantized_rel_norm=0.995,
factor_matrix_method="layered",
factor_matrix_parameter=factor_matrix_parameter,
factor_matrix_columns_per_block=1,
bits_amount_selection="norm-based")
measurements[key] = (rel_error(
data,
st_hosvd.decompress_tucker(
st_hosvd.decompress_orthogonality(
st_hosvd.decompress_quantize(compressed2)))),
st_hosvd.
get_compression_factor_quantize(
data, compressed2))
with open(measurements_temp_path, "w") as f:
json.dump(measurements, f)
os.rename(measurements_temp_path, measurements_path)