def compress(self,
original_path: str,
compressed_path: str,
original_file_info=None):
array = isets.load_array_bsq(file_or_path=original_path,
image_properties_row=original_file_info)
array += self.param_dict["constant"]
isets.dump_array_bsq(array=array, file_or_path=compressed_path)
def test_array_load_dump(self):
"""Test that file properties are correctly obtained and retrieved.
"""
width = 4
height = 2
component_count = 3
for is_float in [True, False]:
bytes_per_sample_values = [2, 4, 8] if is_float else [1, 2, 4, 8]
for bytes_per_sample in bytes_per_sample_values:
for signed in [True, False]:
value_factors = [1] if not signed else [1, -1]
for value_factor in value_factors:
big_endian_values = [True] if (bytes_per_sample == 1 or is_float) \
else [True, False]
for big_endian in big_endian_values:
row = dict(width=width,
height=height,
component_count=component_count,
float=is_float,
signed=signed,
bytes_per_sample=bytes_per_sample,
big_endian=big_endian)
array = np.zeros(
(width, height, component_count),
dtype=isets.iproperties_row_to_numpy_dtype(
row))
i = 0
for z in range(component_count):
for y in range(height):
for x in range(width):
array[x, y, z] = i * value_factor
i += 1
_, temp_file_path = tempfile.mkstemp(
suffix=
f"_{isets.iproperties_row_to_sample_type_tag(row)}.raw"
)
try:
dtype_string = isets.iproperties_row_to_numpy_dtype(
row)
isets.dump_array_bsq(array,
temp_file_path,
mode="wb",
dtype=dtype_string)
loaded_array = isets.load_array_bsq(
file_or_path=temp_file_path,
image_properties_row=row)
assert (array == loaded_array).all(), (
array, loaded_array)
assert os.path.getsize(temp_file_path) \
== width * height * component_count * bytes_per_sample
finally:
try:
os.remove(temp_file_path)
except OSError:
pass
def compress(self,
original_path: str,
compressed_path: str,
original_file_info=None):
# Tested limit: self.max_tested_spatial_size
if original_file_info["width"] <= self.max_dimension_size \
and original_file_info["height"] <= self.max_dimension_size:
return self.compress_one(original_path=original_path,
compressed_path=compressed_path,
original_file_info=original_file_info)
else:
tl_writer = tarlite.TarliteWriter()
img = isets.load_array_bsq(file_or_path=original_path,
image_properties_row=original_file_info)
with tempfile.TemporaryDirectory(
dir=options.base_tmp_dir) as tmp_dir:
compound_size = 0
total_compression_time = 0
for y in range(self.split_height_count):
for x in range(self.split_width_count):
small_array = \
img[x * (original_file_info["width"] // self.split_width_count):
(((x + 1) * (original_file_info[
"width"] // self.split_width_count)) if x < self.split_width_count - 1 else
original_file_info["width"]),
y * (original_file_info["height"] // self.split_height_count):
(((y + 1) * (original_file_info[
"height"] // self.split_height_count)) if y < self.split_height_count - 1 else
original_file_info["height"]),
:]
small_path = os.path.join(tmp_dir, f"{x}_{y}.raw")
small_compressed_path = os.path.join(
tmp_dir, f"{x}_{y}.mcalic")
isets.dump_array_bsq(small_array, small_path)
small_file_info = copy.copy(original_file_info)
small_file_info["width"], small_file_info[
"height"], small_file_info[
"component_count"] = small_array.shape
compression_results = self.compress_one(
original_path=small_path,
compressed_path=small_compressed_path,
original_file_info=small_file_info)
total_compression_time += compression_results.compression_time_seconds
tl_writer.add_file(small_compressed_path)
os.remove(small_path)
compound_size += small_array.size
assert compound_size == original_file_info[
"samples"], f"compound_size = {compound_size} != {original_file_info['samples']} = original samples"
tl_writer.write(output_path=compressed_path)
compression_results = self.compression_results_from_paths(
original_path=original_path,
compressed_path=compressed_path)
compression_results.compression_time_seconds = total_compression_time
return compression_results
def decompress(self, compressed_path, reconstructed_path, original_file_info=None):
tr = tarlite.TarliteReader(tarlite_path=compressed_path)
bands_per_image = min(original_file_info["component_count"],
self.max_dimension_size // original_file_info["height"])
total_decompression_time = 0
recovered_components = []
with tempfile.TemporaryDirectory(dir=options.base_tmp_dir) as tmp_dir:
tr.extract_all(output_dir_path=tmp_dir)
for stack_index, start_band_index in enumerate(
range(0, original_file_info["component_count"], bands_per_image)):
compressed_stack_path = os.path.join(tmp_dir, str(stack_index))
with tempfile.NamedTemporaryFile(dir=options.base_tmp_dir, suffix=".pgm") as stack_path:
decompression_results = super().decompress(
compressed_path=compressed_stack_path, reconstructed_path=stack_path.name)
total_decompression_time += decompression_results.decompression_time_seconds
assert os.path.isfile(stack_path.name)
stack_array = pgm.read_pgm(input_path=stack_path.name)
limits = tuple(range(original_file_info["height"],
stack_array.shape[1],
original_file_info["height"]))
recovered_components.extend(np.hsplit(stack_array, limits))
e_sum = 0
for c in recovered_components:
assert len(c.shape) == 2
e_sum += c.shape[0] * c.shape[1]
assert e_sum == original_file_info['width'] * original_file_info['height'] * original_file_info[
'component_count'], \
f"Wrong number of recovered pixels {e_sum}, " \
f"expected {original_file_info['width'] * original_file_info['height'] * original_file_info['component_count']}."
assert len(recovered_components) == original_file_info["component_count"]
assert recovered_components[0].shape == (original_file_info["width"], original_file_info["height"])
recovered_array = np.dstack(recovered_components)
assert recovered_array.shape == (
original_file_info['width'], original_file_info['height'], original_file_info['component_count'])
if original_file_info["signed"]:
recovered_array = recovered_array.astype(np.int64)
recovered_array -= 2 ** ((8 * original_file_info["bytes_per_sample"]) - 1)
isets.dump_array_bsq(array=recovered_array, file_or_path=reconstructed_path,
dtype=isets.iproperties_row_to_numpy_dtype(image_properties_row=original_file_info))
decompression_results = self.decompression_results_from_paths(
compressed_path=compressed_path, reconstructed_path=reconstructed_path)
decompression_results.decompression_time_seconds = total_decompression_time
return decompression_results
def test_spectral_angle(self):
options.exit_on_error = True
options.sequential = True
for constant_offset in [1, 5, 10]:
width, height, component_count = 2, 3, 4
bytes_per_sample, signed, big_endian = 2, True, True
row = dict(signed=signed,
bytes_per_sample=bytes_per_sample,
big_endian=big_endian,
float=False)
original_array = np.zeros(
(width, height, component_count),
dtype=isets.iproperties_row_to_numpy_dtype(row))
for x in range(width):
for y in range(height):
for z in range(component_count):
original_array[x, y, z] = 100 * z + 10 * y + x
reconstructed_array = original_array + constant_offset
expected_angles = self.get_expected_angles_deg(
original_array, reconstructed_array)
tag = isets.iproperties_to_name_tag(
width=width,
height=height,
component_count=component_count,
big_endian=big_endian,
bytes_per_sample=bytes_per_sample,
signed=signed)
with tempfile.TemporaryDirectory() as tmp_dir:
with tempfile.NamedTemporaryFile(suffix="-" + tag + ".raw",
dir=tmp_dir) as tmp_file:
isets.dump_array_bsq(original_array, tmp_file.name)
sa_exp = icompression.SpectralAngleTable(
codecs=[
trivial_codecs.OffsetLossyCodec(constant_offset)
],
dataset_paths=[tmp_file.name],
csv_experiment_path=os.path.join(
tmp_dir, "exp_persistence.csv"),
csv_dataset_path=os.path.join(
tmp_dir, "dataset_persistence.csv"))
df = sa_exp.get_df()
abs_diff_average_sa = abs(
df.iloc[0]["mean_spectral_angle_deg"] -
(sum(expected_angles) / len(expected_angles)))
abs_diff_max_sa = abs(
df.iloc[0]["max_spectral_angle_deg"] -
max(expected_angles))
assert abs_diff_average_sa < 1e-5, f"Wrong mean spectral angle (diff={abs_diff_average_sa})"
assert abs_diff_max_sa < 1e-5, f"Wrong maximum spectral angle (diff={abs_diff_max_sa})"
def decompress(self,
compressed_path,
reconstructed_path,
original_file_info=None):
dt = isets.iproperties_row_to_numpy_dtype(
image_properties_row=original_file_info)
a = np.full((original_file_info["width"], original_file_info["height"],
original_file_info["component_count"]),
fill_value=self.reconstruct_value,
dtype=dt)
isets.dump_array_bsq(array=a, file_or_path=reconstructed_path)
def test_mse_pae(self):
"""Test MSE and PAE calculation for simple images
for all supported data types.
"""
width = 37
height = 64
component_count = 7
original_constant = 10
for signed in [True, False]:
for bytes_per_sample in [1, 2, 4]:
for big_endian in [True, False]:
if bytes_per_sample == 1 and not big_endian:
continue
dtype = ((">" if big_endian else "<") if bytes_per_sample > 1 else "") \
+ ("i" if signed else "u") \
+ str(bytes_per_sample)
sample_tag = ("s" if signed else "u") \
+ str(8 * bytes_per_sample) \
+ ("be" if big_endian else "le")
fill_values = [original_constant] if not signed else [
-original_constant, original_constant
]
for fill_value in fill_values:
array = np.full((width, height, component_count),
fill_value=fill_value,
dtype=dtype)
for error in [-3, 3]:
with tempfile.TemporaryDirectory() as tmp_dir, \
tempfile.TemporaryDirectory() as persistence_dir, \
tempfile.NamedTemporaryFile(
dir=tmp_dir,
suffix=f"-{component_count}x{height}x{width}_{sample_tag}.raw") as tmp_file:
isets.dump_array_bsq(
array=array, file_or_path=tmp_file.name)
coc = ConstantOutputCodec(
reconstruct_value=fill_value + error)
options.persistence_dir = persistence_dir
ce = icompression.LossyCompressionExperiment(
codecs=[coc],
dataset_paths=[tmp_file.name],
)
df = ce.get_df()
assert (df["pae"] == abs(error)).all(), \
(df["pae"], abs(error))
assert (np.abs(df["mse"] - error ** 2) < (2 * sys.float_info.epsilon)).all(), \
(df["mse"], abs(error))
def decompress(self,
compressed_path,
reconstructed_path,
original_file_info=None):
if original_file_info[
"width"] <= self.max_dimension_size and original_file_info[
"height"] <= self.max_dimension_size:
return self.decompress_one(compressed_path=compressed_path,
reconstructed_path=reconstructed_path,
original_file_info=original_file_info)
else:
tl_reader = tarlite.TarliteReader(tarlite_path=compressed_path)
img = np.zeros(
(original_file_info["width"], original_file_info["height"],
original_file_info["component_count"]),
dtype=isets.iproperties_row_to_numpy_dtype(
image_properties_row=original_file_info))
total_decompression_time = 0
with tempfile.TemporaryDirectory(
dir=options.base_tmp_dir) as tmp_dir:
tl_reader.extract_all(output_dir_path=tmp_dir)
invocation = f"ls -lah {tmp_dir}"
status, output = subprocess.getstatusoutput(invocation)
if status != 0:
raise Exception(
"Status = {} != 0.\nInput=[{}].\nOutput=[{}]".format(
status, invocation, output))
for y in range(self.split_height_count):
for x in range(self.split_width_count):
small_compressed_path = os.path.join(
tmp_dir, f"{x}_{y}.mcalic")
assert os.path.exists(small_compressed_path)
small_path = os.path.join(tmp_dir, f"{x}_{y}.raw")
small_file_info = copy.copy(original_file_info)
small_file_info["height"] = original_file_info["height"] // self.split_height_count \
if y < self.split_height_count - 1 \
else original_file_info["height"] - (self.split_height_count - 1) * (
original_file_info["height"] // self.split_height_count)
small_file_info["width"] = original_file_info["width"] // self.split_width_count \
if x < self.split_width_count - 1 \
else original_file_info["width"] - (self.split_width_count - 1) * (
original_file_info["width"] // self.split_width_count)
dr = self.decompress_one(
compressed_path=small_compressed_path,
reconstructed_path=small_path,
original_file_info=small_file_info)
total_decompression_time += dr.decompression_time_seconds
small_array = isets.load_array_bsq(
file_or_path=small_path,
image_properties_row=small_file_info)
img[x * (original_file_info["width"] //
self.split_width_count):(
((x + 1) * (original_file_info["width"] //
self.split_width_count)
) if x < self.split_width_count -
1 else original_file_info["width"]), y *
(original_file_info["height"] //
self.split_height_count):(
((y + 1) * (original_file_info["height"] //
self.split_height_count)
) if y < self.split_height_count -
1 else original_file_info["height"]
), :] = small_array
isets.dump_array_bsq(array=img,
file_or_path=reconstructed_path)
decompression_results = self.decompression_results_from_paths(
compressed_path=compressed_path,
reconstructed_path=reconstructed_path)
decompression_results.decompression_time_seconds = total_decompression_time
return decompression_results
