def test_array_load_dump(self):
"""Test that file properties are correctly obtained and retrieved.
"""
width = 4
height = 2
component_count = 3
for is_float in [True, False]:
bytes_per_sample_values = [2, 4, 8] if is_float else [1, 2, 4, 8]
for bytes_per_sample in bytes_per_sample_values:
for signed in [True, False]:
value_factors = [1] if not signed else [1, -1]
for value_factor in value_factors:
big_endian_values = [True] if (bytes_per_sample == 1 or is_float) \
else [True, False]
for big_endian in big_endian_values:
row = dict(width=width,
height=height,
component_count=component_count,
float=is_float,
signed=signed,
bytes_per_sample=bytes_per_sample,
big_endian=big_endian)
array = np.zeros(
(width, height, component_count),
dtype=isets.iproperties_row_to_numpy_dtype(
row))
i = 0
for z in range(component_count):
for y in range(height):
for x in range(width):
array[x, y, z] = i * value_factor
i += 1
_, temp_file_path = tempfile.mkstemp(
suffix=
f"_{isets.iproperties_row_to_sample_type_tag(row)}.raw"
)
try:
dtype_string = isets.iproperties_row_to_numpy_dtype(
row)
isets.dump_array_bsq(array,
temp_file_path,
mode="wb",
dtype=dtype_string)
loaded_array = isets.load_array_bsq(
file_or_path=temp_file_path,
image_properties_row=row)
assert (array == loaded_array).all(), (
array, loaded_array)
assert os.path.getsize(temp_file_path) \
== width * height * component_count * bytes_per_sample
finally:
try:
os.remove(temp_file_path)
except OSError:
pass
def decompress(self, compressed_path, reconstructed_path, original_file_info=None):
tr = tarlite.TarliteReader(tarlite_path=compressed_path)
bands_per_image = min(original_file_info["component_count"],
self.max_dimension_size // original_file_info["height"])
total_decompression_time = 0
recovered_components = []
with tempfile.TemporaryDirectory(dir=options.base_tmp_dir) as tmp_dir:
tr.extract_all(output_dir_path=tmp_dir)
for stack_index, start_band_index in enumerate(
range(0, original_file_info["component_count"], bands_per_image)):
compressed_stack_path = os.path.join(tmp_dir, str(stack_index))
with tempfile.NamedTemporaryFile(dir=options.base_tmp_dir, suffix=".pgm") as stack_path:
decompression_results = super().decompress(
compressed_path=compressed_stack_path, reconstructed_path=stack_path.name)
total_decompression_time += decompression_results.decompression_time_seconds
assert os.path.isfile(stack_path.name)
stack_array = pgm.read_pgm(input_path=stack_path.name)
limits = tuple(range(original_file_info["height"],
stack_array.shape[1],
original_file_info["height"]))
recovered_components.extend(np.hsplit(stack_array, limits))
e_sum = 0
for c in recovered_components:
assert len(c.shape) == 2
e_sum += c.shape[0] * c.shape[1]
assert e_sum == original_file_info['width'] * original_file_info['height'] * original_file_info[
'component_count'], \
f"Wrong number of recovered pixels {e_sum}, " \
f"expected {original_file_info['width'] * original_file_info['height'] * original_file_info['component_count']}."
assert len(recovered_components) == original_file_info["component_count"]
assert recovered_components[0].shape == (original_file_info["width"], original_file_info["height"])
recovered_array = np.dstack(recovered_components)
assert recovered_array.shape == (
original_file_info['width'], original_file_info['height'], original_file_info['component_count'])
if original_file_info["signed"]:
recovered_array = recovered_array.astype(np.int64)
recovered_array -= 2 ** ((8 * original_file_info["bytes_per_sample"]) - 1)
isets.dump_array_bsq(array=recovered_array, file_or_path=reconstructed_path,
dtype=isets.iproperties_row_to_numpy_dtype(image_properties_row=original_file_info))
decompression_results = self.decompression_results_from_paths(
compressed_path=compressed_path, reconstructed_path=reconstructed_path)
decompression_results.decompression_time_seconds = total_decompression_time
return decompression_results
def test_spectral_angle(self):
options.exit_on_error = True
options.sequential = True
for constant_offset in [1, 5, 10]:
width, height, component_count = 2, 3, 4
bytes_per_sample, signed, big_endian = 2, True, True
row = dict(signed=signed,
bytes_per_sample=bytes_per_sample,
big_endian=big_endian,
float=False)
original_array = np.zeros(
(width, height, component_count),
dtype=isets.iproperties_row_to_numpy_dtype(row))
for x in range(width):
for y in range(height):
for z in range(component_count):
original_array[x, y, z] = 100 * z + 10 * y + x
reconstructed_array = original_array + constant_offset
expected_angles = self.get_expected_angles_deg(
original_array, reconstructed_array)
tag = isets.iproperties_to_name_tag(
width=width,
height=height,
component_count=component_count,
big_endian=big_endian,
bytes_per_sample=bytes_per_sample,
signed=signed)
with tempfile.TemporaryDirectory() as tmp_dir:
with tempfile.NamedTemporaryFile(suffix="-" + tag + ".raw",
dir=tmp_dir) as tmp_file:
isets.dump_array_bsq(original_array, tmp_file.name)
sa_exp = icompression.SpectralAngleTable(
codecs=[
trivial_codecs.OffsetLossyCodec(constant_offset)
],
dataset_paths=[tmp_file.name],
csv_experiment_path=os.path.join(
tmp_dir, "exp_persistence.csv"),
csv_dataset_path=os.path.join(
tmp_dir, "dataset_persistence.csv"))
df = sa_exp.get_df()
abs_diff_average_sa = abs(
df.iloc[0]["mean_spectral_angle_deg"] -
(sum(expected_angles) / len(expected_angles)))
abs_diff_max_sa = abs(
df.iloc[0]["max_spectral_angle_deg"] -
max(expected_angles))
assert abs_diff_average_sa < 1e-5, f"Wrong mean spectral angle (diff={abs_diff_average_sa})"
assert abs_diff_max_sa < 1e-5, f"Wrong maximum spectral angle (diff={abs_diff_max_sa})"
def decompress(self,
compressed_path,
reconstructed_path,
original_file_info=None):
dt = isets.iproperties_row_to_numpy_dtype(
image_properties_row=original_file_info)
a = np.full((original_file_info["width"], original_file_info["height"],
original_file_info["component_count"]),
fill_value=self.reconstruct_value,
dtype=dt)
isets.dump_array_bsq(array=a, file_or_path=reconstructed_path)
def compress(self,
original_path,
compressed_path,
original_file_info=None):
if self.param_dict['psnr'] is not None:
with tempfile.NamedTemporaryFile() as tmp_file:
br_a = 0
br_b = 32
iteration = 0
psnr_error = float("inf")
while psnr_error > self.psnr_tolerance and iteration <= self.max_search_iterations:
iteration += 1
self.param_dict['bit_rate'] = (br_b + br_a) / 2
compression_results = icompression.WrapperCodec.compress(
self,
original_path,
compressed_path,
original_file_info=original_file_info)
self.decompress(compressed_path,
reconstructed_path=tmp_file.name,
original_file_info=original_file_info)
max_error = (2**(
8 * original_file_info["bytes_per_sample"])) - 1
dtype = isets.iproperties_row_to_numpy_dtype(
original_file_info)
original_array = np.fromfile(original_path,
dtype=dtype).astype(np.int64)
reconstructed_array = np.fromfile(tmp_file.name,
dtype=dtype).astype(
np.int64)
actual_bps = 8 * os.path.getsize(
compressed_path) / original_file_info["samples"]
mse = np.average(
((original_array - reconstructed_array)**2))
psnr = 10 * math.log10(
(max_error**2) / mse) if mse > 0 else float("inf")
if self.param_dict['psnr'] > psnr:
br_a = min(self.param_dict['bit_rate'], actual_bps)
else:
br_b = max(self.param_dict['bit_rate'], actual_bps)
psnr_error = abs(self.param_dict['psnr'] - psnr)
else:
compression_results = icompression.WrapperCodec.compress(
self,
original_path,
compressed_path,
original_file_info=original_file_info)
return compression_results
def decompress_one(self,
compressed_path,
reconstructed_path,
original_file_info=None):
total_decompression_time = 0
with tempfile.NamedTemporaryFile(
dir=options.base_tmp_dir,
prefix=f"reconstructed_{os.path.basename(reconstructed_path)}",
suffix=".raw") as bil_le_file:
offset = 0
if original_file_info["signed"]:
reader = tarlite.TarliteReader(compressed_path)
with tempfile.TemporaryDirectory(
dir=options.base_tmp_dir) as tmp_extract_dir:
reader.extract_all(tmp_extract_dir)
import subprocess
invocation = f"ls -lah {tmp_extract_dir}"
status, output = subprocess.getstatusoutput(invocation)
if status != 0:
raise Exception(
"Status = {} != 0.\nInput=[{}].\nOutput=[{}]".
format(status, invocation, output))
with open(
glob.glob(os.path.join(tmp_extract_dir,
"*.txt"))[0]) as si_file:
offset = int(si_file.read())
assert offset >= 0
os.path.getsize(tmp_extract_dir)
inner_compressed_path = glob.glob(
os.path.join(tmp_extract_dir, "*.mcalic"))[0]
dr = self.decompress_short_names(
compressed_path=inner_compressed_path,
reconstructed_path=bil_le_file.name,
original_file_info=original_file_info)
total_decompression_time += dr.decompression_time_seconds
else:
dr = self.decompress_short_names(
compressed_path=compressed_path,
reconstructed_path=bil_le_file.name,
original_file_info=original_file_info)
total_decompression_time += dr.decompression_time_seconds
original_dtype = isets.iproperties_row_to_numpy_dtype(
image_properties_row=original_file_info)
img = np.fromfile(bil_le_file.name,
dtype=original_dtype.replace(">", "<").replace(
"i", "u")).reshape(
original_file_info["height"],
original_file_info["component_count"],
original_file_info["width"])
if original_file_info["signed"]:
if offset != 0:
img = (img.astype("i4") - offset).astype(original_dtype)
else:
img = img.astype(original_dtype)
img = img.swapaxes(0, 1)
if original_file_info[
"big_endian"] and not original_file_info["signed"]:
# Signed file are already converted back to big_endian if necessary in the previous call to astype()
img = img.astype(original_dtype)
img.tofile(reconstructed_path)
try:
# The decoder always produces this file
os.remove("seqRec")
except FileNotFoundError:
pass
decompression_results = self.decompression_results_from_paths(
compressed_path=compressed_path,
reconstructed_path=reconstructed_path)
decompression_results.decompression_time_seconds = total_decompression_time
return decompression_results
def decompress(self,
compressed_path,
reconstructed_path,
original_file_info=None):
if original_file_info[
"width"] <= self.max_dimension_size and original_file_info[
"height"] <= self.max_dimension_size:
return self.decompress_one(compressed_path=compressed_path,
reconstructed_path=reconstructed_path,
original_file_info=original_file_info)
else:
tl_reader = tarlite.TarliteReader(tarlite_path=compressed_path)
img = np.zeros(
(original_file_info["width"], original_file_info["height"],
original_file_info["component_count"]),
dtype=isets.iproperties_row_to_numpy_dtype(
image_properties_row=original_file_info))
total_decompression_time = 0
with tempfile.TemporaryDirectory(
dir=options.base_tmp_dir) as tmp_dir:
tl_reader.extract_all(output_dir_path=tmp_dir)
invocation = f"ls -lah {tmp_dir}"
status, output = subprocess.getstatusoutput(invocation)
if status != 0:
raise Exception(
"Status = {} != 0.\nInput=[{}].\nOutput=[{}]".format(
status, invocation, output))
for y in range(self.split_height_count):
for x in range(self.split_width_count):
small_compressed_path = os.path.join(
tmp_dir, f"{x}_{y}.mcalic")
assert os.path.exists(small_compressed_path)
small_path = os.path.join(tmp_dir, f"{x}_{y}.raw")
small_file_info = copy.copy(original_file_info)
small_file_info["height"] = original_file_info["height"] // self.split_height_count \
if y < self.split_height_count - 1 \
else original_file_info["height"] - (self.split_height_count - 1) * (
original_file_info["height"] // self.split_height_count)
small_file_info["width"] = original_file_info["width"] // self.split_width_count \
if x < self.split_width_count - 1 \
else original_file_info["width"] - (self.split_width_count - 1) * (
original_file_info["width"] // self.split_width_count)
dr = self.decompress_one(
compressed_path=small_compressed_path,
reconstructed_path=small_path,
original_file_info=small_file_info)
total_decompression_time += dr.decompression_time_seconds
small_array = isets.load_array_bsq(
file_or_path=small_path,
image_properties_row=small_file_info)
img[x * (original_file_info["width"] //
self.split_width_count):(
((x + 1) * (original_file_info["width"] //
self.split_width_count)
) if x < self.split_width_count -
1 else original_file_info["width"]), y *
(original_file_info["height"] //
self.split_height_count):(
((y + 1) * (original_file_info["height"] //
self.split_height_count)
) if y < self.split_height_count -
1 else original_file_info["height"]
), :] = small_array
isets.dump_array_bsq(array=img,
file_or_path=reconstructed_path)
decompression_results = self.decompression_results_from_paths(
compressed_path=compressed_path,
reconstructed_path=reconstructed_path)
decompression_results.decompression_time_seconds = total_decompression_time
return decompression_results
def compress_one(self,
original_path: str,
compressed_path: str,
original_file_info=None):
"""Compress one image tile with M-CALIC.
"""
assert original_file_info["bytes_per_sample"] == 2, \
f"This implementation of M-CALIC ({self.compressor_path}) only supports 16bpp"
assert original_file_info["component_count"] > 1, \
f"This implementation of M-CALIC ({self.compressor_path}) only supports images with more than one component"
with tempfile.NamedTemporaryFile(
dir=options.base_tmp_dir,
prefix=f"bil_le_{os.path.basename(original_path)}"
) as bil_le_file:
# M-Calic implementation requires little endian, unsigned 16bpp BIL format
original_dtype = isets.iproperties_row_to_numpy_dtype(
image_properties_row=original_file_info)
img = np.fromfile(original_path, dtype=original_dtype).reshape(
original_file_info["component_count"],
original_file_info["height"], original_file_info["width"])
offset = None
if original_file_info["signed"]:
offset, original_max = int(img.min()), int(img.max())
offset = min(offset, 0)
assert original_max - offset <= 2 ** 15 - 1, \
f"Invalid dynamic range of signed image ({offset}, {original_max})"
img = (img.astype("i4") - offset).astype(
original_dtype.replace("i", "u"))
if original_file_info["big_endian"]:
img = img.astype(
original_dtype.replace(">", "<").replace("i", "u"))
img.swapaxes(0, 1).tofile(bil_le_file.name)
if original_file_info["signed"]:
with tempfile.NamedTemporaryFile(dir=options.base_tmp_dir,
prefix=f"bil_le_{os.path.basename(original_path)}",
suffix=".mcalic") as tmp_compressed_file, \
tempfile.NamedTemporaryFile(dir=options.base_tmp_dir,
prefix=f"side_info_{os.path.basename(original_path)}",
suffix=".txt", mode="w") as si_file:
si_file.write(f"{abs(offset):d}")
si_file.flush()
compression_results = super().compress(
original_path=bil_le_file.name,
compressed_path=tmp_compressed_file.name,
original_file_info=original_file_info)
tarlite.TarliteWriter(initial_input_paths=[
si_file.name, tmp_compressed_file.name
]).write(compressed_path)
compression_results.original_path = original_path
compression_results.compressed_path = compressed_path
return compression_results
else:
compression_results = super().compress(
original_path=bil_le_file.name,
compressed_path=compressed_path,
original_file_info=original_file_info)
compression_results.original_path = original_path
return compression_results
def numpy_dtype(self):
"""Get the numpy dtype corresponding to the original image's data format
"""
return isets.iproperties_row_to_numpy_dtype(self.image_info_row)
