def compress(self,
original_path: str,
compressed_path: str,
original_file_info=None):
array = isets.load_array_bsq(file_or_path=original_path,
image_properties_row=original_file_info)
array += self.param_dict["constant"]
isets.dump_array_bsq(array=array, file_or_path=compressed_path)
def test_array_load_dump(self):
"""Test that file properties are correctly obtained and retrieved.
"""
width = 4
height = 2
component_count = 3
for is_float in [True, False]:
bytes_per_sample_values = [2, 4, 8] if is_float else [1, 2, 4, 8]
for bytes_per_sample in bytes_per_sample_values:
for signed in [True, False]:
value_factors = [1] if not signed else [1, -1]
for value_factor in value_factors:
big_endian_values = [True] if (bytes_per_sample == 1 or is_float) \
else [True, False]
for big_endian in big_endian_values:
row = dict(width=width,
height=height,
component_count=component_count,
float=is_float,
signed=signed,
bytes_per_sample=bytes_per_sample,
big_endian=big_endian)
array = np.zeros(
(width, height, component_count),
dtype=isets.iproperties_row_to_numpy_dtype(
row))
i = 0
for z in range(component_count):
for y in range(height):
for x in range(width):
array[x, y, z] = i * value_factor
i += 1
_, temp_file_path = tempfile.mkstemp(
suffix=
f"_{isets.iproperties_row_to_sample_type_tag(row)}.raw"
)
try:
dtype_string = isets.iproperties_row_to_numpy_dtype(
row)
isets.dump_array_bsq(array,
temp_file_path,
mode="wb",
dtype=dtype_string)
loaded_array = isets.load_array_bsq(
file_or_path=temp_file_path,
image_properties_row=row)
assert (array == loaded_array).all(), (
array, loaded_array)
assert os.path.getsize(temp_file_path) \
== width * height * component_count * bytes_per_sample
finally:
try:
os.remove(temp_file_path)
except OSError:
pass
def compress(self,
original_path: str,
compressed_path: str,
original_file_info=None):
# Tested limit: self.max_tested_spatial_size
if original_file_info["width"] <= self.max_dimension_size \
and original_file_info["height"] <= self.max_dimension_size:
return self.compress_one(original_path=original_path,
compressed_path=compressed_path,
original_file_info=original_file_info)
else:
tl_writer = tarlite.TarliteWriter()
img = isets.load_array_bsq(file_or_path=original_path,
image_properties_row=original_file_info)
with tempfile.TemporaryDirectory(
dir=options.base_tmp_dir) as tmp_dir:
compound_size = 0
total_compression_time = 0
for y in range(self.split_height_count):
for x in range(self.split_width_count):
small_array = \
img[x * (original_file_info["width"] // self.split_width_count):
(((x + 1) * (original_file_info[
"width"] // self.split_width_count)) if x < self.split_width_count - 1 else
original_file_info["width"]),
y * (original_file_info["height"] // self.split_height_count):
(((y + 1) * (original_file_info[
"height"] // self.split_height_count)) if y < self.split_height_count - 1 else
original_file_info["height"]),
:]
small_path = os.path.join(tmp_dir, f"{x}_{y}.raw")
small_compressed_path = os.path.join(
tmp_dir, f"{x}_{y}.mcalic")
isets.dump_array_bsq(small_array, small_path)
small_file_info = copy.copy(original_file_info)
small_file_info["width"], small_file_info[
"height"], small_file_info[
"component_count"] = small_array.shape
compression_results = self.compress_one(
original_path=small_path,
compressed_path=small_compressed_path,
original_file_info=small_file_info)
total_compression_time += compression_results.compression_time_seconds
tl_writer.add_file(small_compressed_path)
os.remove(small_path)
compound_size += small_array.size
assert compound_size == original_file_info[
"samples"], f"compound_size = {compound_size} != {original_file_info['samples']} = original samples"
tl_writer.write(output_path=compressed_path)
compression_results = self.compression_results_from_paths(
original_path=original_path,
compressed_path=compressed_path)
compression_results.compression_time_seconds = total_compression_time
return compression_results
def compress(self, original_path: str, compressed_path: str, original_file_info=None):
assert original_file_info["bytes_per_sample"] in [1, 2], \
f"Only 1 or 2 bytes per sample, unsigned values {original_file_info['bytes_per_sample']}"
assert original_file_info["width"] <= self.max_dimension_size, \
f"The input path has width {original_file_info['width']} exceeding the maximum {self.max_dimension_size}"
assert original_file_info["height"] <= self.max_dimension_size, \
f"The input path has height {original_file_info['height']} exceeding the maximum {self.max_dimension_size}"
complete_array = isets.load_array_bsq(file_or_path=original_path, image_properties_row=original_file_info)
if original_file_info["signed"]:
complete_array = complete_array.astype(np.int64)
complete_array += 2 ** ((8 * original_file_info['bytes_per_sample']) - 1)
assert (complete_array >= 0).all(), f"Error converting signed into unsigned"
bands_per_image = min(original_file_info["component_count"],
self.max_dimension_size // original_file_info["height"])
tw = tarlite.TarliteWriter()
total_compression_time = 0
with tempfile.TemporaryDirectory(dir=options.base_tmp_dir) as tmp_dir:
stacked_size = 0
for stack_index, start_band_index in enumerate(
range(0, original_file_info["component_count"], bands_per_image)):
end_band_index_not_included = min(original_file_info["component_count"],
start_band_index + bands_per_image)
stacked_array = np.hstack(tuple(complete_array[:, :, z].squeeze()
for z in range(start_band_index, end_band_index_not_included)))
stacked_size += stacked_array.shape[0] * stacked_array.shape[1]
with tempfile.NamedTemporaryFile(dir=options.base_tmp_dir,
suffix=".pgm") as tmp_stack_file:
pgm.write_pgm(array=stacked_array,
bytes_per_sample=original_file_info["bytes_per_sample"],
output_path=tmp_stack_file.name)
stack_compressed_path = os.path.join(tmp_dir, str(stack_index))
compression_results = super().compress(
original_path=tmp_stack_file.name,
compressed_path=stack_compressed_path,
original_file_info=original_file_info)
total_compression_time += compression_results.compression_time_seconds
tw.add_file(input_path=stack_compressed_path)
assert stacked_size == complete_array.shape[0] * complete_array.shape[1] * complete_array.shape[2], \
f"Total stacked size {stacked_size} does not match the expectations. " \
f"({stacked_size} vs {complete_array.shape[0] * complete_array.shape[1] * complete_array.shape[2]}"
tw.write(output_path=compressed_path)
compression_results = self.compression_results_from_paths(
original_path=original_path, compressed_path=compressed_path)
compression_results.compression_time_seconds = total_compression_time
return compression_results
def get_spectral_angles_deg(self, index, row):
"""Return a sequence of spectral angles (in degrees),
one per (x,y) position in the image, flattened in raster order.
"""
# Read original and reconstructed images
original_file_path, task_name = index
image_properties_row = self.get_dataset_info_row(original_file_path)
decompression_results = row.decompression_results
original_array = isets.load_array_bsq(
file_or_path=original_file_path, image_properties_row=image_properties_row)
reconstructed_array = isets.load_array_bsq(
file_or_path=decompression_results.reconstructed_path, image_properties_row=image_properties_row)
# Reshape flattening the x,y axes, and maintaining the z axis for each (x,y) position
original_array = np.reshape(
original_array.swapaxes(0, 1),
(image_properties_row["width"] * image_properties_row["height"], image_properties_row["component_count"]),
"F").astype("i8")
reconstructed_array = np.reshape(
reconstructed_array.swapaxes(0, 1),
(image_properties_row["width"] * image_properties_row["height"], image_properties_row["component_count"]),
"F").astype("i8")
dots = np.einsum("ij,ij->i", original_array, reconstructed_array)
magnitude_a = np.linalg.norm(original_array, axis=1)
magnitude_b = np.linalg.norm(reconstructed_array, axis=1)
for i in range(magnitude_a.shape[0]):
# Avoid division by zero
magnitude_a[i] = max(1e-4, magnitude_a[i])
magnitude_b[i] = max(1e-4, magnitude_b[i])
# Clip, because the dot product can slip past 1 or -1 due to rounding
cosines = np.clip(dots / (magnitude_a * magnitude_b), -1, 1)
angles = np.degrees(np.arccos(cosines))
# Round because two identical images should return an angle of exactly 0
angles = np.round(angles, 5)
return angles.tolist()
def decompress(self,
compressed_path,
reconstructed_path,
original_file_info=None):
if original_file_info[
"width"] <= self.max_dimension_size and original_file_info[
"height"] <= self.max_dimension_size:
return self.decompress_one(compressed_path=compressed_path,
reconstructed_path=reconstructed_path,
original_file_info=original_file_info)
else:
tl_reader = tarlite.TarliteReader(tarlite_path=compressed_path)
img = np.zeros(
(original_file_info["width"], original_file_info["height"],
original_file_info["component_count"]),
dtype=isets.iproperties_row_to_numpy_dtype(
image_properties_row=original_file_info))
total_decompression_time = 0
with tempfile.TemporaryDirectory(
dir=options.base_tmp_dir) as tmp_dir:
tl_reader.extract_all(output_dir_path=tmp_dir)
invocation = f"ls -lah {tmp_dir}"
status, output = subprocess.getstatusoutput(invocation)
if status != 0:
raise Exception(
"Status = {} != 0.\nInput=[{}].\nOutput=[{}]".format(
status, invocation, output))
for y in range(self.split_height_count):
for x in range(self.split_width_count):
small_compressed_path = os.path.join(
tmp_dir, f"{x}_{y}.mcalic")
assert os.path.exists(small_compressed_path)
small_path = os.path.join(tmp_dir, f"{x}_{y}.raw")
small_file_info = copy.copy(original_file_info)
small_file_info["height"] = original_file_info["height"] // self.split_height_count \
if y < self.split_height_count - 1 \
else original_file_info["height"] - (self.split_height_count - 1) * (
original_file_info["height"] // self.split_height_count)
small_file_info["width"] = original_file_info["width"] // self.split_width_count \
if x < self.split_width_count - 1 \
else original_file_info["width"] - (self.split_width_count - 1) * (
original_file_info["width"] // self.split_width_count)
dr = self.decompress_one(
compressed_path=small_compressed_path,
reconstructed_path=small_path,
original_file_info=small_file_info)
total_decompression_time += dr.decompression_time_seconds
small_array = isets.load_array_bsq(
file_or_path=small_path,
image_properties_row=small_file_info)
img[x * (original_file_info["width"] //
self.split_width_count):(
((x + 1) * (original_file_info["width"] //
self.split_width_count)
) if x < self.split_width_count -
1 else original_file_info["width"]), y *
(original_file_info["height"] //
self.split_height_count):(
((y + 1) * (original_file_info["height"] //
self.split_height_count)
) if y < self.split_height_count -
1 else original_file_info["height"]
), :] = small_array
isets.dump_array_bsq(array=img,
file_or_path=reconstructed_path)
decompression_results = self.decompression_results_from_paths(
compressed_path=compressed_path,
reconstructed_path=reconstructed_path)
decompression_results.decompression_time_seconds = total_decompression_time
return decompression_results
def process_row(self, index, column_fun_tuples, row, overwrite, fill):
file_path, codec_name = index
codec = self.codecs_by_name[codec_name]
image_info_row = self.dataset_table_df.loc[indices_to_internal_loc(file_path)]
row_wrapper = self.RowWrapper(
file_path=file_path, codec=codec,
image_info_row=image_info_row,
row=row)
result = super().process_row(index=index, column_fun_tuples=column_fun_tuples,
row=row_wrapper, overwrite=overwrite, fill=fill)
if isinstance(result, Exception):
return result
if self.compressed_copy_dir_path:
output_compressed_path = os.path.join(
self.compressed_copy_dir_path,
codec.name,
os.path.basename(os.path.dirname(file_path)), os.path.basename(file_path))
os.makedirs(os.path.dirname(output_compressed_path), exist_ok=True)
if options.verbose > 1:
print(f"[C]opying {file_path} into {output_compressed_path}")
shutil.copy(row_wrapper.compression_results.compressed_path, output_compressed_path)
if self.reconstructed_dir_path is not None:
output_reconstructed_path = os.path.join(
self.reconstructed_dir_path,
codec.name,
os.path.basename(os.path.dirname(file_path)), os.path.basename(file_path))
os.makedirs(os.path.dirname(output_reconstructed_path), exist_ok=True)
if options.verbose > 1:
print(f"[C]opying {row_wrapper.compression_results.compressed_path} into {output_reconstructed_path}")
shutil.copy(row_wrapper.decompression_results.reconstructed_path,
output_reconstructed_path)
if image_info_row["component_count"] == 3:
rendered_path = f"{output_reconstructed_path}.png"
if not os.path.exists(rendered_path) or options.force:
array = isets.load_array_bsq(file_or_path=row_wrapper.decompression_results.reconstructed_path,
image_properties_row=image_info_row).astype(np.int)
if options.reconstructed_size is not None:
width, height, _ = array.shape
array = array[
width // 2 - options.reconstructed_size // 2:width // 2 + options.reconstructed_size // 2,
height // 2 - options.reconstructed_size // 2:height // 2 + options.reconstructed_size // 2,
:]
cmin = array.min()
cmax = array.max()
array = np.round((255 * (array.astype(np.int) - cmin) / (cmax - cmin))).astype("uint8")
if options.verbose > 1:
print(f"[R]endering {rendered_path}")
numpngw.imwrite(rendered_path, array.swapaxes(0, 1))
else:
full_array = isets.load_array_bsq(
file_or_path=row_wrapper.decompression_results.reconstructed_path,
image_properties_row=image_info_row).astype(np.int)
if options.reconstructed_size is not None:
width, height, _ = full_array.shape
full_array = full_array[
width // 2 - options.reconstructed_size // 2:width // 2 + options.reconstructed_size // 2,
height // 2 - options.reconstructed_size // 2:height // 2 + options.reconstructed_size // 2,
:]
for i, rendered_path in enumerate(f"{output_reconstructed_path}_component{i}.png"
for i in range(image_info_row['component_count'])):
if not os.path.exists(rendered_path) or options.force:
array = full_array[:, :, i].squeeze().swapaxes(0, 1)
cmin = array.min()
cmax = array.max()
array = np.round((255 * (array - cmin) / (cmax - cmin))).astype("uint8")
if options.verbose > 1:
print(f"[R]endering {rendered_path}")
numpngw.imwrite(rendered_path, array)
return row