def compress(self, original_path: str, compressed_path: str, original_file_info=None): array = isets.load_array_bsq(file_or_path=original_path, image_properties_row=original_file_info) array += self.param_dict["constant"] isets.dump_array_bsq(array=array, file_or_path=compressed_path)
def test_array_load_dump(self): """Test that file properties are correctly obtained and retrieved. """ width = 4 height = 2 component_count = 3 for is_float in [True, False]: bytes_per_sample_values = [2, 4, 8] if is_float else [1, 2, 4, 8] for bytes_per_sample in bytes_per_sample_values: for signed in [True, False]: value_factors = [1] if not signed else [1, -1] for value_factor in value_factors: big_endian_values = [True] if (bytes_per_sample == 1 or is_float) \ else [True, False] for big_endian in big_endian_values: row = dict(width=width, height=height, component_count=component_count, float=is_float, signed=signed, bytes_per_sample=bytes_per_sample, big_endian=big_endian) array = np.zeros( (width, height, component_count), dtype=isets.iproperties_row_to_numpy_dtype( row)) i = 0 for z in range(component_count): for y in range(height): for x in range(width): array[x, y, z] = i * value_factor i += 1 _, temp_file_path = tempfile.mkstemp( suffix= f"_{isets.iproperties_row_to_sample_type_tag(row)}.raw" ) try: dtype_string = isets.iproperties_row_to_numpy_dtype( row) isets.dump_array_bsq(array, temp_file_path, mode="wb", dtype=dtype_string) loaded_array = isets.load_array_bsq( file_or_path=temp_file_path, image_properties_row=row) assert (array == loaded_array).all(), ( array, loaded_array) assert os.path.getsize(temp_file_path) \ == width * height * component_count * bytes_per_sample finally: try: os.remove(temp_file_path) except OSError: pass
def compress(self, original_path: str, compressed_path: str, original_file_info=None): # Tested limit: self.max_tested_spatial_size if original_file_info["width"] <= self.max_dimension_size \ and original_file_info["height"] <= self.max_dimension_size: return self.compress_one(original_path=original_path, compressed_path=compressed_path, original_file_info=original_file_info) else: tl_writer = tarlite.TarliteWriter() img = isets.load_array_bsq(file_or_path=original_path, image_properties_row=original_file_info) with tempfile.TemporaryDirectory( dir=options.base_tmp_dir) as tmp_dir: compound_size = 0 total_compression_time = 0 for y in range(self.split_height_count): for x in range(self.split_width_count): small_array = \ img[x * (original_file_info["width"] // self.split_width_count): (((x + 1) * (original_file_info[ "width"] // self.split_width_count)) if x < self.split_width_count - 1 else original_file_info["width"]), y * (original_file_info["height"] // self.split_height_count): (((y + 1) * (original_file_info[ "height"] // self.split_height_count)) if y < self.split_height_count - 1 else original_file_info["height"]), :] small_path = os.path.join(tmp_dir, f"{x}_{y}.raw") small_compressed_path = os.path.join( tmp_dir, f"{x}_{y}.mcalic") isets.dump_array_bsq(small_array, small_path) small_file_info = copy.copy(original_file_info) small_file_info["width"], small_file_info[ "height"], small_file_info[ "component_count"] = small_array.shape compression_results = self.compress_one( original_path=small_path, compressed_path=small_compressed_path, original_file_info=small_file_info) total_compression_time += compression_results.compression_time_seconds tl_writer.add_file(small_compressed_path) os.remove(small_path) compound_size += small_array.size assert compound_size == original_file_info[ "samples"], f"compound_size = {compound_size} != {original_file_info['samples']} = original samples" tl_writer.write(output_path=compressed_path) compression_results = self.compression_results_from_paths( original_path=original_path, compressed_path=compressed_path) compression_results.compression_time_seconds = total_compression_time return compression_results
def decompress(self, compressed_path, reconstructed_path, original_file_info=None): tr = tarlite.TarliteReader(tarlite_path=compressed_path) bands_per_image = min(original_file_info["component_count"], self.max_dimension_size // original_file_info["height"]) total_decompression_time = 0 recovered_components = [] with tempfile.TemporaryDirectory(dir=options.base_tmp_dir) as tmp_dir: tr.extract_all(output_dir_path=tmp_dir) for stack_index, start_band_index in enumerate( range(0, original_file_info["component_count"], bands_per_image)): compressed_stack_path = os.path.join(tmp_dir, str(stack_index)) with tempfile.NamedTemporaryFile(dir=options.base_tmp_dir, suffix=".pgm") as stack_path: decompression_results = super().decompress( compressed_path=compressed_stack_path, reconstructed_path=stack_path.name) total_decompression_time += decompression_results.decompression_time_seconds assert os.path.isfile(stack_path.name) stack_array = pgm.read_pgm(input_path=stack_path.name) limits = tuple(range(original_file_info["height"], stack_array.shape[1], original_file_info["height"])) recovered_components.extend(np.hsplit(stack_array, limits)) e_sum = 0 for c in recovered_components: assert len(c.shape) == 2 e_sum += c.shape[0] * c.shape[1] assert e_sum == original_file_info['width'] * original_file_info['height'] * original_file_info[ 'component_count'], \ f"Wrong number of recovered pixels {e_sum}, " \ f"expected {original_file_info['width'] * original_file_info['height'] * original_file_info['component_count']}." assert len(recovered_components) == original_file_info["component_count"] assert recovered_components[0].shape == (original_file_info["width"], original_file_info["height"]) recovered_array = np.dstack(recovered_components) assert recovered_array.shape == ( original_file_info['width'], original_file_info['height'], original_file_info['component_count']) if original_file_info["signed"]: recovered_array = recovered_array.astype(np.int64) recovered_array -= 2 ** ((8 * original_file_info["bytes_per_sample"]) - 1) isets.dump_array_bsq(array=recovered_array, file_or_path=reconstructed_path, dtype=isets.iproperties_row_to_numpy_dtype(image_properties_row=original_file_info)) decompression_results = self.decompression_results_from_paths( compressed_path=compressed_path, reconstructed_path=reconstructed_path) decompression_results.decompression_time_seconds = total_decompression_time return decompression_results
def test_spectral_angle(self): options.exit_on_error = True options.sequential = True for constant_offset in [1, 5, 10]: width, height, component_count = 2, 3, 4 bytes_per_sample, signed, big_endian = 2, True, True row = dict(signed=signed, bytes_per_sample=bytes_per_sample, big_endian=big_endian, float=False) original_array = np.zeros( (width, height, component_count), dtype=isets.iproperties_row_to_numpy_dtype(row)) for x in range(width): for y in range(height): for z in range(component_count): original_array[x, y, z] = 100 * z + 10 * y + x reconstructed_array = original_array + constant_offset expected_angles = self.get_expected_angles_deg( original_array, reconstructed_array) tag = isets.iproperties_to_name_tag( width=width, height=height, component_count=component_count, big_endian=big_endian, bytes_per_sample=bytes_per_sample, signed=signed) with tempfile.TemporaryDirectory() as tmp_dir: with tempfile.NamedTemporaryFile(suffix="-" + tag + ".raw", dir=tmp_dir) as tmp_file: isets.dump_array_bsq(original_array, tmp_file.name) sa_exp = icompression.SpectralAngleTable( codecs=[ trivial_codecs.OffsetLossyCodec(constant_offset) ], dataset_paths=[tmp_file.name], csv_experiment_path=os.path.join( tmp_dir, "exp_persistence.csv"), csv_dataset_path=os.path.join( tmp_dir, "dataset_persistence.csv")) df = sa_exp.get_df() abs_diff_average_sa = abs( df.iloc[0]["mean_spectral_angle_deg"] - (sum(expected_angles) / len(expected_angles))) abs_diff_max_sa = abs( df.iloc[0]["max_spectral_angle_deg"] - max(expected_angles)) assert abs_diff_average_sa < 1e-5, f"Wrong mean spectral angle (diff={abs_diff_average_sa})" assert abs_diff_max_sa < 1e-5, f"Wrong maximum spectral angle (diff={abs_diff_max_sa})"
def decompress(self, compressed_path, reconstructed_path, original_file_info=None): dt = isets.iproperties_row_to_numpy_dtype( image_properties_row=original_file_info) a = np.full((original_file_info["width"], original_file_info["height"], original_file_info["component_count"]), fill_value=self.reconstruct_value, dtype=dt) isets.dump_array_bsq(array=a, file_or_path=reconstructed_path)
def test_mse_pae(self): """Test MSE and PAE calculation for simple images for all supported data types. """ width = 37 height = 64 component_count = 7 original_constant = 10 for signed in [True, False]: for bytes_per_sample in [1, 2, 4]: for big_endian in [True, False]: if bytes_per_sample == 1 and not big_endian: continue dtype = ((">" if big_endian else "<") if bytes_per_sample > 1 else "") \ + ("i" if signed else "u") \ + str(bytes_per_sample) sample_tag = ("s" if signed else "u") \ + str(8 * bytes_per_sample) \ + ("be" if big_endian else "le") fill_values = [original_constant] if not signed else [ -original_constant, original_constant ] for fill_value in fill_values: array = np.full((width, height, component_count), fill_value=fill_value, dtype=dtype) for error in [-3, 3]: with tempfile.TemporaryDirectory() as tmp_dir, \ tempfile.TemporaryDirectory() as persistence_dir, \ tempfile.NamedTemporaryFile( dir=tmp_dir, suffix=f"-{component_count}x{height}x{width}_{sample_tag}.raw") as tmp_file: isets.dump_array_bsq( array=array, file_or_path=tmp_file.name) coc = ConstantOutputCodec( reconstruct_value=fill_value + error) options.persistence_dir = persistence_dir ce = icompression.LossyCompressionExperiment( codecs=[coc], dataset_paths=[tmp_file.name], ) df = ce.get_df() assert (df["pae"] == abs(error)).all(), \ (df["pae"], abs(error)) assert (np.abs(df["mse"] - error ** 2) < (2 * sys.float_info.epsilon)).all(), \ (df["mse"], abs(error))
def decompress(self, compressed_path, reconstructed_path, original_file_info=None): if original_file_info[ "width"] <= self.max_dimension_size and original_file_info[ "height"] <= self.max_dimension_size: return self.decompress_one(compressed_path=compressed_path, reconstructed_path=reconstructed_path, original_file_info=original_file_info) else: tl_reader = tarlite.TarliteReader(tarlite_path=compressed_path) img = np.zeros( (original_file_info["width"], original_file_info["height"], original_file_info["component_count"]), dtype=isets.iproperties_row_to_numpy_dtype( image_properties_row=original_file_info)) total_decompression_time = 0 with tempfile.TemporaryDirectory( dir=options.base_tmp_dir) as tmp_dir: tl_reader.extract_all(output_dir_path=tmp_dir) invocation = f"ls -lah {tmp_dir}" status, output = subprocess.getstatusoutput(invocation) if status != 0: raise Exception( "Status = {} != 0.\nInput=[{}].\nOutput=[{}]".format( status, invocation, output)) for y in range(self.split_height_count): for x in range(self.split_width_count): small_compressed_path = os.path.join( tmp_dir, f"{x}_{y}.mcalic") assert os.path.exists(small_compressed_path) small_path = os.path.join(tmp_dir, f"{x}_{y}.raw") small_file_info = copy.copy(original_file_info) small_file_info["height"] = original_file_info["height"] // self.split_height_count \ if y < self.split_height_count - 1 \ else original_file_info["height"] - (self.split_height_count - 1) * ( original_file_info["height"] // self.split_height_count) small_file_info["width"] = original_file_info["width"] // self.split_width_count \ if x < self.split_width_count - 1 \ else original_file_info["width"] - (self.split_width_count - 1) * ( original_file_info["width"] // self.split_width_count) dr = self.decompress_one( compressed_path=small_compressed_path, reconstructed_path=small_path, original_file_info=small_file_info) total_decompression_time += dr.decompression_time_seconds small_array = isets.load_array_bsq( file_or_path=small_path, image_properties_row=small_file_info) img[x * (original_file_info["width"] // self.split_width_count):( ((x + 1) * (original_file_info["width"] // self.split_width_count) ) if x < self.split_width_count - 1 else original_file_info["width"]), y * (original_file_info["height"] // self.split_height_count):( ((y + 1) * (original_file_info["height"] // self.split_height_count) ) if y < self.split_height_count - 1 else original_file_info["height"] ), :] = small_array isets.dump_array_bsq(array=img, file_or_path=reconstructed_path) decompression_results = self.decompression_results_from_paths( compressed_path=compressed_path, reconstructed_path=reconstructed_path) decompression_results.decompression_time_seconds = total_decompression_time return decompression_results