def test_array_load_dump(self): """Test that file properties are correctly obtained and retrieved. """ width = 4 height = 2 component_count = 3 for is_float in [True, False]: bytes_per_sample_values = [2, 4, 8] if is_float else [1, 2, 4, 8] for bytes_per_sample in bytes_per_sample_values: for signed in [True, False]: value_factors = [1] if not signed else [1, -1] for value_factor in value_factors: big_endian_values = [True] if (bytes_per_sample == 1 or is_float) \ else [True, False] for big_endian in big_endian_values: row = dict(width=width, height=height, component_count=component_count, float=is_float, signed=signed, bytes_per_sample=bytes_per_sample, big_endian=big_endian) array = np.zeros( (width, height, component_count), dtype=isets.iproperties_row_to_numpy_dtype( row)) i = 0 for z in range(component_count): for y in range(height): for x in range(width): array[x, y, z] = i * value_factor i += 1 _, temp_file_path = tempfile.mkstemp( suffix= f"_{isets.iproperties_row_to_sample_type_tag(row)}.raw" ) try: dtype_string = isets.iproperties_row_to_numpy_dtype( row) isets.dump_array_bsq(array, temp_file_path, mode="wb", dtype=dtype_string) loaded_array = isets.load_array_bsq( file_or_path=temp_file_path, image_properties_row=row) assert (array == loaded_array).all(), ( array, loaded_array) assert os.path.getsize(temp_file_path) \ == width * height * component_count * bytes_per_sample finally: try: os.remove(temp_file_path) except OSError: pass
def decompress(self, compressed_path, reconstructed_path, original_file_info=None): tr = tarlite.TarliteReader(tarlite_path=compressed_path) bands_per_image = min(original_file_info["component_count"], self.max_dimension_size // original_file_info["height"]) total_decompression_time = 0 recovered_components = [] with tempfile.TemporaryDirectory(dir=options.base_tmp_dir) as tmp_dir: tr.extract_all(output_dir_path=tmp_dir) for stack_index, start_band_index in enumerate( range(0, original_file_info["component_count"], bands_per_image)): compressed_stack_path = os.path.join(tmp_dir, str(stack_index)) with tempfile.NamedTemporaryFile(dir=options.base_tmp_dir, suffix=".pgm") as stack_path: decompression_results = super().decompress( compressed_path=compressed_stack_path, reconstructed_path=stack_path.name) total_decompression_time += decompression_results.decompression_time_seconds assert os.path.isfile(stack_path.name) stack_array = pgm.read_pgm(input_path=stack_path.name) limits = tuple(range(original_file_info["height"], stack_array.shape[1], original_file_info["height"])) recovered_components.extend(np.hsplit(stack_array, limits)) e_sum = 0 for c in recovered_components: assert len(c.shape) == 2 e_sum += c.shape[0] * c.shape[1] assert e_sum == original_file_info['width'] * original_file_info['height'] * original_file_info[ 'component_count'], \ f"Wrong number of recovered pixels {e_sum}, " \ f"expected {original_file_info['width'] * original_file_info['height'] * original_file_info['component_count']}." assert len(recovered_components) == original_file_info["component_count"] assert recovered_components[0].shape == (original_file_info["width"], original_file_info["height"]) recovered_array = np.dstack(recovered_components) assert recovered_array.shape == ( original_file_info['width'], original_file_info['height'], original_file_info['component_count']) if original_file_info["signed"]: recovered_array = recovered_array.astype(np.int64) recovered_array -= 2 ** ((8 * original_file_info["bytes_per_sample"]) - 1) isets.dump_array_bsq(array=recovered_array, file_or_path=reconstructed_path, dtype=isets.iproperties_row_to_numpy_dtype(image_properties_row=original_file_info)) decompression_results = self.decompression_results_from_paths( compressed_path=compressed_path, reconstructed_path=reconstructed_path) decompression_results.decompression_time_seconds = total_decompression_time return decompression_results
def test_spectral_angle(self): options.exit_on_error = True options.sequential = True for constant_offset in [1, 5, 10]: width, height, component_count = 2, 3, 4 bytes_per_sample, signed, big_endian = 2, True, True row = dict(signed=signed, bytes_per_sample=bytes_per_sample, big_endian=big_endian, float=False) original_array = np.zeros( (width, height, component_count), dtype=isets.iproperties_row_to_numpy_dtype(row)) for x in range(width): for y in range(height): for z in range(component_count): original_array[x, y, z] = 100 * z + 10 * y + x reconstructed_array = original_array + constant_offset expected_angles = self.get_expected_angles_deg( original_array, reconstructed_array) tag = isets.iproperties_to_name_tag( width=width, height=height, component_count=component_count, big_endian=big_endian, bytes_per_sample=bytes_per_sample, signed=signed) with tempfile.TemporaryDirectory() as tmp_dir: with tempfile.NamedTemporaryFile(suffix="-" + tag + ".raw", dir=tmp_dir) as tmp_file: isets.dump_array_bsq(original_array, tmp_file.name) sa_exp = icompression.SpectralAngleTable( codecs=[ trivial_codecs.OffsetLossyCodec(constant_offset) ], dataset_paths=[tmp_file.name], csv_experiment_path=os.path.join( tmp_dir, "exp_persistence.csv"), csv_dataset_path=os.path.join( tmp_dir, "dataset_persistence.csv")) df = sa_exp.get_df() abs_diff_average_sa = abs( df.iloc[0]["mean_spectral_angle_deg"] - (sum(expected_angles) / len(expected_angles))) abs_diff_max_sa = abs( df.iloc[0]["max_spectral_angle_deg"] - max(expected_angles)) assert abs_diff_average_sa < 1e-5, f"Wrong mean spectral angle (diff={abs_diff_average_sa})" assert abs_diff_max_sa < 1e-5, f"Wrong maximum spectral angle (diff={abs_diff_max_sa})"
def decompress(self, compressed_path, reconstructed_path, original_file_info=None): dt = isets.iproperties_row_to_numpy_dtype( image_properties_row=original_file_info) a = np.full((original_file_info["width"], original_file_info["height"], original_file_info["component_count"]), fill_value=self.reconstruct_value, dtype=dt) isets.dump_array_bsq(array=a, file_or_path=reconstructed_path)
def compress(self, original_path, compressed_path, original_file_info=None): if self.param_dict['psnr'] is not None: with tempfile.NamedTemporaryFile() as tmp_file: br_a = 0 br_b = 32 iteration = 0 psnr_error = float("inf") while psnr_error > self.psnr_tolerance and iteration <= self.max_search_iterations: iteration += 1 self.param_dict['bit_rate'] = (br_b + br_a) / 2 compression_results = icompression.WrapperCodec.compress( self, original_path, compressed_path, original_file_info=original_file_info) self.decompress(compressed_path, reconstructed_path=tmp_file.name, original_file_info=original_file_info) max_error = (2**( 8 * original_file_info["bytes_per_sample"])) - 1 dtype = isets.iproperties_row_to_numpy_dtype( original_file_info) original_array = np.fromfile(original_path, dtype=dtype).astype(np.int64) reconstructed_array = np.fromfile(tmp_file.name, dtype=dtype).astype( np.int64) actual_bps = 8 * os.path.getsize( compressed_path) / original_file_info["samples"] mse = np.average( ((original_array - reconstructed_array)**2)) psnr = 10 * math.log10( (max_error**2) / mse) if mse > 0 else float("inf") if self.param_dict['psnr'] > psnr: br_a = min(self.param_dict['bit_rate'], actual_bps) else: br_b = max(self.param_dict['bit_rate'], actual_bps) psnr_error = abs(self.param_dict['psnr'] - psnr) else: compression_results = icompression.WrapperCodec.compress( self, original_path, compressed_path, original_file_info=original_file_info) return compression_results
def decompress_one(self, compressed_path, reconstructed_path, original_file_info=None): total_decompression_time = 0 with tempfile.NamedTemporaryFile( dir=options.base_tmp_dir, prefix=f"reconstructed_{os.path.basename(reconstructed_path)}", suffix=".raw") as bil_le_file: offset = 0 if original_file_info["signed"]: reader = tarlite.TarliteReader(compressed_path) with tempfile.TemporaryDirectory( dir=options.base_tmp_dir) as tmp_extract_dir: reader.extract_all(tmp_extract_dir) import subprocess invocation = f"ls -lah {tmp_extract_dir}" status, output = subprocess.getstatusoutput(invocation) if status != 0: raise Exception( "Status = {} != 0.\nInput=[{}].\nOutput=[{}]". format(status, invocation, output)) with open( glob.glob(os.path.join(tmp_extract_dir, "*.txt"))[0]) as si_file: offset = int(si_file.read()) assert offset >= 0 os.path.getsize(tmp_extract_dir) inner_compressed_path = glob.glob( os.path.join(tmp_extract_dir, "*.mcalic"))[0] dr = self.decompress_short_names( compressed_path=inner_compressed_path, reconstructed_path=bil_le_file.name, original_file_info=original_file_info) total_decompression_time += dr.decompression_time_seconds else: dr = self.decompress_short_names( compressed_path=compressed_path, reconstructed_path=bil_le_file.name, original_file_info=original_file_info) total_decompression_time += dr.decompression_time_seconds original_dtype = isets.iproperties_row_to_numpy_dtype( image_properties_row=original_file_info) img = np.fromfile(bil_le_file.name, dtype=original_dtype.replace(">", "<").replace( "i", "u")).reshape( original_file_info["height"], original_file_info["component_count"], original_file_info["width"]) if original_file_info["signed"]: if offset != 0: img = (img.astype("i4") - offset).astype(original_dtype) else: img = img.astype(original_dtype) img = img.swapaxes(0, 1) if original_file_info[ "big_endian"] and not original_file_info["signed"]: # Signed file are already converted back to big_endian if necessary in the previous call to astype() img = img.astype(original_dtype) img.tofile(reconstructed_path) try: # The decoder always produces this file os.remove("seqRec") except FileNotFoundError: pass decompression_results = self.decompression_results_from_paths( compressed_path=compressed_path, reconstructed_path=reconstructed_path) decompression_results.decompression_time_seconds = total_decompression_time return decompression_results
def decompress(self, compressed_path, reconstructed_path, original_file_info=None): if original_file_info[ "width"] <= self.max_dimension_size and original_file_info[ "height"] <= self.max_dimension_size: return self.decompress_one(compressed_path=compressed_path, reconstructed_path=reconstructed_path, original_file_info=original_file_info) else: tl_reader = tarlite.TarliteReader(tarlite_path=compressed_path) img = np.zeros( (original_file_info["width"], original_file_info["height"], original_file_info["component_count"]), dtype=isets.iproperties_row_to_numpy_dtype( image_properties_row=original_file_info)) total_decompression_time = 0 with tempfile.TemporaryDirectory( dir=options.base_tmp_dir) as tmp_dir: tl_reader.extract_all(output_dir_path=tmp_dir) invocation = f"ls -lah {tmp_dir}" status, output = subprocess.getstatusoutput(invocation) if status != 0: raise Exception( "Status = {} != 0.\nInput=[{}].\nOutput=[{}]".format( status, invocation, output)) for y in range(self.split_height_count): for x in range(self.split_width_count): small_compressed_path = os.path.join( tmp_dir, f"{x}_{y}.mcalic") assert os.path.exists(small_compressed_path) small_path = os.path.join(tmp_dir, f"{x}_{y}.raw") small_file_info = copy.copy(original_file_info) small_file_info["height"] = original_file_info["height"] // self.split_height_count \ if y < self.split_height_count - 1 \ else original_file_info["height"] - (self.split_height_count - 1) * ( original_file_info["height"] // self.split_height_count) small_file_info["width"] = original_file_info["width"] // self.split_width_count \ if x < self.split_width_count - 1 \ else original_file_info["width"] - (self.split_width_count - 1) * ( original_file_info["width"] // self.split_width_count) dr = self.decompress_one( compressed_path=small_compressed_path, reconstructed_path=small_path, original_file_info=small_file_info) total_decompression_time += dr.decompression_time_seconds small_array = isets.load_array_bsq( file_or_path=small_path, image_properties_row=small_file_info) img[x * (original_file_info["width"] // self.split_width_count):( ((x + 1) * (original_file_info["width"] // self.split_width_count) ) if x < self.split_width_count - 1 else original_file_info["width"]), y * (original_file_info["height"] // self.split_height_count):( ((y + 1) * (original_file_info["height"] // self.split_height_count) ) if y < self.split_height_count - 1 else original_file_info["height"] ), :] = small_array isets.dump_array_bsq(array=img, file_or_path=reconstructed_path) decompression_results = self.decompression_results_from_paths( compressed_path=compressed_path, reconstructed_path=reconstructed_path) decompression_results.decompression_time_seconds = total_decompression_time return decompression_results
def compress_one(self, original_path: str, compressed_path: str, original_file_info=None): """Compress one image tile with M-CALIC. """ assert original_file_info["bytes_per_sample"] == 2, \ f"This implementation of M-CALIC ({self.compressor_path}) only supports 16bpp" assert original_file_info["component_count"] > 1, \ f"This implementation of M-CALIC ({self.compressor_path}) only supports images with more than one component" with tempfile.NamedTemporaryFile( dir=options.base_tmp_dir, prefix=f"bil_le_{os.path.basename(original_path)}" ) as bil_le_file: # M-Calic implementation requires little endian, unsigned 16bpp BIL format original_dtype = isets.iproperties_row_to_numpy_dtype( image_properties_row=original_file_info) img = np.fromfile(original_path, dtype=original_dtype).reshape( original_file_info["component_count"], original_file_info["height"], original_file_info["width"]) offset = None if original_file_info["signed"]: offset, original_max = int(img.min()), int(img.max()) offset = min(offset, 0) assert original_max - offset <= 2 ** 15 - 1, \ f"Invalid dynamic range of signed image ({offset}, {original_max})" img = (img.astype("i4") - offset).astype( original_dtype.replace("i", "u")) if original_file_info["big_endian"]: img = img.astype( original_dtype.replace(">", "<").replace("i", "u")) img.swapaxes(0, 1).tofile(bil_le_file.name) if original_file_info["signed"]: with tempfile.NamedTemporaryFile(dir=options.base_tmp_dir, prefix=f"bil_le_{os.path.basename(original_path)}", suffix=".mcalic") as tmp_compressed_file, \ tempfile.NamedTemporaryFile(dir=options.base_tmp_dir, prefix=f"side_info_{os.path.basename(original_path)}", suffix=".txt", mode="w") as si_file: si_file.write(f"{abs(offset):d}") si_file.flush() compression_results = super().compress( original_path=bil_le_file.name, compressed_path=tmp_compressed_file.name, original_file_info=original_file_info) tarlite.TarliteWriter(initial_input_paths=[ si_file.name, tmp_compressed_file.name ]).write(compressed_path) compression_results.original_path = original_path compression_results.compressed_path = compressed_path return compression_results else: compression_results = super().compress( original_path=bil_le_file.name, compressed_path=compressed_path, original_file_info=original_file_info) compression_results.original_path = original_path return compression_results
def numpy_dtype(self): """Get the numpy dtype corresponding to the original image's data format """ return isets.iproperties_row_to_numpy_dtype(self.image_info_row)