def test_create_tile_region_masks(self): """ test the generation of the raster mask which define the region in the tile """ from iota2.Sampling.VectorFormatting import create_tile_region_masks from iota2.Common.Utils import run from iota2.Tests.UnitTests.TestsUtils import rasterToArray import numpy as np # define inputs test_vector_name = "T31TCJ.sqlite" test_vector = os.path.join(self.test_working_directory, test_vector_name) cmd = "ogr2ogr -nln t31tcj -f SQLite {} {}".format( test_vector, self.ref_region) run(cmd) # launch function create_tile_region_masks(test_vector, "region", "T31TCJ", self.test_working_directory, "MyRegion", self.ref_img) # assert raster_region = fut.FileSearch_AND(self.test_working_directory, True, "MyRegion", ".tif")[0] raster_region_arr = rasterToArray(raster_region) ref_array = np.ones((50, 50)) self.assertTrue(np.allclose(ref_array, raster_region_arr), msg="problem with the normalization by ref")
def test_perform_fusion(self): """ TEST : Fusion.perform_fusion """ from iota2.Classification import Fusion # define inputs sar_raster = os.path.join(self.test_working_directory, "Classif_T31TCJ_model_1_seed_0_SAR.tif") opt_raster = os.path.join(self.test_working_directory, "Classif_T31TCJ_model_1_seed_0.tif") array_to_raster(self.sar_classif, sar_raster) array_to_raster(self.optical_classif, opt_raster) fusion_dic = { "sar_classif": sar_raster, "opt_classif": opt_raster, "sar_model": self.sar_confusion, "opt_model": self.opt_confusion } # launch function ds_fusion_test = Fusion.perform_fusion( fusion_dic, mob="precision", classif_model_pos=self.classif_model_pos, classif_tile_pos=self.classif_tile_pos, classif_seed_pos=self.classif_seed_pos, workingDirectory=None) # assert ds_fusion_test = rasterToArray(ds_fusion_test) self.assertTrue(np.allclose(self.ds_fusion_ref, ds_fusion_test), msg="fusion of classifications failed")
def test_confidence_fusion(self): """ TEST : Fusion.compute_confidence_fusion """ from iota2.Classification import Fusion # define inputs sar_raster = os.path.join(self.test_working_directory, "Classif_T31TCJ_model_1_seed_0_SAR.tif") opt_raster = os.path.join(self.test_working_directory, "Classif_T31TCJ_model_1_seed_0.tif") array_to_raster(self.sar_classif, sar_raster) array_to_raster(self.optical_classif, opt_raster) sar_confid_raster = os.path.join( self.test_working_directory, "T31TCJ_model_1_confidence_seed_0_SAR.tif") opt_confid_raster = os.path.join( self.test_working_directory, "T31TCJ_model_1_confidence_seed_0.tif") array_to_raster(self.sar_confidence, sar_confid_raster, output_format="float") array_to_raster(self.optical_confidence, opt_confid_raster, output_format="float") ds_choice = os.path.join(self.test_working_directory, "choice.tif") array_to_raster(self.choice_map_ref, ds_choice) fusion_dic = { "sar_classif": sar_raster, "opt_classif": opt_raster, "sar_model": self.sar_confusion, "opt_model": self.opt_confusion } workingDirectory = None # Launch function ds_fus_confidence_test = Fusion.compute_confidence_fusion( fusion_dic, ds_choice, self.classif_model_pos, self.classif_tile_pos, self.classif_seed_pos, self.ds_choice_both, self.ds_choice_sar, self.ds_choice_opt, self.ds_no_choice, workingDirectory) # assert ds_fus_confidence_test = rasterToArray(ds_fus_confidence_test) self.assertTrue(np.allclose(self.ds_fus_confidence_ref, ds_fus_confidence_test), msg="fusion of confidences failed")
def test_fusion_choice(self): """ TEST : Fusion.compute_fusion_choice """ from iota2.Classification import Fusion from iota2.Common import IOTA2Directory from iota2.Common import ServiceConfigFile as SCF # define inputs sar_raster = os.path.join(self.test_working_directory, "Classif_T31TCJ_model_1_seed_0_SAR.tif") opt_raster = os.path.join(self.test_working_directory, "Classif_T31TCJ_model_1_seed_0.tif") array_to_raster(self.sar_classif, sar_raster) array_to_raster(self.optical_classif, opt_raster) cfg = SCF.serviceConfigFile(self.config_test) iota2_dir = os.path.join(self.test_working_directory, "fusionTest") cfg.setParam('chain', 'outputPath', iota2_dir) IOTA2Directory.generate_directories(iota2_dir, check_inputs=False) fusion_dic = { "sar_classif": sar_raster, "opt_classif": opt_raster, "sar_model": self.sar_confusion, "opt_model": self.opt_confusion } fusion_class_array = self.ds_fusion_ref fusion_class_raster = os.path.join(self.test_working_directory, "fusionTest.tif") array_to_raster(fusion_class_array, fusion_class_raster) workingDirectory = None # Launch function ds_choice = Fusion.compute_fusion_choice( iota2_dir, fusion_dic, fusion_class_raster, self.classif_model_pos, self.classif_tile_pos, self.classif_seed_pos, self.ds_choice_both, self.ds_choice_sar, self.ds_choice_opt, self.ds_no_choice, workingDirectory) # assert ds_choice_test = rasterToArray(ds_choice) self.assertTrue(np.allclose(self.choice_map_ref, ds_choice_test), msg="compute raster choice failed")
def test_compute_probamap_fusion(self): """ TEST : Fusion.compute_probamap_fusion() """ from iota2.Classification import Fusion from iota2.Common import IOTA2Directory from iota2.Common import ServiceConfigFile as SCF # define inputs sar_raster = os.path.join(self.test_working_directory, "Classif_T31TCJ_model_1_seed_0_SAR.tif") opt_raster = os.path.join(self.test_working_directory, "Classif_T31TCJ_model_1_seed_0.tif") array_to_raster(self.sar_classif, sar_raster) array_to_raster(self.optical_classif, opt_raster) sar_confid_raster = os.path.join( self.test_working_directory, "T31TCJ_model_1_confidence_seed_0_SAR.tif") opt_confid_raster = os.path.join( self.test_working_directory, "T31TCJ_model_1_confidence_seed_0.tif") array_to_raster(self.sar_confidence, sar_confid_raster, output_format="float") array_to_raster(self.optical_confidence, opt_confid_raster, output_format="float") ds_choice = os.path.join(self.test_working_directory, "choice.tif") array_to_raster(self.choice_map_ref, ds_choice) fusion_dic = { "sar_classif": sar_raster, "opt_classif": opt_raster, "sar_model": self.sar_confusion, "opt_model": self.opt_confusion } workingDirectory = None # random probability maps sar_probamap_arr = [ np.array([[253, 874, 600], [947, 812, 941], [580, 94, 192]]), np.array([[541, 711, 326], [273, 915, 698], [296, 1000, 624]]), np.array([[253, 290, 610], [406, 685, 333], [302, 410, 515]]), np.array([[216, 766, 98], [914, 288, 504], [70, 631, 161]]), np.array([[371, 873, 134], [477, 701, 765], [549, 301, 847]]), np.array([[870, 201, 85], [555, 644, 802], [98, 807, 77]]) ] opt_probamap_arr = [ np.array([[268, 528, 131], [514, 299, 252], [725, 427, 731]]), np.array([[119, 241, 543], [974, 629, 626], [3, 37, 819]]), np.array([[409, 534, 710], [916, 43, 993], [207, 68, 282]]), np.array([[820, 169, 423], [710, 626, 525], [377, 777, 461]]), np.array([[475, 116, 395], [838, 297, 262], [650, 828, 595]]), np.array([[940, 261, 20], [339, 934, 278], [444, 326, 219]]) ] # to rasters sar_probamap_path = os.path.join( self.test_working_directory, "PROBAMAP_T31TCJ_model_1_seed_0_SAR.tif") array_to_raster(sar_probamap_arr, sar_probamap_path) opt_probamap_path = os.path.join(self.test_working_directory, "PROBAMAP_T31TCJ_model_1_seed_0.tif") array_to_raster(opt_probamap_arr, opt_probamap_path) # Launch function ds_fus_confidence_test = Fusion.compute_probamap_fusion( fusion_dic, ds_choice, self.classif_model_pos, self.classif_tile_pos, self.classif_seed_pos, self.ds_choice_both, self.ds_choice_sar, self.ds_choice_opt, self.ds_no_choice, workingDirectory) # asserts # length assert from iota2.Common.FileUtils import getRasterNbands self.assertTrue( len(sar_probamap_arr) == len(opt_probamap_arr) == getRasterNbands( ds_fus_confidence_test)) # fusion assert is_ok = [] ds_fus_confidence_test_arr = rasterToArray(ds_fus_confidence_test) for band_num in range(len(ds_fus_confidence_test_arr)): merged_band = ds_fus_confidence_test_arr[band_num] sar_proba_band = sar_probamap_arr[band_num] opt_proba_band = opt_probamap_arr[band_num] for (merged_proba, sar_proba, opt_proba, choice, sar_confi, opt_confi) in zip(merged_band.flat, sar_proba_band.flat, opt_proba_band.flat, self.choice_map_ref.flat, self.sar_confidence.flat, self.optical_confidence.flat): if choice == 1: if sar_confi > opt_confi: is_ok.append(int(merged_proba) == int(sar_proba)) else: is_ok.append(int(merged_proba) == int(opt_proba)) elif choice == 2: is_ok.append(int(merged_proba) == int(sar_proba)) elif choice == 3: is_ok.append(int(merged_proba) == int(opt_proba)) self.assertTrue(all(is_ok))
def apply_function(otb_pipeline: otbApplication, labels: List[str], working_dir: str, function: partial, output_path: Optional[str] = None, mask: Optional[str] = None, mask_value: Optional[int] = 0, chunk_size_mode: Optional[str] = "user_fixed", chunck_size_x: Optional[int] = 10, chunck_size_y: Optional[int] = 10, targeted_chunk: Optional[int] = None, number_of_chunks: Optional[int] = None, output_number_of_bands: Optional[int] = None, ram: Optional[int] = 128, logger=logger) -> Tuple[np.ndarray, List[str], Affine, int]: """Apply a python function to an otb pipeline If a mask is provided (values not to be taken into account are 'mask_value'), then the resulting output could be define as the following : output = output * mask Parameters ---------- otb_pipeline: otbApplication otb application ready to be Execute() labels: List[str] list of input bands names working_dir: str working directory path function: partial function to apply output_path: str output raster path (optional) mask: str input mask path (optional) mask_value: int input mask value to consider (optional) chunk_size_mode : str "user_fixed" / "auto" / "split_number" chunck_size_x: int chunck x size (optional) chunck_size_y: int chunck y size (optional) targeted_chunk : int process only the targeted chunk output_number_of_bands : int use only if targeted_chunk and mask are set ram: int available ram Return ------ tuple (np.array, new_labels, affine transform, epsg code) """ from iota2.Tests.UnitTests.TestsUtils import rasterToArray mosaic = new_labels = None roi_rasters, epsg_code = split_raster(otb_pipeline=otb_pipeline, chunk_size_mode=chunk_size_mode, chunk_size=(chunck_size_x, chunck_size_y), number_of_chunks=number_of_chunks, ram_per_chunk=ram, working_dir=working_dir) if targeted_chunk is not None: roi_rasters = [roi_rasters[targeted_chunk]] mask_array = None if mask: mask_array = rasterToArray(mask) new_arrays = [] chunks_mask = [] for index, roi_raster in enumerate(roi_rasters): start_x = int(roi_raster.GetParameterString("startx")) size_x = int(roi_raster.GetParameterString("sizex")) start_y = int(roi_raster.GetParameterString("starty")) size_y = int(roi_raster.GetParameterString("sizey")) region_info = "processing region start_x : {} size_x : {} start_y : {} size_y : {}".format(start_x, size_x, start_y, size_y) logger.info(region_info) print(region_info) print("memory usage : {}".format(memory_usage_psutil())) (roi_array, proj_geotransform), mask = process_function(roi_raster, function=function, mask_arr=mask_array, mask_value=mask_value, stream_bbox=(start_x, size_x, start_y, size_y)) new_arrays.append((roi_array, proj_geotransform)) chunks_mask.append(mask) all_data_sets = get_rasterio_datasets(new_arrays, mask_value, force_output_shape=(size_y, size_x, output_number_of_bands)) if len(all_data_sets) > 1: mosaic, out_trans = merge(all_data_sets) else: if isinstance(new_arrays[0][0], int): mosaic = np.repeat(mask[np.newaxis, :, :], output_number_of_bands, axis=0) else: mosaic = np.moveaxis(new_arrays[0][0], -1, 0) out_trans = all_data_sets[0].transform if output_path: with rasterio.open(output_path, "w", driver='GTiff', height=mosaic.shape[1], width=mosaic.shape[2], count=mosaic.shape[0], crs="EPSG:{}".format(epsg_code), transform=out_trans, dtype=mosaic.dtype) as dest: dest.write(mosaic) # TODO : new_labels definition return mosaic, new_labels, out_trans, epsg_code, chunks_mask