def test_create_tile_region_masks(self):
"""
test the generation of the raster mask which define the region
in the tile
"""
from iota2.Sampling.VectorFormatting import create_tile_region_masks
from iota2.Common.Utils import run
from iota2.Tests.UnitTests.TestsUtils import rasterToArray
import numpy as np
# define inputs
test_vector_name = "T31TCJ.sqlite"
test_vector = os.path.join(self.test_working_directory,
test_vector_name)
cmd = "ogr2ogr -nln t31tcj -f SQLite {} {}".format(
test_vector, self.ref_region)
run(cmd)
# launch function
create_tile_region_masks(test_vector, "region", "T31TCJ",
self.test_working_directory, "MyRegion",
self.ref_img)
# assert
raster_region = fut.FileSearch_AND(self.test_working_directory, True,
"MyRegion", ".tif")[0]
raster_region_arr = rasterToArray(raster_region)
ref_array = np.ones((50, 50))
self.assertTrue(np.allclose(ref_array, raster_region_arr),
msg="problem with the normalization by ref")
def test_perform_fusion(self):
"""
TEST : Fusion.perform_fusion
"""
from iota2.Classification import Fusion
# define inputs
sar_raster = os.path.join(self.test_working_directory,
"Classif_T31TCJ_model_1_seed_0_SAR.tif")
opt_raster = os.path.join(self.test_working_directory,
"Classif_T31TCJ_model_1_seed_0.tif")
array_to_raster(self.sar_classif, sar_raster)
array_to_raster(self.optical_classif, opt_raster)
fusion_dic = {
"sar_classif": sar_raster,
"opt_classif": opt_raster,
"sar_model": self.sar_confusion,
"opt_model": self.opt_confusion
}
# launch function
ds_fusion_test = Fusion.perform_fusion(
fusion_dic,
mob="precision",
classif_model_pos=self.classif_model_pos,
classif_tile_pos=self.classif_tile_pos,
classif_seed_pos=self.classif_seed_pos,
workingDirectory=None)
# assert
ds_fusion_test = rasterToArray(ds_fusion_test)
self.assertTrue(np.allclose(self.ds_fusion_ref, ds_fusion_test),
msg="fusion of classifications failed")
def test_confidence_fusion(self):
"""
TEST : Fusion.compute_confidence_fusion
"""
from iota2.Classification import Fusion
# define inputs
sar_raster = os.path.join(self.test_working_directory,
"Classif_T31TCJ_model_1_seed_0_SAR.tif")
opt_raster = os.path.join(self.test_working_directory,
"Classif_T31TCJ_model_1_seed_0.tif")
array_to_raster(self.sar_classif, sar_raster)
array_to_raster(self.optical_classif, opt_raster)
sar_confid_raster = os.path.join(
self.test_working_directory,
"T31TCJ_model_1_confidence_seed_0_SAR.tif")
opt_confid_raster = os.path.join(
self.test_working_directory,
"T31TCJ_model_1_confidence_seed_0.tif")
array_to_raster(self.sar_confidence,
sar_confid_raster,
output_format="float")
array_to_raster(self.optical_confidence,
opt_confid_raster,
output_format="float")
ds_choice = os.path.join(self.test_working_directory, "choice.tif")
array_to_raster(self.choice_map_ref, ds_choice)
fusion_dic = {
"sar_classif": sar_raster,
"opt_classif": opt_raster,
"sar_model": self.sar_confusion,
"opt_model": self.opt_confusion
}
workingDirectory = None
# Launch function
ds_fus_confidence_test = Fusion.compute_confidence_fusion(
fusion_dic, ds_choice, self.classif_model_pos,
self.classif_tile_pos, self.classif_seed_pos, self.ds_choice_both,
self.ds_choice_sar, self.ds_choice_opt, self.ds_no_choice,
workingDirectory)
# assert
ds_fus_confidence_test = rasterToArray(ds_fus_confidence_test)
self.assertTrue(np.allclose(self.ds_fus_confidence_ref,
ds_fus_confidence_test),
msg="fusion of confidences failed")
def test_fusion_choice(self):
"""
TEST : Fusion.compute_fusion_choice
"""
from iota2.Classification import Fusion
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
# define inputs
sar_raster = os.path.join(self.test_working_directory,
"Classif_T31TCJ_model_1_seed_0_SAR.tif")
opt_raster = os.path.join(self.test_working_directory,
"Classif_T31TCJ_model_1_seed_0.tif")
array_to_raster(self.sar_classif, sar_raster)
array_to_raster(self.optical_classif, opt_raster)
cfg = SCF.serviceConfigFile(self.config_test)
iota2_dir = os.path.join(self.test_working_directory, "fusionTest")
cfg.setParam('chain', 'outputPath', iota2_dir)
IOTA2Directory.generate_directories(iota2_dir, check_inputs=False)
fusion_dic = {
"sar_classif": sar_raster,
"opt_classif": opt_raster,
"sar_model": self.sar_confusion,
"opt_model": self.opt_confusion
}
fusion_class_array = self.ds_fusion_ref
fusion_class_raster = os.path.join(self.test_working_directory,
"fusionTest.tif")
array_to_raster(fusion_class_array, fusion_class_raster)
workingDirectory = None
# Launch function
ds_choice = Fusion.compute_fusion_choice(
iota2_dir, fusion_dic, fusion_class_raster, self.classif_model_pos,
self.classif_tile_pos, self.classif_seed_pos, self.ds_choice_both,
self.ds_choice_sar, self.ds_choice_opt, self.ds_no_choice,
workingDirectory)
# assert
ds_choice_test = rasterToArray(ds_choice)
self.assertTrue(np.allclose(self.choice_map_ref, ds_choice_test),
msg="compute raster choice failed")
def test_compute_probamap_fusion(self):
"""
TEST : Fusion.compute_probamap_fusion()
"""
from iota2.Classification import Fusion
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
# define inputs
sar_raster = os.path.join(self.test_working_directory,
"Classif_T31TCJ_model_1_seed_0_SAR.tif")
opt_raster = os.path.join(self.test_working_directory,
"Classif_T31TCJ_model_1_seed_0.tif")
array_to_raster(self.sar_classif, sar_raster)
array_to_raster(self.optical_classif, opt_raster)
sar_confid_raster = os.path.join(
self.test_working_directory,
"T31TCJ_model_1_confidence_seed_0_SAR.tif")
opt_confid_raster = os.path.join(
self.test_working_directory,
"T31TCJ_model_1_confidence_seed_0.tif")
array_to_raster(self.sar_confidence,
sar_confid_raster,
output_format="float")
array_to_raster(self.optical_confidence,
opt_confid_raster,
output_format="float")
ds_choice = os.path.join(self.test_working_directory, "choice.tif")
array_to_raster(self.choice_map_ref, ds_choice)
fusion_dic = {
"sar_classif": sar_raster,
"opt_classif": opt_raster,
"sar_model": self.sar_confusion,
"opt_model": self.opt_confusion
}
workingDirectory = None
# random probability maps
sar_probamap_arr = [
np.array([[253, 874, 600], [947, 812, 941], [580, 94, 192]]),
np.array([[541, 711, 326], [273, 915, 698], [296, 1000, 624]]),
np.array([[253, 290, 610], [406, 685, 333], [302, 410, 515]]),
np.array([[216, 766, 98], [914, 288, 504], [70, 631, 161]]),
np.array([[371, 873, 134], [477, 701, 765], [549, 301, 847]]),
np.array([[870, 201, 85], [555, 644, 802], [98, 807, 77]])
]
opt_probamap_arr = [
np.array([[268, 528, 131], [514, 299, 252], [725, 427, 731]]),
np.array([[119, 241, 543], [974, 629, 626], [3, 37, 819]]),
np.array([[409, 534, 710], [916, 43, 993], [207, 68, 282]]),
np.array([[820, 169, 423], [710, 626, 525], [377, 777, 461]]),
np.array([[475, 116, 395], [838, 297, 262], [650, 828, 595]]),
np.array([[940, 261, 20], [339, 934, 278], [444, 326, 219]])
]
# to rasters
sar_probamap_path = os.path.join(
self.test_working_directory,
"PROBAMAP_T31TCJ_model_1_seed_0_SAR.tif")
array_to_raster(sar_probamap_arr, sar_probamap_path)
opt_probamap_path = os.path.join(self.test_working_directory,
"PROBAMAP_T31TCJ_model_1_seed_0.tif")
array_to_raster(opt_probamap_arr, opt_probamap_path)
# Launch function
ds_fus_confidence_test = Fusion.compute_probamap_fusion(
fusion_dic, ds_choice, self.classif_model_pos,
self.classif_tile_pos, self.classif_seed_pos, self.ds_choice_both,
self.ds_choice_sar, self.ds_choice_opt, self.ds_no_choice,
workingDirectory)
# asserts
# length assert
from iota2.Common.FileUtils import getRasterNbands
self.assertTrue(
len(sar_probamap_arr) == len(opt_probamap_arr) == getRasterNbands(
ds_fus_confidence_test))
# fusion assert
is_ok = []
ds_fus_confidence_test_arr = rasterToArray(ds_fus_confidence_test)
for band_num in range(len(ds_fus_confidence_test_arr)):
merged_band = ds_fus_confidence_test_arr[band_num]
sar_proba_band = sar_probamap_arr[band_num]
opt_proba_band = opt_probamap_arr[band_num]
for (merged_proba, sar_proba, opt_proba, choice, sar_confi,
opt_confi) in zip(merged_band.flat, sar_proba_band.flat,
opt_proba_band.flat,
self.choice_map_ref.flat,
self.sar_confidence.flat,
self.optical_confidence.flat):
if choice == 1:
if sar_confi > opt_confi:
is_ok.append(int(merged_proba) == int(sar_proba))
else:
is_ok.append(int(merged_proba) == int(opt_proba))
elif choice == 2:
is_ok.append(int(merged_proba) == int(sar_proba))
elif choice == 3:
is_ok.append(int(merged_proba) == int(opt_proba))
self.assertTrue(all(is_ok))
def apply_function(otb_pipeline: otbApplication,
labels: List[str],
working_dir: str,
function: partial,
output_path: Optional[str] = None,
mask: Optional[str] = None,
mask_value: Optional[int] = 0,
chunk_size_mode: Optional[str] = "user_fixed",
chunck_size_x: Optional[int] = 10,
chunck_size_y: Optional[int] = 10,
targeted_chunk: Optional[int] = None,
number_of_chunks: Optional[int] = None,
output_number_of_bands: Optional[int] = None,
ram: Optional[int] = 128,
logger=logger) -> Tuple[np.ndarray, List[str], Affine, int]:
"""Apply a python function to an otb pipeline
If a mask is provided (values not to be taken into account are 'mask_value'),
then the resulting output could be define as the following :
output = output * mask
Parameters
----------
otb_pipeline: otbApplication
otb application ready to be Execute()
labels: List[str]
list of input bands names
working_dir: str
working directory path
function: partial
function to apply
output_path: str
output raster path (optional)
mask: str
input mask path (optional)
mask_value: int
input mask value to consider (optional)
chunk_size_mode : str
"user_fixed" / "auto" / "split_number"
chunck_size_x: int
chunck x size (optional)
chunck_size_y: int
chunck y size (optional)
targeted_chunk : int
process only the targeted chunk
output_number_of_bands : int
use only if targeted_chunk and mask are set
ram: int
available ram
Return
------
tuple
(np.array, new_labels, affine transform, epsg code)
"""
from iota2.Tests.UnitTests.TestsUtils import rasterToArray
mosaic = new_labels = None
roi_rasters, epsg_code = split_raster(otb_pipeline=otb_pipeline,
chunk_size_mode=chunk_size_mode,
chunk_size=(chunck_size_x,
chunck_size_y),
number_of_chunks=number_of_chunks,
ram_per_chunk=ram,
working_dir=working_dir)
if targeted_chunk is not None:
roi_rasters = [roi_rasters[targeted_chunk]]
mask_array = None
if mask:
mask_array = rasterToArray(mask)
new_arrays = []
chunks_mask = []
for index, roi_raster in enumerate(roi_rasters):
start_x = int(roi_raster.GetParameterString("startx"))
size_x = int(roi_raster.GetParameterString("sizex"))
start_y = int(roi_raster.GetParameterString("starty"))
size_y = int(roi_raster.GetParameterString("sizey"))
region_info = "processing region start_x : {} size_x : {} start_y : {} size_y : {}".format(start_x,
size_x,
start_y,
size_y)
logger.info(region_info)
print(region_info)
print("memory usage : {}".format(memory_usage_psutil()))
(roi_array, proj_geotransform), mask = process_function(roi_raster,
function=function,
mask_arr=mask_array,
mask_value=mask_value,
stream_bbox=(start_x,
size_x,
start_y,
size_y))
new_arrays.append((roi_array, proj_geotransform))
chunks_mask.append(mask)
all_data_sets = get_rasterio_datasets(new_arrays,
mask_value,
force_output_shape=(size_y,
size_x,
output_number_of_bands))
if len(all_data_sets) > 1:
mosaic, out_trans = merge(all_data_sets)
else:
if isinstance(new_arrays[0][0], int):
mosaic = np.repeat(mask[np.newaxis, :, :], output_number_of_bands, axis=0)
else:
mosaic = np.moveaxis(new_arrays[0][0], -1, 0)
out_trans = all_data_sets[0].transform
if output_path:
with rasterio.open(output_path,
"w",
driver='GTiff',
height=mosaic.shape[1],
width=mosaic.shape[2],
count=mosaic.shape[0],
crs="EPSG:{}".format(epsg_code),
transform=out_trans,
dtype=mosaic.dtype) as dest:
dest.write(mosaic)
# TODO : new_labels definition
return mosaic, new_labels, out_trans, epsg_code, chunks_mask