def test_samples_selection(self):
"""
test sampling of a shape file (main function of SamplesSelection.py)
"""
from iota2.Sampling.SamplesSelection import samples_selection
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
from iota2.Tests.UnitTests.Iota2Tests import compareSQLite
from iota2.Common.FileUtils import cpShapeFile
from iota2.Common import FileUtils as fut
# prepare test input
cfg = SCF.serviceConfigFile(self.config_test)
cfg.setParam(
"chain", "outputPath",
os.path.join(self.test_working_directory, "samplesSelTest"))
cfg.setParam("chain", "runs", 2)
cfg.setParam("argTrain", "sampleSelection", {
"sampler": "random",
"strategy": "all"
})
# create IOTA2 directories
IOTA2Directory.generate_directories(os.path.join(
self.test_working_directory, "samplesSelTest"),
check_inputs=False)
shutil.copytree(
self.features_ref,
os.path.join(self.test_working_directory, "samplesSelTest",
"features", "T31TCJ"))
shutil.copy(
self.in_xml,
os.path.join(self.test_working_directory, "samplesSelTest",
"samplesSelection",
"T31TCJ_region_1_seed_0_stats.xml"))
_, in_shape_name = os.path.split(self.in_shape)
in_shape_dir = os.path.join(self.test_working_directory,
"samplesSelTest", "samplesSelection")
in_shape = os.path.join(in_shape_dir, in_shape_name)
cpShapeFile(self.in_shape.replace(".shp", ""),
in_shape.replace(".shp", ""),
extensions=[".prj", ".shp", ".dbf", ".shx"])
#~ # launch function
output_path = cfg.getParam("chain", "outputPath")
runs = cfg.getParam('chain', 'runs')
epsg = cfg.getParam('GlobChain', 'proj')
random_seed = cfg.getParam('chain', 'random_seed')
data_field = cfg.getParam('chain', 'dataField').lower()
parameters = dict(cfg.getParam('argTrain', 'sampleSelection'))
masks_name = "MaskCommunSL.tif"
samples_selection(in_shape, self.test_working_directory, output_path,
runs, epsg, masks_name, parameters, data_field,
random_seed)
#~ # assert
selection_test = fut.FileSearch_AND(
os.path.join(self.test_working_directory, "samplesSelTest"), True,
os.path.basename(self.selection_ref))[0]
same = compareSQLite(self.selection_ref,
selection_test,
CmpMode='coordinates')
self.assertTrue(same, msg="sample selection generation failed")
def test_ds_fusion(self):
"""
TEST : Fusion.dempster_shafer_fusion. Every functions called in Fusion.dempster_shafer_fusion
are tested above. This test check the runnability of Fusion.dempster_shafer_fusion
"""
from iota2.Classification import Fusion
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
# define inputs
cfg = SCF.serviceConfigFile(self.config_test)
iota2_dir = os.path.join(self.test_working_directory, "fusionTest")
cfg.setParam('chain', 'outputPath', iota2_dir)
IOTA2Directory.generate_directories(iota2_dir, check_inputs=False)
sar_raster = os.path.join(iota2_dir, "classif",
"Classif_T31TCJ_model_1_seed_0_SAR.tif")
opt_raster = os.path.join(iota2_dir, "classif",
"Classif_T31TCJ_model_1_seed_0.tif")
array_to_raster(self.sar_classif, sar_raster)
array_to_raster(self.optical_classif, opt_raster)
sar_confid_raster = os.path.join(
iota2_dir, "classif", "T31TCJ_model_1_confidence_seed_0_SAR.tif")
opt_confid_raster = os.path.join(
iota2_dir, "classif", "T31TCJ_model_1_confidence_seed_0.tif")
array_to_raster(self.sar_confidence,
sar_confid_raster,
output_format="float")
array_to_raster(self.optical_confidence,
opt_confid_raster,
output_format="float")
fusion_dic = {
"sar_classif": sar_raster,
"opt_classif": opt_raster,
"sar_model": self.sar_confusion,
"opt_model": self.opt_confusion
}
workingDirectory = None
# Launch function
(fusion_path, confidence_path, probamap_path,
choice_path) = Fusion.dempster_shafer_fusion(
iota2_dir,
fusion_dic,
mob="precision",
workingDirectory=workingDirectory)
self.assertEqual("/classif/Classif_T31TCJ_model_1_seed_0_DS.tif",
fusion_path.replace(iota2_dir, ""))
self.assertEqual("/classif/T31TCJ_model_1_confidence_seed_0_DS.tif",
confidence_path.replace(iota2_dir, ""))
self.assertEqual("/final/TMP/DSchoice_T31TCJ_model_1_seed_0.tif",
choice_path.replace(iota2_dir, ""))
self.assertTrue(probamap_path is None)
def test_fusion_choice(self):
"""
TEST : Fusion.compute_fusion_choice
"""
from iota2.Classification import Fusion
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
# define inputs
sar_raster = os.path.join(self.test_working_directory,
"Classif_T31TCJ_model_1_seed_0_SAR.tif")
opt_raster = os.path.join(self.test_working_directory,
"Classif_T31TCJ_model_1_seed_0.tif")
array_to_raster(self.sar_classif, sar_raster)
array_to_raster(self.optical_classif, opt_raster)
cfg = SCF.serviceConfigFile(self.config_test)
iota2_dir = os.path.join(self.test_working_directory, "fusionTest")
cfg.setParam('chain', 'outputPath', iota2_dir)
IOTA2Directory.generate_directories(iota2_dir, check_inputs=False)
fusion_dic = {
"sar_classif": sar_raster,
"opt_classif": opt_raster,
"sar_model": self.sar_confusion,
"opt_model": self.opt_confusion
}
fusion_class_array = self.ds_fusion_ref
fusion_class_raster = os.path.join(self.test_working_directory,
"fusionTest.tif")
array_to_raster(fusion_class_array, fusion_class_raster)
workingDirectory = None
# Launch function
ds_choice = Fusion.compute_fusion_choice(
iota2_dir, fusion_dic, fusion_class_raster, self.classif_model_pos,
self.classif_tile_pos, self.classif_seed_pos, self.ds_choice_both,
self.ds_choice_sar, self.ds_choice_opt, self.ds_no_choice,
workingDirectory)
# assert
ds_choice_test = rasterToArray(ds_choice)
self.assertTrue(np.allclose(self.choice_map_ref, ds_choice_test),
msg="compute raster choice failed")
def compute_features(output_path: str,
sensors_parameters: sensors_params,
output_raster: str,
tile_name: str,
working_dir: str,
function=smart_scientific_function) -> None:
"""Use a python function to generate features through the use of numpy arrays
Parameters
----------
output_path : str
output directory
sensors_parameters : dict
sensors parameters description
output_raster : str
output raster path
tile_name : str
tile to compute
function : function
function to apply on iota²' stack
"""
IOTA2Directory.generate_directories(output_path, check_inputs=False)
feat_stack, feat_labels, _ = generateFeatures(
working_dir,
tile_name,
sar_optical_post_fusion=False,
output_path=output_path,
sensors_parameters=sensors_parameters)
# Then compute new features
function = partial(function, increment=1)
feat_stack_array, feat_labels = rasterUtils.apply_function(
feat_stack,
feat_labels,
working_dir,
function,
output_raster,
chunck_size_x=10,
chunck_size_y=10,
ram=128)
def test_slic(self):
"""non-regression test, check if SLIC could be performed
"""
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
from iota2.Segmentation import segmentation
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Sensors.Sensors_container import sensors_container
# config file
config_path_test = os.path.join(self.test_working_directory,
"Config_TEST.cfg")
test_path = self.generate_cfg_file(self.config_test, config_path_test)
IOTA2Directory.generate_directories(test_path, check_inputs=False)
slic_working_dir = os.path.join(self.test_working_directory,
"slic_tmp")
iota2_dico = SCF.iota2_parameters(
config_path_test).get_sensors_parameters(self.tile_name)
sensors = sensors_container(self.tile_name, None,
self.test_working_directory, **iota2_dico)
sensors.sensors_preprocess()
# Launch test
segmentation.slicSegmentation(self.tile_name,
test_path,
iota2_dico,
ram=128,
working_dir=slic_working_dir,
force_spw=1)
# as SLIC algorithm contains random variables, the raster's content
# could not be tested
self.assertTrue(len(
FileSearch_AND(
os.path.join(test_path, "features", self.tile_name, "tmp"),
True, "SLIC_{}".format(self.tile_name))) == 1,
msg="SLIC algorithm failed")
def step_execute(self):
"""
Return
------
lambda
the function to execute as a lambda function. The returned object
must be a lambda function.
"""
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.ServiceConfigFile import iota2_parameters
check_inputs = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'check_inputs')
tiles = SCF.serviceConfigFile(self.cfg).getParam('chain',
'listTile').split(" ")
output_path = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
running_parameters = iota2_parameters(
SCF.serviceConfigFile(self.cfg).pathConf)
sensors_parameters = running_parameters.get_sensors_parameters(
tiles[0])
sensor_tile_container = sensors_container(tiles[0], None, output_path,
**sensors_parameters)
sensor_path = sensor_tile_container.get_enabled_sensors_path()[0]
step_function = lambda x: IOTA2_dir.generate_directories(
x, check_inputs,
SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'merge_final_classifications'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'groundTruth'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'regionPath'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'dataField'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'regionField'),
int(
SCF.serviceConfigFile(self.cfg).getParam('GlobChain', 'proj').
split(":")[-1]), sensor_path, tiles)
return step_function
def test_VectorFormatting(self):
"""
test vectorFormatting function
random function is used in Sampling.VectorFormatting.VectorFormatting
we can only check if there is expected number of features with
expected fields and some features values
"""
from iota2.Sampling.VectorFormatting import vector_formatting
from iota2.Common import ServiceConfigFile as SCF
from iota2.Common import IOTA2Directory
from iota2.Common.Utils import run
from iota2.VectorTools.ChangeNameField import changeName
# define inputs
test_output = os.path.join(self.test_working_directory,
"IOTA2_dir_VectorFormatting")
# prepare ground truth
ground_truth = os.path.join(self.test_working_directory,
"groundTruth_test.shp")
cmd = "ogr2ogr -s_srs EPSG:2154 -t_srs EPSG:2154 -dialect 'SQLite' -sql 'select GEOMETRY,code from t31tcj' {} {}".format(
ground_truth, self.in_vector)
run(cmd)
# cfg instance
runs = 2
cfg = SCF.serviceConfigFile(self.config_test)
cfg.setParam('chain', 'outputPath', test_output)
cfg.setParam('chain', 'groundTruth', ground_truth)
cfg.setParam('chain', 'dataField', "code")
cfg.setParam('chain', 'cloud_threshold', 0)
cfg.setParam('chain', 'merge_final_classifications', False)
cfg.setParam('chain', 'runs', runs)
cfg.setParam('GlobChain', 'proj', "EPSG:2154")
cfg.setParam('chain', 'regionPath', self.ref_region)
IOTA2Directory.generate_directories(test_output, check_inputs=False)
# prepare expected function inputs
t31tcj_feat_dir = os.path.join(self.test_working_directory,
"IOTA2_dir_VectorFormatting",
"features", "T31TCJ")
os.mkdir(t31tcj_feat_dir)
# prepare ref img
t31tcj_ref_img = os.path.join(t31tcj_feat_dir, "MaskCommunSL.tif")
shutil.copy(self.ref_img, t31tcj_ref_img)
# prepare envelope
envelope_name = "T31TCJ.shp"
envelope_path = os.path.join(self.test_working_directory,
"IOTA2_dir_VectorFormatting", "envelope",
envelope_name)
fut.cpShapeFile(self.ref_region.replace(".shp", ""),
envelope_path.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
changeName(envelope_path, "region", "FID")
# prepare cloud mask
cloud_name = "CloudThreshold_0.shp"
cloud_path = os.path.join(self.test_working_directory,
"IOTA2_dir_VectorFormatting", "features",
"T31TCJ", cloud_name)
fut.cpShapeFile(self.ref_region.replace(".shp", ""),
cloud_path.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
changeName(cloud_path, "region", "cloud")
# launch function
ratio = cfg.getParam('chain', 'ratio')
random_seed = cfg.getParam('chain', 'random_seed')
enable_cross_validation = cfg.getParam("chain",
"enableCrossValidation")
enable_split_ground_truth = cfg.getParam('chain', 'splitGroundTruth')
fusion_merge_all_validation = cfg.getParam(
'chain', 'fusionOfClassificationAllSamplesValidation')
merge_final_classifications = cfg.getParam(
'chain', 'merge_final_classifications')
merge_final_classifications_ratio = cfg.getParam(
'chain', 'merge_final_classifications_ratio')
region_vec = cfg.getParam('chain', 'regionPath')
epsg = int(cfg.getParam('GlobChain', 'proj').split(":")[-1])
region_field = (cfg.getParam('chain', 'regionField'))
vector_formatting("T31TCJ",
test_output,
ground_truth,
"code",
0,
ratio,
random_seed,
enable_cross_validation,
enable_split_ground_truth,
fusion_merge_all_validation,
runs,
epsg,
region_field,
merge_final_classifications,
merge_final_classifications_ratio,
region_vec,
working_directory=None)
# assert
nb_features_origin = len(
fut.getFieldElement(ground_truth,
driverName="ESRI Shapefile",
field="code",
mode="all",
elemType="str"))
test_vector = fut.FileSearch_AND(
os.path.join(test_output, "formattingVectors"), True,
"T31TCJ.shp")[0]
nb_features_test = len(
fut.getFieldElement(test_vector,
driverName="ESRI Shapefile",
field="code",
mode="all",
elemType="str"))
# check nb features
self.assertTrue(nb_features_origin == nb_features_test,
msg="wrong number of features")
# check fields
origin_fields = fut.get_all_fields_in_shape(ground_truth)
test_fields = fut.get_all_fields_in_shape(test_vector)
new_fields = ['region', 'originfid', 'seed_0', 'seed_1', 'tile_o']
expected_fields = origin_fields + new_fields
self.assertTrue(len(expected_fields) == len(test_fields))
self.assertTrue(all(field in test_fields for field in expected_fields))
def test_fusion_parameters(self):
"""
TEST : Fusion.dempster_shafer_fusion_parameters
"""
from iota2.Classification import Fusion
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
# define inputs
cfg = SCF.serviceConfigFile(self.config_test)
iota2_dir = os.path.join(self.test_working_directory, "fusionTest")
cfg.setParam('chain', 'outputPath', iota2_dir)
IOTA2Directory.generate_directories(iota2_dir, check_inputs=False)
iota2_ds_confusions_dir = os.path.join(iota2_dir, "dataAppVal",
"bymodels")
fut.ensure_dir(iota2_ds_confusions_dir)
# generate some fake data
nb_seed = 2
for i in range(nb_seed):
fake_classif_opt = os.path.join(
iota2_dir, "classif",
"Classif_T31TCJ_model_1_seed_{}.tif".format(i))
fake_classif_sar = os.path.join(
iota2_dir, "classif",
"Classif_T31TCJ_model_1_seed_{}_SAR.tif".format(i))
fake_confidence_opt = os.path.join(
iota2_dir, "classif",
"T31TCJ_model_1_confidence_seed_{}.tif".format(i))
fake_confidence_sar = os.path.join(
iota2_dir, "classif",
"T31TCJ_model_1_confidence_seed_{}_SAR.tif".format(i))
fake_model_confusion_sar = os.path.join(
iota2_ds_confusions_dir, "model_1_seed_{}_SAR.csv".format(i))
fake_model_confusion_opt = os.path.join(
iota2_ds_confusions_dir, "model_1_seed_{}.csv".format(i))
with open(fake_classif_opt, "w") as new_file:
new_file.write("TEST")
with open(fake_classif_sar, "w") as new_file:
new_file.write("TEST")
with open(fake_confidence_opt, "w") as new_file:
new_file.write("TEST")
with open(fake_confidence_sar, "w") as new_file:
new_file.write("TEST")
with open(fake_model_confusion_sar, "w") as new_file:
new_file.write("TEST")
with open(fake_model_confusion_opt, "w") as new_file:
new_file.write("TEST")
parameters_test = Fusion.dempster_shafer_fusion_parameters(iota2_dir)
# parameters_test depend of execution environement, remove local path is necessary
for param_group in parameters_test:
for key, value in list(param_group.items()):
param_group[key] = value.replace(iota2_dir, "")
#reverse parameters_test if necessary
if "seed_1" in parameters_test[0]["sar_model"]:
parameters_test = parameters_test[::-1]
# assert
self.assertTrue(all(param_group_test == param_group_ref
for param_group_test, param_group_ref in zip(
parameters_test, self.parameter_ref)),
msg="input parameters generation failed")
def test_train_and_classify(self):
"""test autoContext training
"""
import re
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.FileUtils import getFieldElement
from iota2.Segmentation import segmentation
from iota2.Learning.trainAutoContext import train_autoContext
from iota2.Classification.ImageClassifier import autoContext_launch_classif
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.ServiceConfigFile import iota2_parameters
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Tests.UnitTests.TestsUtils import test_raster_unique_value
# config file
config_path_test = os.path.join(self.test_working_directory,
"Config_TEST.cfg")
test_path = self.generate_cfg_file(self.config_test, config_path_test)
cfg = SCF.serviceConfigFile(config_path_test)
IOTA2Directory.generate_directories(test_path, check_inputs=False)
autocontext_working_dir = os.path.join(self.test_working_directory,
"autoContext_tmp")
slic_working_dir = os.path.join(self.test_working_directory,
"autoContext_tmp")
iota2_dico = SCF.iota2_parameters(
config_path_test).get_sensors_parameters(self.tile_name)
sensors = sensors_container(self.tile_name, None,
self.test_working_directory, **iota2_dico)
sensors.sensors_preprocess()
running_parameters = iota2_parameters(config_path_test)
sensors_parameters = running_parameters.get_sensors_parameters(
"T31TCJ")
segmentation.slicSegmentation(self.tile_name,
test_path,
sensors_parameters,
ram=128,
working_dir=slic_working_dir,
force_spw=1)
slic_seg = FileSearch_AND(
os.path.join(test_path, "features", self.tile_name, "tmp"), True,
"SLIC_{}".format(self.tile_name))[0]
train_auto_data_ref, superpix_data = self.prepare_autoContext_data_ref(
slic_seg, config_path_test)
parameter_dict = {
"model_name": "1",
"seed": "0",
"list_learning_samples": [train_auto_data_ref],
"list_superPixel_samples": [superpix_data],
"list_tiles": ["T31TCJ"],
"list_slic": [slic_seg]
}
# launch tests
# training
e = None
try:
train_autoContext(parameter_dict,
data_field="code",
output_path=test_path,
sensors_parameters=sensors_parameters,
superpix_data_field="superpix",
iterations=3,
RAM=128,
WORKING_DIR=autocontext_working_dir)
except Exception as e:
print(e)
# Asserts training
self.assertTrue(e is None, msg="train_autoContext failed")
models = FileSearch_AND(os.path.join(test_path, "model"), True,
"model_it_", ".rf")
self.assertTrue(len(models) == 4)
# classification
tile_raster = FileSearch_AND(self.fake_data_dir, True,
"BINARY_MASK.tif")[0]
tile_mask = os.path.join(autocontext_working_dir,
"{}_tile_mask.tif".format(self.tile_name))
CreateBandMathApplication({
"il": [tile_raster],
"out": tile_mask,
"exp": "1"
}).ExecuteAndWriteOutput()
labels = getFieldElement(train_auto_data_ref,
driverName="SQLite",
field="code",
mode="unique",
elemType="str")
parameters_dict = {
"model_name":
"1",
"seed_num":
0,
"tile":
self.tile_name,
"tile_segmentation":
slic_seg,
"tile_mask":
tile_mask,
"labels_list":
labels,
"model_list":
sorted(models,
key=lambda x: int(re.findall("\d", os.path.basename(x))[0]))
}
autoContext_launch_classif(parameters_dict,
cfg.getParam("argTrain", "classifier"),
"T31TCJ",
cfg.getParam("argClassification",
"enable_probability_map"),
cfg.getParam("dimRed", "dimRed"),
"code",
False,
False,
iota2_run_dir=test_path,
sar_optical_post_fusion=False,
nomenclature_path=cfg.getParam(
"chain", "nomenclaturePath"),
sensors_parameters=sensors_parameters,
RAM=128,
WORKING_DIR=autocontext_working_dir)
# Asserts classifications
classif = FileSearch_AND(os.path.join(test_path, "classif"), True,
"Classif_T31TCJ_model_1_seed_0.tif")[0]
confidence = FileSearch_AND(os.path.join(test_path, "classif"), True,
"T31TCJ_model_1_confidence_seed_0.tif")[0]
classif_unique_0 = test_raster_unique_value(classif, 0)
confidence_unique_0 = test_raster_unique_value(confidence, 0)
self.assertTrue(
classif_unique_0 == False,
msg=("AutoContext Classifications failed : classification contains"
" only 0 values"))
self.assertTrue(
confidence_unique_0 == False,
msg=("AutoContext Classifications failed : confidence contains "
"only 0 values"))