def preprocess(tile_name,
config_path,
output_path,
working_directory=None,
RAM=128):
"""
preprocessing input rasters data by tile
Parameters
----------
tile_name [string]
tile's name
config_path [string]
absolute path to the configuration file
output_path : str
iota2 output path
working_directory [string]
absolute path to a working directory
RAM [int]
pipeline's size (Mo)
"""
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.ServiceConfigFile import iota2_parameters
running_parameters = iota2_parameters(config_path)
sensors_parameters = running_parameters.get_sensors_parameters(tile_name)
remote_sensor_container = sensors_container(tile_name, working_directory,
output_path,
**sensors_parameters)
remote_sensor_container.sensors_preprocess(available_ram=RAM)
def commonMasks(tile_name,
output_path,
sensors_parameters,
working_directory=None,
RAM=128):
"""
compute common mask considering all sensors by tile
Parameters
----------
tile_name [string]
tile's name
config_path [string]
absolute path to the configuration file
output_path : str
iota2 output path
working_directory [string]
absolute path to a working directory
RAM [int]
pipeline's size (Mo)
"""
import os
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.Utils import run
from iota2.Common.FileUtils import ensure_dir
# running_parameters = iota2_parameters(config_path)
# sensors_parameters = running_parameters.get_sensors_parameters(tile_name)
remote_sensor_container = sensors_container(tile_name, working_directory,
output_path,
**sensors_parameters)
common_mask, _ = remote_sensor_container.get_common_sensors_footprint(
available_ram=RAM)
common_mask_raster = common_mask.GetParameterValue("out")
if not os.path.exists(common_mask_raster):
ensure_dir(os.path.split(common_mask_raster)[0], raise_exe=False)
common_mask.ExecuteAndWriteOutput()
common_mask_vector = common_mask_raster.replace(".tif", ".shp")
common_mask_vector_cmd = "gdal_polygonize.py -f \"ESRI Shapefile\" -mask {} {} {}".format(
common_mask_raster, common_mask_raster, common_mask_vector)
run(common_mask_vector_cmd)
def test_slic(self):
"""non-regression test, check if SLIC could be performed
"""
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
from iota2.Segmentation import segmentation
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Sensors.Sensors_container import sensors_container
# config file
config_path_test = os.path.join(self.test_working_directory,
"Config_TEST.cfg")
test_path = self.generate_cfg_file(self.config_test, config_path_test)
IOTA2Directory.generate_directories(test_path, check_inputs=False)
slic_working_dir = os.path.join(self.test_working_directory,
"slic_tmp")
iota2_dico = SCF.iota2_parameters(
config_path_test).get_sensors_parameters(self.tile_name)
sensors = sensors_container(self.tile_name, None,
self.test_working_directory, **iota2_dico)
sensors.sensors_preprocess()
# Launch test
segmentation.slicSegmentation(self.tile_name,
test_path,
iota2_dico,
ram=128,
working_dir=slic_working_dir,
force_spw=1)
# as SLIC algorithm contains random variables, the raster's content
# could not be tested
self.assertTrue(len(
FileSearch_AND(
os.path.join(test_path, "features", self.tile_name, "tmp"),
True, "SLIC_{}".format(self.tile_name))) == 1,
msg="SLIC algorithm failed")
def step_execute(self):
"""
Return
------
lambda
the function to execute as a lambda function. The returned object
must be a lambda function.
"""
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.ServiceConfigFile import iota2_parameters
check_inputs = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'check_inputs')
tiles = SCF.serviceConfigFile(self.cfg).getParam('chain',
'listTile').split(" ")
output_path = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
running_parameters = iota2_parameters(
SCF.serviceConfigFile(self.cfg).pathConf)
sensors_parameters = running_parameters.get_sensors_parameters(
tiles[0])
sensor_tile_container = sensors_container(tiles[0], None, output_path,
**sensors_parameters)
sensor_path = sensor_tile_container.get_enabled_sensors_path()[0]
step_function = lambda x: IOTA2_dir.generate_directories(
x, check_inputs,
SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'merge_final_classifications'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'groundTruth'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'regionPath'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'dataField'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'regionField'),
int(
SCF.serviceConfigFile(self.cfg).getParam('GlobChain', 'proj').
split(":")[-1]), sensor_path, tiles)
return step_function
def validity(tile_name,
config_path,
output_path,
maskOut_name,
view_threshold,
workingDirectory=None,
RAM=128):
"""
function dedicated to compute validity raster/vector by tile
Parameters
----------
tile_name [string]
tile's name
config_path [string]
absolute path to the configuration file
maskOut_name [string]
output vector mask's name
view_threshold [int]
threshold
working_directory [string]
absolute path to a working directory
RAM [int]
pipeline's size (Mo)
"""
import os
import shutil
from iota2.Common.ServiceConfigFile import iota2_parameters
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.Utils import run
from iota2.Common.FileUtils import erodeShapeFile
from iota2.Common.FileUtils import removeShape
from iota2.Common.FileUtils import ensure_dir
features_dir = os.path.join(output_path, "features", tile_name)
validity_name = "nbView.tif"
validity_out = os.path.join(features_dir, validity_name)
validity_processing = validity_out
if workingDirectory:
ensure_dir(os.path.join(workingDirectory, tile_name))
validity_processing = os.path.join(workingDirectory, tile_name,
validity_name)
running_parameters = iota2_parameters(config_path)
sensors_parameters = running_parameters.get_sensors_parameters(tile_name)
remote_sensor_container = sensors_container(tile_name, workingDirectory,
output_path,
**sensors_parameters)
sensors_time_series_masks = remote_sensor_container.get_sensors_time_series_masks(
available_ram=RAM)
sensors_masks_size = []
sensors_masks = []
for sensor_name, (time_series_masks, time_series_dep,
nb_bands) in sensors_time_series_masks:
if sensor_name.lower() == "sentinel1":
for _, time_series_masks_app in list(time_series_masks.items()):
time_series_masks_app.Execute()
sensors_masks.append(time_series_masks_app)
else:
time_series_masks.Execute()
sensors_masks.append(time_series_masks)
sensors_masks_size.append(nb_bands)
total_dates = sum(sensors_masks_size)
merge_masks = CreateConcatenateImagesApplication({
"il": sensors_masks,
"ram": str(RAM)
})
merge_masks.Execute()
validity_app = CreateBandMathApplication({
"il":
merge_masks,
"exp":
"{}-({})".format(
total_dates,
"+".join(["im1b{}".format(i + 1) for i in range(total_dates)])),
"ram":
str(0.7 * RAM),
"pixType":
"uint8" if total_dates < 255 else "uint16",
"out":
validity_processing
})
if not os.path.exists(os.path.join(features_dir, validity_name)):
validity_app.ExecuteAndWriteOutput()
if workingDirectory:
shutil.copy(validity_processing,
os.path.join(features_dir, validity_name))
threshold_raster_out = os.path.join(features_dir,
maskOut_name.replace(".shp", ".tif"))
threshold_vector_out_tmp = os.path.join(
features_dir, maskOut_name.replace(".shp", "_TMP.shp"))
threshold_vector_out = os.path.join(features_dir, maskOut_name)
input_threshold = validity_processing if os.path.exists(
validity_processing) else validity_out
threshold_raster = CreateBandMathApplication({
"il":
input_threshold,
"exp":
"im1b1>={}?1:0".format(view_threshold),
"ram":
str(0.7 * RAM),
"pixType":
"uint8",
"out":
threshold_raster_out
})
threshold_raster.ExecuteAndWriteOutput()
cmd_poly = f"gdal_polygonize.py -mask {threshold_raster_out} {threshold_raster_out} -f \"ESRI Shapefile\" {threshold_vector_out_tmp} {os.path.splitext(os.path.basename(threshold_vector_out_tmp))[0]} cloud"
run(cmd_poly)
erodeShapeFile(threshold_vector_out_tmp, threshold_vector_out, 0.1)
os.remove(threshold_raster_out)
removeShape(threshold_vector_out_tmp.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
def test_sensors_container(self):
"""
Test if the sensors_container class enable all required sensors
"""
from iota2.Sensors.Sensors_container import sensors_container
tile_name = "T31TCJ"
args = {
"Sentinel_2": {
"tile_name": tile_name,
"target_proj": 2154,
"all_tiles": tile_name,
"image_directory": self.test_working_directory,
"write_dates_stack": False,
"extract_bands_flag": False,
"output_target_dir": "",
"keep_bands": True,
"i2_output_path": self.test_working_directory,
"temporal_res": 10,
"auto_date_flag": True,
"date_interp_min_user": "",
"date_interp_max_user": "",
"write_outputs_flag": False,
"features": ["NDVI", "NDWI", "Brightness"],
"enable_gapfilling": True,
"hand_features_flag": False,
"hand_features": "",
"copy_input": True,
"rel_refl": False,
"keep_dupl": True,
"vhr_path": "none",
"acorfeat": False
},
"Sentinel_2_S2C": {
"tile_name": "T31TCJ",
"target_proj": 2154,
"all_tiles": "T31TCJ",
"image_directory": self.test_working_directory,
"write_dates_stack": False,
"extract_bands_flag": False,
"output_target_dir": None,
"keep_bands": True,
"i2_output_path": self.test_working_directory,
"temporal_res": 10,
"auto_date_flag": True,
"date_interp_min_user": "",
"date_interp_max_user": "",
"write_outputs_flag": False,
"features": ["NDVI", "NDWI", "Brightness"],
"enable_gapfilling": True,
"hand_features_flag": False,
"hand_features": "",
"copy_input": True,
"rel_refl": False,
"keep_dupl": True,
"vhr_path": "none",
"acorfeat": False
},
"Landsat8": {
"tile_name": "T31TCJ",
"target_proj": 2154,
"all_tiles": "T31TCJ",
"image_directory": self.test_working_directory,
"write_dates_stack": False,
"extract_bands_flag": False,
"output_target_dir": None,
"keep_bands": True,
"i2_output_path": self.test_working_directory,
"temporal_res": 10,
"auto_date_flag": True,
"date_interp_min_user": "",
"date_interp_max_user": "",
"write_outputs_flag": False,
"features": ["NDVI", "NDWI", "Brightness"],
"enable_gapfilling": True,
"hand_features_flag": False,
"hand_features": "",
"copy_input": True,
"rel_refl": False,
"keep_dupl": True,
"vhr_path": "none",
"acorfeat": False
},
"Sentinel_2_L3A": {
"tile_name": "T31TCJ",
"target_proj": 2154,
"all_tiles": "T31TCJ",
"image_directory": self.test_working_directory,
"write_dates_stack": False,
"extract_bands_flag": False,
"output_target_dir": None,
"keep_bands": True,
"i2_output_path": self.test_working_directory,
"temporal_res": 10,
"auto_date_flag": True,
"date_interp_min_user": "",
"date_interp_max_user": "",
"write_outputs_flag": False,
"features": ["NDVI", "NDWI", "Brightness"],
"enable_gapfilling": True,
"hand_features_flag": False,
"hand_features": "",
"copy_input": True,
"rel_refl": False,
"keep_dupl": True,
"vhr_path": "none",
"acorfeat": False
},
"Landsat5_old": {
"tile_name": "T31TCJ",
"target_proj": 2154,
"all_tiles": "T31TCJ",
"image_directory": self.test_working_directory,
"write_dates_stack": False,
"extract_bands_flag": False,
"output_target_dir": None,
"keep_bands": True,
"i2_output_path": self.test_working_directory,
"temporal_res": 10,
"auto_date_flag": True,
"date_interp_min_user": "",
"date_interp_max_user": "",
"write_outputs_flag": False,
"features": ["NDVI", "NDWI", "Brightness"],
"enable_gapfilling": True,
"hand_features_flag": False,
"hand_features": "",
"copy_input": True,
"rel_refl": False,
"keep_dupl": True,
"vhr_path": "none",
"acorfeat": False
},
"Landsat8_old": {
"tile_name": "T31TCJ",
"target_proj": 2154,
"all_tiles": "T31TCJ",
"image_directory": self.test_working_directory,
"write_dates_stack": False,
"extract_bands_flag": False,
"output_target_dir": None,
"keep_bands": True,
"i2_output_path": self.test_working_directory,
"temporal_res": 10,
"auto_date_flag": True,
"date_interp_min_user": "",
"date_interp_max_user": "",
"write_outputs_flag": False,
"features": ["NDVI", "NDWI", "Brightness"],
"enable_gapfilling": True,
"hand_features_flag": False,
"hand_features": "",
"copy_input": True,
"rel_refl": False,
"keep_dupl": True,
"vhr_path": "none",
"acorfeat": False
}
}
sensors = sensors_container(tile_name, self.test_working_directory,
self.test_working_directory, **args)
enabled_sensors = sensors.get_enabled_sensors()
sensors_name = [sensor.name for sensor in enabled_sensors]
self.assertTrue(sensors_name == self.expected_sensors)
def generateFeatures(pathWd: str,
tile: str,
sar_optical_post_fusion: bool,
output_path: str,
sensors_parameters: sensors_params,
mode: Optional[str] = "usually"):
"""
usage : Function use to compute features according to a configuration file
Parameters
----------
pathWd : str
path to a working directory
tile : str
tile's name
sar_optical_post_fusion : bool
flag use to remove SAR data from features
mode : str
'usually' / 'SAR' used to get only sar features
"""
from iota2.Common.OtbAppBank import getInputParameterOutput
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
LOGGER.info(f"prepare features for tile : {tile}")
sensor_tile_container = sensors_container(tile, pathWd, output_path,
**sensors_parameters)
feat_labels = []
dep = []
feat_app = []
if mode == "usually" and sar_optical_post_fusion is False:
sensors_features = sensor_tile_container.get_sensors_features(
available_ram=1000)
for _, ((sensor_features, sensor_features_dep),
features_labels) in sensors_features:
sensor_features.Execute()
feat_app.append(sensor_features)
dep.append(sensor_features_dep)
feat_labels = feat_labels + features_labels
elif mode == "usually" and sar_optical_post_fusion is True:
sensor_tile_container.remove_sensor("Sentinel1")
sensors_features = sensor_tile_container.get_sensors_features(
available_ram=1000)
for _, ((sensor_features, sensor_features_dep),
features_labels) in sensors_features:
sensor_features.Execute()
feat_app.append(sensor_features)
dep.append(sensor_features_dep)
feat_labels = feat_labels + features_labels
elif mode == "SAR":
sensor = sensor_tile_container.get_sensor("Sentinel1")
(sensor_features,
sensor_features_dep), feat_labels = sensor.get_features(ram=1000)
sensor_features.Execute()
feat_app.append(sensor_features)
dep.append(sensor_features_dep)
dep.append(feat_app)
features_name = "{}_Features.tif".format(tile)
features_dir = os.path.join(output_path, "features", tile, "tmp")
features_raster = os.path.join(features_dir, features_name)
if len(feat_app) > 1:
all_features = CreateConcatenateImagesApplication({
"il":
feat_app,
"out":
features_raster
})
else:
all_features = sensor_features
output_param_name = getInputParameterOutput(sensor_features)
all_features.SetParameterString(output_param_name, features_raster)
return all_features, feat_labels, dep
def test_train_and_classify(self):
"""test autoContext training
"""
import re
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.FileUtils import getFieldElement
from iota2.Segmentation import segmentation
from iota2.Learning.trainAutoContext import train_autoContext
from iota2.Classification.ImageClassifier import autoContext_launch_classif
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.ServiceConfigFile import iota2_parameters
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Tests.UnitTests.TestsUtils import test_raster_unique_value
# config file
config_path_test = os.path.join(self.test_working_directory,
"Config_TEST.cfg")
test_path = self.generate_cfg_file(self.config_test, config_path_test)
cfg = SCF.serviceConfigFile(config_path_test)
IOTA2Directory.generate_directories(test_path, check_inputs=False)
autocontext_working_dir = os.path.join(self.test_working_directory,
"autoContext_tmp")
slic_working_dir = os.path.join(self.test_working_directory,
"autoContext_tmp")
iota2_dico = SCF.iota2_parameters(
config_path_test).get_sensors_parameters(self.tile_name)
sensors = sensors_container(self.tile_name, None,
self.test_working_directory, **iota2_dico)
sensors.sensors_preprocess()
running_parameters = iota2_parameters(config_path_test)
sensors_parameters = running_parameters.get_sensors_parameters(
"T31TCJ")
segmentation.slicSegmentation(self.tile_name,
test_path,
sensors_parameters,
ram=128,
working_dir=slic_working_dir,
force_spw=1)
slic_seg = FileSearch_AND(
os.path.join(test_path, "features", self.tile_name, "tmp"), True,
"SLIC_{}".format(self.tile_name))[0]
train_auto_data_ref, superpix_data = self.prepare_autoContext_data_ref(
slic_seg, config_path_test)
parameter_dict = {
"model_name": "1",
"seed": "0",
"list_learning_samples": [train_auto_data_ref],
"list_superPixel_samples": [superpix_data],
"list_tiles": ["T31TCJ"],
"list_slic": [slic_seg]
}
# launch tests
# training
e = None
try:
train_autoContext(parameter_dict,
data_field="code",
output_path=test_path,
sensors_parameters=sensors_parameters,
superpix_data_field="superpix",
iterations=3,
RAM=128,
WORKING_DIR=autocontext_working_dir)
except Exception as e:
print(e)
# Asserts training
self.assertTrue(e is None, msg="train_autoContext failed")
models = FileSearch_AND(os.path.join(test_path, "model"), True,
"model_it_", ".rf")
self.assertTrue(len(models) == 4)
# classification
tile_raster = FileSearch_AND(self.fake_data_dir, True,
"BINARY_MASK.tif")[0]
tile_mask = os.path.join(autocontext_working_dir,
"{}_tile_mask.tif".format(self.tile_name))
CreateBandMathApplication({
"il": [tile_raster],
"out": tile_mask,
"exp": "1"
}).ExecuteAndWriteOutput()
labels = getFieldElement(train_auto_data_ref,
driverName="SQLite",
field="code",
mode="unique",
elemType="str")
parameters_dict = {
"model_name":
"1",
"seed_num":
0,
"tile":
self.tile_name,
"tile_segmentation":
slic_seg,
"tile_mask":
tile_mask,
"labels_list":
labels,
"model_list":
sorted(models,
key=lambda x: int(re.findall("\d", os.path.basename(x))[0]))
}
autoContext_launch_classif(parameters_dict,
cfg.getParam("argTrain", "classifier"),
"T31TCJ",
cfg.getParam("argClassification",
"enable_probability_map"),
cfg.getParam("dimRed", "dimRed"),
"code",
False,
False,
iota2_run_dir=test_path,
sar_optical_post_fusion=False,
nomenclature_path=cfg.getParam(
"chain", "nomenclaturePath"),
sensors_parameters=sensors_parameters,
RAM=128,
WORKING_DIR=autocontext_working_dir)
# Asserts classifications
classif = FileSearch_AND(os.path.join(test_path, "classif"), True,
"Classif_T31TCJ_model_1_seed_0.tif")[0]
confidence = FileSearch_AND(os.path.join(test_path, "classif"), True,
"T31TCJ_model_1_confidence_seed_0.tif")[0]
classif_unique_0 = test_raster_unique_value(classif, 0)
confidence_unique_0 = test_raster_unique_value(confidence, 0)
self.assertTrue(
classif_unique_0 == False,
msg=("AutoContext Classifications failed : classification contains"
" only 0 values"))
self.assertTrue(
confidence_unique_0 == False,
msg=("AutoContext Classifications failed : confidence contains "
"only 0 values"))