def test_config_dictionaries(self):
"""check iota2_parameters class
"""
from iota2.Common.ServiceConfigFile import iota2_parameters
sensors_parameters = iota2_parameters(
self.config_test).get_sensors_parameters("T31TCJ")
self.assertTrue(len(sensors_parameters.keys()) == 3)
def preprocess(tile_name,
config_path,
output_path,
working_directory=None,
RAM=128):
"""
preprocessing input rasters data by tile
Parameters
----------
tile_name [string]
tile's name
config_path [string]
absolute path to the configuration file
output_path : str
iota2 output path
working_directory [string]
absolute path to a working directory
RAM [int]
pipeline's size (Mo)
"""
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.ServiceConfigFile import iota2_parameters
running_parameters = iota2_parameters(config_path)
sensors_parameters = running_parameters.get_sensors_parameters(tile_name)
remote_sensor_container = sensors_container(tile_name, working_directory,
output_path,
**sensors_parameters)
remote_sensor_container.sensors_preprocess(available_ram=RAM)
def step_inputs(self):
"""
Return
------
the return could be and iterable or a callable
"""
from iota2.Common import FileUtils as fut
from iota2.Classification.ImageClassifier import autoContext_classification_param
from iota2.Common.ServiceConfigFile import iota2_parameters
if self.enable_autoContext is False and self.use_scikitlearn is False:
parameters = fut.parseClassifCmd(
os.path.join(self.output_path, "cmd", "cla", "class.txt"))
elif self.enable_autoContext is True:
parameters = autoContext_classification_param(
self.output_path, self.data_field)
elif self.use_scikitlearn:
parameters = fut.parseClassifCmd(
os.path.join(self.output_path, "cmd", "cla", "class.txt"))
running_parameters = iota2_parameters(self.cfg)
parameters = [{
"mask":
param[1],
"model":
param[2],
"stat":
param[3],
"out_classif":
param[4],
"out_confidence":
param[5],
"out_proba":
None,
"working_dir":
param[6],
"tile_name":
param[8],
"sar_optical_post_fusion":
SCF.serviceConfigFile(self.cfg).getParam(
'argTrain', 'dempster_shafer_SAR_Opt_fusion'),
"output_path":
SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath'),
"sensors_parameters":
running_parameters.get_sensors_parameters(tile_name=param[8]),
"pixel_type":
param[17],
"number_of_chunks":
self.scikit_tile_split,
"targeted_chunk":
target_chunk,
"ram":
param[19]
} for param in parameters
for target_chunk in range(self.scikit_tile_split)]
return parameters
def step_execute(self):
"""
Return
------
lambda
the function to execute as a lambda function. The returned object
must be a lambda function.
"""
from iota2.MPI import launch_tasks as tLauncher
from iota2.Learning import TrainSkLearn
from iota2.Learning.TrainSkLearn import cast_config_cv_parameters
from iota2.Learning.trainAutoContext import train_autoContext
from iota2.Common.ServiceConfigFile import iota2_parameters
if self.use_scikitlearn is True and self.enable_autoContext is False:
model_parameters = SCF.serviceConfigFile(
self.cfg).getSection("scikit_models_parameters")
sk_model = model_parameters['model_type']
del model_parameters['model_type']
del model_parameters['standardization']
del model_parameters['cross_validation_grouped']
del model_parameters['cross_validation_folds']
del model_parameters['cross_validation_parameters']
cv_params = SCF.serviceConfigFile(self.cfg).getParam(
"scikit_models_parameters", "cross_validation_parameters")
cv_params = cast_config_cv_parameters(cv_params)
step_function = lambda x: TrainSkLearn.sk_learn(
x["learning_file"], x["feat_labels"], x[
"model_path"], self.data_field, sk_model,
SCF.serviceConfigFile(self.cfg).getParam(
"scikit_models_parameters", "standardization"), cv_params,
SCF.serviceConfigFile(self.cfg).getParam(
"scikit_models_parameters", "cross_validation_grouped"),
SCF.serviceConfigFile(self.cfg).getParam(
"scikit_models_parameters", "cross_validation_folds"), self
.available_ram, **model_parameters)
elif self.enable_autoContext is True:
running_parameters = iota2_parameters(self.cfg)
step_function = lambda x: train_autoContext(
x,
SCF.serviceConfigFile(self.cfg).getParam("chain", "dataField"
), self.output_path,
running_parameters.get_sensors_parameters(self.tiles[
0]), self.superpix_data_field, self.autoContext_iterations,
self.RAM, self.workingDirectory)
else:
bashLauncherFunction = tLauncher.launchBashCmd
step_function = lambda x: bashLauncherFunction(x)
return step_function
def step_execute(self):
"""
Return
------
lambda
the function to execute as a lambda function. The returned object
must be a lambda function.
"""
from iota2.Classification.ImageClassifier import autoContext_launch_classif
from iota2.Classification import ImageClassifier as imageClassifier
from iota2.Common.ServiceConfigFile import iota2_parameters
from iota2.Classification import skClassifier
from iota2.MPI import launch_tasks as tLauncher
if self.enable_autoContext is False and self.use_scikitlearn is False:
launch_py_cmd = tLauncher.launchPythonCmd
step_function = lambda x: launch_py_cmd(
imageClassifier.launchClassification, *x)
elif self.enable_autoContext is True and self.use_scikitlearn is False:
running_parameters = iota2_parameters(self.cfg)
step_function = lambda x: autoContext_launch_classif(
x,
SCF.serviceConfigFile(self.cfg).getParam(
'argTrain', 'classifier'), x["tile"],
SCF.serviceConfigFile(self.cfg).getParam(
'argClassification', 'enable_probability_map'),
SCF.serviceConfigFile(self.cfg).getParam('dimRed', 'dimRed'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'dataField'),
SCF.serviceConfigFile(self.cfg).getParam(
'GlobChain', 'writeOutputs'),
SCF.serviceConfigFile(self.cfg).getParam(
'dimRed', 'reductionMode'),
SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath'),
SCF.serviceConfigFile(self.cfg).getParam(
'argTrain', 'dempster_shafer_SAR_Opt_fusion'),
SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'nomenclaturePath'),
running_parameters.get_sensors_parameters(x["tile"]), self.RAM,
self.workingDirectory)
elif self.enable_autoContext is False and self.use_scikitlearn is True:
step_function = lambda x: skClassifier.predict(**x)
return step_function
def step_execute(self):
"""
Return
------
lambda
the function to execute as a lambda function. The returned object
must be a lambda function.
"""
from iota2.Segmentation import segmentation
from iota2.Common.ServiceConfigFile import iota2_parameters
running_parameters = iota2_parameters(self.cfg)
step_function = lambda x: segmentation.slicSegmentation(
x,
SCF.serviceConfigFile(self.cfg).getParam('chain', 'outputPath'),
running_parameters.get_sensors_parameters(x), self.RAM, self.
workingDirectory)
return step_function
def step_execute(self):
"""
Return
------
lambda
the function to execute as a lambda function. The returned object
must be a lambda function.
"""
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.ServiceConfigFile import iota2_parameters
check_inputs = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'check_inputs')
tiles = SCF.serviceConfigFile(self.cfg).getParam('chain',
'listTile').split(" ")
output_path = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
running_parameters = iota2_parameters(
SCF.serviceConfigFile(self.cfg).pathConf)
sensors_parameters = running_parameters.get_sensors_parameters(
tiles[0])
sensor_tile_container = sensors_container(tiles[0], None, output_path,
**sensors_parameters)
sensor_path = sensor_tile_container.get_enabled_sensors_path()[0]
step_function = lambda x: IOTA2_dir.generate_directories(
x, check_inputs,
SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'merge_final_classifications'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'groundTruth'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'regionPath'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'dataField'),
SCF.serviceConfigFile(self.cfg).getParam('chain', 'regionField'),
int(
SCF.serviceConfigFile(self.cfg).getParam('GlobChain', 'proj').
split(":")[-1]), sensor_path, tiles)
return step_function
def validity(tile_name,
config_path,
output_path,
maskOut_name,
view_threshold,
workingDirectory=None,
RAM=128):
"""
function dedicated to compute validity raster/vector by tile
Parameters
----------
tile_name [string]
tile's name
config_path [string]
absolute path to the configuration file
maskOut_name [string]
output vector mask's name
view_threshold [int]
threshold
working_directory [string]
absolute path to a working directory
RAM [int]
pipeline's size (Mo)
"""
import os
import shutil
from iota2.Common.ServiceConfigFile import iota2_parameters
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.Utils import run
from iota2.Common.FileUtils import erodeShapeFile
from iota2.Common.FileUtils import removeShape
from iota2.Common.FileUtils import ensure_dir
features_dir = os.path.join(output_path, "features", tile_name)
validity_name = "nbView.tif"
validity_out = os.path.join(features_dir, validity_name)
validity_processing = validity_out
if workingDirectory:
ensure_dir(os.path.join(workingDirectory, tile_name))
validity_processing = os.path.join(workingDirectory, tile_name,
validity_name)
running_parameters = iota2_parameters(config_path)
sensors_parameters = running_parameters.get_sensors_parameters(tile_name)
remote_sensor_container = sensors_container(tile_name, workingDirectory,
output_path,
**sensors_parameters)
sensors_time_series_masks = remote_sensor_container.get_sensors_time_series_masks(
available_ram=RAM)
sensors_masks_size = []
sensors_masks = []
for sensor_name, (time_series_masks, time_series_dep,
nb_bands) in sensors_time_series_masks:
if sensor_name.lower() == "sentinel1":
for _, time_series_masks_app in list(time_series_masks.items()):
time_series_masks_app.Execute()
sensors_masks.append(time_series_masks_app)
else:
time_series_masks.Execute()
sensors_masks.append(time_series_masks)
sensors_masks_size.append(nb_bands)
total_dates = sum(sensors_masks_size)
merge_masks = CreateConcatenateImagesApplication({
"il": sensors_masks,
"ram": str(RAM)
})
merge_masks.Execute()
validity_app = CreateBandMathApplication({
"il":
merge_masks,
"exp":
"{}-({})".format(
total_dates,
"+".join(["im1b{}".format(i + 1) for i in range(total_dates)])),
"ram":
str(0.7 * RAM),
"pixType":
"uint8" if total_dates < 255 else "uint16",
"out":
validity_processing
})
if not os.path.exists(os.path.join(features_dir, validity_name)):
validity_app.ExecuteAndWriteOutput()
if workingDirectory:
shutil.copy(validity_processing,
os.path.join(features_dir, validity_name))
threshold_raster_out = os.path.join(features_dir,
maskOut_name.replace(".shp", ".tif"))
threshold_vector_out_tmp = os.path.join(
features_dir, maskOut_name.replace(".shp", "_TMP.shp"))
threshold_vector_out = os.path.join(features_dir, maskOut_name)
input_threshold = validity_processing if os.path.exists(
validity_processing) else validity_out
threshold_raster = CreateBandMathApplication({
"il":
input_threshold,
"exp":
"im1b1>={}?1:0".format(view_threshold),
"ram":
str(0.7 * RAM),
"pixType":
"uint8",
"out":
threshold_raster_out
})
threshold_raster.ExecuteAndWriteOutput()
cmd_poly = f"gdal_polygonize.py -mask {threshold_raster_out} {threshold_raster_out} -f \"ESRI Shapefile\" {threshold_vector_out_tmp} {os.path.splitext(os.path.basename(threshold_vector_out_tmp))[0]} cloud"
run(cmd_poly)
erodeShapeFile(threshold_vector_out_tmp, threshold_vector_out, 0.1)
os.remove(threshold_raster_out)
removeShape(threshold_vector_out_tmp.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
PARSER = argparse.ArgumentParser(description=DESCRIPTION)
PARSER.add_argument("-config",
dest="config",
help="configuration file path",
default=None,
required=True)
PARSER.add_argument("-output",
dest="output",
help="output raster",
default=None,
required=True)
PARSER.add_argument("-tile",
dest="tile_name",
help="tile's name",
default=None,
required=True)
PARSER.add_argument("-working_dir",
dest="working_dir",
help="tile's name",
default=None,
required=True)
ARGS = PARSER.parse_args()
# load configuration file
CFG = SCF.serviceConfigFile(ARGS.pathConf)
PARAMS = iota2_parameters(ARGS.pathConf)
SEN_PARAM = PARAMS.get_sensors_parameters(ARGS.tile)
OUTPUT_PATH = CFG.getParam("chain", "outputPath")
compute_features(OUTPUT_PATH, SEN_PARAM, ARGS.output, ARGS.tile_name,
ARGS.working_dir)
def test_train_and_classify(self):
"""test autoContext training
"""
import re
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.FileUtils import getFieldElement
from iota2.Segmentation import segmentation
from iota2.Learning.trainAutoContext import train_autoContext
from iota2.Classification.ImageClassifier import autoContext_launch_classif
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.ServiceConfigFile import iota2_parameters
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Tests.UnitTests.TestsUtils import test_raster_unique_value
# config file
config_path_test = os.path.join(self.test_working_directory,
"Config_TEST.cfg")
test_path = self.generate_cfg_file(self.config_test, config_path_test)
cfg = SCF.serviceConfigFile(config_path_test)
IOTA2Directory.generate_directories(test_path, check_inputs=False)
autocontext_working_dir = os.path.join(self.test_working_directory,
"autoContext_tmp")
slic_working_dir = os.path.join(self.test_working_directory,
"autoContext_tmp")
iota2_dico = SCF.iota2_parameters(
config_path_test).get_sensors_parameters(self.tile_name)
sensors = sensors_container(self.tile_name, None,
self.test_working_directory, **iota2_dico)
sensors.sensors_preprocess()
running_parameters = iota2_parameters(config_path_test)
sensors_parameters = running_parameters.get_sensors_parameters(
"T31TCJ")
segmentation.slicSegmentation(self.tile_name,
test_path,
sensors_parameters,
ram=128,
working_dir=slic_working_dir,
force_spw=1)
slic_seg = FileSearch_AND(
os.path.join(test_path, "features", self.tile_name, "tmp"), True,
"SLIC_{}".format(self.tile_name))[0]
train_auto_data_ref, superpix_data = self.prepare_autoContext_data_ref(
slic_seg, config_path_test)
parameter_dict = {
"model_name": "1",
"seed": "0",
"list_learning_samples": [train_auto_data_ref],
"list_superPixel_samples": [superpix_data],
"list_tiles": ["T31TCJ"],
"list_slic": [slic_seg]
}
# launch tests
# training
e = None
try:
train_autoContext(parameter_dict,
data_field="code",
output_path=test_path,
sensors_parameters=sensors_parameters,
superpix_data_field="superpix",
iterations=3,
RAM=128,
WORKING_DIR=autocontext_working_dir)
except Exception as e:
print(e)
# Asserts training
self.assertTrue(e is None, msg="train_autoContext failed")
models = FileSearch_AND(os.path.join(test_path, "model"), True,
"model_it_", ".rf")
self.assertTrue(len(models) == 4)
# classification
tile_raster = FileSearch_AND(self.fake_data_dir, True,
"BINARY_MASK.tif")[0]
tile_mask = os.path.join(autocontext_working_dir,
"{}_tile_mask.tif".format(self.tile_name))
CreateBandMathApplication({
"il": [tile_raster],
"out": tile_mask,
"exp": "1"
}).ExecuteAndWriteOutput()
labels = getFieldElement(train_auto_data_ref,
driverName="SQLite",
field="code",
mode="unique",
elemType="str")
parameters_dict = {
"model_name":
"1",
"seed_num":
0,
"tile":
self.tile_name,
"tile_segmentation":
slic_seg,
"tile_mask":
tile_mask,
"labels_list":
labels,
"model_list":
sorted(models,
key=lambda x: int(re.findall("\d", os.path.basename(x))[0]))
}
autoContext_launch_classif(parameters_dict,
cfg.getParam("argTrain", "classifier"),
"T31TCJ",
cfg.getParam("argClassification",
"enable_probability_map"),
cfg.getParam("dimRed", "dimRed"),
"code",
False,
False,
iota2_run_dir=test_path,
sar_optical_post_fusion=False,
nomenclature_path=cfg.getParam(
"chain", "nomenclaturePath"),
sensors_parameters=sensors_parameters,
RAM=128,
WORKING_DIR=autocontext_working_dir)
# Asserts classifications
classif = FileSearch_AND(os.path.join(test_path, "classif"), True,
"Classif_T31TCJ_model_1_seed_0.tif")[0]
confidence = FileSearch_AND(os.path.join(test_path, "classif"), True,
"T31TCJ_model_1_confidence_seed_0.tif")[0]
classif_unique_0 = test_raster_unique_value(classif, 0)
confidence_unique_0 = test_raster_unique_value(confidence, 0)
self.assertTrue(
classif_unique_0 == False,
msg=("AutoContext Classifications failed : classification contains"
" only 0 values"))
self.assertTrue(
confidence_unique_0 == False,
msg=("AutoContext Classifications failed : confidence contains "
"only 0 values"))
def prepare_autoContext_data_ref(self, slic_seg, config_path_test):
"""
"""
from iota2.Common.ServiceConfigFile import iota2_parameters
from iota2.VectorTools.AddField import addField
from iota2.Sampling.SuperPixelsSelection import merge_ref_super_pix
from iota2.Sampling.SplitSamples import split_superpixels_and_reference
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.OtbAppBank import CreateSampleSelectionApplication
from iota2.Common.OtbAppBank import CreatePolygonClassStatisticsApplication
from iota2.Common.OtbAppBank import CreateSampleExtractionApplication
from iota2.Common.GenerateFeatures import generateFeatures
from iota2.Common import ServiceConfigFile as SCF
raster_ref = FileSearch_AND(self.fake_data_dir, True, ".tif")[0]
CreatePolygonClassStatisticsApplication({
"in":
raster_ref,
"vec":
self.ref_data,
"field":
"CODE",
"out":
os.path.join(self.test_working_directory, "stats.xml")
}).ExecuteAndWriteOutput()
ref_sampled = os.path.join(
self.test_working_directory,
"T31TCJ_samples_region_1_seed_0_selection.sqlite")
CreateSampleSelectionApplication({
"in":
raster_ref,
"vec":
self.ref_data,
"field":
"CODE",
"strategy":
"all",
"instats":
os.path.join(self.test_working_directory, "stats.xml"),
"out":
ref_sampled
}).ExecuteAndWriteOutput()
cfg = SCF.serviceConfigFile(config_path_test)
params = iota2_parameters(config_path_test)
sensors_parameters = params.get_sensors_parameters("T31TCJ")
features, feat_labels, dep = generateFeatures(
None,
"T31TCJ",
sar_optical_post_fusion=False,
output_path=cfg.getParam("chain", "outputPath"),
sensors_parameters=sensors_parameters)
extraction = CreateSampleExtractionApplication({
"in": features,
"vec": ref_sampled,
"field": "code",
"outfield.list.names": feat_labels,
"outfield": "list"
})
extraction.ExecuteAndWriteOutput()
addField(ref_sampled,
"newregion",
valueField="1",
valueType=str,
driver_name="SQLite")
os.remove(os.path.join(self.test_working_directory, "stats.xml"))
merge_ref_super_pix(
{
"selection_samples": ref_sampled,
"SLIC": slic_seg
}, "code", "superpix", "is_superpix", "newregion")
learning_vector = os.path.join(
self.test_working_directory,
"T31TCJ_region_1_seed0_Samples_learn.sqlite")
shutil.move(ref_sampled, learning_vector)
ref, superpix = split_superpixels_and_reference(
learning_vector, superpix_column="is_superpix")
return learning_vector, superpix