def compute_features(output_path: str,
sensors_parameters: sensors_params,
output_raster: str,
tile_name: str,
working_dir: str,
function=smart_scientific_function) -> None:
"""Use a python function to generate features through the use of numpy arrays
Parameters
----------
output_path : str
output directory
sensors_parameters : dict
sensors parameters description
output_raster : str
output raster path
tile_name : str
tile to compute
function : function
function to apply on iota²' stack
"""
IOTA2Directory.generate_directories(output_path, check_inputs=False)
feat_stack, feat_labels, _ = generateFeatures(
working_dir,
tile_name,
sar_optical_post_fusion=False,
output_path=output_path,
sensors_parameters=sensors_parameters)
# Then compute new features
function = partial(function, increment=1)
feat_stack_array, feat_labels = rasterUtils.apply_function(
feat_stack,
feat_labels,
working_dir,
function,
output_raster,
chunck_size_x=10,
chunck_size_y=10,
ram=128)
def slicSegmentation(tile_name: str,
output_path: str,
sensors_parameters: sensors_params,
ram: Optional[int] = 128,
working_dir: Optional[Union[str, None]] = None,
force_spw: Optional[Union[int, None]] = None,
logger=LOGGER):
"""generate segmentation using SLIC algorithm
Parameters
----------
tile_name : string
tile's name
output_path : string
iota2 output path
sensors_parameters : dict
sensors parameters description
ram : int
available ram
working_dir : string
directory to store temporary data
force_spw : int
force segments' spatial width
logger : logging
root logger
"""
import math
import shutil
from iota2.Common.GenerateFeatures import generateFeatures
from iota2.Common.OtbAppBank import CreateSLICApplication
from iota2.Common.OtbAppBank import getInputParameterOutput
from iota2.Common.FileUtils import ensure_dir
SLIC_NAME = "SLIC_{}.tif".format(tile_name)
all_features, feat_labels, dep = generateFeatures(
working_dir,
tile_name,
sar_optical_post_fusion=False,
output_path=output_path,
sensors_parameters=sensors_parameters,
mode="usually")
all_features.Execute()
spx, _ = all_features.GetImageSpacing(
getInputParameterOutput(all_features))
tmp_dir = working_dir
if working_dir is None:
tmp_dir = os.path.join(output_path, "features", tile_name, "tmp",
"SLIC_TMPDIR")
else:
tmp_dir = os.path.join(working_dir, tile_name)
ensure_dir(tmp_dir)
slic_seg_path = os.path.join(output_path, "features", tile_name, "tmp",
SLIC_NAME)
features_ram_estimation = all_features.PropagateRequestedRegion(
key="out", region=all_features.GetImageRequestedRegion("out"))
# increase estimation...
features_ram_estimation = features_ram_estimation * 1.5
xy_tiles = math.ceil(
math.sqrt(float(features_ram_estimation) / (float(ram) * 1024**2)))
slic_parameters = {
"in": all_features,
"tmpdir": tmp_dir,
"spw": force_spw if force_spw else int(spx),
"tiling": "manual",
"tiling.manual.ny": int(xy_tiles),
"tiling.manual.nx": int(xy_tiles),
"out": slic_seg_path
}
slic_seg = CreateSLICApplication(slic_parameters)
if not os.path.exists(slic_seg_path):
logger.info("Processing SLIC segmentation : {}\n\t\t\
with parameters : {}".format(tile_name, slic_parameters))
slic_seg.ExecuteAndWriteOutput()
if working_dir is None:
shutil.rmtree(tmp_dir)
def train_autoContext(parameter_dict: Param,
data_field: str,
output_path: str,
sensors_parameters: SENSORS_PARAMS,
superpix_data_field: Optional[str] = "superpix",
iterations: Optional[int] = 3,
RAM: Optional[int] = 128,
WORKING_DIR: Optional[str] = None,
logger: Optional[logging.Logger] = LOGGER):
"""launch autoContext training
Parameters
----------
parameter_dict : dict
dictionnary containing autoContext's input parameters
{"model_name": string,
"seed": integer,
"list_selection": list,
"list_tiles": list,
"list_slic": list}
config_path : string
path to the configuration file
iterations : int
number of auto-context iterations
RAM : integer
available ram
WORKING_DIR : string
path to store temporary data
logger : logging
root logger
"""
import shutil
from iota2.Sampling import GenAnnualSamples
from iota2.Common.GenerateFeatures import generateFeatures
from iota2.Common.OtbAppBank import CreateTrainAutoContext
from iota2.Common.FileUtils import ensure_dir
tiles = parameter_dict["list_tiles"]
model_name = parameter_dict["model_name"]
seed_num = parameter_dict["seed"]
data_ref = parameter_dict["list_learning_samples"]
data_segmented = parameter_dict["list_superPixel_samples"]
field = data_field.lower()
models_path = os.path.join(output_path, "model")
model_path = os.path.join(models_path,
"model_{}_seed_{}".format(model_name, seed_num))
ensure_dir(model_path)
if WORKING_DIR is None:
tmp_dir = os.path.join(model_path, "tmp")
else:
tmp_dir = os.path.join(
WORKING_DIR, "model_{}_seed_{}_tmp".format(model_name, seed_num))
ensure_dir(tmp_dir)
_, feat_labels, _ = generateFeatures(pathWd=WORKING_DIR,
tile=tiles[0],
sar_optical_post_fusion=False,
output_path=output_path,
sensors_parameters=sensors_parameters)
feat_labels = [elem.lower() for elem in feat_labels]
train_autoContext_app = CreateTrainAutoContext({
"refdata":
data_ref,
"reffield":
field,
"superpixdata":
data_segmented,
"superpixdatafield":
superpix_data_field,
"feat":
feat_labels,
"nit":
iterations,
"out":
"{}/".format(model_path)
})
logger.info("Start training autoContext, produce model {}, seed {}".format(
model_name, seed_num))
train_autoContext_app.ExecuteAndWriteOutput()
logger.info("training autoContext DONE")
shutil.rmtree(tmp_dir)
def predict(mask: str,
model: str,
stat: str,
out_classif: str,
out_confidence: str,
out_proba: str,
working_dir: str,
tile_name: str,
sar_optical_post_fusion: bool,
output_path: str,
sensors_parameters: sensors_params,
pixel_type: str,
number_of_chunks: Optional[int] = None,
targeted_chunk: Optional[int] = None,
ram: Optional[int] = 128,
logger=logger) -> None:
"""perform scikit-learn prediction
This function will produce 2 rasters, the classification map and the
confidence map. The confidence map represent the classifier confidence in
the chosen class.
Parameters
----------
mask: str
raster mask path
model: str
input model path
stat: str
statistics path
out_classif: str
output classification raster path
out_confidence: str
output confidence raster path
out_proba: str
output confidence raster path
working_dir: str
path to a working direction to store temporary data
tile_name: str
tile's name
sar_optical_post_fusion: bool
flag to use post classification sar optical workflow
output_path: str
iota2 output path
sensors_parameters: sensors_params
sensors description
pixel_type: str
output pixel type
number_of_chunks: int
The prediction process can be done by strips. This parameter
set the number of strips
targeted_chunk: int
If this parameter is provided, only the targeted strip will be compute (parallelization).
ram: int
ram (in mb) available
logger : logging
root logger
"""
import os
import shutil
import pickle
from functools import partial
from iota2.Common import rasterUtils as rasterU
from iota2.Common import ServiceConfigFile as serviceConf
from iota2.Common.GenerateFeatures import generateFeatures
from iota2.Common.FileUtils import findCurrentTileInString
mode = "usually" if "SAR.txt" not in model else "SAR"
with open(model, 'rb') as model_file:
model, scaler = pickle.load(model_file)
# if hasattr(model, "n_jobs"):
# model.n_jobs = -1
if number_of_chunks and targeted_chunk:
if targeted_chunk > number_of_chunks - 1:
raise ValueError(
"targeted_chunk must be inferior to the number of chunks")
function_partial = partial(do_predict, model=model, scaler=scaler)
classification_dir = os.path.join(output_path, "classif")
feat_stack, feat_labels, _ = generateFeatures(
working_dir,
tile_name,
sar_optical_post_fusion=sar_optical_post_fusion,
output_path=output_path,
sensors_parameters=sensors_parameters,
mode=mode)
logger.info("producing {}".format(out_classif))
# ~ sk-learn provide only methods 'predict' and 'predict_proba', no proba_max.
# ~ Then we have to compute the full probability vector to get the maximum
# ~ confidence and generate the confidence map
predicted_proba, _, transform, epsg, masks = rasterU.apply_function(
feat_stack,
feat_labels,
working_dir,
function_partial,
out_proba,
mask=mask,
mask_value=0,
chunk_size_mode="split_number",
number_of_chunks=number_of_chunks,
targeted_chunk=targeted_chunk,
output_number_of_bands=len(model.classes_),
ram=ram)
logger.info("predictions done")
if len(masks) > 1:
raise ValueError("Only one mask is expected")
if targeted_chunk is not None:
out_classif = out_classif.replace(
".tif", "_SUBREGION_{}.tif".format(targeted_chunk))
out_confidence = out_confidence.replace(
".tif", "_SUBREGION_{}.tif".format(targeted_chunk))
proba_to_label(predicted_proba, out_classif, model.classes_, transform,
epsg, masks[0])
probabilities_to_max_proba(predicted_proba, transform, epsg,
out_confidence)
if working_dir:
shutil.copy(out_classif, classification_dir)
shutil.copy(out_confidence, classification_dir)
os.remove(out_classif)
os.remove(out_confidence)
if out_proba:
shutil.copy(out_proba, classification_dir)
os.remove(out_proba)
def prepare_autoContext_data_ref(self, slic_seg, config_path_test):
"""
"""
from iota2.Common.ServiceConfigFile import iota2_parameters
from iota2.VectorTools.AddField import addField
from iota2.Sampling.SuperPixelsSelection import merge_ref_super_pix
from iota2.Sampling.SplitSamples import split_superpixels_and_reference
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.OtbAppBank import CreateSampleSelectionApplication
from iota2.Common.OtbAppBank import CreatePolygonClassStatisticsApplication
from iota2.Common.OtbAppBank import CreateSampleExtractionApplication
from iota2.Common.GenerateFeatures import generateFeatures
from iota2.Common import ServiceConfigFile as SCF
raster_ref = FileSearch_AND(self.fake_data_dir, True, ".tif")[0]
CreatePolygonClassStatisticsApplication({
"in":
raster_ref,
"vec":
self.ref_data,
"field":
"CODE",
"out":
os.path.join(self.test_working_directory, "stats.xml")
}).ExecuteAndWriteOutput()
ref_sampled = os.path.join(
self.test_working_directory,
"T31TCJ_samples_region_1_seed_0_selection.sqlite")
CreateSampleSelectionApplication({
"in":
raster_ref,
"vec":
self.ref_data,
"field":
"CODE",
"strategy":
"all",
"instats":
os.path.join(self.test_working_directory, "stats.xml"),
"out":
ref_sampled
}).ExecuteAndWriteOutput()
cfg = SCF.serviceConfigFile(config_path_test)
params = iota2_parameters(config_path_test)
sensors_parameters = params.get_sensors_parameters("T31TCJ")
features, feat_labels, dep = generateFeatures(
None,
"T31TCJ",
sar_optical_post_fusion=False,
output_path=cfg.getParam("chain", "outputPath"),
sensors_parameters=sensors_parameters)
extraction = CreateSampleExtractionApplication({
"in": features,
"vec": ref_sampled,
"field": "code",
"outfield.list.names": feat_labels,
"outfield": "list"
})
extraction.ExecuteAndWriteOutput()
addField(ref_sampled,
"newregion",
valueField="1",
valueType=str,
driver_name="SQLite")
os.remove(os.path.join(self.test_working_directory, "stats.xml"))
merge_ref_super_pix(
{
"selection_samples": ref_sampled,
"SLIC": slic_seg
}, "code", "superpix", "is_superpix", "newregion")
learning_vector = os.path.join(
self.test_working_directory,
"T31TCJ_region_1_seed0_Samples_learn.sqlite")
shutil.move(ref_sampled, learning_vector)
ref, superpix = split_superpixels_and_reference(
learning_vector, superpix_column="is_superpix")
return learning_vector, superpix