def get_features(self, ram=128):
"""
get features
"""
import os
import multiprocessing as mp
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import computeUserFeatures
from iota2.Common.OtbAppBank import CreateIota2FeatureExtractionApplication
from iota2.Common.OtbAppBank import getInputParameterOutput
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.OtbAppBank import executeApp
features_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(features_dir, raise_exe=False)
features_out = os.path.join(features_dir, self.features_names)
# ~ features = self.cfg_IOTA2.getParam("GlobChain", "features")
# ~ enable_gapFilling = self.cfg_IOTA2.getParam("GlobChain",
# ~ "useGapFilling")
# ~ hand_features_flag = self.cfg_IOTA2.getParam('GlobChain',
# ~ 'useAdditionalFeatures')
# input
(in_stack, in_stack_dep
), in_stack_features_labels = self.get_time_series_gapfilling()
_, dates_enabled = self.write_interpolation_dates_file()
if not self.enable_gapfilling:
(in_stack,
in_stack_dep), in_stack_features_labels = self.get_time_series()
_, dates_enabled = self.write_dates_file()
if self.write_outputs_flag is False:
in_stack.Execute()
else:
in_stack_raster = in_stack.GetParameterValue(
getInputParameterOutput(in_stack))
if not os.path.exists(in_stack_raster):
# in_stack.ExecuteAndWriteOutput()
multi_proc = mp.Process(target=executeApp, args=[in_stack])
multi_proc.start()
multi_proc.join()
if os.path.exists(in_stack_raster):
in_stack = in_stack_raster
# output
app_dep = []
if self.hand_features_flag:
# ~ hand_features = self.cfg_IOTA2.getParam("Sentinel_2",
# ~ "additionalFeatures")
comp = len(
self.stack_band_position) if not self.extracted_bands else len(
self.extracted_bands)
(user_date_features, fields_userfeat, user_feat_date,
stack) = computeUserFeatures(in_stack, dates_enabled, comp,
self.hand_features.split(","))
user_date_features.Execute()
app_dep.append([user_date_features, user_feat_date, stack])
if self.features:
bands_avail = self.stack_band_position
if self.extracted_bands:
bands_avail = [
band_name for band_name, _ in self.extracted_bands
]
# check mandatory bands
if "B4" not in bands_avail:
raise Exception(
"red band (B4) is needed to compute features")
if "B8" not in bands_avail:
raise Exception(
"nir band (B8) is needed to compute features")
if "B11" not in bands_avail:
raise Exception(
"swir band (B11) is needed to compute features")
feat_parameters = {
"in": in_stack,
"out": features_out,
"comp": len(bands_avail),
"red": bands_avail.index("B4") + 1,
"nir": bands_avail.index("B8") + 1,
"swir": bands_avail.index("B11") + 1,
"copyinput": self.copy_input,
"relrefl": self.rel_refl,
"keepduplicates": self.keep_dupl,
"acorfeat": self.acorfeat,
"pixType": "int16",
"ram": str(ram)
}
features_app = CreateIota2FeatureExtractionApplication(
feat_parameters)
if self.copy_input is False:
in_stack_features_labels = []
features_labels = (
in_stack_features_labels +
self.get_features_labels(dates_enabled, self.rel_refl,
self.keep_dupl, self.copy_input))
else:
features_app = in_stack
features_labels = in_stack_features_labels
app_dep.append([in_stack, in_stack_dep])
if self.hand_features_flag:
features_app.Execute()
app_dep.append(features_app)
features_app = CreateConcatenateImagesApplication({
"il": [features_app, user_date_features],
"out":
features_out,
"ram":
str(ram)
})
features_labels += fields_userfeat
return (features_app, app_dep), features_labels
def slicSegmentation(tile_name: str,
output_path: str,
sensors_parameters: sensors_params,
ram: Optional[int] = 128,
working_dir: Optional[Union[str, None]] = None,
force_spw: Optional[Union[int, None]] = None,
logger=LOGGER):
"""generate segmentation using SLIC algorithm
Parameters
----------
tile_name : string
tile's name
output_path : string
iota2 output path
sensors_parameters : dict
sensors parameters description
ram : int
available ram
working_dir : string
directory to store temporary data
force_spw : int
force segments' spatial width
logger : logging
root logger
"""
import math
import shutil
from iota2.Common.GenerateFeatures import generateFeatures
from iota2.Common.OtbAppBank import CreateSLICApplication
from iota2.Common.OtbAppBank import getInputParameterOutput
from iota2.Common.FileUtils import ensure_dir
SLIC_NAME = "SLIC_{}.tif".format(tile_name)
all_features, feat_labels, dep = generateFeatures(
working_dir,
tile_name,
sar_optical_post_fusion=False,
output_path=output_path,
sensors_parameters=sensors_parameters,
mode="usually")
all_features.Execute()
spx, _ = all_features.GetImageSpacing(
getInputParameterOutput(all_features))
tmp_dir = working_dir
if working_dir is None:
tmp_dir = os.path.join(output_path, "features", tile_name, "tmp",
"SLIC_TMPDIR")
else:
tmp_dir = os.path.join(working_dir, tile_name)
ensure_dir(tmp_dir)
slic_seg_path = os.path.join(output_path, "features", tile_name, "tmp",
SLIC_NAME)
features_ram_estimation = all_features.PropagateRequestedRegion(
key="out", region=all_features.GetImageRequestedRegion("out"))
# increase estimation...
features_ram_estimation = features_ram_estimation * 1.5
xy_tiles = math.ceil(
math.sqrt(float(features_ram_estimation) / (float(ram) * 1024**2)))
slic_parameters = {
"in": all_features,
"tmpdir": tmp_dir,
"spw": force_spw if force_spw else int(spx),
"tiling": "manual",
"tiling.manual.ny": int(xy_tiles),
"tiling.manual.nx": int(xy_tiles),
"out": slic_seg_path
}
slic_seg = CreateSLICApplication(slic_parameters)
if not os.path.exists(slic_seg_path):
logger.info("Processing SLIC segmentation : {}\n\t\t\
with parameters : {}".format(tile_name, slic_parameters))
slic_seg.ExecuteAndWriteOutput()
if working_dir is None:
shutil.rmtree(tmp_dir)
def get_time_series_gapfilling(self, ram=128):
"""
get_time_series_gapfilling
"""
import os
import multiprocessing as mp
from iota2.Common.OtbAppBank import CreateImageTimeSeriesGapFillingApplication
from iota2.Common.OtbAppBank import getInputParameterOutput
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.OtbAppBank import executeApp
gap_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(gap_dir, raise_exe=False)
gap_out = os.path.join(gap_dir, self.time_series_gapfilling_name)
dates_interp_file, dates_interp = self.write_interpolation_dates_file()
dates_in_file, _ = self.write_dates_file()
masks, masks_dep, _ = self.get_time_series_masks()
(time_series, time_series_dep), _ = self.get_time_series()
# inputs
if self.write_outputs_flag is False:
time_series.Execute()
masks.Execute()
else:
time_series_raster = time_series.GetParameterValue(
getInputParameterOutput(time_series))
masks_raster = masks.GetParameterValue(
getInputParameterOutput(masks))
if not os.path.exists(masks_raster):
multi_proc = mp.Process(target=executeApp, args=[masks])
multi_proc.start()
multi_proc.join()
if not os.path.exists(time_series_raster):
# time_series.ExecuteAndWriteOutput()
multi_proc = mp.Process(target=executeApp, args=[time_series])
multi_proc.start()
multi_proc.join()
if os.path.exists(masks_raster):
masks = masks_raster
if os.path.exists(time_series_raster):
time_series = time_series_raster
comp = len(
self.stack_band_position) if not self.extracted_bands else len(
self.extracted_bands)
gap = CreateImageTimeSeriesGapFillingApplication({
"in": time_series,
"mask": masks,
"comp": str(comp),
"it": "linear",
"id": dates_in_file,
"od": dates_interp_file,
"out": gap_out,
"ram": str(ram),
"pixType": "int16"
})
app_dep = [time_series, masks, masks_dep, time_series_dep]
bands = self.stack_band_position
if self.extracted_bands:
bands = [band_name for band_name, band_pos in self.extracted_bands]
features_labels = [
"{}_{}_{}".format(self.__class__.name, band_name, date)
for date in dates_interp for band_name in bands
]
return (gap, app_dep), features_labels
def launchClassification(tempFolderSerie,
Classifmask,
model,
stats,
outputClassif,
confmap,
pathWd,
classifier_type: str,
tile: str,
proba_map_expected: bool,
dimred,
sar_optical_post_fusion,
output_path: str,
data_field: str,
write_features: bool,
reduction_mode,
sensors_parameters,
pixType,
MaximizeCPU=True,
RAM=500,
auto_context={},
logger=LOGGER):
"""
"""
from iota2.Common import GenerateFeatures as genFeatures
from iota2.Sampling import DimensionalityReduction as DR
from iota2.Common.OtbAppBank import getInputParameterOutput
output_directory = os.path.join(output_path, "classif")
# wMode = cfg.getParam('GlobChain', 'writeOutputs')
featuresPath = os.path.join(output_path, "features")
wd = pathWd
if not pathWd:
wd = featuresPath
wd = os.path.join(featuresPath, tile)
if pathWd:
wd = os.path.join(pathWd, tile)
if not os.path.exists(wd):
try:
os.mkdir(wd)
except Exception:
logger.warning(f"{wd} Allready exists")
mode = "usually"
if "SAR.tif" in outputClassif:
mode = "SAR"
AllFeatures, _, dep_features = genFeatures.generateFeatures(
pathWd=wd,
tile=tile,
sar_optical_post_fusion=sar_optical_post_fusion,
output_path=output_path,
sensors_parameters=sensors_parameters,
mode=mode)
feature_raster = AllFeatures.GetParameterValue(
getInputParameterOutput(AllFeatures))
if write_features:
if not os.path.exists(feature_raster):
AllFeatures.ExecuteAndWriteOutput()
AllFeatures = feature_raster
else:
AllFeatures.Execute()
ClassifInput = AllFeatures
if dimred:
logger.debug("Classification model : {}".format(model))
dimRedModelList = DR.get_dim_red_models_from_classification_model(
model)
logger.debug("Dim red models : {}".format(dimRedModelList))
[ClassifInput,
other_dep] = DR.apply_dimensionality_reduction_to_feature_stack(
reduction_mode, output_path, AllFeatures, dimRedModelList)
if write_features:
ClassifInput.ExecuteAndWriteOutput()
else:
ClassifInput.Execute()
iota2_samples_dir = os.path.join(output_path, "learningSamples")
models_class = get_class_by_models(
iota2_samples_dir,
data_field,
model=model if proba_map_expected else None)
classif = iota2Classification(ClassifInput,
classifier_type,
model,
tile,
output_directory,
models_class,
proba_map=proba_map_expected,
working_directory=pathWd,
classif_mask=Classifmask,
pixType=pixType,
stat_norm=stats,
RAM=RAM,
mode=mode,
auto_context=auto_context)
classif.generate()
def get_features(self, ram=128, logger=LOGGER):
"""get sar features
"""
import configparser
from iota2.Common.FileUtils import getNbDateInTile, FileSearch_AND
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import generateSARFeat_dates
from iota2.Common.OtbAppBank import getInputParameterOutput
if self.use_gapfilling:
(s1_data,
dependancies), s1_labels = self.get_time_series_gapFilling(ram)
else:
(s1_data, dependancies), s1_labels = self.get_time_series(ram)
config = configparser.ConfigParser()
config.read(self.s1_cfg)
sar_features_expr = None
if config.has_option("Features", "expression"):
sar_features_expr_cfg = config.get("Features", "expression")
if not "none" in sar_features_expr_cfg.lower():
sar_features_expr = sar_features_expr_cfg.split(",")
dependancies = [dependancies]
s1_features = []
sar_time_series = {
"asc": {
"vv": {
"App": None,
"availDates": None
},
"vh": {
"App": None,
"availDates": None
}
},
"des": {
"vv": {
"App": None,
"availDates": None
},
"vh": {
"App": None,
"availDates": None
}
}
}
for sensor_mode, time_series_app in list(s1_data.items()):
_, polarisation, orbit = sensor_mode.split("_")
# inputs
if self.write_outputs_flag is False:
time_series_app.Execute()
else:
time_series_raster = time_series_app.GetParameterValue(
getInputParameterOutput(time_series_app))
if not os.path.exists(time_series_raster):
time_series_app.ExecuteAndWriteOutput()
if os.path.exists(time_series_raster):
time_series_app = time_series_raster
sar_time_series[orbit.lower()][
polarisation.lower()]["App"] = time_series_app
s1_features.append(time_series_app)
dependancies.append(time_series_app)
if self.use_gapfilling:
date_file = FileSearch_AND(
self.features_dir, True,
"{}_{}_dates_interpolation.txt".format(
polarisation.lower(), orbit.upper()))[0]
else:
tar_dir = os.path.join(config.get("Paths", "output"),
self.tile_name[1:])
date_file = FileSearch_AND(
tar_dir, True,
"{}_{}_dates_input.txt".format(polarisation.lower(),
orbit.upper()))[0]
sar_time_series[orbit.lower()][
polarisation.lower()]["availDates"] = getNbDateInTile(
date_file, display=False, raw_dates=True)
features_labels = []
for sensor_mode, features in list(s1_labels.items()):
features_labels += features
if sar_features_expr:
sar_user_features_raster = os.path.join(
self.features_dir, "tmp", self.user_sar_features_name)
user_sar_features, user_sar_features_lab = generateSARFeat_dates(
sar_features_expr, sar_time_series, sar_user_features_raster)
if self.write_outputs_flag is False:
user_sar_features.Execute()
else:
if not os.path.exists(sar_user_features_raster):
user_sar_features.ExecuteAndWriteOutput()
if os.path.exists(sar_user_features_raster):
user_sar_features = sar_user_features_raster
dependancies.append(user_sar_features)
s1_features.append(user_sar_features)
features_labels += user_sar_features_lab
sar_features_raster = os.path.join(self.features_dir, "tmp",
self.sar_features_name)
sar_features = CreateConcatenateImagesApplication({
"il": s1_features,
"out": sar_features_raster,
"ram": str(ram)
})
return (sar_features, dependancies), features_labels
def get_time_series_gapFilling(self, ram=128):
"""
Due to the SAR data, time series must be split by polarisation
and orbit (ascending / descending)
"""
import configparser
from iota2.Common.FileUtils import getNbDateInTile
from iota2.Common.OtbAppBank import getSARstack
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import CreateImageTimeSeriesGapFillingApplication
from iota2.Common.OtbAppBank import getInputParameterOutput
(all_filtered, all_masks, interp_date_files,
input_date_files) = getSARstack(self.s1_cfg,
self.tile_name,
self.all_tiles.split(" "),
os.path.join(self.i2_output_path,
"features"),
workingDirectory=None)
# to be clearer
s1_data = OrderedDict()
s1_labels = OrderedDict()
config = configparser.ConfigParser()
config.read(self.s1_cfg)
interpolation_method = "linear"
if config.has_option("Processing", "gapFilling_interpolation"):
interpolation_method = config.get("Processing",
"gapFilling_interpolation")
dependancies = []
for filtered, masks, interp_dates, in_dates in zip(
all_filtered, all_masks, interp_date_files, input_date_files):
sar_mode = os.path.basename(
filtered.GetParameterValue("outputstack"))
sar_mode = "_".join(os.path.splitext(sar_mode)[0].split("_")[0:-1])
polarisation = sar_mode.split("_")[1]
orbit = sar_mode.split("_")[2]
gapfilling_orbit_pol_name_masks = f"{self.gapfilling_orbit_pol_name_mask}_{orbit}_{polarisation}.tif"
gapfilling_raster_mask = os.path.join(
self.features_dir, "tmp", gapfilling_orbit_pol_name_masks)
masks_stack = CreateConcatenateImagesApplication({
"il": masks,
"out": gapfilling_raster_mask,
"ram": str(ram)
})
if self.write_outputs_flag is False:
filtered.Execute()
masks_stack.Execute()
else:
filtered_raster = filtered.GetParameterValue(
getInputParameterOutput(filtered))
masks_stack_raster = masks_stack.GetParameterValue(
getInputParameterOutput(masks_stack))
if not os.path.exists(masks_stack_raster):
masks_stack.ExecuteAndWriteOutput()
if not os.path.exists(filtered_raster):
filtered.ExecuteAndWriteOutput()
if os.path.exists(masks_stack_raster):
masks_stack = masks_stack_raster
if os.path.exists(filtered_raster):
filtered = filtered_raster
dependancies.append((filtered, masks_stack))
gapfilling_orbit_pol_name = f"{self.gapfilling_orbit_pol_name}_{orbit}_{polarisation}.tif"
gapfilling_raster = os.path.join(self.features_dir, "tmp",
gapfilling_orbit_pol_name)
gap_app = CreateImageTimeSeriesGapFillingApplication({
"in":
filtered,
"mask":
masks_stack,
"it":
interpolation_method,
"id":
in_dates,
"od":
interp_dates,
"comp":
str(1),
"out":
gapfilling_raster
})
s1_data[sar_mode] = gap_app
sar_dates = sorted(getNbDateInTile(interp_dates,
display=False,
raw_dates=True),
key=lambda x: int(x))
labels = [
"{}_{}_{}_{}".format(self.__class__.name, orbit, polarisation,
date).lower() for date in sar_dates
]
s1_labels[sar_mode] = labels
return (s1_data, dependancies), s1_labels
def generateFeatures(pathWd: str,
tile: str,
sar_optical_post_fusion: bool,
output_path: str,
sensors_parameters: sensors_params,
mode: Optional[str] = "usually"):
"""
usage : Function use to compute features according to a configuration file
Parameters
----------
pathWd : str
path to a working directory
tile : str
tile's name
sar_optical_post_fusion : bool
flag use to remove SAR data from features
mode : str
'usually' / 'SAR' used to get only sar features
"""
from iota2.Common.OtbAppBank import getInputParameterOutput
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
LOGGER.info(f"prepare features for tile : {tile}")
sensor_tile_container = sensors_container(tile, pathWd, output_path,
**sensors_parameters)
feat_labels = []
dep = []
feat_app = []
if mode == "usually" and sar_optical_post_fusion is False:
sensors_features = sensor_tile_container.get_sensors_features(
available_ram=1000)
for _, ((sensor_features, sensor_features_dep),
features_labels) in sensors_features:
sensor_features.Execute()
feat_app.append(sensor_features)
dep.append(sensor_features_dep)
feat_labels = feat_labels + features_labels
elif mode == "usually" and sar_optical_post_fusion is True:
sensor_tile_container.remove_sensor("Sentinel1")
sensors_features = sensor_tile_container.get_sensors_features(
available_ram=1000)
for _, ((sensor_features, sensor_features_dep),
features_labels) in sensors_features:
sensor_features.Execute()
feat_app.append(sensor_features)
dep.append(sensor_features_dep)
feat_labels = feat_labels + features_labels
elif mode == "SAR":
sensor = sensor_tile_container.get_sensor("Sentinel1")
(sensor_features,
sensor_features_dep), feat_labels = sensor.get_features(ram=1000)
sensor_features.Execute()
feat_app.append(sensor_features)
dep.append(sensor_features_dep)
dep.append(feat_app)
features_name = "{}_Features.tif".format(tile)
features_dir = os.path.join(output_path, "features", tile, "tmp")
features_raster = os.path.join(features_dir, features_name)
if len(feat_app) > 1:
all_features = CreateConcatenateImagesApplication({
"il":
feat_app,
"out":
features_raster
})
else:
all_features = sensor_features
output_param_name = getInputParameterOutput(sensor_features)
all_features.SetParameterString(output_param_name, features_raster)
return all_features, feat_labels, dep