def compute_probamap_fusion(fusion_dic,
ds_choice,
classif_model_pos,
classif_tile_pos,
classif_seed_pos,
ds_choice_both,
ds_choice_sar,
ds_choice_opt,
ds_no_choice,
workingDirectory,
ram=128,
logger=LOGGER):
"""
from the fusion of classification's raster choice compute the
fusion of confidence map
Parameters
----------
fusion_dic : dict
dictionnary containing keys : "sar_classif", "opt_classif", "sar_model"
"opt_model"
ds_choice : string
path to the fusion of classifications choice map
classif_model_pos : int
position of the model's name in classification's name if
splited by '_'
classif_tile_pos : int
position of the tile's name in classification's name if
splited by '_'
classif_seed_pos : int
position of the seed number in classification's name if
splited by '_'
ds_choice_both : int
output value if fusion of classifications get the same label than
SAR classification and optical classification
ds_choice_sar : int
output value if fusion of classifications get the same label than
SAR classification
ds_choice_opt : int
output value if fusion of classifications get the same label than
optical classification
ds_no_choice : int
default case
workingDirectory : string
path to a working directory
LOGGER : logging
root logger
Notes
-----
fusion rules are :
- If SAR's label is chosen by the DS method then SAR confidence
is chosen.
- If Optical's label is chosen by the DS method then optical
confidence is chosen.
- If the same label is chosen by SAR and optical models, then the
output pixel vector is given my the model which is more
confidence in it's choice.
Return
------
string
output path
"""
from iota2.Common import OtbAppBank
from iota2.Common.FileUtils import getRasterNbands
classif_dir, _ = os.path.split(fusion_dic["sar_classif"])
model = os.path.basename(
fusion_dic["sar_classif"]).split("_")[classif_model_pos]
seed = os.path.basename(
fusion_dic["sar_classif"]).split("_")[classif_seed_pos]
tile = os.path.basename(
fusion_dic["sar_classif"]).split("_")[classif_tile_pos]
sar_proba_map = fu.fileSearchRegEx(
os.path.join(
classif_dir,
"PROBAMAP_{}_model_{}_seed_{}_SAR.tif".format(tile, model,
seed)))[0]
opt_proba_map = fu.fileSearchRegEx(
os.path.join(
classif_dir,
"PROBAMAP_{}_model_{}_seed_{}.tif".format(tile, model, seed)))[0]
sar_confidence = fu.fileSearchRegEx(
os.path.join(
classif_dir,
"{}_model_{}_confidence_seed_{}_SAR.tif".format(tile, model,
seed)))[0]
opt_confidence = fu.fileSearchRegEx(
os.path.join(
classif_dir,
"{}_model_{}_confidence_seed_{}.tif".format(tile, model, seed)))[0]
im_list = [
ds_choice, sar_proba_map, opt_proba_map, sar_confidence, opt_confidence
]
nb_bands_probamap_opt = getRasterNbands(opt_proba_map)
nb_bands_probamap_sar = getRasterNbands(sar_proba_map)
if nb_bands_probamap_opt != nb_bands_probamap_sar:
raise Exception(
"SAR probality map and Optical probality map must have the same number of bands"
)
exp = ("im1b1=={ds_choice_both} and im4b1>im5b1?im2:"
"im1b1=={ds_choice_both} and im4b1<im5b1?im3:"
"im1b1=={ds_choice_sar}?im2:"
"im1b1=={ds_choice_opt}?im3:"
"{ds_no_choice}").format(
ds_choice_both=ds_choice_both,
ds_choice_sar=ds_choice_sar,
ds_choice_opt=ds_choice_opt,
ds_no_choice="{" +
",".join([str(ds_no_choice)] * nb_bands_probamap_opt) + "}")
ds_probamap_name = "PROBAMAP_{}_model_{}_seed_{}_DS.tif".format(
tile, model, seed)
ds_probamap_dir = classif_dir
ds_probamap = os.path.join(ds_probamap_dir, ds_probamap_name)
if workingDirectory:
ds_probamap = os.path.join(workingDirectory, ds_probamap_name)
probamap_param = {
"il": im_list,
"out": ds_probamap,
"ram": str(ram),
"exp": exp
}
probamap_app = OtbAppBank.CreateBandMathXApplication(probamap_param)
if not os.path.exists(os.path.join(ds_probamap_dir, ds_probamap_name)):
logger.info(f"computing : {ds_probamap}")
probamap_app.ExecuteAndWriteOutput()
logger.debug(f"{ds_probamap} : DONE")
if workingDirectory:
# copy confidence
shutil.copy(
ds_probamap,
os.path.join(os.path.join(ds_probamap_dir, ds_probamap_name)))
# remove
os.remove(ds_probamap)
return os.path.join(os.path.join(ds_probamap_dir, ds_probamap_name))
def fusion(pathClassif: str, N: int, allTiles: List[str], fusionOptions: str,
nomenclature_path: str, region_vec: str, ds_sar_opt: bool,
pathWd: str) -> List[str]:
"""generate otb fusion commands by parsing the iota2 classifications
directory
Parameters
----------
pathClassif: str
path to the iota2 classification directory
N: int
number of random seeds to split learnging / validation
samples
allTiles: list
list of tiles to consider
fusionOptions: str
fusion options of FusionOfClassifications otb application
nomenclature_path: str
nomenclature file
region_vec: str
region shapeFile database
ds_sar_opt: bool
flag to inform if the sar optical post classification
workflow is enable
pathWd: str
working directory path
Return
------
list
list of commands as strings
"""
pathWd = None
pix_type = fu.getOutputPixType(nomenclature_path)
classification_suffix_pattern = ""
if ds_sar_opt:
classification_suffix_pattern = "_DS"
if region_vec:
all_classif = fu.fileSearchRegEx(pathClassif +
"/Classif_*_model_*f*_seed_*" +
classification_suffix_pattern +
".tif")
allTiles = []
models = []
for classif in all_classif:
mod = classif.split("/")[-1].split("_")[3].split("f")[0]
tile = classif.split("/")[-1].split("_")[1]
if mod not in models:
models.append(mod)
if tile not in allTiles:
allTiles.append(tile)
all_cmd = []
for seed in range(N):
for tile in allTiles:
directory_out = pathClassif
if pathWd != None:
directory_out = "$TMPDIR"
if region_vec is None:
classifPath = fu.FileSearch_AND(
pathClassif, True, "Classif_" + tile, "seed_" + str(seed) +
classification_suffix_pattern + ".tif")
allPathFusion = " ".join(classifPath)
cmd = "otbcli_FusionOfClassifications -il " + allPathFusion + " " + fusionOptions + " -out " + directory_out + "/" + tile + "_FUSION_seed_" + str(
seed) + ".tif"
all_cmd.append(cmd)
else:
for mod in models:
classifPath = fu.fileSearchRegEx(
pathClassif + "/Classif_" + tile + "_model_" + mod +
"f*_seed_" + str(seed) +
classification_suffix_pattern + ".tif")
if len(classifPath) != 0:
allPathFusion = " ".join(classifPath)
cmd = "otbcli_FusionOfClassifications -il " + allPathFusion + " " + fusionOptions + " -out " + directory_out + "/" + tile + "_FUSION_model_" + mod + "_seed_" + str(
seed) + ".tif " + pix_type
all_cmd.append(cmd)
tmp = pathClassif.split("/")
if pathClassif[-1] == "/":
del tmp[-1]
tmp[-1] = "cmd/fusion"
path_to_cmd_fusion = "/".join(tmp)
fu.writeCmds(path_to_cmd_fusion + "/fusion.txt", all_cmd)
return all_cmd
def dempster_shafer_fusion_parameters(iota2_dir):
"""
use to feed dempster_shafer_fusion function
from the iota2 output directory, return parameter needed to compute
a fusion of classifcations by dempster-shafer method
Parameters
----------
iota2_dir : string
iota2 output directory
Return
------
list
list of dict containing keys {"sar_classif", "opt_classif",
"sar_model", "opt_model"}
"""
classif_seed_pos = 5
classif_tile_pos = 1
classif_model_pos = 3
iota2_ds_confusions_dir = os.path.join(iota2_dir, "dataAppVal", "bymodels")
iota2_classif_dir = os.path.join(iota2_dir, "classif")
classifications = fu.FileSearch_AND(iota2_classif_dir, True, "Classif",
".tif")
# group by models
model_group = []
for classif in classifications:
classif_name = os.path.basename(classif)
classif_seed = classif_name.split("_")[classif_seed_pos].replace(
".tif", "")
classif_model = classif_name.split("_")[classif_model_pos]
classif_tile = classif_name.split("_")[classif_tile_pos]
key_param = (classif_seed, classif_model, classif_tile)
model_group.append((key_param, classif))
raster_group = [param for key, param in fu.sortByFirstElem(model_group)]
out_parameters = []
for raster_model in raster_group:
for raster in raster_model:
raster_name = os.path.basename(raster)
classif_seed = raster_name.split("_")[classif_seed_pos].replace(
".tif", "")
classif_model = raster_name.split("_")[classif_model_pos]
if "SAR.tif" in raster_name:
sar_model = fu.fileSearchRegEx(
os.path.join(
iota2_ds_confusions_dir,
"model_{}_seed_{}_SAR.csv".format(
classif_model, classif_seed)))[0]
sar_classif = raster
else:
opt_model = fu.fileSearchRegEx(
os.path.join(
iota2_ds_confusions_dir,
"model_{}_seed_{}.csv".format(classif_model,
classif_seed)))[0]
opt_classif = raster
out_parameters.append({
"sar_classif": sar_classif,
"opt_classif": opt_classif,
"sar_model": sar_model,
"opt_model": opt_model
})
return out_parameters
def compute_confidence_fusion(fusion_dic,
ds_choice,
classif_model_pos,
classif_tile_pos,
classif_seed_pos,
ds_choice_both,
ds_choice_sar,
ds_choice_opt,
ds_no_choice,
workingDirectory,
logger=LOGGER):
"""
from the fusion of classification's raster choice compute the fusion of confidence map
Parameters
----------
fusion_dic : dict
dictionnary containing keys : "sar_classif", "opt_classif", "sar_model"
"opt_model"
ds_choice : string
path to the fusion of classifications choice map
classif_model_pos : int
position of the model's name in classification's name if
splited by '_'
classif_tile_pos : int
position of the tile's name in classification's name if
splited by '_'
classif_seed_pos : int
position of the seed number in classification's name if
splited by '_'
ds_choice_both : int
output value if fusion of classifications get the same label than
SAR classification and optical classification
ds_choice_sar : int
output value if fusion of classifications get the same label than
SAR classification
ds_choice_opt : int
output value if fusion of classifications get the same label than
optical classification
ds_no_choice : int
default case
workingDirectory : string
path to a working directory
logger : logging
root logger
Notes
-----
confidence fusion rules are :
If SAR's label is chosen by the DS method then SAR confidence is chosen.
If Optical's label is chosen by the DS method then optical confidence is chosen.
If the same label is chosen by SAR and optical models, then the
maximum confidence is chosen.
Return
------
string
output path
"""
from iota2.Common import OtbAppBank
classif_dir, _ = os.path.split(fusion_dic["sar_classif"])
model = os.path.basename(
fusion_dic["sar_classif"]).split("_")[classif_model_pos]
seed = os.path.basename(
fusion_dic["sar_classif"]).split("_")[classif_seed_pos]
tile = os.path.basename(
fusion_dic["sar_classif"]).split("_")[classif_tile_pos]
sar_confidence = fu.fileSearchRegEx(
os.path.join(
classif_dir,
"{}_model_{}_confidence_seed_{}_SAR.tif".format(tile, model,
seed)))[0]
opt_confidence = fu.fileSearchRegEx(
os.path.join(
classif_dir,
"{}_model_{}_confidence_seed_{}.tif".format(tile, model, seed)))[0]
im_list = [ds_choice, sar_confidence, opt_confidence]
confidence_exp = "im1b1=={ds_choice_both}?max(im2b1, im3b1):im1b1=={ds_choice_sar}?im2b1:im1b1=={ds_choice_opt}?im3b1:{ds_no_choice}".format(
ds_choice_both=ds_choice_both,
ds_choice_sar=ds_choice_sar,
ds_choice_opt=ds_choice_opt,
ds_no_choice=ds_no_choice)
ds_confidence_name = "{}_model_{}_confidence_seed_{}_DS.tif".format(
tile, model, seed)
ds_confidence_dir = classif_dir
ds_confidence = os.path.join(ds_confidence_dir, ds_confidence_name)
if workingDirectory:
ds_confidence = os.path.join(workingDirectory, ds_confidence_name)
confidence_param = {
"il": im_list,
"out": ds_confidence,
"exp": confidence_exp
}
confidence = OtbAppBank.CreateBandMathApplication(confidence_param)
if not os.path.exists(os.path.join(ds_confidence_dir, ds_confidence_name)):
logger.info(f"computing : {ds_confidence}")
confidence.ExecuteAndWriteOutput()
logger.debug(f"{ds_choice} : DONE")
if workingDirectory:
# copy confidence
shutil.copy(ds_confidence,
os.path.join(ds_confidence_dir, ds_confidence_name))
# remove
os.remove(ds_confidence)
return os.path.join(ds_confidence_dir, ds_confidence_name)
def undecision_management(path_test: str,
path_fusion: str,
field_region: str,
path_to_img: str,
path_to_region: str,
no_label_management: str,
path_wd: str,
list_indices: List[str],
user_feat_path: str,
pix_type: str,
region_vec: Optional[str] = None,
user_feat_pattern: Optional[str] = None,
ds_sar_opt: Optional[bool] = False) -> None:
"""
manage undecision comming from fusion of classifications
Parameters
----------
path_test: string
path_fusion: string
field_region: string
path_to_img: string
path_to_region: string
region_vec: string
no_label_management: string
path_wd: string
list_indices: List[string],
user_feat_path: string
pix_type: string
user_feat_pattern: string
ds_sar_opt: bool
Return
------
None
"""
stack_ind = fu.get_feat_stack_name(list_indices, user_feat_path,
user_feat_pattern)
suffix_pattern = ""
if ds_sar_opt:
suffix_pattern = "_DS"
if region_vec:
current_model = path_fusion.split("/")[-1].split("_")[3]
config_model = os.path.join(path_test, "config_model",
"configModel.cfg")
n_fold = get_nb_split_shape(current_model, config_model)
path_directory = path_test + "/classif"
if path_wd is not None:
working_dir = path_wd
path_directory = path_wd
else:
working_dir = path_test + "/classif/MASK"
current_tile = path_fusion.split("/")[-1].split("_")[0]
shp_rname = path_to_region.split("/")[-1].replace(".shp", "")
all_model = fu.FileSearch_AND(os.path.join(path_test, "model"), True,
"model", ".txt")
if region_vec is None:
model_tile = gen_mask_region_by_tile(
field_region, stack_ind, working_dir, current_tile, all_model,
shp_rname, path_to_img, path_test,
os.path.join(path_test, "config_model",
"configModel.cfg"), path_wd)
elif region_vec and no_label_management == "maxConfidence":
model_tile = path_fusion.split("/")[-1].split("_")[3]
elif region_vec and no_label_management == "learningPriority":
model_tile_tmp = path_fusion.split("/")[-1].split("_")[3]
model_tile = []
for i in range(n_fold):
model_tile.append(f"{model_tile_tmp} f{i + 1}")
if len(model_tile) == 0 or no_label_management == "maxConfidence":
seed = os.path.split(path_fusion)[-1].split("_")[-1].split(".")[0]
img_confidence = fu.FileSearch_AND(
path_test + "/classif", True,
"confidence_seed_" + str(seed) + suffix_pattern + ".tif",
current_tile)
img_classif = fu.FileSearch_AND(os.path.join(path_test, "classif"),
True, "Classif_" + current_tile,
f"seed_{seed}", suffix_pattern)
img_data = os.path.join(
path_directory, f"{current_tile}_FUSION_NODATA_seed{seed}.tif")
if region_vec:
img_confidence = fu.fileSearchRegEx(
f"{path_test+os.sep}classif{os.sep}"
f"{current_tile}_model_{model_tile}"
f"f*_confidence_seed_{seed}{suffix_pattern}.tif")
img_classif = fu.fileSearchRegEx(
f"{path_test+os.sep}classif{os.sep}"
f"Classif_{current_tile}_model_"
f"{model_tile}f*_seed_"
f"{seed}{suffix_pattern}.tif")
img_data = (f"{path_directory+os.sep}Classif_"
f"{current_tile}_model_{model_tile}"
f"_seed_{seed}{suffix_pattern}.tif")
img_confidence.sort()
img_classif.sort()
exp, il_str = build_confidence_exp(path_fusion, img_confidence,
img_classif)
cmd = (f"otbcli_BandMath -il {il_str} -out {img_data} {pix_type} "
f"-exp '{exp}' ")
run(cmd)
if path_wd is not None:
run(f"cp {img_data} {os.path.join(path_test, 'classif')}")
elif len(model_tile) != 0 and no_label_management == "learningPriority":
# Concaténation des classifications pour une tuile (qui a ou non
# plusieurs régions) et Concaténation des masques de régions pour
# une tuile (qui a ou non plusieurs régions)
seed = os.path.split(path_fusion)[-1].split("_")[-1].split(".")[0]
concat_out = (f"{os.path.join(path_test, 'classif')}"
f"{current_tile}_FUSION_concat_seed{seed}.tif")
if region_vec:
concat_out = (
f"{os.path.join(path_test, 'classif')}"
f"{current_tile}_FUSION_model_{model_tile[0].split('f')[0]}"
f"concat_seed{seed}.tif")
path_to_classif_concat = concat_classifs_one_tile(
seed, current_tile, path_test, model_tile, concat_out, path_wd)
pattern_mask = "*region_*_{current_tile}_NODATA.tif"
classif_fusion_mask = fu.fileSearchRegEx(
os.path.join(path_test, "classif", "MASK", pattern_mask))
out_concat_mask = os.path.join(path_test, "classif",
f"{current_tile}_MASK.tif")
if region_vec:
pattern_mask = ("*region_{model_tile[0].split('f')[0]}"
f"_{current_tile}.tif")
classif_fusion_mask_tmp = fu.fileSearchRegEx(
os.path.join(path_test, "classif", "MASK", pattern_mask))
out_concat_mask = (f"{os.path.join(path_test, 'classif')}"
f"{current_tile}_MASK_model_"
f"{model_tile[0].split('f')[0]}.tif")
classif_fusion_mask = []
for i in range(n_fold):
classif_fusion_mask.append(classif_fusion_mask_tmp[0])
path_to_region_mask_concat = concat_region_one_tile(
path_test, classif_fusion_mask, path_wd, out_concat_mask)
# construction de la commande
exp = ""
im1 = path_fusion
im2 = path_to_region_mask_concat
im3 = path_to_classif_concat
for i in range(len(classif_fusion_mask)):
if i + 1 < len(classif_fusion_mask):
exp = exp + "im2b" + str(i + 1) + ">=1?im3b" + str(i + 1) + ":"
else:
exp = exp + "im2b" + str(i + 1) + ">=1?im3b" + str(i +
1) + ":0"
exp = "im1b1!=0?im1b1:(" + exp + ")"
img_data = (os.path.join(
path_directory, f"{current_tile}_FUSION_"
f"NODATA_seed{seed}.tif"))
if region_vec:
img_data = (f"{path_directory+os.sep}Classif_{current_tile}_model_"
f"{model_tile[0].split('f')[0]}_seed_{seed}.tif")
cmd = (f'otbcli_BandMath -il {im1} {im2} {im3} -out + {img_data} '
f'{pix_type} -exp "{exp}"')
run(cmd)
if path_wd is not None:
run("cp {img_data} {path_test+os.sep}classif")
def classification_shaping(path_classif: str, runs: int, path_out: str,
path_wd: str, classif_mode: str, path_test: str,
ds_sar_opt: bool, proj: int, nomenclature_path: str,
output_statistics: bool, spatial_resolution: float,
proba_map_flag: bool, region_shape: str,
color_path: str) -> None:
"""function use to mosaic rasters and to produce final maps
path_classif: str
directory where as classifications
runs: int
number of random learning/validation samples-set
path_out: str
output directory
path_wd: str
working directory
classif_mode: str
fusion of classifications ?
path_test: str
iota2 output directory
ds_sar_opt: bool
flag to inform if SAR and optical post-classification workflow
is enable
proj: int
epsg code
nomenclature_path: str
nomenclature path
output_statistics: bool
flag to enable output statistics
spatial_resolution: float
output's spatial resolution
proba_map_flag: bool
flag to inform if probability map was produce
region_shape: str
region shapeFile path
color_path: str
color table file
"""
if path_wd is None:
tmp = path_out + "/TMP"
if not os.path.exists(path_out + "/TMP"):
os.mkdir(tmp)
else:
tmp = path_wd
if not os.path.exists(path_out + "/TMP"):
os.mkdir(path_out + "/TMP")
all_tiles = list(
set([
classif.split("_")[1] for classif in fu.FileSearch_AND(
path_test + "/classif", False, "Classif", ".tif")
]))
pix_type = fu.getOutputPixType(nomenclature_path)
features_path = os.path.join(path_test, "features")
all_tmp_folder = fu.fileSearchRegEx(path_test + "/TMPFOLDER*")
if all_tmp_folder:
for tmp_folder in all_tmp_folder:
shutil.rmtree(tmp_folder)
suffix = "*"
if ds_sar_opt:
suffix = "*DS*"
genGlobalConfidence(runs, path_wd, spatial_resolution, proj, path_test,
classif_mode, all_tiles, region_shape, ds_sar_opt,
proba_map_flag)
if region_shape and classif_mode == "fusion":
old_classif = fu.fileSearchRegEx(path_test +
"/classif/Classif_*_model_*f*_seed_" +
suffix + ".tif")
for rm in old_classif:
if not os.path.exists(path_test + "/final/TMP/OLDCLASSIF"):
os.mkdir(path_test + "/final/TMP/OLDCLASSIF")
run("mv " + rm + " " + path_test + "/final/TMP/OLDCLASSIF")
classification = []
confidence = []
proba_map = []
cloud = []
for seed in range(runs):
classification.append([])
confidence.append([])
cloud.append([])
sort = []
if proba_map_flag:
proba_map_list = fu.fileSearchRegEx(
path_test + "/classif/PROBAMAP_*_model_*_seed_" + str(seed) +
suffix + ".tif")
proba_map_list = removeInListByRegEx(
proba_map_list, ".*model_.*f.*_seed." + suffix)
proba_map.append(proba_map_list)
if classif_mode == "separate" or region_shape:
all_classif_seed = fu.FileSearch_AND(path_classif, True, ".tif",
"Classif",
"seed_" + str(seed))
if ds_sar_opt:
all_classif_seed = fu.FileSearch_AND(path_classif, True,
".tif", "Classif",
"seed_" + str(seed),
"DS.tif")
ind = 1
elif classif_mode == "fusion":
all_classif_seed = fu.FileSearch_AND(
path_classif, True, "_FUSION_NODATA_seed" + str(seed) + ".tif")
if ds_sar_opt:
all_classif_seed = fu.FileSearch_AND(
path_classif, True,
"_FUSION_NODATA_seed" + str(seed) + "_DS.tif")
ind = 0
for tile in all_classif_seed:
sort.append((tile.split("/")[-1].split("_")[ind], tile))
sort = fu.sortByFirstElem(sort)
for tile, paths in sort:
exp = ""
all_cl = ""
all_cl_rm = []
for i in range(len(paths)):
all_cl = all_cl + paths[i] + " "
all_cl_rm.append(paths[i])
if i < len(paths) - 1:
exp = exp + "im" + str(i + 1) + "b1 + "
else:
exp = exp + "im" + str(i + 1) + "b1"
path_cl_final = tmp + "/" + tile + "_seed_" + str(seed) + ".tif"
classification[seed].append(path_cl_final)
cmd = 'otbcli_BandMath -il ' + all_cl + '-out ' + path_cl_final + ' ' + pix_type + ' -exp "' + exp + '"'
run(cmd)
tile_confidence = path_out + "/TMP/" + tile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
confidence[seed].append(tile_confidence)
cloud_tile = fu.FileSearch_AND(features_path + "/" + tile, True,
"nbView.tif")[0]
classif_tile = tmp + "/" + tile + "_seed_" + str(seed) + ".tif"
cloud_tile_priority = path_test + "/final/TMP/" + tile + "_Cloud.tif"
cloud_tile_priority_tmp = tmp + "/" + tile + "_Cloud.tif"
cloud_tile_priority_stats_ok = path_test + "/final/TMP/" + tile + "_Cloud_StatsOK.tif"
cloud_tile_priority_tmp_stats_ok = tmp + "/" + tile + "_Cloud_StatsOK.tif"
cloud[seed].append(cloud_tile_priority)
if not os.path.exists(cloud_tile_priority):
cmd_cloud = f'otbcli_BandMath -il {cloud_tile} {classif_tile} -out {cloud_tile_priority_tmp} int16 -exp "im2b1>0?im1b1:0"'
run(cmd_cloud)
if output_statistics:
cmd_cloud = 'otbcli_BandMath -il ' + cloud_tile + ' ' + classif_tile + ' -out ' + cloud_tile_priority_tmp_stats_ok + ' int16 -exp "im2b1>0?im1b1:-1"'
run(cmd_cloud)
if path_wd:
shutil.copy(cloud_tile_priority_tmp_stats_ok,
cloud_tile_priority_stats_ok)
os.remove(cloud_tile_priority_tmp_stats_ok)
if path_wd:
shutil.copy(cloud_tile_priority_tmp, cloud_tile_priority)
os.remove(cloud_tile_priority_tmp)
if path_wd is not None:
run("cp -a " + tmp + "/* " + path_out + "/TMP")
for seed in range(runs):
assemble_folder = path_test + "/final"
if path_wd:
assemble_folder = path_wd
fu.assembleTile_Merge(classification[seed],
spatial_resolution,
"{}/Classif_Seed_{}.tif".format(
assemble_folder, seed),
"Byte" if pix_type == "uint8" else "Int16",
co={
"COMPRESS": "LZW",
"BIGTIFF": "YES"
})
if path_wd:
shutil.copy(path_wd + "/Classif_Seed_" + str(seed) + ".tif",
path_test + "/final")
os.remove(path_wd + "/Classif_Seed_" + str(seed) + ".tif")
fu.assembleTile_Merge(confidence[seed],
spatial_resolution,
assemble_folder + "/Confidence_Seed_" +
str(seed) + ".tif",
"Byte",
co={
"COMPRESS": "LZW",
"BIGTIFF": "YES"
})
if path_wd:
shutil.copy(path_wd + "/Confidence_Seed_" + str(seed) + ".tif",
path_test + "/final")
os.remove(path_wd + "/Confidence_Seed_" + str(seed) + ".tif")
color.CreateIndexedColorImage(
path_test + "/final/Classif_Seed_" + str(seed) + ".tif",
color_path,
output_pix_type=gdal.GDT_Byte
if pix_type == "uint8" else gdal.GDT_UInt16)
if proba_map_flag:
proba_map_mosaic = os.path.join(
assemble_folder, "ProbabilityMap_seed_{}.tif".format(seed))
fu.assembleTile_Merge(proba_map[seed],
spatial_resolution,
proba_map_mosaic,
"Int16",
co={
"COMPRESS": "LZW",
"BIGTIFF": "YES"
})
if path_wd:
shutil.copy(proba_map_mosaic, path_test + "/final")
os.remove(proba_map_mosaic)
fu.assembleTile_Merge(cloud[0],
spatial_resolution,
assemble_folder + "/PixelsValidity.tif",
"Byte",
co={
"COMPRESS": "LZW",
"BIGTIFF": "YES"
})
if path_wd:
shutil.copy(path_wd + "/PixelsValidity.tif", path_test + "/final")
os.remove(path_wd + "/PixelsValidity.tif")
def genGlobalConfidence(N, pathWd, spatialRes, proj, pathTest, classifMode,
AllTile: List[str], shapeRegion, ds_sar_opt,
proba_map_flag):
"""generate confidences ready to be mosaic
"""
PROBAMAP_PATTERN = "PROBAMAP"
tmpClassif = pathTest + "/classif/tmpClassif"
pathToClassif = pathTest + "/classif"
if pathWd:
tmpClassif = pathWd + "/tmpClassif"
if not os.path.exists(tmpClassif):
run("mkdir " + tmpClassif)
for seed in range(N):
for tuile in AllTile:
if shapeRegion is None:
if classifMode == "separate":
confidence_pattern = os.path.join(
pathToClassif,
"{}*model*confidence_seed_{}*.tif".format(tuile, seed))
if ds_sar_opt:
confidence_pattern = os.path.join(
pathToClassif,
"{}*model*confidence_seed_{}*_DS.tif".format(
tuile, seed))
confidence = fu.fileSearchRegEx(confidence_pattern)
globalConf = tmpClassif + "/" + tuile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
globalConf_f = pathTest + "/final/TMP/" + tuile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
cmd = 'otbcli_BandMath -il ' + confidence[
0] + ' -out ' + globalConf + ' uint8 -exp "100*im1b1"'
run(cmd)
shutil.copyfile(globalConf, globalConf_f)
os.remove(globalConf)
else:
raise Exception((
"if there is no region shape specify in the "
"configuration file, argClassification.classifMode must be set to 'separate'"
))
else: #output Mode
suffix = "*"
if ds_sar_opt:
suffix = "*DS*"
if classifMode != "separate":
classifTile = fu.fileSearchRegEx(
pathToClassif + "/Classif_" + tuile +
"*model_*f*_seed_" + str(seed) + suffix
) # tmp tile (produce by each classifier, without nodata)
splitModel = []
for classif in classifTile:
model = classif.split("/")[-1].split("_")[3].split(
"f")[0]
try:
ind = splitModel.index(model)
except ValueError:
splitModel.append(model)
splitConfidence = []
confidence_all = fu.fileSearchRegEx(
pathToClassif + "/" + tuile +
"*model_*_confidence_seed_" + str(seed) + suffix)
confidence_withoutSplit = removeInListByRegEx(
confidence_all, ".*model_.*f.*_confidence." + suffix)
for model in splitModel:
classifTile = fu.fileSearchRegEx(
pathToClassif + "/Classif_" + tuile + "*model_" +
model + "f*_seed_" + str(seed) + suffix
) # tmp tile (produce by each classifier, without nodata)
finalTile = pathToClassif + "/Classif_" + tuile + "_model_" + model + "_seed_" + str(
seed) + ".tif"
if ds_sar_opt:
finalTile = pathToClassif + "/Classif_" + tuile + "_model_" + model + "_seed_" + str(
seed) + "_DS.tif"
confidence = fu.fileSearchRegEx(pathToClassif + "/" +
tuile + "*model_" +
model +
"f*_confidence_seed_" +
str(seed) + suffix)
if proba_map_flag:
proba_map_fusion(proba_map_list=fu.fileSearchRegEx(
"{}/{}_{}_model_{}f*_seed_{}{}.tif".format(
pathToClassif, PROBAMAP_PATTERN, tuile,
model, seed, suffix)),
working_directory=pathWd,
ram=2000)
classifTile = sorted(classifTile)
confidence = sorted(confidence)
OutPutConfidence = tmpClassif + "/" + tuile + "_model_" + model + "_confidence_seed_" + str(
seed) + ".tif"
if ds_sar_opt:
OutPutConfidence = tmpClassif + "/" + tuile + "_model_" + model + "_confidence_seed_" + str(
seed) + "_DS.tif"
cmd = BuildConfidenceCmd(finalTile,
classifTile,
confidence,
OutPutConfidence,
fact=100,
pixType="uint8")
run(cmd)
splitConfidence.append(OutPutConfidence)
i = 0 #init
j = 0
exp1 = "+".join([
"im" + str(i + 1) + "b1"
for i in range(len(splitConfidence))
]) #-> confidence from splited models are from 0 to 100
exp2 = "+".join([
"(100*im" + str(j + 1) + "b1)" for j in np.arange(
len(splitConfidence),
len(splitConfidence) +
len(confidence_withoutSplit))
]) #-> confidence from NO-splited models are from 0 to 1
if not splitConfidence:
exp2 = "+".join([
"100*im" + str(j + 1) + "b1"
for j in range(len(confidence_withoutSplit))
])
if exp1 and exp2:
exp = exp1 + "+" + exp2
if exp1 and not exp2:
exp = exp1
if not exp1 and exp2:
exp = exp2
confidence_list = splitConfidence + confidence_withoutSplit
AllConfidence = " ".join(confidence_list)
OutPutConfidence = tmpClassif + "/" + tuile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
cmd = 'otbcli_BandMath -il ' + AllConfidence + ' -out ' + OutPutConfidence + ' uint8 -exp "' + exp + '"'
run(cmd)
shutil.copy(OutPutConfidence, pathTest + "/final/TMP")
os.remove(OutPutConfidence)
#shutil.rmtree(tmpClassif)
else:
confidence = fu.fileSearchRegEx(pathToClassif + "/" +
tuile +
"*model*confidence_seed_" +
str(seed) + suffix)
exp = "+".join([
"im" + str(i + 1) + "b1"
for i in range(len(confidence))
])
AllConfidence = " ".join(confidence)
#for currentConf in confidence:
globalConf = tmpClassif + "/" + tuile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
globalConf_f = pathTest + "/final/TMP/" + tuile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
cmd = 'otbcli_BandMath -il ' + AllConfidence + ' -out ' + globalConf + ' uint8 -exp "100*(' + exp + ')"'
#print confidence
run(cmd)
shutil.copyfile(globalConf, globalConf_f)
os.remove(globalConf)
def merge_output_statistics(i2_output_path: str, runs: int) -> None:
"""merge tile's statistics to a gloable one
Parameters
----------
i2_output_path : str
iota2 output directory
runs : int
nomber of random learning/validation samples-set
"""
for seed in range(runs):
seed_stats = fu.fileSearchRegEx(i2_output_path +
"/final/TMP/*_stats_seed_" +
str(seed) + ".cfg")
all_diff_test = Config(open(seed_stats[0])).AllDiffStats
all_diff_test = all_diff_test.split(",")
vok_buff = []
vnok_buff = []
aok_buff = []
anok_buff = []
validity_buff = []
for current_tile_stats in seed_stats:
histo_vok, bins_vok = get_valid_ok(current_tile_stats)
histo_vnok, bins_vnok = get_valid_nok(current_tile_stats)
histo_aok, bins_aok = get_app_ok(current_tile_stats)
histo_anok, bins_anok = get_app_nok(current_tile_stats)
histo_validity, bins_validity = get_validity(current_tile_stats)
vok_buff.append(histo_vok)
vnok_buff.append(histo_vnok)
aok_buff.append(histo_aok)
anok_buff.append(histo_anok)
validity_buff.append(histo_validity)
sum_vok = sum_in_list(vok_buff)
sum_vnok = sum_in_list(vnok_buff)
sum_aok = sum_in_list(aok_buff)
sum_anok = sum_in_list(anok_buff)
sum_validity = sum_in_list(validity_buff)
mean_vok, std_vok = compute_mean_std(sum_vok, bins_vok)
mean_vnok, std_vnok = compute_mean_std(sum_vnok, bins_vnok)
plt.plot(bins_vok,
sum_vok,
label="Valid OK\nmean: " + "{0:.2f}".format(mean_vok) +
"\nstd: " + "{0:.2f}".format(std_vok) + "\n",
color="green")
plt.plot(bins_vnok,
sum_vnok,
label="Valid NOK\nmean: " + "{0:.2f}".format(mean_vnok) +
"\nstd: " + "{0:.2f}".format(std_vnok) + "\n",
color="red")
plt.ylabel("Nb pix")
plt.xlabel("Confidence")
lgd = plt.legend(loc="center left",
bbox_to_anchor=(1, 0.8),
numpoints=1)
plt.title('Histogram')
plt.savefig(i2_output_path + "/final/Stats_VOK_VNOK.png",
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
# We clear the buffer and close the figure
plt.clf()
plt.close()
save_histogram(i2_output_path + "/final/Stats_VNOK.txt", sum_vnok,
bins_vnok)
save_histogram(i2_output_path + "/final/Stats_VOK.txt", sum_vok,
bins_vok)
plt.figure()
mean_aok, std_aok = compute_mean_std(sum_aok, bins_aok)
mean_anok, std_anok = compute_mean_std(sum_anok, bins_anok)
plt.plot(bins_aok,
sum_aok,
label="Learning OK\nmean: " + "{0:.2f}".format(mean_aok) +
"\nstd: " + "{0:.2f}".format(std_aok) + "\n",
color="yellow")
plt.plot(bins_anok,
sum_anok,
label="Learning NOK\nmean: " + "{0:.2f}".format(mean_anok) +
"\nstd: " + "{0:.2f}".format(std_anok),
color="blue")
plt.ylabel("Nb pix")
plt.xlabel("Confidence")
lgd = plt.legend(loc="center left",
bbox_to_anchor=(1, 0.8),
numpoints=1)
plt.title('Histogram')
plt.savefig(i2_output_path + "/final/Stats_LOK_LNOK.png",
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
# We clear the buffer and close the figure
plt.clf()
plt.close()
save_histogram(i2_output_path + "/final/Stats_LNOK.txt", sum_anok,
bins_anok)
save_histogram(i2_output_path + "/final/Stats_LOK.txt", sum_aok,
bins_aok)
plt.figure()
plt.bar(bins_validity,
sum_validity,
label="pixels validity",
color="red",
align="center")
plt.ylabel("Nb pix")
plt.xlabel("Validity")
plt.gca().yaxis.grid(True)
plt.legend()
plt.title('Histogram')
plt.xticks(bins_validity, bins_validity)
plt.xlim((0, max(bins_validity) + 1))
plt.savefig(i2_output_path + "/final/Validity.png",
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
# We clear the buffer and close the figure
plt.clf()
plt.close()
save_histogram(i2_output_path + "/final/Validity.txt", sum_validity,
bins_validity)
def launchClassification(model, path_cfg, output_path: str,
classifier_name: str, classification_mode: str,
nomenclature_path: str, stat, pathToRT, pathToImg,
pathToRegion, fieldRegion, pathToCmdClassif, pathOut,
RAM, pathWd):
"""
Parameters
----------
output_path : str
iota2 output directory
"""
scriptPath = os.path.join(fu.get_iota2_project_dir(), "iota2")
pixType = fu.getOutputPixType(nomenclature_path)
AllCmd = []
maskFiles = pathOut + "/MASK"
if not os.path.exists(maskFiles):
run("mkdir " + maskFiles)
if pathToRegion is None:
pathToRegion = os.path.join(output_path, "MyRegion.shp")
shpRName = pathToRegion.split("/")[-1].replace(".shp", "")
AllModel_tmp = fu.FileSearch_AND(model, True, "model", ".txt")
AllModel = fu.fileSearchRegEx(model + "/*model*.txt")
for currentFile in AllModel_tmp:
if currentFile not in AllModel:
os.remove(currentFile)
for path in AllModel:
model = path.split("/")[-1].split("_")[1]
tiles = fu.getListTileFromModel(
model, output_path + "/config_model/configModel.cfg")
model_Mask = model
if re.search('model_.*f.*_', path.split("/")[-1]):
model_Mask = path.split("/")[-1].split("_")[1].split("f")[0]
seed = path.split("/")[-1].split("_")[-1].replace(".txt", "")
suffix = ""
if "SAR.txt" in os.path.basename(path):
seed = path.split("/")[-1].split("_")[-2]
suffix = "_SAR"
tilesToEvaluate = tiles
#construction du string de sortie
for tile in tilesToEvaluate:
pathToFeat = fu.FileSearch_AND(pathToImg + "/" + tile + "/tmp/",
True, "MaskCommunSL", ".tif")[0]
maskSHP = pathToRT + "/" + shpRName + "_region_" + model_Mask + "_" + tile + ".shp"
maskTif = shpRName + "_region_" + model_Mask + "_" + tile + ".tif"
CmdConfidenceMap = ""
confidenceMap_name = "{}_model_{}_confidence_seed_{}{}.tif".format(
tile, model, seed, suffix)
CmdConfidenceMap = " -confmap " + os.path.join(
pathOut, confidenceMap_name)
if not os.path.exists(maskFiles + "/" + maskTif):
pathToMaskCommun = pathToImg + "/" + tile + "/tmp/" + "MaskCommunSL" + ".shp"
#cas cluster
if pathWd != None:
maskFiles = pathWd
nameOut = fu.ClipVectorData(maskSHP, pathToMaskCommun,
maskFiles,
maskTif.replace(".tif", ""))
cmdRaster = "otbcli_Rasterization -in "+nameOut+" -mode attribute -mode.attribute.field "+\
fieldRegion+" -im "+pathToFeat+" -out "+maskFiles+"/"+maskTif
if "fusion" in classification_mode:
cmdRaster = "otbcli_Rasterization -in "+nameOut+" -mode binary -mode.binary.foreground 1 -im "+\
pathToFeat+" -out "+maskFiles+"/"+maskTif
run(cmdRaster)
if pathWd != None:
run("cp " + pathWd + "/" + maskTif + " " + pathOut +
"/MASK")
os.remove(pathWd + "/" + maskTif)
out = pathOut + "/Classif_" + tile + "_model_" + model + "_seed_" + seed + suffix + ".tif"
#hpc case
if pathWd != None:
out = "$TMPDIR/Classif_" + tile + "_model_" + model + "_seed_" + seed + suffix + ".tif"
CmdConfidenceMap = " -confmap $TMPDIR/" + confidenceMap_name
appli = "python " + scriptPath + "/Classification/ImageClassifier.py -conf " + path_cfg + " "
pixType_cmd = " -pixType " + pixType
if pathWd != None:
pixType_cmd = pixType_cmd + " --wd $TMPDIR "
cmd = appli + " -in " + pathToFeat + " -model " + path + " -mask " + pathOut + "/MASK/" + maskTif + " -out " + out + " " + pixType_cmd + " -ram " + str(
RAM) + " " + CmdConfidenceMap
# add stats if svm
if "svm" in classifier_name.lower():
model_statistics = os.path.join(
stat, "Model_{}_seed_{}.xml".format(model, seed))
cmd = "{} -imstat {}".format(cmd, model_statistics)
AllCmd.append(cmd)
fu.writeCmds(pathToCmdClassif + "/class.txt", AllCmd)
return AllCmd
