def gen_conf_matrix(path_classif: str, path_valid: str, runs: int,
data_field: str, path_to_cmd_confusion: str, path_wd: str,
path_test: str, spatial_res: int, list_tiles: List[str],
enable_cross_validation: bool) -> List[str]:
"""
Parameters
----------
path_classif: string
path_valid: string
runs: int
data_field: string
path_to_cmd_confusion: string
path_wd: string
Return
------
list(string)
"""
all_cmd = []
path_tmp = os.path.join(path_classif, "TMP")
# path_Test = cfg.getParam('chain', 'outputPath')
# spatial_Res = cfg.getParam('chain', 'spatialResolution')
# enable_Cross_Validation = cfg.getParam('chain', 'enableCrossValidation')
working_directory = os.path.join(path_classif, "TMP")
if path_wd:
working_directory = path_wd
all_tiles = []
validation_files = fu.FileSearch_AND(path_valid, True, "_val.sqlite")
for valid in validation_files:
current_tile = valid.split("/")[-1].split("_")[0]
try:
all_tiles.index(current_tile)
except ValueError:
all_tiles.append(current_tile)
for seed in range(runs):
# recherche de tout les shapeFiles par seed, par tuiles pour
# les fusionner
for tile in all_tiles:
seed_val = seed
if enable_cross_validation:
seed_val = runs - 1
if enable_cross_validation and seed == runs - 1:
continue
val_tile = fu.FileSearch_AND(
path_valid, True, tile,
"_seed_" + str(seed_val) + "_val.sqlite")[0]
learn_tile = fu.FileSearch_AND(
path_valid, True, tile,
"_seed_" + str(seed) + "_learn.sqlite")[0]
path_directory = path_tmp
cmd = (f'otbcli_ComputeConfusionMatrix -in {path_classif}/"'
f'"Classif_Seed_{seed}.tif -out {path_directory}/'
f'{tile}_seed_{seed}.csv'
f' -ref.vector.field {data_field.lower()} -ref vector '
f'-ref.vector.in {val_tile}')
all_cmd.append(cmd)
classif = path_tmp + "/" + tile + "_seed_" + str(seed) + ".tif"
diff = path_tmp + "/" + tile + "_seed_" + str(
seed) + "_CompRef.tif"
compare_ref(val_tile, learn_tile, classif, diff, working_directory,
path_wd, data_field, spatial_res)
fu.writeCmds(path_to_cmd_confusion + "/confusion.txt", all_cmd)
if enable_cross_validation:
runs = runs - 1
for seed in range(runs):
all_diff = fu.FileSearch_AND(path_tmp, True,
f"_seed_{seed}_CompRef.tif")
diff_seed = os.path.join(path_test, "final", f"diff_seed_{seed}.tif")
if path_wd:
diff_seed = os.path.join(working_directory,
f"diff_seed_{seed}.tif")
fu.assembleTile_Merge(all_diff, spatial_res, diff_seed, ot="Byte")
if path_wd:
shutil.copy(
working_directory + f"/diff_seed_{seed}.tif",
os.path.join(path_test, "final", f"diff_seed_{seed}.tif"))
# Create dummy rasters if necessary
tile_asked = list_tiles.split()
missing_tiles = [elem for elem in tile_asked if elem not in all_tiles]
create_dummy_rasters(missing_tiles, runs, path_test)
return all_cmd
def classification_shaping(path_classif: str, runs: int, path_out: str,
path_wd: str, classif_mode: str, path_test: str,
ds_sar_opt: bool, proj: int, nomenclature_path: str,
output_statistics: bool, spatial_resolution: float,
proba_map_flag: bool, region_shape: str,
color_path: str) -> None:
"""function use to mosaic rasters and to produce final maps
path_classif: str
directory where as classifications
runs: int
number of random learning/validation samples-set
path_out: str
output directory
path_wd: str
working directory
classif_mode: str
fusion of classifications ?
path_test: str
iota2 output directory
ds_sar_opt: bool
flag to inform if SAR and optical post-classification workflow
is enable
proj: int
epsg code
nomenclature_path: str
nomenclature path
output_statistics: bool
flag to enable output statistics
spatial_resolution: float
output's spatial resolution
proba_map_flag: bool
flag to inform if probability map was produce
region_shape: str
region shapeFile path
color_path: str
color table file
"""
if path_wd is None:
tmp = path_out + "/TMP"
if not os.path.exists(path_out + "/TMP"):
os.mkdir(tmp)
else:
tmp = path_wd
if not os.path.exists(path_out + "/TMP"):
os.mkdir(path_out + "/TMP")
all_tiles = list(
set([
classif.split("_")[1] for classif in fu.FileSearch_AND(
path_test + "/classif", False, "Classif", ".tif")
]))
pix_type = fu.getOutputPixType(nomenclature_path)
features_path = os.path.join(path_test, "features")
all_tmp_folder = fu.fileSearchRegEx(path_test + "/TMPFOLDER*")
if all_tmp_folder:
for tmp_folder in all_tmp_folder:
shutil.rmtree(tmp_folder)
suffix = "*"
if ds_sar_opt:
suffix = "*DS*"
genGlobalConfidence(runs, path_wd, spatial_resolution, proj, path_test,
classif_mode, all_tiles, region_shape, ds_sar_opt,
proba_map_flag)
if region_shape and classif_mode == "fusion":
old_classif = fu.fileSearchRegEx(path_test +
"/classif/Classif_*_model_*f*_seed_" +
suffix + ".tif")
for rm in old_classif:
if not os.path.exists(path_test + "/final/TMP/OLDCLASSIF"):
os.mkdir(path_test + "/final/TMP/OLDCLASSIF")
run("mv " + rm + " " + path_test + "/final/TMP/OLDCLASSIF")
classification = []
confidence = []
proba_map = []
cloud = []
for seed in range(runs):
classification.append([])
confidence.append([])
cloud.append([])
sort = []
if proba_map_flag:
proba_map_list = fu.fileSearchRegEx(
path_test + "/classif/PROBAMAP_*_model_*_seed_" + str(seed) +
suffix + ".tif")
proba_map_list = removeInListByRegEx(
proba_map_list, ".*model_.*f.*_seed." + suffix)
proba_map.append(proba_map_list)
if classif_mode == "separate" or region_shape:
all_classif_seed = fu.FileSearch_AND(path_classif, True, ".tif",
"Classif",
"seed_" + str(seed))
if ds_sar_opt:
all_classif_seed = fu.FileSearch_AND(path_classif, True,
".tif", "Classif",
"seed_" + str(seed),
"DS.tif")
ind = 1
elif classif_mode == "fusion":
all_classif_seed = fu.FileSearch_AND(
path_classif, True, "_FUSION_NODATA_seed" + str(seed) + ".tif")
if ds_sar_opt:
all_classif_seed = fu.FileSearch_AND(
path_classif, True,
"_FUSION_NODATA_seed" + str(seed) + "_DS.tif")
ind = 0
for tile in all_classif_seed:
sort.append((tile.split("/")[-1].split("_")[ind], tile))
sort = fu.sortByFirstElem(sort)
for tile, paths in sort:
exp = ""
all_cl = ""
all_cl_rm = []
for i in range(len(paths)):
all_cl = all_cl + paths[i] + " "
all_cl_rm.append(paths[i])
if i < len(paths) - 1:
exp = exp + "im" + str(i + 1) + "b1 + "
else:
exp = exp + "im" + str(i + 1) + "b1"
path_cl_final = tmp + "/" + tile + "_seed_" + str(seed) + ".tif"
classification[seed].append(path_cl_final)
cmd = 'otbcli_BandMath -il ' + all_cl + '-out ' + path_cl_final + ' ' + pix_type + ' -exp "' + exp + '"'
run(cmd)
tile_confidence = path_out + "/TMP/" + tile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
confidence[seed].append(tile_confidence)
cloud_tile = fu.FileSearch_AND(features_path + "/" + tile, True,
"nbView.tif")[0]
classif_tile = tmp + "/" + tile + "_seed_" + str(seed) + ".tif"
cloud_tile_priority = path_test + "/final/TMP/" + tile + "_Cloud.tif"
cloud_tile_priority_tmp = tmp + "/" + tile + "_Cloud.tif"
cloud_tile_priority_stats_ok = path_test + "/final/TMP/" + tile + "_Cloud_StatsOK.tif"
cloud_tile_priority_tmp_stats_ok = tmp + "/" + tile + "_Cloud_StatsOK.tif"
cloud[seed].append(cloud_tile_priority)
if not os.path.exists(cloud_tile_priority):
cmd_cloud = f'otbcli_BandMath -il {cloud_tile} {classif_tile} -out {cloud_tile_priority_tmp} int16 -exp "im2b1>0?im1b1:0"'
run(cmd_cloud)
if output_statistics:
cmd_cloud = 'otbcli_BandMath -il ' + cloud_tile + ' ' + classif_tile + ' -out ' + cloud_tile_priority_tmp_stats_ok + ' int16 -exp "im2b1>0?im1b1:-1"'
run(cmd_cloud)
if path_wd:
shutil.copy(cloud_tile_priority_tmp_stats_ok,
cloud_tile_priority_stats_ok)
os.remove(cloud_tile_priority_tmp_stats_ok)
if path_wd:
shutil.copy(cloud_tile_priority_tmp, cloud_tile_priority)
os.remove(cloud_tile_priority_tmp)
if path_wd is not None:
run("cp -a " + tmp + "/* " + path_out + "/TMP")
for seed in range(runs):
assemble_folder = path_test + "/final"
if path_wd:
assemble_folder = path_wd
fu.assembleTile_Merge(classification[seed],
spatial_resolution,
"{}/Classif_Seed_{}.tif".format(
assemble_folder, seed),
"Byte" if pix_type == "uint8" else "Int16",
co={
"COMPRESS": "LZW",
"BIGTIFF": "YES"
})
if path_wd:
shutil.copy(path_wd + "/Classif_Seed_" + str(seed) + ".tif",
path_test + "/final")
os.remove(path_wd + "/Classif_Seed_" + str(seed) + ".tif")
fu.assembleTile_Merge(confidence[seed],
spatial_resolution,
assemble_folder + "/Confidence_Seed_" +
str(seed) + ".tif",
"Byte",
co={
"COMPRESS": "LZW",
"BIGTIFF": "YES"
})
if path_wd:
shutil.copy(path_wd + "/Confidence_Seed_" + str(seed) + ".tif",
path_test + "/final")
os.remove(path_wd + "/Confidence_Seed_" + str(seed) + ".tif")
color.CreateIndexedColorImage(
path_test + "/final/Classif_Seed_" + str(seed) + ".tif",
color_path,
output_pix_type=gdal.GDT_Byte
if pix_type == "uint8" else gdal.GDT_UInt16)
if proba_map_flag:
proba_map_mosaic = os.path.join(
assemble_folder, "ProbabilityMap_seed_{}.tif".format(seed))
fu.assembleTile_Merge(proba_map[seed],
spatial_resolution,
proba_map_mosaic,
"Int16",
co={
"COMPRESS": "LZW",
"BIGTIFF": "YES"
})
if path_wd:
shutil.copy(proba_map_mosaic, path_test + "/final")
os.remove(proba_map_mosaic)
fu.assembleTile_Merge(cloud[0],
spatial_resolution,
assemble_folder + "/PixelsValidity.tif",
"Byte",
co={
"COMPRESS": "LZW",
"BIGTIFF": "YES"
})
if path_wd:
shutil.copy(path_wd + "/PixelsValidity.tif", path_test + "/final")
os.remove(path_wd + "/PixelsValidity.tif")
def mergeTileRaster(path, clipfile, fieldclip, valueclip, tiles, tilesfolder,
tileId, tileNamePrefix, out):
if not os.path.exists(
os.path.join(out,
"tile_" + fieldclip + "_" + str(valueclip) + '.tif')):
timeinit = time.time()
localenv = os.path.join(path, "tmp%s" % (str(valueclip)))
if os.path.exists(localenv): shutil.rmtree(localenv)
os.mkdir(localenv)
# Find vector tiles concerned by the given zone
listTilesFiles = getTilesFiles(clipfile, tiles, tilesfolder, tileId,
tileNamePrefix, localenv, fieldclip,
valueclip)
print(listTilesFiles)
outraster = os.path.join(
localenv, "tile_" + fieldclip + "_" + str(valueclip) + '.tif')
logger.info('Raster mosaic of zone %s' % (str(valueclip)))
print('Raster mosaic of zone %s' % (str(valueclip)))
if len(listTilesFiles) != 0:
tomerge = []
for rasttile in listTilesFiles:
shutil.copy(rasttile, localenv)
tmptile = os.path.join(localenv, os.path.basename(rasttile))
rasttiletmp = os.path.join(
localenv,
os.path.splitext(os.path.basename(rasttile))[0] +
'_nd.tif')
bmappli = oa.CreateBandMathApplication({
"il":
tmptile,
"out":
rasttiletmp,
"exp":
'im1b1 < 223 ? im1b1 : 0'
})
bmappli.ExecuteAndWriteOutput()
tomerge.append(rasttiletmp)
if len(tomerge) != 0:
sx, sy = fut.getRasterResolution(tomerge[0])
fut.assembleTile_Merge(tomerge, sx, outraster, "Byte")
logger.info('Raster mosaic "%s" done for zone %s' %
(outraster, str(valueclip)))
for rasttile in tomerge:
os.remove(rasttile)
if os.path.exists(out):
if os.path.isdir(out):
shutil.copy(outraster, out)
else:
logger.info(
'Output folder %s for mosaic storage does not exist' %
(out))
timemerge = time.time()
print(" ".join([
" : ".join(["Merge Tiles",
str(timemerge - timeinit)]), "seconds"
]))
return outraster
else:
print(
"No tiles or crown tile rasters does not exist for the area %s of the clip file %s"
% (valueclip, os.path.basename(clipfile)))
else:
logger.info('Raster mosaic "%s" already exists' % (str(valueclip)))
print('Raster mosaic "%s" already exists' % (str(valueclip)))
