def extract_poi(tile_vector: str,
region: str,
seed: int,
region_field: str,
poi: str,
poi_val: str,
force_seed_field: Optional[bool] = None) -> None:
"""
Extract Polygon Of Interest
Parameters
----------
tile_vector: string
region: string
seed: int
region_field: str
poi: str
poi_val: str
force_seed_field: bool
Return
------
None
"""
from iota2.Common.Utils import run
learn_flag = "learn"
validation_flag = "validation"
seed_field = "seed_{}".format(seed)
cmd = (f"ogr2ogr -where \"{region_field}='{region}' AND {seed_field}"
f"='{learn_flag}'\" {poi} {tile_vector}")
run(cmd)
if poi_val:
if force_seed_field:
seed_field = force_seed_field
cmd = (f"ogr2ogr -where \"{region_field}='{region}' AND {seed_field}="
f"'{validation_flag}'\" {poi_val} {tile_vector}")
run(cmd)
def shape_reference_vector(ref_vector: str, output_name: str) -> str:
"""
modify reference vector (add field, rename...)
Parameters
----------
ref_vector : string
output_name : string
Return
------
string
"""
import os
from iota2.Common.Utils import run
from iota2.Common import FileUtils as fut
from iota2.VectorTools.AddField import addField
path, _ = os.path.split(ref_vector)
tmp = os.path.join(path, output_name + "_TMP")
fut.cpShapeFile(ref_vector.replace(".shp", ""), tmp,
[".prj", ".shp", ".dbf", ".shx"])
addField(tmp + ".shp", "region", "1", str)
addField(tmp + ".shp", "seed_0", "learn", str)
cmd = (f"ogr2ogr -dialect 'SQLite' -sql 'select GEOMETRY,seed_0, "
f"region, CODE as code from {output_name}_TMP' "
f"{path}/{output_name}.shp {tmp}.shp")
run(cmd)
os.remove(tmp + ".shp")
os.remove(tmp + ".shx")
os.remove(tmp + ".prj")
os.remove(tmp + ".dbf")
return path + "/" + output_name + ".shp"
def extract_meta_data_fields(input_sample_file_name: str,
reduced_output_file_name: str) -> None:
"""
Parameters
----------
input_sample_file_name: string
reduced_output_file_name: string
Return
------
None
Notes
-----
Extract MetaDataFields from input vector file in order to append reduced
fields
"""
from iota2.Common.Utils import run
reduced_output_file_name_table = "output"
(_, meta_data_fields) = get_available_features(input_sample_file_name,
'global')
fields = ",".join(meta_data_fields)
cmd = (f"ogr2ogr -dialect 'SQLITE' -nln {reduced_output_file_name_table} "
f"-f 'SQLite' -select '{fields}' "
f"{reduced_output_file_name} {input_sample_file_name}")
run(cmd)
def test_create_tile_region_masks(self):
"""
test the generation of the raster mask which define the region
in the tile
"""
from iota2.Sampling.VectorFormatting import create_tile_region_masks
from iota2.Common.Utils import run
from iota2.Tests.UnitTests.TestsUtils import rasterToArray
import numpy as np
# define inputs
test_vector_name = "T31TCJ.sqlite"
test_vector = os.path.join(self.test_working_directory,
test_vector_name)
cmd = "ogr2ogr -nln t31tcj -f SQLite {} {}".format(
test_vector, self.ref_region)
run(cmd)
# launch function
create_tile_region_masks(test_vector, "region", "T31TCJ",
self.test_working_directory, "MyRegion",
self.ref_img)
# assert
raster_region = fut.FileSearch_AND(self.test_working_directory, True,
"MyRegion", ".tif")[0]
raster_region_arr = rasterToArray(raster_region)
ref_array = np.ones((50, 50))
self.assertTrue(np.allclose(ref_array, raster_region_arr),
msg="problem with the normalization by ref")
def extract_class(vec_in: str, vec_out: str, target_class: List[str],
data_field: str) -> int:
"""
Extract class
IN:
vec_in: str
vec_out: str
target_class:List[str]
data_field: str
"""
from iota2.Common.Utils import run
if type(target_class) != type(list()):
target_class = target_class.data
where = " OR ".join(
["{}={}".format(data_field.lower(), klass) for klass in target_class])
cmd = (f"ogr2ogr -f 'SQLite' -nln output -where '{where}' "
f"{vec_out} {vec_in}")
run(cmd)
return len(
fu.getFieldElement(vec_out,
driverName="SQLite",
field=data_field.lower(),
mode="all",
elemType="int"))
def create_dummy_rasters(missing_tiles: List[str], runs: int,
output_path: str) -> None:
"""
Parameters
----------
missing_tiles: list(string)
runs: int
output_path: string
Return
------
None
Notes
-----
use when mode is 'one_region' but there is no validations / learning
samples into a specific tile
"""
classifications_dir = os.path.join(output_path, "classif")
final_dir = os.path.join(output_path, "final", "TMP")
for tile in missing_tiles:
classif_tile = fu.FileSearch_AND(classifications_dir, True,
"Classif_" + str(tile))[0]
for seed in range(runs):
dummy_raster_name = tile + "_seed_" + str(seed) + "_CompRef.tif"
dummy_raster = final_dir + "/" + dummy_raster_name
dummy_raster_cmd = (f"gdal_merge.py -ot Byte -n 0 -createonly -o "
f"{ dummy_raster} {classif_tile}")
run(dummy_raster_cmd)
def test_split_vector_by_region(self):
"""
test : split a vector by the region he belongs to
"""
from iota2.Sampling.VectorFormatting import split_vector_by_region
from iota2.Common.Utils import run
from iota2.Tests.UnitTests.Iota2Tests import random_update
# define inputs
nb_features_origin = len(
fut.getFieldElement(self.in_vector,
driverName="ESRI shapefile",
field="region",
mode="all",
elemType="str"))
nb_features_new_region = 5
test_vector_name = "T31TCJ_Samples.sqlite"
test_vector = os.path.join(self.test_working_directory,
test_vector_name)
cmd = "ogr2ogr -nln output -f SQLite {} {}".format(
test_vector, self.in_vector)
run(cmd)
random_update(test_vector, "output", "seed_0", "learn",
nb_features_origin)
random_update(test_vector, "output", "region", "2",
nb_features_new_region)
output_dir = self.test_working_directory
region_field = "region"
# launch function
split_vector_by_region(test_vector,
output_dir,
region_field,
runs=1,
driver="SQLite")
# assert
vector_reg_1 = fut.FileSearch_AND(self.test_working_directory, True,
"region_1")[0]
vector_reg_2 = fut.FileSearch_AND(self.test_working_directory, True,
"region_2")[0]
feat_vect_reg_1 = len(
fut.getFieldElement(vector_reg_1,
driverName="SQLite",
field="region",
mode="all",
elemType="str"))
feat_vect_reg_2 = len(
fut.getFieldElement(vector_reg_2,
driverName="SQLite",
field="region",
mode="all",
elemType="str"))
self.assertTrue(nb_features_new_region == feat_vect_reg_2)
self.assertTrue(nb_features_origin == feat_vect_reg_1 +
feat_vect_reg_2)
def gen_mask_region_by_tile(field_region: str,
stack_ind: str,
working_dir: str,
current_tile: str,
all_model: str,
shp_rname: str,
path_to_img: str,
path_test: str,
path_to_config: str,
path_wd: Optional[str] = None) -> List[str]:
"""
Parameters
----------
field_region: string
stack_ind: string
working_dir: string
current_tile: string
all_model: string
shp_rname: string
path_to_image: string
path_test: string
path_wd: string
path_to_config: string
config file containing which model is associated to a given tile
Return
------
list(str)
"""
model_tile = []
for path in all_model:
current_model = path.split("/")[-1].split("_")[1]
tiles = fu.getListTileFromModel(current_model, path_to_config)
model = path.split("/")[-1].split("_")[1]
for tile in tiles:
# get the model which learn the current tile
if tile == current_tile:
model_tile.append(model)
path_to_feat = os.path.join(path_to_img, tile, "Final", stack_ind)
mask_shp = os.path.join(path_test, "shapeRegion",
f"{shp_rname}_region_{model}_{tile}.shp")
mask_tif = os.path.join(
working_dir, f"{shp_rname}_region_{model}_{tile}_NODATA.tif")
mask_tif_f = os.path.join(
path_test, "classif", "MASK",
f"{shp_rname}_region_{model}_{tile}_NODATA.tif")
# Create mask
if not os.path.exists(mask_tif_f):
cmd_raster = (
f"otbcli_Rasterization -in {mask_shp} -mode attribute "
f"-mode.attribute.field {field_region} -im {path_to_feat}"
f" -out {mask_tif}")
run(cmd_raster)
if path_wd is not None:
cmd = "cp " + mask_tif + " " + path_test + "/classif/MASK"
run(cmd)
return model_tile
def split_sel(model_selection, tiles, working_directory, epsg):
"""
split a SQLite file containing points by tiles
Parameters
----------
model_selection : string
path to samplesSelection's output to a given model
tiles : list
list of tiles intersected by the model
epsg : string
epsg's projection. ie : epsg="EPSG:2154"
working_directory : string
path to a working directory
"""
import sqlite3 as lite
from iota2.Common.Utils import run
tile_field_name = "tile_o"
out_tiles = []
for tile in tiles:
mod_sel_name = os.path.splitext(os.path.basename(model_selection))[0]
tile_mod_sel_name_tmp = "{}_{}_tmp".format(tile, mod_sel_name)
tile_mod_sel_tmp = os.path.join(working_directory,
tile_mod_sel_name_tmp + ".sqlite")
if os.path.exists(tile_mod_sel_tmp):
os.remove(tile_mod_sel_tmp)
conn = lite.connect(tile_mod_sel_tmp)
cursor = conn.cursor()
cursor.execute("ATTACH '{}' AS db".format(model_selection))
cursor.execute(
"CREATE TABLE {} as SELECT * FROM db.output WHERE {}='{}'".format(
tile_mod_sel_name_tmp.lower(), tile_field_name, tile))
conn.commit()
tile_mod_sel_name = "{}_{}".format(tile, mod_sel_name)
tile_mod_sel = os.path.join(working_directory,
tile_mod_sel_name + ".sqlite")
clause = "SELECT * FROM {}".format(tile_mod_sel_name_tmp)
cmd = (f'ogr2ogr -sql "{clause}" -dialect "SQLite" -f "SQLite" -s_srs '
f'{epsg} -t_srs {epsg} -nln {tile_mod_sel_name.lower()}'
f' {tile_mod_sel} {tile_mod_sel_tmp}')
run(cmd)
os.remove(tile_mod_sel_tmp)
out_tiles.append(tile_mod_sel)
conn = cursor = None
return out_tiles
def commonMasks(tile_name,
output_path,
sensors_parameters,
working_directory=None,
RAM=128):
"""
compute common mask considering all sensors by tile
Parameters
----------
tile_name [string]
tile's name
config_path [string]
absolute path to the configuration file
output_path : str
iota2 output path
working_directory [string]
absolute path to a working directory
RAM [int]
pipeline's size (Mo)
"""
import os
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.Utils import run
from iota2.Common.FileUtils import ensure_dir
# running_parameters = iota2_parameters(config_path)
# sensors_parameters = running_parameters.get_sensors_parameters(tile_name)
remote_sensor_container = sensors_container(tile_name, working_directory,
output_path,
**sensors_parameters)
common_mask, _ = remote_sensor_container.get_common_sensors_footprint(
available_ram=RAM)
common_mask_raster = common_mask.GetParameterValue("out")
if not os.path.exists(common_mask_raster):
ensure_dir(os.path.split(common_mask_raster)[0], raise_exe=False)
common_mask.ExecuteAndWriteOutput()
common_mask_vector = common_mask_raster.replace(".tif", ".shp")
common_mask_vector_cmd = "gdal_polygonize.py -f \"ESRI Shapefile\" -mask {} {} {}".format(
common_mask_raster, common_mask_raster, common_mask_vector)
run(common_mask_vector_cmd)
def get_region_model_in_tile(current_tile: str, current_region: str,
output_path: str, path_wd: str, ref_img: str,
field_region: str, test_mode: bool,
test_path: str, test_output_folder: str) -> str:
"""
usage : rasterize region shape.
IN:
currentTile [string] : tile to compute
currentRegion [string] : current region in tile
output_path [str] : output path
pathWd [string] : working directory
refImg [string] : reference image
testMode [bool] : flag to enable test mode
testPath [string] : path to the vector shape
testOutputFolder [string] : path to the output folder
OUT:
rasterMask [string] : path to the output raster
"""
from iota2.Common.Utils import run
working_directory = os.path.join(output_path, "learningSamples")
if path_wd:
working_directory = path_wd
name_out = f"Mask_region_{current_region}_{current_tile}.tif"
if test_mode:
mask_shp = test_path
working_directory = test_output_folder
else:
mask_shp = fu.FileSearch_AND(output_path + "/shapeRegion/", True,
current_tile,
f"region_{current_region.split('f')[0]}",
".shp")[0]
raster_mask = os.path.join(working_directory, name_out)
cmd_raster = (f"otbcli_Rasterization -in {mask_shp} -mode attribute "
f"-mode.attribute.field {field_region} -im {ref_img} "
f"-out {raster_mask}")
run(cmd_raster)
return raster_mask
def keep_fields(vec_in: str,
vec_out: str,
fields: Optional[List[str]] = [],
proj_in: Optional[int] = 2154,
proj_out: Optional[int] = 2154) -> None:
"""
use to extract fields of an input SQLite file
Parameters
----------
vec_in : string
input SQLite vector File
vec_out : string
output SQLite vector File
fields : list
list of fields to keep
proj_in : int
input projection
proj_out : int
output projection
"""
from iota2.VectorTools.vector_functions import get_geom_column_name
from iota2.Common.Utils import run
table_in = (os.path.splitext(os.path.split(vec_in)[-1])[0]).lower()
table_out = (os.path.splitext(os.path.split(vec_out)[-1])[0]).lower()
_, ext = os.path.splitext(vec_in)
driver_vec_in = "ESRI Shapefile"
if "sqlite" in ext:
driver_vec_in = "SQLite"
geom_column_name = get_geom_column_name(vec_in, driver=driver_vec_in)
sql_clause = (f"select {geom_column_name},"
f"{','.join(fields)} from {table_in}")
cmd = (f"ogr2ogr -s_srs EPSG:{proj_in} -t_srs EPSG:{proj_out} -dialect "
f"'SQLite' -f 'SQLite' -nln {table_out} -sql '{sql_clause}' "
f"{vec_out} {vec_in}")
run(cmd)
def extract_roi(raster: str,
current_tile: str,
path_wd: str,
output_path: str,
name: str,
ref: str,
test_mode: Optional[bool] = None,
test_output: Optional[str] = None) -> str:
"""
usage : extract ROI in raster
IN:
raster [string] : path to the input raster
currentTile [string] : current tile to compute
output_path [string] : the output path
path_wd [string] : path to the working directory
name [string] : output name
ref [strin] : raster reference use to get it's extent
test_mode [bool] : activate test mode
test_output [str] : path to test output data
OUT:
raterROI [string] : path to the extracted raster.
"""
from iota2.Common.Utils import run
working_directory = output_path + "/learningSamples/"
if path_wd:
working_directory = path_wd
if test_mode:
working_directory = test_output
current_tile_raster = ref
min_x, max_x, min_y, max_y = fu.getRasterExtent(current_tile_raster)
raster_roi = os.path.join(working_directory,
current_tile + "_" + name + ".tif")
cmd = (f"gdalwarp -of GTiff -te {min_x} {min_y} {max_x} {max_y} "
f"-ot Byte {raster} {raster_roi}")
run(cmd)
return raster_roi
def concat_region_one_tile(path_test: str, classif_fusion_mask: str,
path_wd: str, tile_mask_concat: str) -> str:
"""
Parameters
----------
path_test: string
classif_fusion_mask: string
path_wd: string
tile_mask_concat: string
Return
------
string
"""
# TileMask_concat = pathTest+"/classif/"+currentTile+"_MASK.tif"
if len(classif_fusion_mask) == 1 and not os.path.exists(tile_mask_concat):
shutil.copy(classif_fusion_mask[0], tile_mask_concat)
elif len(classif_fusion_mask) != 1 and not os.path.exists(
tile_mask_concat):
# in order to match images and their mask
classif_fusion_mask_sort = sorted(classif_fusion_mask,
key=get_model_in_mask)
string_cl_fus = " ".join(classif_fusion_mask_sort)
path_directory = os.path.join(path_test, "classif")
if path_wd is not None:
path_directory = path_wd
cmd = (
f"otbcli_ConcatenateImages -ram 128 -il {string_cl_fus} -out "
f"{os.path.join(path_directory, tile_mask_concat.split('/')[-1])}")
run(cmd)
if path_wd is not None:
cmd = (
f"cp {os.path.join(path_wd, tile_mask_concat.split('/')[-1])}"
f"{os.path.join(path_test, 'classif')}")
run(cmd)
return tile_mask_concat
def transform_to_shape(sqlite_vectors: List[str],
formatting_vectors_dir: str) -> List[str]:
"""
Parameters
----------
sqlite_vectors: list(str)
formatting_vectors_dir: str
Return
------
list(str)
"""
from iota2.Common import FileUtils as fut
from iota2.Common.Utils import run
out = []
for sqlite_vector in sqlite_vectors:
out_name = os.path.splitext(os.path.basename(sqlite_vector))[0]
out_path = os.path.join(formatting_vectors_dir, f"{out_name}.shp")
if os.path.exists(out_path):
fut.removeShape(out_path.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
cmd = f"ogr2ogr -f 'ESRI Shapefile' {out_path} {sqlite_vector}"
run(cmd)
out.append(out_path)
return out
def gen_raster_ref(vec, output_path, masks_name, working_directory):
"""
generate the reference image needed to sampleSelection application
Parameters
----------
vec : string
path to the shapeFile containing all polygons dedicated to learn
a model.
cfg : ServiceConfigFile object
working_directory : string
Path to a working directory
"""
from iota2.Common.Utils import run
tile_field_name = "tile_o"
# iota2_dir = cfg.getParam('chain', 'outputPath')
features_directory = os.path.join(output_path, "features")
tiles = fut.getFieldElement(vec,
driverName="ESRI Shapefile",
field=tile_field_name,
mode="unique",
elemType="str")
# masks_name = fut.getCommonMaskName(cfg) + ".tif"
rasters_tiles = [
fut.FileSearch_AND(os.path.join(features_directory, tile_name), True,
masks_name)[0] for tile_name in tiles
]
raster_ref_name = "ref_raster_{}.tif".format(
os.path.splitext(os.path.basename(vec))[0])
raster_ref = os.path.join(working_directory, raster_ref_name)
raster_ref_cmd = "gdal_merge.py -ot Byte -n 0 -createonly -o {} {}".format(
raster_ref, " ".join(rasters_tiles))
run(raster_ref_cmd)
return raster_ref, tiles
def concat_classifs_one_tile(seed: str,
current_tile: str,
path_test: str,
model_tile: str,
concat_out: str,
path_wd: Optional[str] = None) -> str:
"""
Parameters
----------
path_wd: string
seed: string
current_tile: string
path_test: string
model_tile: string
concat_out: string
Return
------
string
"""
classif_fusion = []
for model in model_tile:
classif_fusion.append(
os.path.join(
path_test, "classif", f"Classif_{current_tile}_model_"
f"{model}_seed_{seed}.tif"))
if len(classif_fusion) == 1:
cmd = f"cp {classif_fusion[0]} {concat_out}"
run(cmd)
else:
# in order to match images and their mask
classif_fusion_sort = sorted(classif_fusion, key=get_model_in_classif)
string_cl_fus = " ".join(classif_fusion_sort)
path_to_directory = os.path.join(path_test, "classif")
if path_wd is not None:
path_to_directory = path_wd
cmd = (f"otbcli_ConcatenateImages -ram 128 -il {string_cl_fus} -out "
f"{os.path.join(path_to_directory,concat_out.split('/')[-1])}")
run(cmd)
if not os.path.exists(concat_out):
cmd = (f"cp {os.path.join(path_wd, concat_out.split('/')[-1])} "
f"{os.path.join(path_test , 'classif')}")
run(cmd)
return concat_out
def validity(tile_name,
config_path,
output_path,
maskOut_name,
view_threshold,
workingDirectory=None,
RAM=128):
"""
function dedicated to compute validity raster/vector by tile
Parameters
----------
tile_name [string]
tile's name
config_path [string]
absolute path to the configuration file
maskOut_name [string]
output vector mask's name
view_threshold [int]
threshold
working_directory [string]
absolute path to a working directory
RAM [int]
pipeline's size (Mo)
"""
import os
import shutil
from iota2.Common.ServiceConfigFile import iota2_parameters
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.Utils import run
from iota2.Common.FileUtils import erodeShapeFile
from iota2.Common.FileUtils import removeShape
from iota2.Common.FileUtils import ensure_dir
features_dir = os.path.join(output_path, "features", tile_name)
validity_name = "nbView.tif"
validity_out = os.path.join(features_dir, validity_name)
validity_processing = validity_out
if workingDirectory:
ensure_dir(os.path.join(workingDirectory, tile_name))
validity_processing = os.path.join(workingDirectory, tile_name,
validity_name)
running_parameters = iota2_parameters(config_path)
sensors_parameters = running_parameters.get_sensors_parameters(tile_name)
remote_sensor_container = sensors_container(tile_name, workingDirectory,
output_path,
**sensors_parameters)
sensors_time_series_masks = remote_sensor_container.get_sensors_time_series_masks(
available_ram=RAM)
sensors_masks_size = []
sensors_masks = []
for sensor_name, (time_series_masks, time_series_dep,
nb_bands) in sensors_time_series_masks:
if sensor_name.lower() == "sentinel1":
for _, time_series_masks_app in list(time_series_masks.items()):
time_series_masks_app.Execute()
sensors_masks.append(time_series_masks_app)
else:
time_series_masks.Execute()
sensors_masks.append(time_series_masks)
sensors_masks_size.append(nb_bands)
total_dates = sum(sensors_masks_size)
merge_masks = CreateConcatenateImagesApplication({
"il": sensors_masks,
"ram": str(RAM)
})
merge_masks.Execute()
validity_app = CreateBandMathApplication({
"il":
merge_masks,
"exp":
"{}-({})".format(
total_dates,
"+".join(["im1b{}".format(i + 1) for i in range(total_dates)])),
"ram":
str(0.7 * RAM),
"pixType":
"uint8" if total_dates < 255 else "uint16",
"out":
validity_processing
})
if not os.path.exists(os.path.join(features_dir, validity_name)):
validity_app.ExecuteAndWriteOutput()
if workingDirectory:
shutil.copy(validity_processing,
os.path.join(features_dir, validity_name))
threshold_raster_out = os.path.join(features_dir,
maskOut_name.replace(".shp", ".tif"))
threshold_vector_out_tmp = os.path.join(
features_dir, maskOut_name.replace(".shp", "_TMP.shp"))
threshold_vector_out = os.path.join(features_dir, maskOut_name)
input_threshold = validity_processing if os.path.exists(
validity_processing) else validity_out
threshold_raster = CreateBandMathApplication({
"il":
input_threshold,
"exp":
"im1b1>={}?1:0".format(view_threshold),
"ram":
str(0.7 * RAM),
"pixType":
"uint8",
"out":
threshold_raster_out
})
threshold_raster.ExecuteAndWriteOutput()
cmd_poly = f"gdal_polygonize.py -mask {threshold_raster_out} {threshold_raster_out} -f \"ESRI Shapefile\" {threshold_vector_out_tmp} {os.path.splitext(os.path.basename(threshold_vector_out_tmp))[0]} cloud"
run(cmd_poly)
erodeShapeFile(threshold_vector_out_tmp, threshold_vector_out, 0.1)
os.remove(threshold_raster_out)
removeShape(threshold_vector_out_tmp.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
def launchClassification(model, path_cfg, output_path: str,
classifier_name: str, classification_mode: str,
nomenclature_path: str, stat, pathToRT, pathToImg,
pathToRegion, fieldRegion, pathToCmdClassif, pathOut,
RAM, pathWd):
"""
Parameters
----------
output_path : str
iota2 output directory
"""
scriptPath = os.path.join(fu.get_iota2_project_dir(), "iota2")
pixType = fu.getOutputPixType(nomenclature_path)
AllCmd = []
maskFiles = pathOut + "/MASK"
if not os.path.exists(maskFiles):
run("mkdir " + maskFiles)
if pathToRegion is None:
pathToRegion = os.path.join(output_path, "MyRegion.shp")
shpRName = pathToRegion.split("/")[-1].replace(".shp", "")
AllModel_tmp = fu.FileSearch_AND(model, True, "model", ".txt")
AllModel = fu.fileSearchRegEx(model + "/*model*.txt")
for currentFile in AllModel_tmp:
if currentFile not in AllModel:
os.remove(currentFile)
for path in AllModel:
model = path.split("/")[-1].split("_")[1]
tiles = fu.getListTileFromModel(
model, output_path + "/config_model/configModel.cfg")
model_Mask = model
if re.search('model_.*f.*_', path.split("/")[-1]):
model_Mask = path.split("/")[-1].split("_")[1].split("f")[0]
seed = path.split("/")[-1].split("_")[-1].replace(".txt", "")
suffix = ""
if "SAR.txt" in os.path.basename(path):
seed = path.split("/")[-1].split("_")[-2]
suffix = "_SAR"
tilesToEvaluate = tiles
#construction du string de sortie
for tile in tilesToEvaluate:
pathToFeat = fu.FileSearch_AND(pathToImg + "/" + tile + "/tmp/",
True, "MaskCommunSL", ".tif")[0]
maskSHP = pathToRT + "/" + shpRName + "_region_" + model_Mask + "_" + tile + ".shp"
maskTif = shpRName + "_region_" + model_Mask + "_" + tile + ".tif"
CmdConfidenceMap = ""
confidenceMap_name = "{}_model_{}_confidence_seed_{}{}.tif".format(
tile, model, seed, suffix)
CmdConfidenceMap = " -confmap " + os.path.join(
pathOut, confidenceMap_name)
if not os.path.exists(maskFiles + "/" + maskTif):
pathToMaskCommun = pathToImg + "/" + tile + "/tmp/" + "MaskCommunSL" + ".shp"
#cas cluster
if pathWd != None:
maskFiles = pathWd
nameOut = fu.ClipVectorData(maskSHP, pathToMaskCommun,
maskFiles,
maskTif.replace(".tif", ""))
cmdRaster = "otbcli_Rasterization -in "+nameOut+" -mode attribute -mode.attribute.field "+\
fieldRegion+" -im "+pathToFeat+" -out "+maskFiles+"/"+maskTif
if "fusion" in classification_mode:
cmdRaster = "otbcli_Rasterization -in "+nameOut+" -mode binary -mode.binary.foreground 1 -im "+\
pathToFeat+" -out "+maskFiles+"/"+maskTif
run(cmdRaster)
if pathWd != None:
run("cp " + pathWd + "/" + maskTif + " " + pathOut +
"/MASK")
os.remove(pathWd + "/" + maskTif)
out = pathOut + "/Classif_" + tile + "_model_" + model + "_seed_" + seed + suffix + ".tif"
#hpc case
if pathWd != None:
out = "$TMPDIR/Classif_" + tile + "_model_" + model + "_seed_" + seed + suffix + ".tif"
CmdConfidenceMap = " -confmap $TMPDIR/" + confidenceMap_name
appli = "python " + scriptPath + "/Classification/ImageClassifier.py -conf " + path_cfg + " "
pixType_cmd = " -pixType " + pixType
if pathWd != None:
pixType_cmd = pixType_cmd + " --wd $TMPDIR "
cmd = appli + " -in " + pathToFeat + " -model " + path + " -mask " + pathOut + "/MASK/" + maskTif + " -out " + out + " " + pixType_cmd + " -ram " + str(
RAM) + " " + CmdConfidenceMap
# add stats if svm
if "svm" in classifier_name.lower():
model_statistics = os.path.join(
stat, "Model_{}_seed_{}.xml".format(model, seed))
cmd = "{} -imstat {}".format(cmd, model_statistics)
AllCmd.append(cmd)
fu.writeCmds(pathToCmdClassif + "/class.txt", AllCmd)
return AllCmd
def split_by_sets(vector: str,
seeds: int,
split_directory: str,
proj_in: int,
proj_out: int,
tile_name: str,
cross_valid: Optional[bool] = False,
split_ground_truth: Optional[bool] = True) -> List[str]:
"""
use to create new vector file by learning / validation sets
Parameters
----------
vector : string
path to a shape file containg ground truth
seeds : int
number of run
split_directory : string
output directory
proj_in : int
input projection
proj_out : int
output projection
tile_name : string
tile's name
crossValid : bool
flag to enable cross validation
splitGroundTruth : bool
flat to split ground truth
"""
from iota2.Common import FileUtils as fut
from iota2.Common.Utils import run
out_vectors = []
valid_flag = "validation"
learn_flag = "learn"
tile_origin_field_name = "tile_o"
vector_layer_name = (os.path.splitext(
os.path.split(vector)[-1])[0]).lower()
# predict fields to keep
fields_to_rm = ["seed_" + str(seed) for seed in range(seeds)]
fields_to_rm.append(tile_origin_field_name)
all_fields = fut.get_all_fields_in_shape(vector)
fields = [
field_name for field_name in all_fields
if field_name not in fields_to_rm
]
# start split
for seed in range(seeds):
valid_clause = f"seed_{seed}='{valid_flag}'"
learn_clause = f"seed_{seed}='{learn_flag}'"
sql_cmd_valid = (f"select * FROM {vector_layer_name}"
f" WHERE {valid_clause}")
output_vec_valid_name = "_".join(
[tile_name, "seed_" + str(seed), "val"])
output_vec_valid_name_tmp = "_".join(
[tile_name, "seed_" + str(seed), "val", "tmp"])
output_vec_valid_tmp = os.path.join(
split_directory, output_vec_valid_name_tmp + ".sqlite")
output_vec_valid = os.path.join(split_directory,
output_vec_valid_name + ".sqlite")
cmd_valid = (f'ogr2ogr -t_srs EPSG:{proj_out} -s_srs EPSG:{proj_in} '
f'-nln {output_vec_valid_name_tmp} -f "SQLite" -sql '
f'"{sql_cmd_valid}" {output_vec_valid_tmp} {vector}')
sql_cmd_learn = "select * FROM {} WHERE {}".format(
vector_layer_name, learn_clause)
output_vec_learn_name = "_".join(
[tile_name, "seed_" + str(seed), "learn"])
output_vec_learn_name_tmp = "_".join(
[tile_name, "seed_" + str(seed), "learn", "tmp"])
output_vec_learn_tmp = os.path.join(
split_directory, output_vec_learn_name_tmp + ".sqlite")
output_vec_learn = os.path.join(split_directory,
output_vec_learn_name + ".sqlite")
cmd_learn = (
f'ogr2ogr -t_srs EPSG:{proj_out} -s_srs EPSG:{proj_in} '
f'-nln {output_vec_learn_name_tmp} -f "SQLite" -sql "{sql_cmd_learn}"'
f' {output_vec_learn_tmp} {vector}')
if cross_valid is False:
if split_ground_truth:
run(cmd_valid)
# remove useless fields
keep_fields(output_vec_valid_tmp,
output_vec_valid,
fields=fields,
proj_in=proj_in,
proj_out=proj_out)
os.remove(output_vec_valid_tmp)
run(cmd_learn)
# remove useless fields
keep_fields(output_vec_learn_tmp,
output_vec_learn,
fields=fields,
proj_in=proj_in,
proj_out=proj_out)
os.remove(output_vec_learn_tmp)
if split_ground_truth is False:
shutil.copy(output_vec_learn, output_vec_valid)
out_vectors.append(output_vec_valid)
out_vectors.append(output_vec_learn)
else:
if seed < seeds - 1:
run(cmd_learn)
keep_fields(output_vec_learn_tmp,
output_vec_learn,
fields=fields,
proj_in=proj_in,
proj_out=proj_out)
out_vectors.append(output_vec_learn)
os.remove(output_vec_learn_tmp)
elif seed == seeds - 1:
run(cmd_valid)
keep_fields(output_vec_valid_tmp,
output_vec_valid,
fields=fields,
proj_in=proj_in,
proj_out=proj_out)
out_vectors.append(output_vec_valid)
os.remove(output_vec_valid_tmp)
return out_vectors
def CreateModelShapeFromTiles(tilesModel, pathTiles, pathOut, OutSHPname,
fieldOut, pathWd):
"""
create one shapeFile where all features belong to a model number according to the model description
IN :
- tilesModel : a list of list which describe which tile belong to which model
ex : for 3 models
tile model 1 : D0H0,D0H1
tile model 2 : D0H2,D0H3
tile model 3 : D0H4,D0H5,D0H6
tilesModel = [["D0H0","D0H1"],["D0H2","D0H3"],["D0H4","D0H5","D0H6"]]
- pathTiles : path to the tile's envelope with priority consideration
ex : /xx/x/xxx/x
/!\ the folder which contain the envelopes must contain only the envelopes <========
- pathOut : path to store the resulting shapeFile
ex : x/x/x/xxx
- OutSHPname : the name of the resulting shapeFile
ex : "model"
- fieldOut : the name of the field which will contain the model number
ex : "Mod"
- pathWd : path to working directory (not mandatory, due to cluster's architecture default = None)
OUT :
a shapeFile which contains for all feature the model number which it belong to
"""
if pathWd is None:
pathToTMP = pathOut + "/AllTMP"
else:
# HPC case
pathToTMP = pathWd
if not os.path.exists(pathToTMP):
run("mkdir " + pathToTMP)
to_remove = []
for i in range(len(tilesModel)):
for j in range(len(tilesModel[i])):
to_remove.append(
fu.renameShapefile(pathTiles, tilesModel[i][j], "", "",
pathToTMP))
AllTilePath = []
AllTilePath_ER = []
for i in range(len(tilesModel)):
for j in range(len(tilesModel[i])):
try:
ind = AllTilePath.index(pathTiles + "/" + tilesModel[i][j] +
".shp")
except ValueError:
AllTilePath.append(pathToTMP + "/" + tilesModel[i][j] + ".shp")
AllTilePath_ER.append(pathToTMP + "/" + tilesModel[i][j] +
"_ERODE.shp")
for i in range(len(tilesModel)):
for j in range(len(tilesModel[i])):
currentTile = pathToTMP + "/" + tilesModel[i][j] + ".shp"
AddFieldModel(currentTile, i + 1, fieldOut)
for path in AllTilePath:
fu.erodeShapeFile(path, path.replace(".shp", "_ERODE.shp"), 0.1)
fu.mergeVectors(OutSHPname, pathOut, AllTilePath_ER)
if not pathWd:
run("rm -r " + pathToTMP)
else:
for rm in to_remove:
fu.removeShape(rm.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
def split_vector_by_region(in_vect: str,
output_dir: str,
region_field: str,
runs: Optional[int] = 1,
driver: Optional[str] = "ESRI shapefile",
proj_in: Optional[str] = "EPSG:2154",
proj_out: Optional[str] = "EPSG:2154",
mode: Optional[str] = "usually") -> List[str]:
"""
create new files by regions in input vector.
Parameters
----------
in_vect : string
input vector path
output_dir : string
path to output directory
region_field : string
field in in_vect describing regions
driver : string
ogr driver
proj_in : string
input projection
proj_out : string
output projection
mode : string
define if we split SAR sensor to the other
Return
------
list
paths to new output vectors
"""
from iota2.Common import FileUtils as fut
from iota2.Common.Utils import run
output_paths = []
# const
tile_pos = 0
learn_flag = "learn"
table_name = "output"
vec_name = os.path.split(in_vect)[-1]
tile = vec_name.split("_")[tile_pos]
extent = os.path.splitext(vec_name)[-1]
regions = fut.getFieldElement(in_vect,
driverName=driver,
field=region_field,
mode="unique",
elemType="str")
table = vec_name.split(".")[0]
if driver != "ESRI shapefile":
table = "output"
# split vector
for seed in range(runs):
fields_to_keep = ",".join([
elem for elem in fut.get_all_fields_in_shape(in_vect, "SQLite")
if "seed_" not in elem
])
for region in regions:
out_vec_name_learn = "_".join([
tile, "region", region, "seed" + str(seed), "Samples_learn_tmp"
])
if mode != "usually":
out_vec_name_learn = "_".join([
tile, "region", region, "seed" + str(seed), "Samples",
"SAR", "learn_tmp"
])
output_vec_learn = os.path.join(output_dir,
out_vec_name_learn + extent)
seed_clause_learn = f"seed_{seed}='{learn_flag}'"
region_clause = f"{region_field}='{region}'"
# split vectors by runs and learning sets
sql_cmd_learn = (f"select * FROM {table} WHERE {seed_clause_learn}"
f" AND {region_clause}")
cmd = (f'ogr2ogr -t_srs {proj_out} -s_srs {proj_in} -nln {table}'
f' -f "{driver}" -sql "{sql_cmd_learn}" {output_vec_learn} '
f'{in_vect}')
run(cmd)
# drop useless column
sql_clause = f"select GEOMETRY,{fields_to_keep} from {table_name}"
output_vec_learn_out = output_vec_learn.replace("_tmp", "")
cmd = (
f"ogr2ogr -s_srs {proj_in} -t_srs {proj_out} -dialect "
f"'SQLite' -f 'SQLite' -nln {table_name} -sql '{sql_clause}' "
f"{output_vec_learn_out} {output_vec_learn}")
run(cmd)
output_paths.append(output_vec_learn_out)
os.remove(output_vec_learn)
return output_paths
def extract_maj_vote_samples(vec_in: str,
vec_out: str,
ratio_to_keep: float,
data_field: str,
region_field: str,
driver_name: Optional[str] = "ESRI Shapefile"):
"""
dedicated to extract samples by class according to a ratio.
Samples are remove from vec_in and place in vec_out
Parameters
----------
vec_in : string
path to a shapeFile (.shp)
vec_out : string
path to a sqlite (.sqlite)
ratio_to_keep [float]
percentage of samples to extract ratio_to_keep = 0.1
mean extract 10% of each class in each regions
dataField : string
field containing class labels
regionField : string
field containing regions labels
driver_name : string
OGR driver
"""
from osgeo import ogr
from iota2.Common import FileUtils as fut
from iota2.Sampling import SplitInSubSets as subset
from iota2.Common.Utils import run
class_avail = fut.getFieldElement(vec_in,
driverName=driver_name,
field=data_field,
mode="unique",
elemType="int")
region_avail = fut.getFieldElement(vec_in,
driverName=driver_name,
field=region_field,
mode="unique",
elemType="str")
driver = ogr.GetDriverByName(driver_name)
source = driver.Open(vec_in, 1)
layer = source.GetLayer(0)
sample_id_to_extract, _ = subset.get_random_poly(layer, data_field,
class_avail,
ratio_to_keep,
region_field,
region_avail)
# Create new file with targeted FID
fid_samples_in = built_where_sql_exp(sample_id_to_extract, clause="in")
cmd = f"ogr2ogr -where '{fid_samples_in}' -f 'SQLite' {vec_out} {vec_in}"
run(cmd)
# remove in vec_in targeted FID
vec_in_rm = vec_in.replace(".shp", "_tmp.shp")
fid_samples_not_in = built_where_sql_exp(sample_id_to_extract,
clause="not in")
cmd = f"ogr2ogr -where '{fid_samples_not_in}' {vec_in_rm} {vec_in}"
run(cmd)
fut.removeShape(vec_in.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
cmd = f"ogr2ogr {vec_in} {vec_in_rm}"
run(cmd)
fut.removeShape(vec_in_rm.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
def test_VectorFormatting(self):
"""
test vectorFormatting function
random function is used in Sampling.VectorFormatting.VectorFormatting
we can only check if there is expected number of features with
expected fields and some features values
"""
from iota2.Sampling.VectorFormatting import vector_formatting
from iota2.Common import ServiceConfigFile as SCF
from iota2.Common import IOTA2Directory
from iota2.Common.Utils import run
from iota2.VectorTools.ChangeNameField import changeName
# define inputs
test_output = os.path.join(self.test_working_directory,
"IOTA2_dir_VectorFormatting")
# prepare ground truth
ground_truth = os.path.join(self.test_working_directory,
"groundTruth_test.shp")
cmd = "ogr2ogr -s_srs EPSG:2154 -t_srs EPSG:2154 -dialect 'SQLite' -sql 'select GEOMETRY,code from t31tcj' {} {}".format(
ground_truth, self.in_vector)
run(cmd)
# cfg instance
runs = 2
cfg = SCF.serviceConfigFile(self.config_test)
cfg.setParam('chain', 'outputPath', test_output)
cfg.setParam('chain', 'groundTruth', ground_truth)
cfg.setParam('chain', 'dataField', "code")
cfg.setParam('chain', 'cloud_threshold', 0)
cfg.setParam('chain', 'merge_final_classifications', False)
cfg.setParam('chain', 'runs', runs)
cfg.setParam('GlobChain', 'proj', "EPSG:2154")
cfg.setParam('chain', 'regionPath', self.ref_region)
IOTA2Directory.generate_directories(test_output, check_inputs=False)
# prepare expected function inputs
t31tcj_feat_dir = os.path.join(self.test_working_directory,
"IOTA2_dir_VectorFormatting",
"features", "T31TCJ")
os.mkdir(t31tcj_feat_dir)
# prepare ref img
t31tcj_ref_img = os.path.join(t31tcj_feat_dir, "MaskCommunSL.tif")
shutil.copy(self.ref_img, t31tcj_ref_img)
# prepare envelope
envelope_name = "T31TCJ.shp"
envelope_path = os.path.join(self.test_working_directory,
"IOTA2_dir_VectorFormatting", "envelope",
envelope_name)
fut.cpShapeFile(self.ref_region.replace(".shp", ""),
envelope_path.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
changeName(envelope_path, "region", "FID")
# prepare cloud mask
cloud_name = "CloudThreshold_0.shp"
cloud_path = os.path.join(self.test_working_directory,
"IOTA2_dir_VectorFormatting", "features",
"T31TCJ", cloud_name)
fut.cpShapeFile(self.ref_region.replace(".shp", ""),
cloud_path.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
changeName(cloud_path, "region", "cloud")
# launch function
ratio = cfg.getParam('chain', 'ratio')
random_seed = cfg.getParam('chain', 'random_seed')
enable_cross_validation = cfg.getParam("chain",
"enableCrossValidation")
enable_split_ground_truth = cfg.getParam('chain', 'splitGroundTruth')
fusion_merge_all_validation = cfg.getParam(
'chain', 'fusionOfClassificationAllSamplesValidation')
merge_final_classifications = cfg.getParam(
'chain', 'merge_final_classifications')
merge_final_classifications_ratio = cfg.getParam(
'chain', 'merge_final_classifications_ratio')
region_vec = cfg.getParam('chain', 'regionPath')
epsg = int(cfg.getParam('GlobChain', 'proj').split(":")[-1])
region_field = (cfg.getParam('chain', 'regionField'))
vector_formatting("T31TCJ",
test_output,
ground_truth,
"code",
0,
ratio,
random_seed,
enable_cross_validation,
enable_split_ground_truth,
fusion_merge_all_validation,
runs,
epsg,
region_field,
merge_final_classifications,
merge_final_classifications_ratio,
region_vec,
working_directory=None)
# assert
nb_features_origin = len(
fut.getFieldElement(ground_truth,
driverName="ESRI Shapefile",
field="code",
mode="all",
elemType="str"))
test_vector = fut.FileSearch_AND(
os.path.join(test_output, "formattingVectors"), True,
"T31TCJ.shp")[0]
nb_features_test = len(
fut.getFieldElement(test_vector,
driverName="ESRI Shapefile",
field="code",
mode="all",
elemType="str"))
# check nb features
self.assertTrue(nb_features_origin == nb_features_test,
msg="wrong number of features")
# check fields
origin_fields = fut.get_all_fields_in_shape(ground_truth)
test_fields = fut.get_all_fields_in_shape(test_vector)
new_fields = ['region', 'originfid', 'seed_0', 'seed_1', 'tile_o']
expected_fields = origin_fields + new_fields
self.assertTrue(len(expected_fields) == len(test_fields))
self.assertTrue(all(field in test_fields for field in expected_fields))
def test_undecision_management(self):
"""
Test undecision_management
"""
import filecmp
from iota2.Classification import undecision_management as UM
from iota2.Classification import Fusion as FUS
from iota2.Common.Utils import run
from iota2.Common import FileUtils as fut
shutil.copytree(
os.path.join(IOTA2DIR, "data", "references", "NoData", "Input",
"Classif"), self.test_working_directory)
shutil.copytree(
os.path.join(IOTA2DIR, "data", "references", "NoData", "Input",
"config_model"),
os.path.join(self.test_working_directory, "config_model"))
if not os.path.exists(
os.path.join(self.test_working_directory, "cmd", "fusion")):
os.makedirs(
os.path.join(self.test_working_directory, "cmd", "fusion"))
shutil.copy(
os.path.join(self.test_working_directory, "classif",
"Classif_D0005H0002_model_1_seed_0.tif"),
os.path.join(self.test_working_directory, "classif",
"Classif_D0005H0002_model_1f2_seed_0.tif"))
shutil.copy(
os.path.join(self.test_working_directory, "classif",
"Classif_D0005H0002_model_1_seed_0.tif"),
os.path.join(self.test_working_directory, "classif",
"Classif_D0005H0002_model_1f1_seed_0.tif"))
shutil.copy(
os.path.join(self.test_working_directory, "classif",
"D0005H0002_model_1_confidence_seed_0.tif"),
os.path.join(self.test_working_directory, "classif",
"D0005H0002_model_1f2_confidence_seed_0.tif"))
shutil.copy(
os.path.join(self.test_working_directory, "classif",
"D0005H0002_model_1_confidence_seed_0.tif"),
os.path.join(self.test_working_directory, "classif",
"D0005H0002_model_1f1_confidence_seed_0.tif"))
cmd_fus = FUS.fusion(
os.path.join(self.test_working_directory, "classif"), 1,
["D0005H0002"], '-nodatalabel 0 -method majorityvoting',
os.path.join(IOTA2DIR, "data", "references", "nomenclature.txt"),
os.path.join(IOTA2DIR, "data", "regionShape",
"4Tiles.shp"), False, None)
for cmd in cmd_fus:
run(cmd)
fusion_files = fut.FileSearch_AND(
os.path.join(self.test_working_directory, "classif"), True,
"_FUSION_")
print(fusion_files)
pixtype = fut.getOutputPixType(
os.path.join(IOTA2DIR, "data", "references", "nomenclature.txt"))
for fusionpath in fusion_files:
UM.undecision_management(
self.test_working_directory, fusionpath, "region",
os.path.join(IOTA2DIR, "data", "references", "features"),
os.path.join(IOTA2DIR, "data", "references",
"GenerateRegionShape", "region_need_To_env.shp"),
"maxConfidence", None, ["NDVI", "NDWI", "Brightness"],
'../../../../MNT_L8Grid', pixtype,
os.path.join(IOTA2DIR, "data", "regionShape",
"4Tiles.shp"), "ALT,ASP,SLP", False)
# TODO: test intelligent
self.assertTrue(
filecmp.cmp(
os.path.join(self.test_working_directory, "classif",
"D0005H0002_FUSION_model_1_seed_0.tif"),
os.path.join(IOTA2DIR, "data", "references", "NoData",
"Output",
"D0005H0002_FUSION_model_1_seed_0.tif")))
def undecision_management(path_test: str,
path_fusion: str,
field_region: str,
path_to_img: str,
path_to_region: str,
no_label_management: str,
path_wd: str,
list_indices: List[str],
user_feat_path: str,
pix_type: str,
region_vec: Optional[str] = None,
user_feat_pattern: Optional[str] = None,
ds_sar_opt: Optional[bool] = False) -> None:
"""
manage undecision comming from fusion of classifications
Parameters
----------
path_test: string
path_fusion: string
field_region: string
path_to_img: string
path_to_region: string
region_vec: string
no_label_management: string
path_wd: string
list_indices: List[string],
user_feat_path: string
pix_type: string
user_feat_pattern: string
ds_sar_opt: bool
Return
------
None
"""
stack_ind = fu.get_feat_stack_name(list_indices, user_feat_path,
user_feat_pattern)
suffix_pattern = ""
if ds_sar_opt:
suffix_pattern = "_DS"
if region_vec:
current_model = path_fusion.split("/")[-1].split("_")[3]
config_model = os.path.join(path_test, "config_model",
"configModel.cfg")
n_fold = get_nb_split_shape(current_model, config_model)
path_directory = path_test + "/classif"
if path_wd is not None:
working_dir = path_wd
path_directory = path_wd
else:
working_dir = path_test + "/classif/MASK"
current_tile = path_fusion.split("/")[-1].split("_")[0]
shp_rname = path_to_region.split("/")[-1].replace(".shp", "")
all_model = fu.FileSearch_AND(os.path.join(path_test, "model"), True,
"model", ".txt")
if region_vec is None:
model_tile = gen_mask_region_by_tile(
field_region, stack_ind, working_dir, current_tile, all_model,
shp_rname, path_to_img, path_test,
os.path.join(path_test, "config_model",
"configModel.cfg"), path_wd)
elif region_vec and no_label_management == "maxConfidence":
model_tile = path_fusion.split("/")[-1].split("_")[3]
elif region_vec and no_label_management == "learningPriority":
model_tile_tmp = path_fusion.split("/")[-1].split("_")[3]
model_tile = []
for i in range(n_fold):
model_tile.append(f"{model_tile_tmp} f{i + 1}")
if len(model_tile) == 0 or no_label_management == "maxConfidence":
seed = os.path.split(path_fusion)[-1].split("_")[-1].split(".")[0]
img_confidence = fu.FileSearch_AND(
path_test + "/classif", True,
"confidence_seed_" + str(seed) + suffix_pattern + ".tif",
current_tile)
img_classif = fu.FileSearch_AND(os.path.join(path_test, "classif"),
True, "Classif_" + current_tile,
f"seed_{seed}", suffix_pattern)
img_data = os.path.join(
path_directory, f"{current_tile}_FUSION_NODATA_seed{seed}.tif")
if region_vec:
img_confidence = fu.fileSearchRegEx(
f"{path_test+os.sep}classif{os.sep}"
f"{current_tile}_model_{model_tile}"
f"f*_confidence_seed_{seed}{suffix_pattern}.tif")
img_classif = fu.fileSearchRegEx(
f"{path_test+os.sep}classif{os.sep}"
f"Classif_{current_tile}_model_"
f"{model_tile}f*_seed_"
f"{seed}{suffix_pattern}.tif")
img_data = (f"{path_directory+os.sep}Classif_"
f"{current_tile}_model_{model_tile}"
f"_seed_{seed}{suffix_pattern}.tif")
img_confidence.sort()
img_classif.sort()
exp, il_str = build_confidence_exp(path_fusion, img_confidence,
img_classif)
cmd = (f"otbcli_BandMath -il {il_str} -out {img_data} {pix_type} "
f"-exp '{exp}' ")
run(cmd)
if path_wd is not None:
run(f"cp {img_data} {os.path.join(path_test, 'classif')}")
elif len(model_tile) != 0 and no_label_management == "learningPriority":
# Concaténation des classifications pour une tuile (qui a ou non
# plusieurs régions) et Concaténation des masques de régions pour
# une tuile (qui a ou non plusieurs régions)
seed = os.path.split(path_fusion)[-1].split("_")[-1].split(".")[0]
concat_out = (f"{os.path.join(path_test, 'classif')}"
f"{current_tile}_FUSION_concat_seed{seed}.tif")
if region_vec:
concat_out = (
f"{os.path.join(path_test, 'classif')}"
f"{current_tile}_FUSION_model_{model_tile[0].split('f')[0]}"
f"concat_seed{seed}.tif")
path_to_classif_concat = concat_classifs_one_tile(
seed, current_tile, path_test, model_tile, concat_out, path_wd)
pattern_mask = "*region_*_{current_tile}_NODATA.tif"
classif_fusion_mask = fu.fileSearchRegEx(
os.path.join(path_test, "classif", "MASK", pattern_mask))
out_concat_mask = os.path.join(path_test, "classif",
f"{current_tile}_MASK.tif")
if region_vec:
pattern_mask = ("*region_{model_tile[0].split('f')[0]}"
f"_{current_tile}.tif")
classif_fusion_mask_tmp = fu.fileSearchRegEx(
os.path.join(path_test, "classif", "MASK", pattern_mask))
out_concat_mask = (f"{os.path.join(path_test, 'classif')}"
f"{current_tile}_MASK_model_"
f"{model_tile[0].split('f')[0]}.tif")
classif_fusion_mask = []
for i in range(n_fold):
classif_fusion_mask.append(classif_fusion_mask_tmp[0])
path_to_region_mask_concat = concat_region_one_tile(
path_test, classif_fusion_mask, path_wd, out_concat_mask)
# construction de la commande
exp = ""
im1 = path_fusion
im2 = path_to_region_mask_concat
im3 = path_to_classif_concat
for i in range(len(classif_fusion_mask)):
if i + 1 < len(classif_fusion_mask):
exp = exp + "im2b" + str(i + 1) + ">=1?im3b" + str(i + 1) + ":"
else:
exp = exp + "im2b" + str(i + 1) + ">=1?im3b" + str(i +
1) + ":0"
exp = "im1b1!=0?im1b1:(" + exp + ")"
img_data = (os.path.join(
path_directory, f"{current_tile}_FUSION_"
f"NODATA_seed{seed}.tif"))
if region_vec:
img_data = (f"{path_directory+os.sep}Classif_{current_tile}_model_"
f"{model_tile[0].split('f')[0]}_seed_{seed}.tif")
cmd = (f'otbcli_BandMath -il {im1} {im2} {im3} -out + {img_data} '
f'{pix_type} -exp "{exp}"')
run(cmd)
if path_wd is not None:
run("cp {img_data} {path_test+os.sep}classif")
def classification_shaping(path_classif: str, runs: int, path_out: str,
path_wd: str, classif_mode: str, path_test: str,
ds_sar_opt: bool, proj: int, nomenclature_path: str,
output_statistics: bool, spatial_resolution: float,
proba_map_flag: bool, region_shape: str,
color_path: str) -> None:
"""function use to mosaic rasters and to produce final maps
path_classif: str
directory where as classifications
runs: int
number of random learning/validation samples-set
path_out: str
output directory
path_wd: str
working directory
classif_mode: str
fusion of classifications ?
path_test: str
iota2 output directory
ds_sar_opt: bool
flag to inform if SAR and optical post-classification workflow
is enable
proj: int
epsg code
nomenclature_path: str
nomenclature path
output_statistics: bool
flag to enable output statistics
spatial_resolution: float
output's spatial resolution
proba_map_flag: bool
flag to inform if probability map was produce
region_shape: str
region shapeFile path
color_path: str
color table file
"""
if path_wd is None:
tmp = path_out + "/TMP"
if not os.path.exists(path_out + "/TMP"):
os.mkdir(tmp)
else:
tmp = path_wd
if not os.path.exists(path_out + "/TMP"):
os.mkdir(path_out + "/TMP")
all_tiles = list(
set([
classif.split("_")[1] for classif in fu.FileSearch_AND(
path_test + "/classif", False, "Classif", ".tif")
]))
pix_type = fu.getOutputPixType(nomenclature_path)
features_path = os.path.join(path_test, "features")
all_tmp_folder = fu.fileSearchRegEx(path_test + "/TMPFOLDER*")
if all_tmp_folder:
for tmp_folder in all_tmp_folder:
shutil.rmtree(tmp_folder)
suffix = "*"
if ds_sar_opt:
suffix = "*DS*"
genGlobalConfidence(runs, path_wd, spatial_resolution, proj, path_test,
classif_mode, all_tiles, region_shape, ds_sar_opt,
proba_map_flag)
if region_shape and classif_mode == "fusion":
old_classif = fu.fileSearchRegEx(path_test +
"/classif/Classif_*_model_*f*_seed_" +
suffix + ".tif")
for rm in old_classif:
if not os.path.exists(path_test + "/final/TMP/OLDCLASSIF"):
os.mkdir(path_test + "/final/TMP/OLDCLASSIF")
run("mv " + rm + " " + path_test + "/final/TMP/OLDCLASSIF")
classification = []
confidence = []
proba_map = []
cloud = []
for seed in range(runs):
classification.append([])
confidence.append([])
cloud.append([])
sort = []
if proba_map_flag:
proba_map_list = fu.fileSearchRegEx(
path_test + "/classif/PROBAMAP_*_model_*_seed_" + str(seed) +
suffix + ".tif")
proba_map_list = removeInListByRegEx(
proba_map_list, ".*model_.*f.*_seed." + suffix)
proba_map.append(proba_map_list)
if classif_mode == "separate" or region_shape:
all_classif_seed = fu.FileSearch_AND(path_classif, True, ".tif",
"Classif",
"seed_" + str(seed))
if ds_sar_opt:
all_classif_seed = fu.FileSearch_AND(path_classif, True,
".tif", "Classif",
"seed_" + str(seed),
"DS.tif")
ind = 1
elif classif_mode == "fusion":
all_classif_seed = fu.FileSearch_AND(
path_classif, True, "_FUSION_NODATA_seed" + str(seed) + ".tif")
if ds_sar_opt:
all_classif_seed = fu.FileSearch_AND(
path_classif, True,
"_FUSION_NODATA_seed" + str(seed) + "_DS.tif")
ind = 0
for tile in all_classif_seed:
sort.append((tile.split("/")[-1].split("_")[ind], tile))
sort = fu.sortByFirstElem(sort)
for tile, paths in sort:
exp = ""
all_cl = ""
all_cl_rm = []
for i in range(len(paths)):
all_cl = all_cl + paths[i] + " "
all_cl_rm.append(paths[i])
if i < len(paths) - 1:
exp = exp + "im" + str(i + 1) + "b1 + "
else:
exp = exp + "im" + str(i + 1) + "b1"
path_cl_final = tmp + "/" + tile + "_seed_" + str(seed) + ".tif"
classification[seed].append(path_cl_final)
cmd = 'otbcli_BandMath -il ' + all_cl + '-out ' + path_cl_final + ' ' + pix_type + ' -exp "' + exp + '"'
run(cmd)
tile_confidence = path_out + "/TMP/" + tile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
confidence[seed].append(tile_confidence)
cloud_tile = fu.FileSearch_AND(features_path + "/" + tile, True,
"nbView.tif")[0]
classif_tile = tmp + "/" + tile + "_seed_" + str(seed) + ".tif"
cloud_tile_priority = path_test + "/final/TMP/" + tile + "_Cloud.tif"
cloud_tile_priority_tmp = tmp + "/" + tile + "_Cloud.tif"
cloud_tile_priority_stats_ok = path_test + "/final/TMP/" + tile + "_Cloud_StatsOK.tif"
cloud_tile_priority_tmp_stats_ok = tmp + "/" + tile + "_Cloud_StatsOK.tif"
cloud[seed].append(cloud_tile_priority)
if not os.path.exists(cloud_tile_priority):
cmd_cloud = f'otbcli_BandMath -il {cloud_tile} {classif_tile} -out {cloud_tile_priority_tmp} int16 -exp "im2b1>0?im1b1:0"'
run(cmd_cloud)
if output_statistics:
cmd_cloud = 'otbcli_BandMath -il ' + cloud_tile + ' ' + classif_tile + ' -out ' + cloud_tile_priority_tmp_stats_ok + ' int16 -exp "im2b1>0?im1b1:-1"'
run(cmd_cloud)
if path_wd:
shutil.copy(cloud_tile_priority_tmp_stats_ok,
cloud_tile_priority_stats_ok)
os.remove(cloud_tile_priority_tmp_stats_ok)
if path_wd:
shutil.copy(cloud_tile_priority_tmp, cloud_tile_priority)
os.remove(cloud_tile_priority_tmp)
if path_wd is not None:
run("cp -a " + tmp + "/* " + path_out + "/TMP")
for seed in range(runs):
assemble_folder = path_test + "/final"
if path_wd:
assemble_folder = path_wd
fu.assembleTile_Merge(classification[seed],
spatial_resolution,
"{}/Classif_Seed_{}.tif".format(
assemble_folder, seed),
"Byte" if pix_type == "uint8" else "Int16",
co={
"COMPRESS": "LZW",
"BIGTIFF": "YES"
})
if path_wd:
shutil.copy(path_wd + "/Classif_Seed_" + str(seed) + ".tif",
path_test + "/final")
os.remove(path_wd + "/Classif_Seed_" + str(seed) + ".tif")
fu.assembleTile_Merge(confidence[seed],
spatial_resolution,
assemble_folder + "/Confidence_Seed_" +
str(seed) + ".tif",
"Byte",
co={
"COMPRESS": "LZW",
"BIGTIFF": "YES"
})
if path_wd:
shutil.copy(path_wd + "/Confidence_Seed_" + str(seed) + ".tif",
path_test + "/final")
os.remove(path_wd + "/Confidence_Seed_" + str(seed) + ".tif")
color.CreateIndexedColorImage(
path_test + "/final/Classif_Seed_" + str(seed) + ".tif",
color_path,
output_pix_type=gdal.GDT_Byte
if pix_type == "uint8" else gdal.GDT_UInt16)
if proba_map_flag:
proba_map_mosaic = os.path.join(
assemble_folder, "ProbabilityMap_seed_{}.tif".format(seed))
fu.assembleTile_Merge(proba_map[seed],
spatial_resolution,
proba_map_mosaic,
"Int16",
co={
"COMPRESS": "LZW",
"BIGTIFF": "YES"
})
if path_wd:
shutil.copy(proba_map_mosaic, path_test + "/final")
os.remove(proba_map_mosaic)
fu.assembleTile_Merge(cloud[0],
spatial_resolution,
assemble_folder + "/PixelsValidity.tif",
"Byte",
co={
"COMPRESS": "LZW",
"BIGTIFF": "YES"
})
if path_wd:
shutil.copy(path_wd + "/PixelsValidity.tif", path_test + "/final")
os.remove(path_wd + "/PixelsValidity.tif")
def genGlobalConfidence(N, pathWd, spatialRes, proj, pathTest, classifMode,
AllTile: List[str], shapeRegion, ds_sar_opt,
proba_map_flag):
"""generate confidences ready to be mosaic
"""
PROBAMAP_PATTERN = "PROBAMAP"
tmpClassif = pathTest + "/classif/tmpClassif"
pathToClassif = pathTest + "/classif"
if pathWd:
tmpClassif = pathWd + "/tmpClassif"
if not os.path.exists(tmpClassif):
run("mkdir " + tmpClassif)
for seed in range(N):
for tuile in AllTile:
if shapeRegion is None:
if classifMode == "separate":
confidence_pattern = os.path.join(
pathToClassif,
"{}*model*confidence_seed_{}*.tif".format(tuile, seed))
if ds_sar_opt:
confidence_pattern = os.path.join(
pathToClassif,
"{}*model*confidence_seed_{}*_DS.tif".format(
tuile, seed))
confidence = fu.fileSearchRegEx(confidence_pattern)
globalConf = tmpClassif + "/" + tuile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
globalConf_f = pathTest + "/final/TMP/" + tuile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
cmd = 'otbcli_BandMath -il ' + confidence[
0] + ' -out ' + globalConf + ' uint8 -exp "100*im1b1"'
run(cmd)
shutil.copyfile(globalConf, globalConf_f)
os.remove(globalConf)
else:
raise Exception((
"if there is no region shape specify in the "
"configuration file, argClassification.classifMode must be set to 'separate'"
))
else: #output Mode
suffix = "*"
if ds_sar_opt:
suffix = "*DS*"
if classifMode != "separate":
classifTile = fu.fileSearchRegEx(
pathToClassif + "/Classif_" + tuile +
"*model_*f*_seed_" + str(seed) + suffix
) # tmp tile (produce by each classifier, without nodata)
splitModel = []
for classif in classifTile:
model = classif.split("/")[-1].split("_")[3].split(
"f")[0]
try:
ind = splitModel.index(model)
except ValueError:
splitModel.append(model)
splitConfidence = []
confidence_all = fu.fileSearchRegEx(
pathToClassif + "/" + tuile +
"*model_*_confidence_seed_" + str(seed) + suffix)
confidence_withoutSplit = removeInListByRegEx(
confidence_all, ".*model_.*f.*_confidence." + suffix)
for model in splitModel:
classifTile = fu.fileSearchRegEx(
pathToClassif + "/Classif_" + tuile + "*model_" +
model + "f*_seed_" + str(seed) + suffix
) # tmp tile (produce by each classifier, without nodata)
finalTile = pathToClassif + "/Classif_" + tuile + "_model_" + model + "_seed_" + str(
seed) + ".tif"
if ds_sar_opt:
finalTile = pathToClassif + "/Classif_" + tuile + "_model_" + model + "_seed_" + str(
seed) + "_DS.tif"
confidence = fu.fileSearchRegEx(pathToClassif + "/" +
tuile + "*model_" +
model +
"f*_confidence_seed_" +
str(seed) + suffix)
if proba_map_flag:
proba_map_fusion(proba_map_list=fu.fileSearchRegEx(
"{}/{}_{}_model_{}f*_seed_{}{}.tif".format(
pathToClassif, PROBAMAP_PATTERN, tuile,
model, seed, suffix)),
working_directory=pathWd,
ram=2000)
classifTile = sorted(classifTile)
confidence = sorted(confidence)
OutPutConfidence = tmpClassif + "/" + tuile + "_model_" + model + "_confidence_seed_" + str(
seed) + ".tif"
if ds_sar_opt:
OutPutConfidence = tmpClassif + "/" + tuile + "_model_" + model + "_confidence_seed_" + str(
seed) + "_DS.tif"
cmd = BuildConfidenceCmd(finalTile,
classifTile,
confidence,
OutPutConfidence,
fact=100,
pixType="uint8")
run(cmd)
splitConfidence.append(OutPutConfidence)
i = 0 #init
j = 0
exp1 = "+".join([
"im" + str(i + 1) + "b1"
for i in range(len(splitConfidence))
]) #-> confidence from splited models are from 0 to 100
exp2 = "+".join([
"(100*im" + str(j + 1) + "b1)" for j in np.arange(
len(splitConfidence),
len(splitConfidence) +
len(confidence_withoutSplit))
]) #-> confidence from NO-splited models are from 0 to 1
if not splitConfidence:
exp2 = "+".join([
"100*im" + str(j + 1) + "b1"
for j in range(len(confidence_withoutSplit))
])
if exp1 and exp2:
exp = exp1 + "+" + exp2
if exp1 and not exp2:
exp = exp1
if not exp1 and exp2:
exp = exp2
confidence_list = splitConfidence + confidence_withoutSplit
AllConfidence = " ".join(confidence_list)
OutPutConfidence = tmpClassif + "/" + tuile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
cmd = 'otbcli_BandMath -il ' + AllConfidence + ' -out ' + OutPutConfidence + ' uint8 -exp "' + exp + '"'
run(cmd)
shutil.copy(OutPutConfidence, pathTest + "/final/TMP")
os.remove(OutPutConfidence)
#shutil.rmtree(tmpClassif)
else:
confidence = fu.fileSearchRegEx(pathToClassif + "/" +
tuile +
"*model*confidence_seed_" +
str(seed) + suffix)
exp = "+".join([
"im" + str(i + 1) + "b1"
for i in range(len(confidence))
])
AllConfidence = " ".join(confidence)
#for currentConf in confidence:
globalConf = tmpClassif + "/" + tuile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
globalConf_f = pathTest + "/final/TMP/" + tuile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
cmd = 'otbcli_BandMath -il ' + AllConfidence + ' -out ' + globalConf + ' uint8 -exp "100*(' + exp + ')"'
#print confidence
run(cmd)
shutil.copyfile(globalConf, globalConf_f)
os.remove(globalConf)
def compare_ref(shape_ref: str, shape_learn: str, classif: str, diff: str,
working_directory: str, path_wd: str, data_field: str,
spatial_res: int):
"""
Parameters
----------
shape_ref: string
shape_learn: string
classif: string
diff: string
working_directory: string
path_wd: string
data_field: string
spatial_res: string
Return
------
string
"""
min_x, max_x, min_y, max_y = fu.getRasterExtent(classif)
shape_raster_val = working_directory + os.sep + shape_ref.split(
"/")[-1].replace(".sqlite", ".tif")
shape_raster_learn = working_directory + os.sep + shape_learn.split(
"/")[-1].replace(".sqlite", ".tif")
# Rasterise val
shape_ref_table_name = os.path.splitext(
os.path.split(shape_ref)[-1])[0].lower()
cmd = (f"gdal_rasterize -l {shape_ref_table_name} -a {data_field} -init 0 "
f"-tr {spatial_res} {spatial_res} {shape_ref} {shape_raster_val} "
f"-te {min_x} {min_y} {max_x} {max_y}")
run(cmd)
# Rasterise learn
shape_learn_table_name = os.path.splitext(
os.path.split(shape_learn)[-1])[0].lower()
cmd = (
f"gdal_rasterize -l {shape_learn_table_name} -a {data_field} -init "
f"0 -tr {spatial_res} {spatial_res} {shape_learn} {shape_raster_learn}"
f" -te {min_x} {min_y} {max_x} {max_y}")
run(cmd)
# diff val
diff_val = working_directory + "/" + diff.split("/")[-1].replace(
".tif", "_val.tif")
# reference identique -> 2 | reference != -> 1 | pas de reference -> 0
cmd_val = (f'otbcli_BandMath -il {shape_raster_val} {classif} -out '
f'{diff_val} uint8 -exp "im1b1==0?0:im1b1==im2b1?2:1"')
run(cmd_val)
os.remove(shape_raster_val)
# diff learn
diff_learn = working_directory + "/" + diff.split("/")[-1].replace(
".tif", "_learn.tif")
# reference identique -> 4 | reference != -> 3 | pas de reference -> 0
cmd_learn = (f'otbcli_BandMath -il {shape_raster_learn} {classif} -out '
f'{diff_learn} uint8 -exp "im1b1==0?0:im1b1==im2b1?4:3"')
run(cmd_learn)
os.remove(shape_raster_learn)
# sum diff val + learn
diff_tmp = working_directory + "/" + diff.split("/")[-1]
cmd_sum = (f'otbcli_BandMath -il {diff_val} {diff_learn} -out {diff_tmp}'
f' uint8 -exp "im1b1+im2b1"')
run(cmd_sum)
os.remove(diff_val)
os.remove(diff_learn)
if path_wd and not os.path.exists(diff):
shutil.copy(diff_tmp, diff)
os.remove(diff_tmp)
return diff
def sqlite_to_geojson(input_db: str, output_db: str, logger=LOGGER) -> None:
"""sqlite database to geojson database
"""
from iota2.Common.Utils import run
logger.info(f"changin input data format {input_db} to {output_db}")
run(f'ogr2ogr -f "GeoJSON" {output_db} {input_db}')
