def extract_poi(tile_vector: str, region: str, seed: int, region_field: str, poi: str, poi_val: str, force_seed_field: Optional[bool] = None) -> None: """ Extract Polygon Of Interest Parameters ---------- tile_vector: string region: string seed: int region_field: str poi: str poi_val: str force_seed_field: bool Return ------ None """ from iota2.Common.Utils import run learn_flag = "learn" validation_flag = "validation" seed_field = "seed_{}".format(seed) cmd = (f"ogr2ogr -where \"{region_field}='{region}' AND {seed_field}" f"='{learn_flag}'\" {poi} {tile_vector}") run(cmd) if poi_val: if force_seed_field: seed_field = force_seed_field cmd = (f"ogr2ogr -where \"{region_field}='{region}' AND {seed_field}=" f"'{validation_flag}'\" {poi_val} {tile_vector}") run(cmd)
def shape_reference_vector(ref_vector: str, output_name: str) -> str: """ modify reference vector (add field, rename...) Parameters ---------- ref_vector : string output_name : string Return ------ string """ import os from iota2.Common.Utils import run from iota2.Common import FileUtils as fut from iota2.VectorTools.AddField import addField path, _ = os.path.split(ref_vector) tmp = os.path.join(path, output_name + "_TMP") fut.cpShapeFile(ref_vector.replace(".shp", ""), tmp, [".prj", ".shp", ".dbf", ".shx"]) addField(tmp + ".shp", "region", "1", str) addField(tmp + ".shp", "seed_0", "learn", str) cmd = (f"ogr2ogr -dialect 'SQLite' -sql 'select GEOMETRY,seed_0, " f"region, CODE as code from {output_name}_TMP' " f"{path}/{output_name}.shp {tmp}.shp") run(cmd) os.remove(tmp + ".shp") os.remove(tmp + ".shx") os.remove(tmp + ".prj") os.remove(tmp + ".dbf") return path + "/" + output_name + ".shp"
def extract_meta_data_fields(input_sample_file_name: str, reduced_output_file_name: str) -> None: """ Parameters ---------- input_sample_file_name: string reduced_output_file_name: string Return ------ None Notes ----- Extract MetaDataFields from input vector file in order to append reduced fields """ from iota2.Common.Utils import run reduced_output_file_name_table = "output" (_, meta_data_fields) = get_available_features(input_sample_file_name, 'global') fields = ",".join(meta_data_fields) cmd = (f"ogr2ogr -dialect 'SQLITE' -nln {reduced_output_file_name_table} " f"-f 'SQLite' -select '{fields}' " f"{reduced_output_file_name} {input_sample_file_name}") run(cmd)
def test_create_tile_region_masks(self): """ test the generation of the raster mask which define the region in the tile """ from iota2.Sampling.VectorFormatting import create_tile_region_masks from iota2.Common.Utils import run from iota2.Tests.UnitTests.TestsUtils import rasterToArray import numpy as np # define inputs test_vector_name = "T31TCJ.sqlite" test_vector = os.path.join(self.test_working_directory, test_vector_name) cmd = "ogr2ogr -nln t31tcj -f SQLite {} {}".format( test_vector, self.ref_region) run(cmd) # launch function create_tile_region_masks(test_vector, "region", "T31TCJ", self.test_working_directory, "MyRegion", self.ref_img) # assert raster_region = fut.FileSearch_AND(self.test_working_directory, True, "MyRegion", ".tif")[0] raster_region_arr = rasterToArray(raster_region) ref_array = np.ones((50, 50)) self.assertTrue(np.allclose(ref_array, raster_region_arr), msg="problem with the normalization by ref")
def extract_class(vec_in: str, vec_out: str, target_class: List[str], data_field: str) -> int: """ Extract class IN: vec_in: str vec_out: str target_class:List[str] data_field: str """ from iota2.Common.Utils import run if type(target_class) != type(list()): target_class = target_class.data where = " OR ".join( ["{}={}".format(data_field.lower(), klass) for klass in target_class]) cmd = (f"ogr2ogr -f 'SQLite' -nln output -where '{where}' " f"{vec_out} {vec_in}") run(cmd) return len( fu.getFieldElement(vec_out, driverName="SQLite", field=data_field.lower(), mode="all", elemType="int"))
def create_dummy_rasters(missing_tiles: List[str], runs: int, output_path: str) -> None: """ Parameters ---------- missing_tiles: list(string) runs: int output_path: string Return ------ None Notes ----- use when mode is 'one_region' but there is no validations / learning samples into a specific tile """ classifications_dir = os.path.join(output_path, "classif") final_dir = os.path.join(output_path, "final", "TMP") for tile in missing_tiles: classif_tile = fu.FileSearch_AND(classifications_dir, True, "Classif_" + str(tile))[0] for seed in range(runs): dummy_raster_name = tile + "_seed_" + str(seed) + "_CompRef.tif" dummy_raster = final_dir + "/" + dummy_raster_name dummy_raster_cmd = (f"gdal_merge.py -ot Byte -n 0 -createonly -o " f"{ dummy_raster} {classif_tile}") run(dummy_raster_cmd)
def test_split_vector_by_region(self): """ test : split a vector by the region he belongs to """ from iota2.Sampling.VectorFormatting import split_vector_by_region from iota2.Common.Utils import run from iota2.Tests.UnitTests.Iota2Tests import random_update # define inputs nb_features_origin = len( fut.getFieldElement(self.in_vector, driverName="ESRI shapefile", field="region", mode="all", elemType="str")) nb_features_new_region = 5 test_vector_name = "T31TCJ_Samples.sqlite" test_vector = os.path.join(self.test_working_directory, test_vector_name) cmd = "ogr2ogr -nln output -f SQLite {} {}".format( test_vector, self.in_vector) run(cmd) random_update(test_vector, "output", "seed_0", "learn", nb_features_origin) random_update(test_vector, "output", "region", "2", nb_features_new_region) output_dir = self.test_working_directory region_field = "region" # launch function split_vector_by_region(test_vector, output_dir, region_field, runs=1, driver="SQLite") # assert vector_reg_1 = fut.FileSearch_AND(self.test_working_directory, True, "region_1")[0] vector_reg_2 = fut.FileSearch_AND(self.test_working_directory, True, "region_2")[0] feat_vect_reg_1 = len( fut.getFieldElement(vector_reg_1, driverName="SQLite", field="region", mode="all", elemType="str")) feat_vect_reg_2 = len( fut.getFieldElement(vector_reg_2, driverName="SQLite", field="region", mode="all", elemType="str")) self.assertTrue(nb_features_new_region == feat_vect_reg_2) self.assertTrue(nb_features_origin == feat_vect_reg_1 + feat_vect_reg_2)
def gen_mask_region_by_tile(field_region: str, stack_ind: str, working_dir: str, current_tile: str, all_model: str, shp_rname: str, path_to_img: str, path_test: str, path_to_config: str, path_wd: Optional[str] = None) -> List[str]: """ Parameters ---------- field_region: string stack_ind: string working_dir: string current_tile: string all_model: string shp_rname: string path_to_image: string path_test: string path_wd: string path_to_config: string config file containing which model is associated to a given tile Return ------ list(str) """ model_tile = [] for path in all_model: current_model = path.split("/")[-1].split("_")[1] tiles = fu.getListTileFromModel(current_model, path_to_config) model = path.split("/")[-1].split("_")[1] for tile in tiles: # get the model which learn the current tile if tile == current_tile: model_tile.append(model) path_to_feat = os.path.join(path_to_img, tile, "Final", stack_ind) mask_shp = os.path.join(path_test, "shapeRegion", f"{shp_rname}_region_{model}_{tile}.shp") mask_tif = os.path.join( working_dir, f"{shp_rname}_region_{model}_{tile}_NODATA.tif") mask_tif_f = os.path.join( path_test, "classif", "MASK", f"{shp_rname}_region_{model}_{tile}_NODATA.tif") # Create mask if not os.path.exists(mask_tif_f): cmd_raster = ( f"otbcli_Rasterization -in {mask_shp} -mode attribute " f"-mode.attribute.field {field_region} -im {path_to_feat}" f" -out {mask_tif}") run(cmd_raster) if path_wd is not None: cmd = "cp " + mask_tif + " " + path_test + "/classif/MASK" run(cmd) return model_tile
def split_sel(model_selection, tiles, working_directory, epsg): """ split a SQLite file containing points by tiles Parameters ---------- model_selection : string path to samplesSelection's output to a given model tiles : list list of tiles intersected by the model epsg : string epsg's projection. ie : epsg="EPSG:2154" working_directory : string path to a working directory """ import sqlite3 as lite from iota2.Common.Utils import run tile_field_name = "tile_o" out_tiles = [] for tile in tiles: mod_sel_name = os.path.splitext(os.path.basename(model_selection))[0] tile_mod_sel_name_tmp = "{}_{}_tmp".format(tile, mod_sel_name) tile_mod_sel_tmp = os.path.join(working_directory, tile_mod_sel_name_tmp + ".sqlite") if os.path.exists(tile_mod_sel_tmp): os.remove(tile_mod_sel_tmp) conn = lite.connect(tile_mod_sel_tmp) cursor = conn.cursor() cursor.execute("ATTACH '{}' AS db".format(model_selection)) cursor.execute( "CREATE TABLE {} as SELECT * FROM db.output WHERE {}='{}'".format( tile_mod_sel_name_tmp.lower(), tile_field_name, tile)) conn.commit() tile_mod_sel_name = "{}_{}".format(tile, mod_sel_name) tile_mod_sel = os.path.join(working_directory, tile_mod_sel_name + ".sqlite") clause = "SELECT * FROM {}".format(tile_mod_sel_name_tmp) cmd = (f'ogr2ogr -sql "{clause}" -dialect "SQLite" -f "SQLite" -s_srs ' f'{epsg} -t_srs {epsg} -nln {tile_mod_sel_name.lower()}' f' {tile_mod_sel} {tile_mod_sel_tmp}') run(cmd) os.remove(tile_mod_sel_tmp) out_tiles.append(tile_mod_sel) conn = cursor = None return out_tiles
def commonMasks(tile_name, output_path, sensors_parameters, working_directory=None, RAM=128): """ compute common mask considering all sensors by tile Parameters ---------- tile_name [string] tile's name config_path [string] absolute path to the configuration file output_path : str iota2 output path working_directory [string] absolute path to a working directory RAM [int] pipeline's size (Mo) """ import os from iota2.Sensors.Sensors_container import sensors_container from iota2.Common.Utils import run from iota2.Common.FileUtils import ensure_dir # running_parameters = iota2_parameters(config_path) # sensors_parameters = running_parameters.get_sensors_parameters(tile_name) remote_sensor_container = sensors_container(tile_name, working_directory, output_path, **sensors_parameters) common_mask, _ = remote_sensor_container.get_common_sensors_footprint( available_ram=RAM) common_mask_raster = common_mask.GetParameterValue("out") if not os.path.exists(common_mask_raster): ensure_dir(os.path.split(common_mask_raster)[0], raise_exe=False) common_mask.ExecuteAndWriteOutput() common_mask_vector = common_mask_raster.replace(".tif", ".shp") common_mask_vector_cmd = "gdal_polygonize.py -f \"ESRI Shapefile\" -mask {} {} {}".format( common_mask_raster, common_mask_raster, common_mask_vector) run(common_mask_vector_cmd)
def get_region_model_in_tile(current_tile: str, current_region: str, output_path: str, path_wd: str, ref_img: str, field_region: str, test_mode: bool, test_path: str, test_output_folder: str) -> str: """ usage : rasterize region shape. IN: currentTile [string] : tile to compute currentRegion [string] : current region in tile output_path [str] : output path pathWd [string] : working directory refImg [string] : reference image testMode [bool] : flag to enable test mode testPath [string] : path to the vector shape testOutputFolder [string] : path to the output folder OUT: rasterMask [string] : path to the output raster """ from iota2.Common.Utils import run working_directory = os.path.join(output_path, "learningSamples") if path_wd: working_directory = path_wd name_out = f"Mask_region_{current_region}_{current_tile}.tif" if test_mode: mask_shp = test_path working_directory = test_output_folder else: mask_shp = fu.FileSearch_AND(output_path + "/shapeRegion/", True, current_tile, f"region_{current_region.split('f')[0]}", ".shp")[0] raster_mask = os.path.join(working_directory, name_out) cmd_raster = (f"otbcli_Rasterization -in {mask_shp} -mode attribute " f"-mode.attribute.field {field_region} -im {ref_img} " f"-out {raster_mask}") run(cmd_raster) return raster_mask
def keep_fields(vec_in: str, vec_out: str, fields: Optional[List[str]] = [], proj_in: Optional[int] = 2154, proj_out: Optional[int] = 2154) -> None: """ use to extract fields of an input SQLite file Parameters ---------- vec_in : string input SQLite vector File vec_out : string output SQLite vector File fields : list list of fields to keep proj_in : int input projection proj_out : int output projection """ from iota2.VectorTools.vector_functions import get_geom_column_name from iota2.Common.Utils import run table_in = (os.path.splitext(os.path.split(vec_in)[-1])[0]).lower() table_out = (os.path.splitext(os.path.split(vec_out)[-1])[0]).lower() _, ext = os.path.splitext(vec_in) driver_vec_in = "ESRI Shapefile" if "sqlite" in ext: driver_vec_in = "SQLite" geom_column_name = get_geom_column_name(vec_in, driver=driver_vec_in) sql_clause = (f"select {geom_column_name}," f"{','.join(fields)} from {table_in}") cmd = (f"ogr2ogr -s_srs EPSG:{proj_in} -t_srs EPSG:{proj_out} -dialect " f"'SQLite' -f 'SQLite' -nln {table_out} -sql '{sql_clause}' " f"{vec_out} {vec_in}") run(cmd)
def extract_roi(raster: str, current_tile: str, path_wd: str, output_path: str, name: str, ref: str, test_mode: Optional[bool] = None, test_output: Optional[str] = None) -> str: """ usage : extract ROI in raster IN: raster [string] : path to the input raster currentTile [string] : current tile to compute output_path [string] : the output path path_wd [string] : path to the working directory name [string] : output name ref [strin] : raster reference use to get it's extent test_mode [bool] : activate test mode test_output [str] : path to test output data OUT: raterROI [string] : path to the extracted raster. """ from iota2.Common.Utils import run working_directory = output_path + "/learningSamples/" if path_wd: working_directory = path_wd if test_mode: working_directory = test_output current_tile_raster = ref min_x, max_x, min_y, max_y = fu.getRasterExtent(current_tile_raster) raster_roi = os.path.join(working_directory, current_tile + "_" + name + ".tif") cmd = (f"gdalwarp -of GTiff -te {min_x} {min_y} {max_x} {max_y} " f"-ot Byte {raster} {raster_roi}") run(cmd) return raster_roi
def concat_region_one_tile(path_test: str, classif_fusion_mask: str, path_wd: str, tile_mask_concat: str) -> str: """ Parameters ---------- path_test: string classif_fusion_mask: string path_wd: string tile_mask_concat: string Return ------ string """ # TileMask_concat = pathTest+"/classif/"+currentTile+"_MASK.tif" if len(classif_fusion_mask) == 1 and not os.path.exists(tile_mask_concat): shutil.copy(classif_fusion_mask[0], tile_mask_concat) elif len(classif_fusion_mask) != 1 and not os.path.exists( tile_mask_concat): # in order to match images and their mask classif_fusion_mask_sort = sorted(classif_fusion_mask, key=get_model_in_mask) string_cl_fus = " ".join(classif_fusion_mask_sort) path_directory = os.path.join(path_test, "classif") if path_wd is not None: path_directory = path_wd cmd = ( f"otbcli_ConcatenateImages -ram 128 -il {string_cl_fus} -out " f"{os.path.join(path_directory, tile_mask_concat.split('/')[-1])}") run(cmd) if path_wd is not None: cmd = ( f"cp {os.path.join(path_wd, tile_mask_concat.split('/')[-1])}" f"{os.path.join(path_test, 'classif')}") run(cmd) return tile_mask_concat
def transform_to_shape(sqlite_vectors: List[str], formatting_vectors_dir: str) -> List[str]: """ Parameters ---------- sqlite_vectors: list(str) formatting_vectors_dir: str Return ------ list(str) """ from iota2.Common import FileUtils as fut from iota2.Common.Utils import run out = [] for sqlite_vector in sqlite_vectors: out_name = os.path.splitext(os.path.basename(sqlite_vector))[0] out_path = os.path.join(formatting_vectors_dir, f"{out_name}.shp") if os.path.exists(out_path): fut.removeShape(out_path.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) cmd = f"ogr2ogr -f 'ESRI Shapefile' {out_path} {sqlite_vector}" run(cmd) out.append(out_path) return out
def gen_raster_ref(vec, output_path, masks_name, working_directory): """ generate the reference image needed to sampleSelection application Parameters ---------- vec : string path to the shapeFile containing all polygons dedicated to learn a model. cfg : ServiceConfigFile object working_directory : string Path to a working directory """ from iota2.Common.Utils import run tile_field_name = "tile_o" # iota2_dir = cfg.getParam('chain', 'outputPath') features_directory = os.path.join(output_path, "features") tiles = fut.getFieldElement(vec, driverName="ESRI Shapefile", field=tile_field_name, mode="unique", elemType="str") # masks_name = fut.getCommonMaskName(cfg) + ".tif" rasters_tiles = [ fut.FileSearch_AND(os.path.join(features_directory, tile_name), True, masks_name)[0] for tile_name in tiles ] raster_ref_name = "ref_raster_{}.tif".format( os.path.splitext(os.path.basename(vec))[0]) raster_ref = os.path.join(working_directory, raster_ref_name) raster_ref_cmd = "gdal_merge.py -ot Byte -n 0 -createonly -o {} {}".format( raster_ref, " ".join(rasters_tiles)) run(raster_ref_cmd) return raster_ref, tiles
def concat_classifs_one_tile(seed: str, current_tile: str, path_test: str, model_tile: str, concat_out: str, path_wd: Optional[str] = None) -> str: """ Parameters ---------- path_wd: string seed: string current_tile: string path_test: string model_tile: string concat_out: string Return ------ string """ classif_fusion = [] for model in model_tile: classif_fusion.append( os.path.join( path_test, "classif", f"Classif_{current_tile}_model_" f"{model}_seed_{seed}.tif")) if len(classif_fusion) == 1: cmd = f"cp {classif_fusion[0]} {concat_out}" run(cmd) else: # in order to match images and their mask classif_fusion_sort = sorted(classif_fusion, key=get_model_in_classif) string_cl_fus = " ".join(classif_fusion_sort) path_to_directory = os.path.join(path_test, "classif") if path_wd is not None: path_to_directory = path_wd cmd = (f"otbcli_ConcatenateImages -ram 128 -il {string_cl_fus} -out " f"{os.path.join(path_to_directory,concat_out.split('/')[-1])}") run(cmd) if not os.path.exists(concat_out): cmd = (f"cp {os.path.join(path_wd, concat_out.split('/')[-1])} " f"{os.path.join(path_test , 'classif')}") run(cmd) return concat_out
def validity(tile_name, config_path, output_path, maskOut_name, view_threshold, workingDirectory=None, RAM=128): """ function dedicated to compute validity raster/vector by tile Parameters ---------- tile_name [string] tile's name config_path [string] absolute path to the configuration file maskOut_name [string] output vector mask's name view_threshold [int] threshold working_directory [string] absolute path to a working directory RAM [int] pipeline's size (Mo) """ import os import shutil from iota2.Common.ServiceConfigFile import iota2_parameters from iota2.Sensors.Sensors_container import sensors_container from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.Utils import run from iota2.Common.FileUtils import erodeShapeFile from iota2.Common.FileUtils import removeShape from iota2.Common.FileUtils import ensure_dir features_dir = os.path.join(output_path, "features", tile_name) validity_name = "nbView.tif" validity_out = os.path.join(features_dir, validity_name) validity_processing = validity_out if workingDirectory: ensure_dir(os.path.join(workingDirectory, tile_name)) validity_processing = os.path.join(workingDirectory, tile_name, validity_name) running_parameters = iota2_parameters(config_path) sensors_parameters = running_parameters.get_sensors_parameters(tile_name) remote_sensor_container = sensors_container(tile_name, workingDirectory, output_path, **sensors_parameters) sensors_time_series_masks = remote_sensor_container.get_sensors_time_series_masks( available_ram=RAM) sensors_masks_size = [] sensors_masks = [] for sensor_name, (time_series_masks, time_series_dep, nb_bands) in sensors_time_series_masks: if sensor_name.lower() == "sentinel1": for _, time_series_masks_app in list(time_series_masks.items()): time_series_masks_app.Execute() sensors_masks.append(time_series_masks_app) else: time_series_masks.Execute() sensors_masks.append(time_series_masks) sensors_masks_size.append(nb_bands) total_dates = sum(sensors_masks_size) merge_masks = CreateConcatenateImagesApplication({ "il": sensors_masks, "ram": str(RAM) }) merge_masks.Execute() validity_app = CreateBandMathApplication({ "il": merge_masks, "exp": "{}-({})".format( total_dates, "+".join(["im1b{}".format(i + 1) for i in range(total_dates)])), "ram": str(0.7 * RAM), "pixType": "uint8" if total_dates < 255 else "uint16", "out": validity_processing }) if not os.path.exists(os.path.join(features_dir, validity_name)): validity_app.ExecuteAndWriteOutput() if workingDirectory: shutil.copy(validity_processing, os.path.join(features_dir, validity_name)) threshold_raster_out = os.path.join(features_dir, maskOut_name.replace(".shp", ".tif")) threshold_vector_out_tmp = os.path.join( features_dir, maskOut_name.replace(".shp", "_TMP.shp")) threshold_vector_out = os.path.join(features_dir, maskOut_name) input_threshold = validity_processing if os.path.exists( validity_processing) else validity_out threshold_raster = CreateBandMathApplication({ "il": input_threshold, "exp": "im1b1>={}?1:0".format(view_threshold), "ram": str(0.7 * RAM), "pixType": "uint8", "out": threshold_raster_out }) threshold_raster.ExecuteAndWriteOutput() cmd_poly = f"gdal_polygonize.py -mask {threshold_raster_out} {threshold_raster_out} -f \"ESRI Shapefile\" {threshold_vector_out_tmp} {os.path.splitext(os.path.basename(threshold_vector_out_tmp))[0]} cloud" run(cmd_poly) erodeShapeFile(threshold_vector_out_tmp, threshold_vector_out, 0.1) os.remove(threshold_raster_out) removeShape(threshold_vector_out_tmp.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"])
def launchClassification(model, path_cfg, output_path: str, classifier_name: str, classification_mode: str, nomenclature_path: str, stat, pathToRT, pathToImg, pathToRegion, fieldRegion, pathToCmdClassif, pathOut, RAM, pathWd): """ Parameters ---------- output_path : str iota2 output directory """ scriptPath = os.path.join(fu.get_iota2_project_dir(), "iota2") pixType = fu.getOutputPixType(nomenclature_path) AllCmd = [] maskFiles = pathOut + "/MASK" if not os.path.exists(maskFiles): run("mkdir " + maskFiles) if pathToRegion is None: pathToRegion = os.path.join(output_path, "MyRegion.shp") shpRName = pathToRegion.split("/")[-1].replace(".shp", "") AllModel_tmp = fu.FileSearch_AND(model, True, "model", ".txt") AllModel = fu.fileSearchRegEx(model + "/*model*.txt") for currentFile in AllModel_tmp: if currentFile not in AllModel: os.remove(currentFile) for path in AllModel: model = path.split("/")[-1].split("_")[1] tiles = fu.getListTileFromModel( model, output_path + "/config_model/configModel.cfg") model_Mask = model if re.search('model_.*f.*_', path.split("/")[-1]): model_Mask = path.split("/")[-1].split("_")[1].split("f")[0] seed = path.split("/")[-1].split("_")[-1].replace(".txt", "") suffix = "" if "SAR.txt" in os.path.basename(path): seed = path.split("/")[-1].split("_")[-2] suffix = "_SAR" tilesToEvaluate = tiles #construction du string de sortie for tile in tilesToEvaluate: pathToFeat = fu.FileSearch_AND(pathToImg + "/" + tile + "/tmp/", True, "MaskCommunSL", ".tif")[0] maskSHP = pathToRT + "/" + shpRName + "_region_" + model_Mask + "_" + tile + ".shp" maskTif = shpRName + "_region_" + model_Mask + "_" + tile + ".tif" CmdConfidenceMap = "" confidenceMap_name = "{}_model_{}_confidence_seed_{}{}.tif".format( tile, model, seed, suffix) CmdConfidenceMap = " -confmap " + os.path.join( pathOut, confidenceMap_name) if not os.path.exists(maskFiles + "/" + maskTif): pathToMaskCommun = pathToImg + "/" + tile + "/tmp/" + "MaskCommunSL" + ".shp" #cas cluster if pathWd != None: maskFiles = pathWd nameOut = fu.ClipVectorData(maskSHP, pathToMaskCommun, maskFiles, maskTif.replace(".tif", "")) cmdRaster = "otbcli_Rasterization -in "+nameOut+" -mode attribute -mode.attribute.field "+\ fieldRegion+" -im "+pathToFeat+" -out "+maskFiles+"/"+maskTif if "fusion" in classification_mode: cmdRaster = "otbcli_Rasterization -in "+nameOut+" -mode binary -mode.binary.foreground 1 -im "+\ pathToFeat+" -out "+maskFiles+"/"+maskTif run(cmdRaster) if pathWd != None: run("cp " + pathWd + "/" + maskTif + " " + pathOut + "/MASK") os.remove(pathWd + "/" + maskTif) out = pathOut + "/Classif_" + tile + "_model_" + model + "_seed_" + seed + suffix + ".tif" #hpc case if pathWd != None: out = "$TMPDIR/Classif_" + tile + "_model_" + model + "_seed_" + seed + suffix + ".tif" CmdConfidenceMap = " -confmap $TMPDIR/" + confidenceMap_name appli = "python " + scriptPath + "/Classification/ImageClassifier.py -conf " + path_cfg + " " pixType_cmd = " -pixType " + pixType if pathWd != None: pixType_cmd = pixType_cmd + " --wd $TMPDIR " cmd = appli + " -in " + pathToFeat + " -model " + path + " -mask " + pathOut + "/MASK/" + maskTif + " -out " + out + " " + pixType_cmd + " -ram " + str( RAM) + " " + CmdConfidenceMap # add stats if svm if "svm" in classifier_name.lower(): model_statistics = os.path.join( stat, "Model_{}_seed_{}.xml".format(model, seed)) cmd = "{} -imstat {}".format(cmd, model_statistics) AllCmd.append(cmd) fu.writeCmds(pathToCmdClassif + "/class.txt", AllCmd) return AllCmd
def split_by_sets(vector: str, seeds: int, split_directory: str, proj_in: int, proj_out: int, tile_name: str, cross_valid: Optional[bool] = False, split_ground_truth: Optional[bool] = True) -> List[str]: """ use to create new vector file by learning / validation sets Parameters ---------- vector : string path to a shape file containg ground truth seeds : int number of run split_directory : string output directory proj_in : int input projection proj_out : int output projection tile_name : string tile's name crossValid : bool flag to enable cross validation splitGroundTruth : bool flat to split ground truth """ from iota2.Common import FileUtils as fut from iota2.Common.Utils import run out_vectors = [] valid_flag = "validation" learn_flag = "learn" tile_origin_field_name = "tile_o" vector_layer_name = (os.path.splitext( os.path.split(vector)[-1])[0]).lower() # predict fields to keep fields_to_rm = ["seed_" + str(seed) for seed in range(seeds)] fields_to_rm.append(tile_origin_field_name) all_fields = fut.get_all_fields_in_shape(vector) fields = [ field_name for field_name in all_fields if field_name not in fields_to_rm ] # start split for seed in range(seeds): valid_clause = f"seed_{seed}='{valid_flag}'" learn_clause = f"seed_{seed}='{learn_flag}'" sql_cmd_valid = (f"select * FROM {vector_layer_name}" f" WHERE {valid_clause}") output_vec_valid_name = "_".join( [tile_name, "seed_" + str(seed), "val"]) output_vec_valid_name_tmp = "_".join( [tile_name, "seed_" + str(seed), "val", "tmp"]) output_vec_valid_tmp = os.path.join( split_directory, output_vec_valid_name_tmp + ".sqlite") output_vec_valid = os.path.join(split_directory, output_vec_valid_name + ".sqlite") cmd_valid = (f'ogr2ogr -t_srs EPSG:{proj_out} -s_srs EPSG:{proj_in} ' f'-nln {output_vec_valid_name_tmp} -f "SQLite" -sql ' f'"{sql_cmd_valid}" {output_vec_valid_tmp} {vector}') sql_cmd_learn = "select * FROM {} WHERE {}".format( vector_layer_name, learn_clause) output_vec_learn_name = "_".join( [tile_name, "seed_" + str(seed), "learn"]) output_vec_learn_name_tmp = "_".join( [tile_name, "seed_" + str(seed), "learn", "tmp"]) output_vec_learn_tmp = os.path.join( split_directory, output_vec_learn_name_tmp + ".sqlite") output_vec_learn = os.path.join(split_directory, output_vec_learn_name + ".sqlite") cmd_learn = ( f'ogr2ogr -t_srs EPSG:{proj_out} -s_srs EPSG:{proj_in} ' f'-nln {output_vec_learn_name_tmp} -f "SQLite" -sql "{sql_cmd_learn}"' f' {output_vec_learn_tmp} {vector}') if cross_valid is False: if split_ground_truth: run(cmd_valid) # remove useless fields keep_fields(output_vec_valid_tmp, output_vec_valid, fields=fields, proj_in=proj_in, proj_out=proj_out) os.remove(output_vec_valid_tmp) run(cmd_learn) # remove useless fields keep_fields(output_vec_learn_tmp, output_vec_learn, fields=fields, proj_in=proj_in, proj_out=proj_out) os.remove(output_vec_learn_tmp) if split_ground_truth is False: shutil.copy(output_vec_learn, output_vec_valid) out_vectors.append(output_vec_valid) out_vectors.append(output_vec_learn) else: if seed < seeds - 1: run(cmd_learn) keep_fields(output_vec_learn_tmp, output_vec_learn, fields=fields, proj_in=proj_in, proj_out=proj_out) out_vectors.append(output_vec_learn) os.remove(output_vec_learn_tmp) elif seed == seeds - 1: run(cmd_valid) keep_fields(output_vec_valid_tmp, output_vec_valid, fields=fields, proj_in=proj_in, proj_out=proj_out) out_vectors.append(output_vec_valid) os.remove(output_vec_valid_tmp) return out_vectors
def CreateModelShapeFromTiles(tilesModel, pathTiles, pathOut, OutSHPname, fieldOut, pathWd): """ create one shapeFile where all features belong to a model number according to the model description IN : - tilesModel : a list of list which describe which tile belong to which model ex : for 3 models tile model 1 : D0H0,D0H1 tile model 2 : D0H2,D0H3 tile model 3 : D0H4,D0H5,D0H6 tilesModel = [["D0H0","D0H1"],["D0H2","D0H3"],["D0H4","D0H5","D0H6"]] - pathTiles : path to the tile's envelope with priority consideration ex : /xx/x/xxx/x /!\ the folder which contain the envelopes must contain only the envelopes <======== - pathOut : path to store the resulting shapeFile ex : x/x/x/xxx - OutSHPname : the name of the resulting shapeFile ex : "model" - fieldOut : the name of the field which will contain the model number ex : "Mod" - pathWd : path to working directory (not mandatory, due to cluster's architecture default = None) OUT : a shapeFile which contains for all feature the model number which it belong to """ if pathWd is None: pathToTMP = pathOut + "/AllTMP" else: # HPC case pathToTMP = pathWd if not os.path.exists(pathToTMP): run("mkdir " + pathToTMP) to_remove = [] for i in range(len(tilesModel)): for j in range(len(tilesModel[i])): to_remove.append( fu.renameShapefile(pathTiles, tilesModel[i][j], "", "", pathToTMP)) AllTilePath = [] AllTilePath_ER = [] for i in range(len(tilesModel)): for j in range(len(tilesModel[i])): try: ind = AllTilePath.index(pathTiles + "/" + tilesModel[i][j] + ".shp") except ValueError: AllTilePath.append(pathToTMP + "/" + tilesModel[i][j] + ".shp") AllTilePath_ER.append(pathToTMP + "/" + tilesModel[i][j] + "_ERODE.shp") for i in range(len(tilesModel)): for j in range(len(tilesModel[i])): currentTile = pathToTMP + "/" + tilesModel[i][j] + ".shp" AddFieldModel(currentTile, i + 1, fieldOut) for path in AllTilePath: fu.erodeShapeFile(path, path.replace(".shp", "_ERODE.shp"), 0.1) fu.mergeVectors(OutSHPname, pathOut, AllTilePath_ER) if not pathWd: run("rm -r " + pathToTMP) else: for rm in to_remove: fu.removeShape(rm.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"])
def split_vector_by_region(in_vect: str, output_dir: str, region_field: str, runs: Optional[int] = 1, driver: Optional[str] = "ESRI shapefile", proj_in: Optional[str] = "EPSG:2154", proj_out: Optional[str] = "EPSG:2154", mode: Optional[str] = "usually") -> List[str]: """ create new files by regions in input vector. Parameters ---------- in_vect : string input vector path output_dir : string path to output directory region_field : string field in in_vect describing regions driver : string ogr driver proj_in : string input projection proj_out : string output projection mode : string define if we split SAR sensor to the other Return ------ list paths to new output vectors """ from iota2.Common import FileUtils as fut from iota2.Common.Utils import run output_paths = [] # const tile_pos = 0 learn_flag = "learn" table_name = "output" vec_name = os.path.split(in_vect)[-1] tile = vec_name.split("_")[tile_pos] extent = os.path.splitext(vec_name)[-1] regions = fut.getFieldElement(in_vect, driverName=driver, field=region_field, mode="unique", elemType="str") table = vec_name.split(".")[0] if driver != "ESRI shapefile": table = "output" # split vector for seed in range(runs): fields_to_keep = ",".join([ elem for elem in fut.get_all_fields_in_shape(in_vect, "SQLite") if "seed_" not in elem ]) for region in regions: out_vec_name_learn = "_".join([ tile, "region", region, "seed" + str(seed), "Samples_learn_tmp" ]) if mode != "usually": out_vec_name_learn = "_".join([ tile, "region", region, "seed" + str(seed), "Samples", "SAR", "learn_tmp" ]) output_vec_learn = os.path.join(output_dir, out_vec_name_learn + extent) seed_clause_learn = f"seed_{seed}='{learn_flag}'" region_clause = f"{region_field}='{region}'" # split vectors by runs and learning sets sql_cmd_learn = (f"select * FROM {table} WHERE {seed_clause_learn}" f" AND {region_clause}") cmd = (f'ogr2ogr -t_srs {proj_out} -s_srs {proj_in} -nln {table}' f' -f "{driver}" -sql "{sql_cmd_learn}" {output_vec_learn} ' f'{in_vect}') run(cmd) # drop useless column sql_clause = f"select GEOMETRY,{fields_to_keep} from {table_name}" output_vec_learn_out = output_vec_learn.replace("_tmp", "") cmd = ( f"ogr2ogr -s_srs {proj_in} -t_srs {proj_out} -dialect " f"'SQLite' -f 'SQLite' -nln {table_name} -sql '{sql_clause}' " f"{output_vec_learn_out} {output_vec_learn}") run(cmd) output_paths.append(output_vec_learn_out) os.remove(output_vec_learn) return output_paths
def extract_maj_vote_samples(vec_in: str, vec_out: str, ratio_to_keep: float, data_field: str, region_field: str, driver_name: Optional[str] = "ESRI Shapefile"): """ dedicated to extract samples by class according to a ratio. Samples are remove from vec_in and place in vec_out Parameters ---------- vec_in : string path to a shapeFile (.shp) vec_out : string path to a sqlite (.sqlite) ratio_to_keep [float] percentage of samples to extract ratio_to_keep = 0.1 mean extract 10% of each class in each regions dataField : string field containing class labels regionField : string field containing regions labels driver_name : string OGR driver """ from osgeo import ogr from iota2.Common import FileUtils as fut from iota2.Sampling import SplitInSubSets as subset from iota2.Common.Utils import run class_avail = fut.getFieldElement(vec_in, driverName=driver_name, field=data_field, mode="unique", elemType="int") region_avail = fut.getFieldElement(vec_in, driverName=driver_name, field=region_field, mode="unique", elemType="str") driver = ogr.GetDriverByName(driver_name) source = driver.Open(vec_in, 1) layer = source.GetLayer(0) sample_id_to_extract, _ = subset.get_random_poly(layer, data_field, class_avail, ratio_to_keep, region_field, region_avail) # Create new file with targeted FID fid_samples_in = built_where_sql_exp(sample_id_to_extract, clause="in") cmd = f"ogr2ogr -where '{fid_samples_in}' -f 'SQLite' {vec_out} {vec_in}" run(cmd) # remove in vec_in targeted FID vec_in_rm = vec_in.replace(".shp", "_tmp.shp") fid_samples_not_in = built_where_sql_exp(sample_id_to_extract, clause="not in") cmd = f"ogr2ogr -where '{fid_samples_not_in}' {vec_in_rm} {vec_in}" run(cmd) fut.removeShape(vec_in.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) cmd = f"ogr2ogr {vec_in} {vec_in_rm}" run(cmd) fut.removeShape(vec_in_rm.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"])
def test_VectorFormatting(self): """ test vectorFormatting function random function is used in Sampling.VectorFormatting.VectorFormatting we can only check if there is expected number of features with expected fields and some features values """ from iota2.Sampling.VectorFormatting import vector_formatting from iota2.Common import ServiceConfigFile as SCF from iota2.Common import IOTA2Directory from iota2.Common.Utils import run from iota2.VectorTools.ChangeNameField import changeName # define inputs test_output = os.path.join(self.test_working_directory, "IOTA2_dir_VectorFormatting") # prepare ground truth ground_truth = os.path.join(self.test_working_directory, "groundTruth_test.shp") cmd = "ogr2ogr -s_srs EPSG:2154 -t_srs EPSG:2154 -dialect 'SQLite' -sql 'select GEOMETRY,code from t31tcj' {} {}".format( ground_truth, self.in_vector) run(cmd) # cfg instance runs = 2 cfg = SCF.serviceConfigFile(self.config_test) cfg.setParam('chain', 'outputPath', test_output) cfg.setParam('chain', 'groundTruth', ground_truth) cfg.setParam('chain', 'dataField', "code") cfg.setParam('chain', 'cloud_threshold', 0) cfg.setParam('chain', 'merge_final_classifications', False) cfg.setParam('chain', 'runs', runs) cfg.setParam('GlobChain', 'proj', "EPSG:2154") cfg.setParam('chain', 'regionPath', self.ref_region) IOTA2Directory.generate_directories(test_output, check_inputs=False) # prepare expected function inputs t31tcj_feat_dir = os.path.join(self.test_working_directory, "IOTA2_dir_VectorFormatting", "features", "T31TCJ") os.mkdir(t31tcj_feat_dir) # prepare ref img t31tcj_ref_img = os.path.join(t31tcj_feat_dir, "MaskCommunSL.tif") shutil.copy(self.ref_img, t31tcj_ref_img) # prepare envelope envelope_name = "T31TCJ.shp" envelope_path = os.path.join(self.test_working_directory, "IOTA2_dir_VectorFormatting", "envelope", envelope_name) fut.cpShapeFile(self.ref_region.replace(".shp", ""), envelope_path.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) changeName(envelope_path, "region", "FID") # prepare cloud mask cloud_name = "CloudThreshold_0.shp" cloud_path = os.path.join(self.test_working_directory, "IOTA2_dir_VectorFormatting", "features", "T31TCJ", cloud_name) fut.cpShapeFile(self.ref_region.replace(".shp", ""), cloud_path.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) changeName(cloud_path, "region", "cloud") # launch function ratio = cfg.getParam('chain', 'ratio') random_seed = cfg.getParam('chain', 'random_seed') enable_cross_validation = cfg.getParam("chain", "enableCrossValidation") enable_split_ground_truth = cfg.getParam('chain', 'splitGroundTruth') fusion_merge_all_validation = cfg.getParam( 'chain', 'fusionOfClassificationAllSamplesValidation') merge_final_classifications = cfg.getParam( 'chain', 'merge_final_classifications') merge_final_classifications_ratio = cfg.getParam( 'chain', 'merge_final_classifications_ratio') region_vec = cfg.getParam('chain', 'regionPath') epsg = int(cfg.getParam('GlobChain', 'proj').split(":")[-1]) region_field = (cfg.getParam('chain', 'regionField')) vector_formatting("T31TCJ", test_output, ground_truth, "code", 0, ratio, random_seed, enable_cross_validation, enable_split_ground_truth, fusion_merge_all_validation, runs, epsg, region_field, merge_final_classifications, merge_final_classifications_ratio, region_vec, working_directory=None) # assert nb_features_origin = len( fut.getFieldElement(ground_truth, driverName="ESRI Shapefile", field="code", mode="all", elemType="str")) test_vector = fut.FileSearch_AND( os.path.join(test_output, "formattingVectors"), True, "T31TCJ.shp")[0] nb_features_test = len( fut.getFieldElement(test_vector, driverName="ESRI Shapefile", field="code", mode="all", elemType="str")) # check nb features self.assertTrue(nb_features_origin == nb_features_test, msg="wrong number of features") # check fields origin_fields = fut.get_all_fields_in_shape(ground_truth) test_fields = fut.get_all_fields_in_shape(test_vector) new_fields = ['region', 'originfid', 'seed_0', 'seed_1', 'tile_o'] expected_fields = origin_fields + new_fields self.assertTrue(len(expected_fields) == len(test_fields)) self.assertTrue(all(field in test_fields for field in expected_fields))
def test_undecision_management(self): """ Test undecision_management """ import filecmp from iota2.Classification import undecision_management as UM from iota2.Classification import Fusion as FUS from iota2.Common.Utils import run from iota2.Common import FileUtils as fut shutil.copytree( os.path.join(IOTA2DIR, "data", "references", "NoData", "Input", "Classif"), self.test_working_directory) shutil.copytree( os.path.join(IOTA2DIR, "data", "references", "NoData", "Input", "config_model"), os.path.join(self.test_working_directory, "config_model")) if not os.path.exists( os.path.join(self.test_working_directory, "cmd", "fusion")): os.makedirs( os.path.join(self.test_working_directory, "cmd", "fusion")) shutil.copy( os.path.join(self.test_working_directory, "classif", "Classif_D0005H0002_model_1_seed_0.tif"), os.path.join(self.test_working_directory, "classif", "Classif_D0005H0002_model_1f2_seed_0.tif")) shutil.copy( os.path.join(self.test_working_directory, "classif", "Classif_D0005H0002_model_1_seed_0.tif"), os.path.join(self.test_working_directory, "classif", "Classif_D0005H0002_model_1f1_seed_0.tif")) shutil.copy( os.path.join(self.test_working_directory, "classif", "D0005H0002_model_1_confidence_seed_0.tif"), os.path.join(self.test_working_directory, "classif", "D0005H0002_model_1f2_confidence_seed_0.tif")) shutil.copy( os.path.join(self.test_working_directory, "classif", "D0005H0002_model_1_confidence_seed_0.tif"), os.path.join(self.test_working_directory, "classif", "D0005H0002_model_1f1_confidence_seed_0.tif")) cmd_fus = FUS.fusion( os.path.join(self.test_working_directory, "classif"), 1, ["D0005H0002"], '-nodatalabel 0 -method majorityvoting', os.path.join(IOTA2DIR, "data", "references", "nomenclature.txt"), os.path.join(IOTA2DIR, "data", "regionShape", "4Tiles.shp"), False, None) for cmd in cmd_fus: run(cmd) fusion_files = fut.FileSearch_AND( os.path.join(self.test_working_directory, "classif"), True, "_FUSION_") print(fusion_files) pixtype = fut.getOutputPixType( os.path.join(IOTA2DIR, "data", "references", "nomenclature.txt")) for fusionpath in fusion_files: UM.undecision_management( self.test_working_directory, fusionpath, "region", os.path.join(IOTA2DIR, "data", "references", "features"), os.path.join(IOTA2DIR, "data", "references", "GenerateRegionShape", "region_need_To_env.shp"), "maxConfidence", None, ["NDVI", "NDWI", "Brightness"], '../../../../MNT_L8Grid', pixtype, os.path.join(IOTA2DIR, "data", "regionShape", "4Tiles.shp"), "ALT,ASP,SLP", False) # TODO: test intelligent self.assertTrue( filecmp.cmp( os.path.join(self.test_working_directory, "classif", "D0005H0002_FUSION_model_1_seed_0.tif"), os.path.join(IOTA2DIR, "data", "references", "NoData", "Output", "D0005H0002_FUSION_model_1_seed_0.tif")))
def undecision_management(path_test: str, path_fusion: str, field_region: str, path_to_img: str, path_to_region: str, no_label_management: str, path_wd: str, list_indices: List[str], user_feat_path: str, pix_type: str, region_vec: Optional[str] = None, user_feat_pattern: Optional[str] = None, ds_sar_opt: Optional[bool] = False) -> None: """ manage undecision comming from fusion of classifications Parameters ---------- path_test: string path_fusion: string field_region: string path_to_img: string path_to_region: string region_vec: string no_label_management: string path_wd: string list_indices: List[string], user_feat_path: string pix_type: string user_feat_pattern: string ds_sar_opt: bool Return ------ None """ stack_ind = fu.get_feat_stack_name(list_indices, user_feat_path, user_feat_pattern) suffix_pattern = "" if ds_sar_opt: suffix_pattern = "_DS" if region_vec: current_model = path_fusion.split("/")[-1].split("_")[3] config_model = os.path.join(path_test, "config_model", "configModel.cfg") n_fold = get_nb_split_shape(current_model, config_model) path_directory = path_test + "/classif" if path_wd is not None: working_dir = path_wd path_directory = path_wd else: working_dir = path_test + "/classif/MASK" current_tile = path_fusion.split("/")[-1].split("_")[0] shp_rname = path_to_region.split("/")[-1].replace(".shp", "") all_model = fu.FileSearch_AND(os.path.join(path_test, "model"), True, "model", ".txt") if region_vec is None: model_tile = gen_mask_region_by_tile( field_region, stack_ind, working_dir, current_tile, all_model, shp_rname, path_to_img, path_test, os.path.join(path_test, "config_model", "configModel.cfg"), path_wd) elif region_vec and no_label_management == "maxConfidence": model_tile = path_fusion.split("/")[-1].split("_")[3] elif region_vec and no_label_management == "learningPriority": model_tile_tmp = path_fusion.split("/")[-1].split("_")[3] model_tile = [] for i in range(n_fold): model_tile.append(f"{model_tile_tmp} f{i + 1}") if len(model_tile) == 0 or no_label_management == "maxConfidence": seed = os.path.split(path_fusion)[-1].split("_")[-1].split(".")[0] img_confidence = fu.FileSearch_AND( path_test + "/classif", True, "confidence_seed_" + str(seed) + suffix_pattern + ".tif", current_tile) img_classif = fu.FileSearch_AND(os.path.join(path_test, "classif"), True, "Classif_" + current_tile, f"seed_{seed}", suffix_pattern) img_data = os.path.join( path_directory, f"{current_tile}_FUSION_NODATA_seed{seed}.tif") if region_vec: img_confidence = fu.fileSearchRegEx( f"{path_test+os.sep}classif{os.sep}" f"{current_tile}_model_{model_tile}" f"f*_confidence_seed_{seed}{suffix_pattern}.tif") img_classif = fu.fileSearchRegEx( f"{path_test+os.sep}classif{os.sep}" f"Classif_{current_tile}_model_" f"{model_tile}f*_seed_" f"{seed}{suffix_pattern}.tif") img_data = (f"{path_directory+os.sep}Classif_" f"{current_tile}_model_{model_tile}" f"_seed_{seed}{suffix_pattern}.tif") img_confidence.sort() img_classif.sort() exp, il_str = build_confidence_exp(path_fusion, img_confidence, img_classif) cmd = (f"otbcli_BandMath -il {il_str} -out {img_data} {pix_type} " f"-exp '{exp}' ") run(cmd) if path_wd is not None: run(f"cp {img_data} {os.path.join(path_test, 'classif')}") elif len(model_tile) != 0 and no_label_management == "learningPriority": # Concaténation des classifications pour une tuile (qui a ou non # plusieurs régions) et Concaténation des masques de régions pour # une tuile (qui a ou non plusieurs régions) seed = os.path.split(path_fusion)[-1].split("_")[-1].split(".")[0] concat_out = (f"{os.path.join(path_test, 'classif')}" f"{current_tile}_FUSION_concat_seed{seed}.tif") if region_vec: concat_out = ( f"{os.path.join(path_test, 'classif')}" f"{current_tile}_FUSION_model_{model_tile[0].split('f')[0]}" f"concat_seed{seed}.tif") path_to_classif_concat = concat_classifs_one_tile( seed, current_tile, path_test, model_tile, concat_out, path_wd) pattern_mask = "*region_*_{current_tile}_NODATA.tif" classif_fusion_mask = fu.fileSearchRegEx( os.path.join(path_test, "classif", "MASK", pattern_mask)) out_concat_mask = os.path.join(path_test, "classif", f"{current_tile}_MASK.tif") if region_vec: pattern_mask = ("*region_{model_tile[0].split('f')[0]}" f"_{current_tile}.tif") classif_fusion_mask_tmp = fu.fileSearchRegEx( os.path.join(path_test, "classif", "MASK", pattern_mask)) out_concat_mask = (f"{os.path.join(path_test, 'classif')}" f"{current_tile}_MASK_model_" f"{model_tile[0].split('f')[0]}.tif") classif_fusion_mask = [] for i in range(n_fold): classif_fusion_mask.append(classif_fusion_mask_tmp[0]) path_to_region_mask_concat = concat_region_one_tile( path_test, classif_fusion_mask, path_wd, out_concat_mask) # construction de la commande exp = "" im1 = path_fusion im2 = path_to_region_mask_concat im3 = path_to_classif_concat for i in range(len(classif_fusion_mask)): if i + 1 < len(classif_fusion_mask): exp = exp + "im2b" + str(i + 1) + ">=1?im3b" + str(i + 1) + ":" else: exp = exp + "im2b" + str(i + 1) + ">=1?im3b" + str(i + 1) + ":0" exp = "im1b1!=0?im1b1:(" + exp + ")" img_data = (os.path.join( path_directory, f"{current_tile}_FUSION_" f"NODATA_seed{seed}.tif")) if region_vec: img_data = (f"{path_directory+os.sep}Classif_{current_tile}_model_" f"{model_tile[0].split('f')[0]}_seed_{seed}.tif") cmd = (f'otbcli_BandMath -il {im1} {im2} {im3} -out + {img_data} ' f'{pix_type} -exp "{exp}"') run(cmd) if path_wd is not None: run("cp {img_data} {path_test+os.sep}classif")
def classification_shaping(path_classif: str, runs: int, path_out: str, path_wd: str, classif_mode: str, path_test: str, ds_sar_opt: bool, proj: int, nomenclature_path: str, output_statistics: bool, spatial_resolution: float, proba_map_flag: bool, region_shape: str, color_path: str) -> None: """function use to mosaic rasters and to produce final maps path_classif: str directory where as classifications runs: int number of random learning/validation samples-set path_out: str output directory path_wd: str working directory classif_mode: str fusion of classifications ? path_test: str iota2 output directory ds_sar_opt: bool flag to inform if SAR and optical post-classification workflow is enable proj: int epsg code nomenclature_path: str nomenclature path output_statistics: bool flag to enable output statistics spatial_resolution: float output's spatial resolution proba_map_flag: bool flag to inform if probability map was produce region_shape: str region shapeFile path color_path: str color table file """ if path_wd is None: tmp = path_out + "/TMP" if not os.path.exists(path_out + "/TMP"): os.mkdir(tmp) else: tmp = path_wd if not os.path.exists(path_out + "/TMP"): os.mkdir(path_out + "/TMP") all_tiles = list( set([ classif.split("_")[1] for classif in fu.FileSearch_AND( path_test + "/classif", False, "Classif", ".tif") ])) pix_type = fu.getOutputPixType(nomenclature_path) features_path = os.path.join(path_test, "features") all_tmp_folder = fu.fileSearchRegEx(path_test + "/TMPFOLDER*") if all_tmp_folder: for tmp_folder in all_tmp_folder: shutil.rmtree(tmp_folder) suffix = "*" if ds_sar_opt: suffix = "*DS*" genGlobalConfidence(runs, path_wd, spatial_resolution, proj, path_test, classif_mode, all_tiles, region_shape, ds_sar_opt, proba_map_flag) if region_shape and classif_mode == "fusion": old_classif = fu.fileSearchRegEx(path_test + "/classif/Classif_*_model_*f*_seed_" + suffix + ".tif") for rm in old_classif: if not os.path.exists(path_test + "/final/TMP/OLDCLASSIF"): os.mkdir(path_test + "/final/TMP/OLDCLASSIF") run("mv " + rm + " " + path_test + "/final/TMP/OLDCLASSIF") classification = [] confidence = [] proba_map = [] cloud = [] for seed in range(runs): classification.append([]) confidence.append([]) cloud.append([]) sort = [] if proba_map_flag: proba_map_list = fu.fileSearchRegEx( path_test + "/classif/PROBAMAP_*_model_*_seed_" + str(seed) + suffix + ".tif") proba_map_list = removeInListByRegEx( proba_map_list, ".*model_.*f.*_seed." + suffix) proba_map.append(proba_map_list) if classif_mode == "separate" or region_shape: all_classif_seed = fu.FileSearch_AND(path_classif, True, ".tif", "Classif", "seed_" + str(seed)) if ds_sar_opt: all_classif_seed = fu.FileSearch_AND(path_classif, True, ".tif", "Classif", "seed_" + str(seed), "DS.tif") ind = 1 elif classif_mode == "fusion": all_classif_seed = fu.FileSearch_AND( path_classif, True, "_FUSION_NODATA_seed" + str(seed) + ".tif") if ds_sar_opt: all_classif_seed = fu.FileSearch_AND( path_classif, True, "_FUSION_NODATA_seed" + str(seed) + "_DS.tif") ind = 0 for tile in all_classif_seed: sort.append((tile.split("/")[-1].split("_")[ind], tile)) sort = fu.sortByFirstElem(sort) for tile, paths in sort: exp = "" all_cl = "" all_cl_rm = [] for i in range(len(paths)): all_cl = all_cl + paths[i] + " " all_cl_rm.append(paths[i]) if i < len(paths) - 1: exp = exp + "im" + str(i + 1) + "b1 + " else: exp = exp + "im" + str(i + 1) + "b1" path_cl_final = tmp + "/" + tile + "_seed_" + str(seed) + ".tif" classification[seed].append(path_cl_final) cmd = 'otbcli_BandMath -il ' + all_cl + '-out ' + path_cl_final + ' ' + pix_type + ' -exp "' + exp + '"' run(cmd) tile_confidence = path_out + "/TMP/" + tile + "_GlobalConfidence_seed_" + str( seed) + ".tif" confidence[seed].append(tile_confidence) cloud_tile = fu.FileSearch_AND(features_path + "/" + tile, True, "nbView.tif")[0] classif_tile = tmp + "/" + tile + "_seed_" + str(seed) + ".tif" cloud_tile_priority = path_test + "/final/TMP/" + tile + "_Cloud.tif" cloud_tile_priority_tmp = tmp + "/" + tile + "_Cloud.tif" cloud_tile_priority_stats_ok = path_test + "/final/TMP/" + tile + "_Cloud_StatsOK.tif" cloud_tile_priority_tmp_stats_ok = tmp + "/" + tile + "_Cloud_StatsOK.tif" cloud[seed].append(cloud_tile_priority) if not os.path.exists(cloud_tile_priority): cmd_cloud = f'otbcli_BandMath -il {cloud_tile} {classif_tile} -out {cloud_tile_priority_tmp} int16 -exp "im2b1>0?im1b1:0"' run(cmd_cloud) if output_statistics: cmd_cloud = 'otbcli_BandMath -il ' + cloud_tile + ' ' + classif_tile + ' -out ' + cloud_tile_priority_tmp_stats_ok + ' int16 -exp "im2b1>0?im1b1:-1"' run(cmd_cloud) if path_wd: shutil.copy(cloud_tile_priority_tmp_stats_ok, cloud_tile_priority_stats_ok) os.remove(cloud_tile_priority_tmp_stats_ok) if path_wd: shutil.copy(cloud_tile_priority_tmp, cloud_tile_priority) os.remove(cloud_tile_priority_tmp) if path_wd is not None: run("cp -a " + tmp + "/* " + path_out + "/TMP") for seed in range(runs): assemble_folder = path_test + "/final" if path_wd: assemble_folder = path_wd fu.assembleTile_Merge(classification[seed], spatial_resolution, "{}/Classif_Seed_{}.tif".format( assemble_folder, seed), "Byte" if pix_type == "uint8" else "Int16", co={ "COMPRESS": "LZW", "BIGTIFF": "YES" }) if path_wd: shutil.copy(path_wd + "/Classif_Seed_" + str(seed) + ".tif", path_test + "/final") os.remove(path_wd + "/Classif_Seed_" + str(seed) + ".tif") fu.assembleTile_Merge(confidence[seed], spatial_resolution, assemble_folder + "/Confidence_Seed_" + str(seed) + ".tif", "Byte", co={ "COMPRESS": "LZW", "BIGTIFF": "YES" }) if path_wd: shutil.copy(path_wd + "/Confidence_Seed_" + str(seed) + ".tif", path_test + "/final") os.remove(path_wd + "/Confidence_Seed_" + str(seed) + ".tif") color.CreateIndexedColorImage( path_test + "/final/Classif_Seed_" + str(seed) + ".tif", color_path, output_pix_type=gdal.GDT_Byte if pix_type == "uint8" else gdal.GDT_UInt16) if proba_map_flag: proba_map_mosaic = os.path.join( assemble_folder, "ProbabilityMap_seed_{}.tif".format(seed)) fu.assembleTile_Merge(proba_map[seed], spatial_resolution, proba_map_mosaic, "Int16", co={ "COMPRESS": "LZW", "BIGTIFF": "YES" }) if path_wd: shutil.copy(proba_map_mosaic, path_test + "/final") os.remove(proba_map_mosaic) fu.assembleTile_Merge(cloud[0], spatial_resolution, assemble_folder + "/PixelsValidity.tif", "Byte", co={ "COMPRESS": "LZW", "BIGTIFF": "YES" }) if path_wd: shutil.copy(path_wd + "/PixelsValidity.tif", path_test + "/final") os.remove(path_wd + "/PixelsValidity.tif")
def genGlobalConfidence(N, pathWd, spatialRes, proj, pathTest, classifMode, AllTile: List[str], shapeRegion, ds_sar_opt, proba_map_flag): """generate confidences ready to be mosaic """ PROBAMAP_PATTERN = "PROBAMAP" tmpClassif = pathTest + "/classif/tmpClassif" pathToClassif = pathTest + "/classif" if pathWd: tmpClassif = pathWd + "/tmpClassif" if not os.path.exists(tmpClassif): run("mkdir " + tmpClassif) for seed in range(N): for tuile in AllTile: if shapeRegion is None: if classifMode == "separate": confidence_pattern = os.path.join( pathToClassif, "{}*model*confidence_seed_{}*.tif".format(tuile, seed)) if ds_sar_opt: confidence_pattern = os.path.join( pathToClassif, "{}*model*confidence_seed_{}*_DS.tif".format( tuile, seed)) confidence = fu.fileSearchRegEx(confidence_pattern) globalConf = tmpClassif + "/" + tuile + "_GlobalConfidence_seed_" + str( seed) + ".tif" globalConf_f = pathTest + "/final/TMP/" + tuile + "_GlobalConfidence_seed_" + str( seed) + ".tif" cmd = 'otbcli_BandMath -il ' + confidence[ 0] + ' -out ' + globalConf + ' uint8 -exp "100*im1b1"' run(cmd) shutil.copyfile(globalConf, globalConf_f) os.remove(globalConf) else: raise Exception(( "if there is no region shape specify in the " "configuration file, argClassification.classifMode must be set to 'separate'" )) else: #output Mode suffix = "*" if ds_sar_opt: suffix = "*DS*" if classifMode != "separate": classifTile = fu.fileSearchRegEx( pathToClassif + "/Classif_" + tuile + "*model_*f*_seed_" + str(seed) + suffix ) # tmp tile (produce by each classifier, without nodata) splitModel = [] for classif in classifTile: model = classif.split("/")[-1].split("_")[3].split( "f")[0] try: ind = splitModel.index(model) except ValueError: splitModel.append(model) splitConfidence = [] confidence_all = fu.fileSearchRegEx( pathToClassif + "/" + tuile + "*model_*_confidence_seed_" + str(seed) + suffix) confidence_withoutSplit = removeInListByRegEx( confidence_all, ".*model_.*f.*_confidence." + suffix) for model in splitModel: classifTile = fu.fileSearchRegEx( pathToClassif + "/Classif_" + tuile + "*model_" + model + "f*_seed_" + str(seed) + suffix ) # tmp tile (produce by each classifier, without nodata) finalTile = pathToClassif + "/Classif_" + tuile + "_model_" + model + "_seed_" + str( seed) + ".tif" if ds_sar_opt: finalTile = pathToClassif + "/Classif_" + tuile + "_model_" + model + "_seed_" + str( seed) + "_DS.tif" confidence = fu.fileSearchRegEx(pathToClassif + "/" + tuile + "*model_" + model + "f*_confidence_seed_" + str(seed) + suffix) if proba_map_flag: proba_map_fusion(proba_map_list=fu.fileSearchRegEx( "{}/{}_{}_model_{}f*_seed_{}{}.tif".format( pathToClassif, PROBAMAP_PATTERN, tuile, model, seed, suffix)), working_directory=pathWd, ram=2000) classifTile = sorted(classifTile) confidence = sorted(confidence) OutPutConfidence = tmpClassif + "/" + tuile + "_model_" + model + "_confidence_seed_" + str( seed) + ".tif" if ds_sar_opt: OutPutConfidence = tmpClassif + "/" + tuile + "_model_" + model + "_confidence_seed_" + str( seed) + "_DS.tif" cmd = BuildConfidenceCmd(finalTile, classifTile, confidence, OutPutConfidence, fact=100, pixType="uint8") run(cmd) splitConfidence.append(OutPutConfidence) i = 0 #init j = 0 exp1 = "+".join([ "im" + str(i + 1) + "b1" for i in range(len(splitConfidence)) ]) #-> confidence from splited models are from 0 to 100 exp2 = "+".join([ "(100*im" + str(j + 1) + "b1)" for j in np.arange( len(splitConfidence), len(splitConfidence) + len(confidence_withoutSplit)) ]) #-> confidence from NO-splited models are from 0 to 1 if not splitConfidence: exp2 = "+".join([ "100*im" + str(j + 1) + "b1" for j in range(len(confidence_withoutSplit)) ]) if exp1 and exp2: exp = exp1 + "+" + exp2 if exp1 and not exp2: exp = exp1 if not exp1 and exp2: exp = exp2 confidence_list = splitConfidence + confidence_withoutSplit AllConfidence = " ".join(confidence_list) OutPutConfidence = tmpClassif + "/" + tuile + "_GlobalConfidence_seed_" + str( seed) + ".tif" cmd = 'otbcli_BandMath -il ' + AllConfidence + ' -out ' + OutPutConfidence + ' uint8 -exp "' + exp + '"' run(cmd) shutil.copy(OutPutConfidence, pathTest + "/final/TMP") os.remove(OutPutConfidence) #shutil.rmtree(tmpClassif) else: confidence = fu.fileSearchRegEx(pathToClassif + "/" + tuile + "*model*confidence_seed_" + str(seed) + suffix) exp = "+".join([ "im" + str(i + 1) + "b1" for i in range(len(confidence)) ]) AllConfidence = " ".join(confidence) #for currentConf in confidence: globalConf = tmpClassif + "/" + tuile + "_GlobalConfidence_seed_" + str( seed) + ".tif" globalConf_f = pathTest + "/final/TMP/" + tuile + "_GlobalConfidence_seed_" + str( seed) + ".tif" cmd = 'otbcli_BandMath -il ' + AllConfidence + ' -out ' + globalConf + ' uint8 -exp "100*(' + exp + ')"' #print confidence run(cmd) shutil.copyfile(globalConf, globalConf_f) os.remove(globalConf)
def compare_ref(shape_ref: str, shape_learn: str, classif: str, diff: str, working_directory: str, path_wd: str, data_field: str, spatial_res: int): """ Parameters ---------- shape_ref: string shape_learn: string classif: string diff: string working_directory: string path_wd: string data_field: string spatial_res: string Return ------ string """ min_x, max_x, min_y, max_y = fu.getRasterExtent(classif) shape_raster_val = working_directory + os.sep + shape_ref.split( "/")[-1].replace(".sqlite", ".tif") shape_raster_learn = working_directory + os.sep + shape_learn.split( "/")[-1].replace(".sqlite", ".tif") # Rasterise val shape_ref_table_name = os.path.splitext( os.path.split(shape_ref)[-1])[0].lower() cmd = (f"gdal_rasterize -l {shape_ref_table_name} -a {data_field} -init 0 " f"-tr {spatial_res} {spatial_res} {shape_ref} {shape_raster_val} " f"-te {min_x} {min_y} {max_x} {max_y}") run(cmd) # Rasterise learn shape_learn_table_name = os.path.splitext( os.path.split(shape_learn)[-1])[0].lower() cmd = ( f"gdal_rasterize -l {shape_learn_table_name} -a {data_field} -init " f"0 -tr {spatial_res} {spatial_res} {shape_learn} {shape_raster_learn}" f" -te {min_x} {min_y} {max_x} {max_y}") run(cmd) # diff val diff_val = working_directory + "/" + diff.split("/")[-1].replace( ".tif", "_val.tif") # reference identique -> 2 | reference != -> 1 | pas de reference -> 0 cmd_val = (f'otbcli_BandMath -il {shape_raster_val} {classif} -out ' f'{diff_val} uint8 -exp "im1b1==0?0:im1b1==im2b1?2:1"') run(cmd_val) os.remove(shape_raster_val) # diff learn diff_learn = working_directory + "/" + diff.split("/")[-1].replace( ".tif", "_learn.tif") # reference identique -> 4 | reference != -> 3 | pas de reference -> 0 cmd_learn = (f'otbcli_BandMath -il {shape_raster_learn} {classif} -out ' f'{diff_learn} uint8 -exp "im1b1==0?0:im1b1==im2b1?4:3"') run(cmd_learn) os.remove(shape_raster_learn) # sum diff val + learn diff_tmp = working_directory + "/" + diff.split("/")[-1] cmd_sum = (f'otbcli_BandMath -il {diff_val} {diff_learn} -out {diff_tmp}' f' uint8 -exp "im1b1+im2b1"') run(cmd_sum) os.remove(diff_val) os.remove(diff_learn) if path_wd and not os.path.exists(diff): shutil.copy(diff_tmp, diff) os.remove(diff_tmp) return diff
def sqlite_to_geojson(input_db: str, output_db: str, logger=LOGGER) -> None: """sqlite database to geojson database """ from iota2.Common.Utils import run logger.info(f"changin input data format {input_db} to {output_db}") run(f'ogr2ogr -f "GeoJSON" {output_db} {input_db}')