def samples_stats(region_seed_tile: Tuple[str, str, str], iota2_directory: str, data_field: str, working_directory: Optional[Union[str, None]] = None, logger=LOGGER) -> str: """generate samples statistics by tiles Parameters ---------- region_seed_tile: tuple iota2_directory: str iota2 output directory data_field: str data field in region database working_directory: str path to a working directory logger: logging root logger Return ------ str file containing statistics from otbcli_PolygonClassStatistics """ region, seed, tile = region_seed_tile samples_selection_dir = os.path.join(iota2_directory, "samplesSelection") tile_region_dir = os.path.join(iota2_directory, "shapeRegion") w_dir = samples_selection_dir if working_directory: w_dir = working_directory raster_mask = fut.FileSearch_AND(tile_region_dir, True, "region_" + region.split("f")[0] + "_", ".tif", tile)[0] region_vec = fut.FileSearch_AND(samples_selection_dir, True, "_region_" + region, "seed_" + seed, ".shp")[0] logger.info( f"Launch statistics on tile {tile} in region {region} run {seed}") region_tile_stats_name = f"{tile}_region_{region}_seed_{seed}_stats.xml" region_tile_stats = os.path.join(w_dir, region_tile_stats_name) polygon_stats_app = otb.CreatePolygonClassStatisticsApplication({ "in": raster_mask, "mask": raster_mask, "vec": region_vec, "field": data_field, "out": region_tile_stats }) polygon_stats_app.ExecuteAndWriteOutput() if working_directory: shutil.copy(region_tile_stats, samples_selection_dir) return region_tile_stats
def test_split_vector_by_region(self): """ test : split a vector by the region he belongs to """ from iota2.Sampling.VectorFormatting import split_vector_by_region from iota2.Common.Utils import run from iota2.Tests.UnitTests.Iota2Tests import random_update # define inputs nb_features_origin = len( fut.getFieldElement(self.in_vector, driverName="ESRI shapefile", field="region", mode="all", elemType="str")) nb_features_new_region = 5 test_vector_name = "T31TCJ_Samples.sqlite" test_vector = os.path.join(self.test_working_directory, test_vector_name) cmd = "ogr2ogr -nln output -f SQLite {} {}".format( test_vector, self.in_vector) run(cmd) random_update(test_vector, "output", "seed_0", "learn", nb_features_origin) random_update(test_vector, "output", "region", "2", nb_features_new_region) output_dir = self.test_working_directory region_field = "region" # launch function split_vector_by_region(test_vector, output_dir, region_field, runs=1, driver="SQLite") # assert vector_reg_1 = fut.FileSearch_AND(self.test_working_directory, True, "region_1")[0] vector_reg_2 = fut.FileSearch_AND(self.test_working_directory, True, "region_2")[0] feat_vect_reg_1 = len( fut.getFieldElement(vector_reg_1, driverName="SQLite", field="region", mode="all", elemType="str")) feat_vect_reg_2 = len( fut.getFieldElement(vector_reg_2, driverName="SQLite", field="region", mode="all", elemType="str")) self.assertTrue(nb_features_new_region == feat_vect_reg_2) self.assertTrue(nb_features_origin == feat_vect_reg_1 + feat_vect_reg_2)
def config_model(outputPath, region_field): """ usage : determine which model will class which tile """ #const output = None pos_tile = 0 formatting_vec_dir = os.path.join(outputPath, "formattingVectors") samples = fu.FileSearch_AND(formatting_vec_dir, True, ".shp") #init all_regions = [] for sample in samples: tile_name = os.path.splitext( os.path.basename(sample))[0].split("_")[pos_tile] regions = fu.getFieldElement(sample, driverName="ESRI Shapefile", field=region_field, mode="unique", elemType="str") for region in regions: all_regions.append((region, tile_name)) #{'model_name':[TileName, TileName...],'...':...,...} model_tiles = dict(fu.sortByFirstElem(all_regions)) #add tiles if they are missing by checking in /shapeRegion/ directory shape_region_dir = os.path.join(outputPath, "shapeRegion") shape_region_path = fu.FileSearch_AND(shape_region_dir, True, ".shp") #check if there is actually polygons shape_regions = [ elem for elem in shape_region_path if len( fu.getFieldElement(elem, driverName="ESRI Shapefile", field=region_field, mode="all", elemType="str")) >= 1 ] for shape_region in shape_regions: tile = os.path.splitext( os.path.basename(shape_region))[0].split("_")[-1] region = os.path.splitext( os.path.basename(shape_region))[0].split("_")[-2] for model_name, tiles_model in list(model_tiles.items()): if model_name.split("f")[0] == region and tile not in tiles_model: tiles_model.append(tile) #Construct output file string output = "AllModel:\n[" for model_name, tiles_model in list(model_tiles.items()): output_tmp = "\n\tmodelName:'{}'\n\ttilesList:'{}'".format( model_name, "_".join(tiles_model)) output = output + "\n\t{" + output_tmp + "\n\t}" output += "\n]" return output
def split_samples(output_path: str, data_field: str, enable_cross_validation: bool, region_threshold: Union[str, float], region_field: str, ratio: float, random_seed: int, runs: int, epsg: Union[str, int], workingDirectory: Optional[str] = None, logger: Optional[Logger] = LOGGER): """ """ from iota2.Common import FileUtils as fut # const regions_pos = -2 if isinstance(epsg, str): epsg = int(epsg.split(":")[-1]) if isinstance(region_threshold, str): region_threshold = float(region_threshold) formatting_vectors_dir = os.path.join(output_path, "formattingVectors") shape_region_dir = os.path.join(output_path, "shapeRegion") vectors = fut.FileSearch_AND(formatting_vectors_dir, True, ".shp") # get All possible regions by parsing shapeFile's name shapes_region = fut.FileSearch_AND(shape_region_dir, True, ".shp") regions = list( set([ os.path.split(shape)[-1].split("_")[regions_pos] for shape in shapes_region ])) # compute region's area areas, regions_tiles, data_to_rm = get_regions_area( vectors, regions, formatting_vectors_dir, workingDirectory, region_field) # get how many sub-regions must be created by too huge regions. regions_split = get_splits_regions(areas, region_threshold) for region_name, area in list(areas.items()): logger.info(f"region : {region_name} , area : {area}") updated_vectors = split(regions_split, regions_tiles, data_field, region_field) # transform sqlites to shape file, according to input data format new_regions_shapes = transform_to_shape(updated_vectors, formatting_vectors_dir) for data in data_to_rm: os.remove(data) data_app_val_dir = os.path.join(output_path, "dataAppVal") update_learning_validation_sets(new_regions_shapes, data_app_val_dir, data_field, region_field, ratio, runs, epsg, enable_cross_validation, random_seed)
def getVectorsChunks(inpath, inbase="dept_"): if not fut.FileSearch_AND(inpath, True, inbase, ".shp", "chk"): listout = fut.FileSearch_AND(inpath, True, inbase, ".shp", "stats") else: listout = fut.FileSearch_AND(inpath, True, inbase, ".shp", "chk") listofchkofzones = fut.sortByFirstElem([ ("_".join(x.split('_')[0:len(x.split('_')) - 1]), x) for x in listout ]) return listofchkofzones
def getListVectToSimplify(path): simplified = [os.path.splitext(x)[0].split('_')[len(os.path.splitext(x)[0].split('_')) - 2] \ for x in fut.FileSearch_AND(path, True, ".shp", "douglas") \ if "hermite" not in x] polygonized = [[x, os.path.splitext(x)[0].split('_')[len(os.path.splitext(x)[0].split('_')) - 1]] \ for x in fut.FileSearch_AND(path, True, "tile_", ".shp") \ if "douglas" not in x and "hermite" not in x] return [ os.path.join(path, x) for x, y in polygonized if y not in simplified ]
def generate_region_shape(envelope_directory: str, output_region_file: str, out_field_name: str, i2_output_path: str, working_directory: str) -> None: """generate regions shape envelope_directory: str directory containing all iota2 tile's envelope output_region_file: str output file out_field_name: str output field containing region i2_output_path: str iota2 output path working_directory: str path to a working directory """ region = [] all_tiles = fu.FileSearch_AND(envelope_directory, False, ".shp") region.append(all_tiles) if not output_region_file: output_region_file = os.path.join(i2_output_path, "MyRegion.shp") p_f = output_region_file.replace(" ", "").split("/") out_name = p_f[-1].split(".")[0] path_mod = "" for i in range(1, len(p_f) - 1): path_mod = path_mod + "/" + p_f[i] CreateModelShapeFromTiles(region, envelope_directory, path_mod, out_name, out_field_name, working_directory)
def create_dummy_rasters(missing_tiles: List[str], runs: int, output_path: str) -> None: """ Parameters ---------- missing_tiles: list(string) runs: int output_path: string Return ------ None Notes ----- use when mode is 'one_region' but there is no validations / learning samples into a specific tile """ classifications_dir = os.path.join(output_path, "classif") final_dir = os.path.join(output_path, "final", "TMP") for tile in missing_tiles: classif_tile = fu.FileSearch_AND(classifications_dir, True, "Classif_" + str(tile))[0] for seed in range(runs): dummy_raster_name = tile + "_seed_" + str(seed) + "_CompRef.tif" dummy_raster = final_dir + "/" + dummy_raster_name dummy_raster_cmd = (f"gdal_merge.py -ot Byte -n 0 -createonly -o " f"{ dummy_raster} {classif_tile}") run(dummy_raster_cmd)
def get_vectors_to_sample( iota2_formatting_dir: str, ds_fusion_sar_opt: Optional[bool] = False) -> List[Dict[str, str]]: """ get vectors to sample IN : iota2_formatting_dir : str path to shapefiles ds_fusion_sar_opt : bool activate sar mode OUT: List of dictionary containing all shapefiles """ formatting_tiles = fu.FileSearch_AND(iota2_formatting_dir, True, ".shp") # parameters generation tiles_vectors = [{"usually": vector} for vector in formatting_tiles] # parameters dedicated to SAR tiles_vectors_sar = [{"SAR": vector} for vector in formatting_tiles] tiles_vectors_to_sample = tiles_vectors # if we want to have SAR classification and Optical classification belong # we have to double the number of parameters in generateSamples if ds_fusion_sar_opt: tiles_vectors_to_sample = tiles_vectors + tiles_vectors_sar return tiles_vectors_to_sample
def test_vector_splits(self): from iota2.Sampling import SplitInSubSets as VS from iota2.Common import FileUtils as fu from iota2.Tests.UnitTests import TestsUtils # We execute the function splitInSubSets() for new_region_shape in self.new_regions_shapes: tile_name = os.path.splitext(os.path.basename(new_region_shape))[0] vectors_to_rm = fu.FileSearch_AND(self.data_app_val_dir, True, tile_name) for vect in vectors_to_rm: os.remove(vect) VS.splitInSubSets(new_region_shape, self.data_field, self.region_field, self.ratio, self.seeds, "ESRI Shapefile", random_seed=0) print(new_region_shape) # We check the output self.assertTrue( TestsUtils.compareVectorFile(self.ref_split_shp, self.out_split_shp, 'coordinates', 'polygon', "ESRI Shapefile"), "Split vector output are different")
def test_samples_selection(self): """ test sampling of a shape file (main function of SamplesSelection.py) """ from iota2.Sampling.SamplesSelection import samples_selection from iota2.Common import IOTA2Directory from iota2.Common import ServiceConfigFile as SCF from iota2.Tests.UnitTests.Iota2Tests import compareSQLite from iota2.Common.FileUtils import cpShapeFile from iota2.Common import FileUtils as fut # prepare test input cfg = SCF.serviceConfigFile(self.config_test) cfg.setParam( "chain", "outputPath", os.path.join(self.test_working_directory, "samplesSelTest")) cfg.setParam("chain", "runs", 2) cfg.setParam("argTrain", "sampleSelection", { "sampler": "random", "strategy": "all" }) # create IOTA2 directories IOTA2Directory.generate_directories(os.path.join( self.test_working_directory, "samplesSelTest"), check_inputs=False) shutil.copytree( self.features_ref, os.path.join(self.test_working_directory, "samplesSelTest", "features", "T31TCJ")) shutil.copy( self.in_xml, os.path.join(self.test_working_directory, "samplesSelTest", "samplesSelection", "T31TCJ_region_1_seed_0_stats.xml")) _, in_shape_name = os.path.split(self.in_shape) in_shape_dir = os.path.join(self.test_working_directory, "samplesSelTest", "samplesSelection") in_shape = os.path.join(in_shape_dir, in_shape_name) cpShapeFile(self.in_shape.replace(".shp", ""), in_shape.replace(".shp", ""), extensions=[".prj", ".shp", ".dbf", ".shx"]) #~ # launch function output_path = cfg.getParam("chain", "outputPath") runs = cfg.getParam('chain', 'runs') epsg = cfg.getParam('GlobChain', 'proj') random_seed = cfg.getParam('chain', 'random_seed') data_field = cfg.getParam('chain', 'dataField').lower() parameters = dict(cfg.getParam('argTrain', 'sampleSelection')) masks_name = "MaskCommunSL.tif" samples_selection(in_shape, self.test_working_directory, output_path, runs, epsg, masks_name, parameters, data_field, random_seed) #~ # assert selection_test = fut.FileSearch_AND( os.path.join(self.test_working_directory, "samplesSelTest"), True, os.path.basename(self.selection_ref))[0] same = compareSQLite(self.selection_ref, selection_test, CmpMode='coordinates') self.assertTrue(same, msg="sample selection generation failed")
def test_create_tile_region_masks(self): """ test the generation of the raster mask which define the region in the tile """ from iota2.Sampling.VectorFormatting import create_tile_region_masks from iota2.Common.Utils import run from iota2.Tests.UnitTests.TestsUtils import rasterToArray import numpy as np # define inputs test_vector_name = "T31TCJ.sqlite" test_vector = os.path.join(self.test_working_directory, test_vector_name) cmd = "ogr2ogr -nln t31tcj -f SQLite {} {}".format( test_vector, self.ref_region) run(cmd) # launch function create_tile_region_masks(test_vector, "region", "T31TCJ", self.test_working_directory, "MyRegion", self.ref_img) # assert raster_region = fut.FileSearch_AND(self.test_working_directory, True, "MyRegion", ".tif")[0] raster_region_arr = rasterToArray(raster_region) ref_array = np.ones((50, 50)) self.assertTrue(np.allclose(ref_array, raster_region_arr), msg="problem with the normalization by ref")
def confusion_models_merge_parameters(iota2_dir: str): """ function use to feed confusion_models_merge function Parameter --------- iota2_dir : string path to the iota2 running directory Return ------ list list containing all sub confusion matrix which must be merged. """ ds_sar_opt_conf_dir = os.path.join(iota2_dir, "dataAppVal", "bymodels") csv_seed_pos = 4 csv_model_pos = 2 all_csv = fu.FileSearch_AND(ds_sar_opt_conf_dir, True, "samples", ".csv") # group by models model_group = [] for csv in all_csv: _, csv_name = os.path.split(csv) csv_seed = csv_name.split("_")[csv_seed_pos] csv_model = csv_name.split("_")[csv_model_pos] # csv_mode = "SAR.csv" or "val.csv", use to descriminate models csv_mode = csv_name.split("_")[-1] key_param = (csv_model, csv_seed, csv_mode) model_group.append((key_param, csv)) groups_param = [param for key, param in fu.sortByFirstElem(model_group)] return groups_param
def step_inputs(self): """ Return ------ the return could be and iterable or a callable """ from iota2.Common import FileUtils as fut return fut.FileSearch_AND(os.path.join(self.output_path, "classif"), True, "_FUSION_")
def generate_shape_tile(tiles: List[str], pathWd: str, output_path: str, proj: int) -> None: """generate tile's envelope with priority management Parameters ---------- tiles : list list of tiles envelopes to generate pathOut : str output directory pathWd : str working directory output_path : str iota2 output directory proj : int epsg code of target projection """ pathOut = os.path.join(output_path, "envelope") if not os.path.exists(pathOut): os.mkdir(pathOut) featuresPath = os.path.join(output_path, "features") cMaskName = "MaskCommunSL" for tile in tiles: if not os.path.exists(featuresPath + "/" + tile): os.mkdir(featuresPath + "/" + tile) os.mkdir(featuresPath + "/" + tile + "/tmp") commonDirectory = pathOut + "/commonMasks/" if not os.path.exists(commonDirectory): os.mkdir(commonDirectory) common = [ featuresPath + "/" + Ctile + "/tmp/" + cMaskName + ".tif" for Ctile in tiles ] ObjListTile = [ Tile(currentTile, name) for currentTile, name in zip(common, tiles) ] ObjListTile_sort = sorted(ObjListTile, key=priorityKey) tmpFile = pathOut + "/TMP" if pathWd: tmpFile = pathWd + "/TMP" if not os.path.exists(tmpFile): os.mkdir(tmpFile) genTileEnvPrio(ObjListTile_sort, pathOut, tmpFile, proj) AllPRIO = fu.FileSearch_AND(tmpFile, True, "_PRIO.shp") for prioTile in AllPRIO: tileName = prioTile.split("/")[-1].split("_")[0] fu.cpShapeFile(prioTile.replace(".shp", ""), pathOut + "/" + tileName, [".prj", ".shp", ".dbf", ".shx"]) shutil.rmtree(tmpFile) shutil.rmtree(commonDirectory)
def getListVectToClip(path, fieldclip, vectorpath): listtoclip = [] for filetoclip in fut.FileSearch_AND(path, True, ".shp", "hermite"): listtoclip.append( (filetoclip, os.path.basename(filetoclip.replace(fieldclip, '')).split('_')[2])) return listtoclip
def step_inputs(self): """ Return ------ the return could be and iterable or a callable """ from iota2.Common import FileUtils as fut selected_polygons = fut.FileSearch_AND( os.path.join(self.output_path, "samplesSelection"), True, ".shp") if self.enable_cross_validation: selected_polygons = sorted(selected_polygons, key=self.sort_by_seed)[:-1] return selected_polygons
def tile_vectors_to_models(iota2_learning_samples_dir: str) -> List[str]: """ use to feed vector_samples_merge function Parameters ---------- iota2_learning_samples_dir : string path to "learningSamples" iota² directory sep_sar_opt : bool flag use to inform if SAR data has to be computed separately Return ------ list list of list of vectors to be merged to form a vector by model """ vectors = fu.FileSearch_AND(iota2_learning_samples_dir, True, "Samples_learn.sqlite") vectors_sar = fu.FileSearch_AND(iota2_learning_samples_dir, True, "Samples_SAR_learn.sqlite") vect_to_model = split_vectors_by_regions( vectors) + split_vectors_by_regions(vectors_sar) return vect_to_model
def get_models(formatting_vector_directory: str, region_field: str, runs: int) -> List[Tuple[str, List[str], int]]: """ usage : describe samples spatial repartition function use to determine with shapeFile as to be merged in order to compute statistics thanks to otb_SampleSelection OUT: regions_tiles_seed [list] : example regions_tiles_seed = [('1', ['T1', 'T2'], 0), ('1', ['T1', T2], 1), ('2', ['T2', 'T3], 0), ('2', ['T2', 'T3], 1)] mean the region '1' is present in tiles 'T1' and 'T2' in run 0 and 1 and region '2' in 'T2', 'T3' in runs 0 and 1 """ # the way of getting region could be improve ? tiles = fut.FileSearch_AND(formatting_vector_directory, True, ".shp") region_tile = [] all_regions_in_run = [] for tile in tiles: all_regions = [] tile_name = os.path.splitext(os.path.basename(tile))[0] r_tmp = fut.getFieldElement(tile, driverName="ESRI Shapefile", field=region_field, mode="unique", elemType="str") for r_tile in r_tmp: if r_tile not in all_regions: all_regions.append(r_tile) for region in all_regions: if region not in all_regions_in_run: all_regions_in_run.append(region) region_tile.append((region, tile_name)) region_tile_tmp = dict(fut.sortByFirstElem(region_tile)) region_tile_dic = {} for region, region_tiles in list(region_tile_tmp.items()): region_tile_dic[region] = list(set(region_tiles)) all_regions_in_run = sorted(all_regions_in_run) regions_tiles_seed = [(region, region_tile_dic[region], run) for run in range(runs) for region in all_regions_in_run] return regions_tiles_seed
def update_learning_validation_sets(new_regions_shapes: List[str], data_app_val_dir: str, data_field: str, region_field: str, ratio: float, seeds: int, epsg: str, enable_cross_validation: bool, random_seed: int) -> None: """ Parameters ---------- new_regions_shapes: list(string) data_app_val_dir: string data_field: string region_field: string ratio: float seeds: intersect epsg: string enable_cross_validation: bool random_seed: int Return ------ """ from iota2.Sampling.VectorFormatting import split_by_sets from iota2.Sampling import SplitInSubSets as subset from iota2.Common import FileUtils as fut for new_region_shape in new_regions_shapes: tile_name = os.path.splitext(os.path.basename(new_region_shape))[0] vectors_to_rm = fut.FileSearch_AND(data_app_val_dir, True, tile_name) for vect in vectors_to_rm: os.remove(vect) # remove seeds fields subset.splitInSubSets(new_region_shape, data_field, region_field, ratio, seeds, "ESRI Shapefile", crossValidation=enable_cross_validation, random_seed=random_seed) split_by_sets(new_region_shape, seeds, data_app_val_dir, epsg, epsg, tile_name, cross_valid=enable_cross_validation)
def getListToPolygonize(path): if path[len(path) - 1] == "/": path = path[:-1] listmos = fut.FileSearch_AND(path, True, ".tif") listVect = [] for root, dirs, files in os.walk(path): for filename in files: if ".shp" in filename and "douglas" not in filename and "hermite" not in filename: fileToSearch = os.path.join( root, os.path.splitext(filename)[0] + ".tif") if fileToSearch in listmos: listmos.remove(fileToSearch) print(listmos) input("payse") return listmos
def prepare_annual_features(working_directory, reference_directory, pattern, rename=None): """ double all rasters's pixels rename must be a tuple """ import iota2.Common.FileUtils as fut import shutil for dirname, dirnames, filenames in os.walk(reference_directory): # print path to all subdirectories first. for subdirname in dirnames: os.mkdir( os.path.join(dirname, subdirname).replace( reference_directory, working_directory).replace(rename[0], rename[1])) # print path to all filenames. for filename in filenames: shutil.copy( os.path.join(dirname, filename), os.path.join(dirname, filename).replace( reference_directory, working_directory).replace(rename[0], rename[1])) rasters_path = fut.FileSearch_AND(working_directory, True, pattern) for raster in rasters_path: cmd = ('otbcli_BandMathX -il ' + raster + ' -out ' + raster + ' -exp "im1+im1"') print(cmd) os.system(cmd) if rename: all_content = [] for dirname, dirnames, filenames in os.walk(working_directory): # print path to all subdirectories first. for subdirname in dirnames: all_content.append(os.path.join(dirname, subdirname)) # print path to all filenames. for filename in filenames: all_content.append(os.path.join(dirname, filename))
def prepare_selection(sample_sel_directory: str, tile_name: str, workingDirectory: Optional[str] = None): """ this function is dedicated to merge selection comming from different models by tiles. It is necessary in order to prepare sampleExtraction in the step call 'generate samples' Parameters ---------- sample_sel_directory : string path to the IOTA² directory containing selections by models tile_name : string tile's name workingDirectory : string path to a working directory LOGGER : logging object root logger """ wd = sample_sel_directory if workingDirectory: wd = workingDirectory vectors = fut.FileSearch_AND(sample_sel_directory, True, tile_name, "selection.sqlite") merge_selection_name = "{}_selection_merge".format(tile_name) output_selection_merge = os.path.join(wd, merge_selection_name + ".sqlite") if not os.path.exists(output_selection_merge): fut.mergeVectors(merge_selection_name, wd, vectors, ext="sqlite", out_Tbl_name="output") if workingDirectory: shutil.copy( output_selection_merge, os.path.join(sample_sel_directory, merge_selection_name + ".sqlite")) return os.path.join(sample_sel_directory, merge_selection_name + ".sqlite")
def region_tile(sample_sel_dir: str): """ """ tile_field_name = "tile_o" region_vectors = fut.FileSearch_AND(sample_sel_dir, True, ".shp") output = [] region_vectors = sorted(region_vectors) for region_vector in region_vectors: tiles = fut.getFieldElement(region_vector, driverName="ESRI Shapefile", field=tile_field_name, mode="unique", elemType="str") region_name = os.path.splitext( os.path.basename(region_vector))[0].split("_")[2] seed = os.path.splitext( os.path.basename(region_vector))[0].split("_")[4] tiles = sorted(tiles) for tile in tiles: output.append((region_name, seed, tile)) return output
def get_region_model_in_tile(current_tile: str, current_region: str, output_path: str, path_wd: str, ref_img: str, field_region: str, test_mode: bool, test_path: str, test_output_folder: str) -> str: """ usage : rasterize region shape. IN: currentTile [string] : tile to compute currentRegion [string] : current region in tile output_path [str] : output path pathWd [string] : working directory refImg [string] : reference image testMode [bool] : flag to enable test mode testPath [string] : path to the vector shape testOutputFolder [string] : path to the output folder OUT: rasterMask [string] : path to the output raster """ from iota2.Common.Utils import run working_directory = os.path.join(output_path, "learningSamples") if path_wd: working_directory = path_wd name_out = f"Mask_region_{current_region}_{current_tile}.tif" if test_mode: mask_shp = test_path working_directory = test_output_folder else: mask_shp = fu.FileSearch_AND(output_path + "/shapeRegion/", True, current_tile, f"region_{current_region.split('f')[0]}", ".shp")[0] raster_mask = os.path.join(working_directory, name_out) cmd_raster = (f"otbcli_Rasterization -in {mask_shp} -mode attribute " f"-mode.attribute.field {field_region} -im {ref_img} " f"-out {raster_mask}") run(cmd_raster) return raster_mask
def test_iota2_regularisation(self): """Test regularization """ rules = mr.getMaskRegularisation(self.nomenclature) for rule in rules: mr.adaptRegularization(self.wd, self.raster10m, os.path.join(self.tmp, rule[2]), "1000", rule, 2) rasters = fut.FileSearch_AND(self.tmp, True, "mask", ".tif") mr.mergeRegularization(self.tmp, rasters, 10, self.outfile, "1000") # test outtest = testutils.rasterToArray(self.outfile) outref = testutils.rasterToArray(self.rasterregref) self.assertTrue(np.array_equal(outtest, outref)) # remove temporary folders if os.path.exists(self.wd): shutil.rmtree(self.wd, ignore_errors=True) if os.path.exists(self.out): shutil.rmtree(self.out, ignore_errors=True)
def gen_raster_ref(vec, output_path, masks_name, working_directory): """ generate the reference image needed to sampleSelection application Parameters ---------- vec : string path to the shapeFile containing all polygons dedicated to learn a model. cfg : ServiceConfigFile object working_directory : string Path to a working directory """ from iota2.Common.Utils import run tile_field_name = "tile_o" # iota2_dir = cfg.getParam('chain', 'outputPath') features_directory = os.path.join(output_path, "features") tiles = fut.getFieldElement(vec, driverName="ESRI Shapefile", field=tile_field_name, mode="unique", elemType="str") # masks_name = fut.getCommonMaskName(cfg) + ".tif" rasters_tiles = [ fut.FileSearch_AND(os.path.join(features_directory, tile_name), True, masks_name)[0] for tile_name in tiles ] raster_ref_name = "ref_raster_{}.tif".format( os.path.splitext(os.path.basename(vec))[0]) raster_ref = os.path.join(working_directory, raster_ref_name) raster_ref_cmd = "gdal_merge.py -ot Byte -n 0 -createonly -o {} {}".format( raster_ref, " ".join(rasters_tiles)) run(raster_ref_cmd) return raster_ref, tiles
def get_features_application( train_shape: str, working_directory: str, samples: str, data_field: str, output_path: str, sar_optical_post_fusion: bool, sensors_parameters: sensors_params_type, ram: Optional[int] = 128, mode: Optional[str] = "usually", only_mask_comm: Optional[bool] = False, only_sensors_masks: Optional[bool] = False, logger: Optional[Logger] = LOGGER ) -> Tuple[otb_app_type, List[otb_app_type]]: """ usage : compute from a stack of data -> gapFilling -> features computation -> sampleExtractions thanks to OTB's applications' IN: trainShape [string] : path to a vector shape containing points workingDirectory [string] : working directory path samples [string] : output sqlite file dataField [string] : data's field in trainShape tile [string] : actual tile to compute. (ex : T31TCJ) output_path [string] : output_path onlyMaskComm [bool] : flag to stop the script after common Mask computation onlySensorsMasks [bool] : compute only masks OUT: sampleExtr [SampleExtraction OTB's object]: """ # const # seed_position = -1 from iota2.Common import GenerateFeatures as genFeatures from iota2.Sensors.ProcessLauncher import commonMasks tile = train_shape.split("/")[-1].split(".")[0].split("_")[0] working_directory_features = os.path.join(working_directory, tile) c_mask_directory = os.path.join(output_path, "features", tile, "tmp") if not os.path.exists(working_directory_features): try: os.mkdir(working_directory_features) except OSError: logger.warning(f"{working_directory_features} allready exists") (all_features, feat_labels, dep_features) = genFeatures.generateFeatures(working_directory_features, tile, sar_optical_post_fusion, output_path, sensors_parameters, mode=mode) if only_sensors_masks: # return AllRefl,AllMask,datesInterp,realDates return dep_features[1], dep_features[2], dep_features[3], dep_features[ 4] all_features.Execute() ref = fu.FileSearch_AND(c_mask_directory, True, "MaskCommunSL.tif") if not ref: commonMasks(tile, output_path, sensors_parameters) ref = fu.FileSearch_AND(c_mask_directory, True, "MaskCommunSL.tif")[0] if only_mask_comm: return ref sample_extr = otb.Registry.CreateApplication("SampleExtraction") sample_extr.SetParameterString("ram", str(0.7 * ram)) sample_extr.SetParameterString("vec", train_shape) sample_extr.SetParameterInputImage( "in", all_features.GetParameterOutputImage("out")) sample_extr.SetParameterString("out", samples) sample_extr.SetParameterString("outfield", "list") sample_extr.SetParameterStringList("outfield.list.names", feat_labels) sample_extr.UpdateParameters() sample_extr.SetParameterStringList("field", [data_field.lower()]) all_dep = [all_features, dep_features] return sample_extr, all_dep
def generate_samples_classif_mix(folder_sample: str, working_directory: str, train_shape: str, path_wd: str, output_path: str, annual_crop: List[Union[str, int]], all_class: List[Union[str, int]], data_field: str, previous_classif_path: str, proj: int, runs: Union[str, int], enable_cross_validation: bool, region_field: str, validity_threshold: int, target_resolution: int, sar_optical_post_fusion: bool, sensors_parameters: sensors_params_type, folder_features: Optional[str] = None, ram: Optional[int] = 128, w_mode: Optional[bool] = False, test_mode: Optional[bool] = False, test_shape_region: Optional[str] = None, sample_sel: Optional[str] = None, mode: Optional[str] = "usually", logger: Optional[Logger] = LOGGER): """ usage : from one classification, chose randomly annual sample merge with non annual sample and extract features. IN: folderSample [string] : output folder workingDirectory [string] : computation folder trainShape [string] : vector shape (polygons) to sample pathWd [string] : if different from None, enable HPC mode (copy at ending) featuresPath [string] : path to all stack annualCrop [list of string/int] : list containing annual crops ex : [11,12] AllClass [list of string/int] : list containing all classes in vector shape ex : [11,12,51..] cfg [string] : configuration file class previousClassifPath [string] : path to the iota2 output directory which generate previous classification dataField [string] : data's field into vector shape testMode [bool] : enable testMode -> iota2tests.py testPrevConfig [string] : path to the configuration file which generate previous classification testShapeRegion [string] : path to the shapefile representing region in the tile. testFeaturePath [string] : path to the stack of data OUT: samples [string] : vector shape containing points """ from iota2.Sampling.SamplesSelection import prepare_selection from iota2.Sampling import GenAnnualSamples as genAS if os.path.exists( os.path.join( folder_sample, train_shape.split("/")[-1].replace(".shp", "_Samples.sqlite"))): return None if enable_cross_validation: runs = runs - 1 features_path = os.path.join(output_path, "features") sample_sel_directory = os.path.join(output_path, "samplesSelection") work_dir = sample_sel_directory if working_directory: work_dir = working_directory data_field = data_field.lower() current_tile = (os.path.splitext(os.path.basename(train_shape))[0]) if sample_sel: sample_selection = sample_sel else: sample_selection = prepare_selection(sample_sel_directory, current_tile) non_annual_shape = os.path.join( work_dir, "{}_nonAnnual_selection.sqlite".format(current_tile)) annual_shape = os.path.join( work_dir, "{}_annual_selection.sqlite".format(current_tile)) # garde toutes les classes pérennes nb_feat_nannu = extract_class(sample_selection, non_annual_shape, all_class, data_field) regions = fu.getFieldElement(train_shape, driverName="ESRI Shapefile", field=region_field, mode="unique", elemType="str") print(sample_selection) print(train_shape) # avoir la répartition des classes anuelles par seed et par region # -> pouvoir faire annu_repartition[11][R][S] annu_repartition = get_repartition(sample_selection, annual_crop, data_field, region_field, regions, runs) nb_feat_annu = get_number_annual_sample(annu_repartition) # raster ref (in order to extract ROIs) ref = fu.FileSearch_AND(os.path.join(features_path, current_tile), True, "MaskCommunSL.tif")[0] if nb_feat_nannu > 0: all_coord = get_points_coord_in_shape(non_annual_shape, "SQLite") else: all_coord = [0] classification_raster = extract_roi(os.path.join(previous_classif_path, "final", "Classif_Seed_0.tif"), current_tile, path_wd, output_path, f"Classif_{current_tile}", ref, test_mode, test_output=folder_sample) validity_raster = extract_roi(os.path.join(previous_classif_path, "final", "PixelsValidity.tif"), current_tile, path_wd, output_path, f"Cloud{current_tile}", ref, test_mode, test_output=folder_sample) # build regions mask into the tile masks = [ get_region_model_in_tile(current_tile, current_region, output_path, path_wd, classification_raster, region_field, test_mode, test_shape_region, test_output_folder=folder_sample) for current_region in regions ] if nb_feat_annu > 0: annual_points = genAS.genAnnualShapePoints( all_coord, "SQLite", working_directory, target_resolution, annual_crop, data_field, current_tile, validity_threshold, validity_raster, classification_raster, masks, train_shape, annual_shape, proj, region_field, runs, annu_repartition) merge_name = train_shape.split("/")[-1].replace(".shp", "_selectionMerge") sample_selection = os.path.join(working_directory, f"{merge_name}.sqlite") if (nb_feat_nannu > 0) and (nb_feat_annu > 0 and annual_points): fu.mergeSQLite(merge_name, working_directory, [non_annual_shape, annual_shape]) elif (nb_feat_nannu > 0) and not (nb_feat_annu > 0 and annual_points): # If not annual samples can be added then annual classes are ignored shutil.copy(non_annual_shape, sample_selection) elif not (nb_feat_nannu > 0) and (nb_feat_annu > 0 and annual_points): # If not non annual samples are found then use all annual samples shutil.copy(annual_shape, sample_selection) samples = os.path.join( working_directory, train_shape.split("/")[-1].replace(".shp", "_Samples.sqlite")) sample_extr, dep_tmp = get_features_application( sample_selection, working_directory, samples, data_field, output_path, sar_optical_post_fusion, sensors_parameters, ram, mode) # sampleExtr.ExecuteAndWriteOutput() multi_proc = mp.Process(target=executeApp, args=[sample_extr]) multi_proc.start() multi_proc.join() split_vectors = split_vector_by_region(in_vect=samples, output_dir=working_directory, region_field=region_field, runs=int(runs), driver="SQLite", proj_in="EPSG:" + str(proj), proj_out="EPSG:" + str(proj)) if test_mode: split_vectors = None if path_wd and os.path.exists(samples): for sample in split_vectors: shutil.copy(sample, folder_sample) if os.path.exists(non_annual_shape): os.remove(non_annual_shape) if os.path.exists(annual_shape): os.remove(annual_shape) if w_mode: target_directory = os.path.join(folder_features, current_tile) if not os.path.exists(target_directory): try: os.mkdir(target_directory) except OSError: logger.warning(f"{target_directory} allready exists") try: os.mkdir(os.path.join(target_directory, "tmp")) except OSError: logger.warning(f"{target_directory}/tmp allready exists") from_dir = os.path.join(working_directory, current_tile, "tmp") to_dir = os.path.join(target_directory, "tmp") if os.path.exists(from_dir): fu.updateDirectory(from_dir, to_dir) os.remove(samples) os.remove(classification_raster) os.remove(validity_raster) for mask in masks: os.remove(mask) return split_vectors
def test_VectorFormatting(self): """ test vectorFormatting function random function is used in Sampling.VectorFormatting.VectorFormatting we can only check if there is expected number of features with expected fields and some features values """ from iota2.Sampling.VectorFormatting import vector_formatting from iota2.Common import ServiceConfigFile as SCF from iota2.Common import IOTA2Directory from iota2.Common.Utils import run from iota2.VectorTools.ChangeNameField import changeName # define inputs test_output = os.path.join(self.test_working_directory, "IOTA2_dir_VectorFormatting") # prepare ground truth ground_truth = os.path.join(self.test_working_directory, "groundTruth_test.shp") cmd = "ogr2ogr -s_srs EPSG:2154 -t_srs EPSG:2154 -dialect 'SQLite' -sql 'select GEOMETRY,code from t31tcj' {} {}".format( ground_truth, self.in_vector) run(cmd) # cfg instance runs = 2 cfg = SCF.serviceConfigFile(self.config_test) cfg.setParam('chain', 'outputPath', test_output) cfg.setParam('chain', 'groundTruth', ground_truth) cfg.setParam('chain', 'dataField', "code") cfg.setParam('chain', 'cloud_threshold', 0) cfg.setParam('chain', 'merge_final_classifications', False) cfg.setParam('chain', 'runs', runs) cfg.setParam('GlobChain', 'proj', "EPSG:2154") cfg.setParam('chain', 'regionPath', self.ref_region) IOTA2Directory.generate_directories(test_output, check_inputs=False) # prepare expected function inputs t31tcj_feat_dir = os.path.join(self.test_working_directory, "IOTA2_dir_VectorFormatting", "features", "T31TCJ") os.mkdir(t31tcj_feat_dir) # prepare ref img t31tcj_ref_img = os.path.join(t31tcj_feat_dir, "MaskCommunSL.tif") shutil.copy(self.ref_img, t31tcj_ref_img) # prepare envelope envelope_name = "T31TCJ.shp" envelope_path = os.path.join(self.test_working_directory, "IOTA2_dir_VectorFormatting", "envelope", envelope_name) fut.cpShapeFile(self.ref_region.replace(".shp", ""), envelope_path.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) changeName(envelope_path, "region", "FID") # prepare cloud mask cloud_name = "CloudThreshold_0.shp" cloud_path = os.path.join(self.test_working_directory, "IOTA2_dir_VectorFormatting", "features", "T31TCJ", cloud_name) fut.cpShapeFile(self.ref_region.replace(".shp", ""), cloud_path.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) changeName(cloud_path, "region", "cloud") # launch function ratio = cfg.getParam('chain', 'ratio') random_seed = cfg.getParam('chain', 'random_seed') enable_cross_validation = cfg.getParam("chain", "enableCrossValidation") enable_split_ground_truth = cfg.getParam('chain', 'splitGroundTruth') fusion_merge_all_validation = cfg.getParam( 'chain', 'fusionOfClassificationAllSamplesValidation') merge_final_classifications = cfg.getParam( 'chain', 'merge_final_classifications') merge_final_classifications_ratio = cfg.getParam( 'chain', 'merge_final_classifications_ratio') region_vec = cfg.getParam('chain', 'regionPath') epsg = int(cfg.getParam('GlobChain', 'proj').split(":")[-1]) region_field = (cfg.getParam('chain', 'regionField')) vector_formatting("T31TCJ", test_output, ground_truth, "code", 0, ratio, random_seed, enable_cross_validation, enable_split_ground_truth, fusion_merge_all_validation, runs, epsg, region_field, merge_final_classifications, merge_final_classifications_ratio, region_vec, working_directory=None) # assert nb_features_origin = len( fut.getFieldElement(ground_truth, driverName="ESRI Shapefile", field="code", mode="all", elemType="str")) test_vector = fut.FileSearch_AND( os.path.join(test_output, "formattingVectors"), True, "T31TCJ.shp")[0] nb_features_test = len( fut.getFieldElement(test_vector, driverName="ESRI Shapefile", field="code", mode="all", elemType="str")) # check nb features self.assertTrue(nb_features_origin == nb_features_test, msg="wrong number of features") # check fields origin_fields = fut.get_all_fields_in_shape(ground_truth) test_fields = fut.get_all_fields_in_shape(test_vector) new_fields = ['region', 'originfid', 'seed_0', 'seed_1', 'tile_o'] expected_fields = origin_fields + new_fields self.assertTrue(len(expected_fields) == len(test_fields)) self.assertTrue(all(field in test_fields for field in expected_fields))