def test_vector_splits_cross_validation(self): from iota2.Sampling import SplitInSubSets as VS from iota2.Common import FileUtils as fut # We execute the function splitInSubSets() new_region_shape = self.new_regions_shapes[0] VS.splitInSubSets(new_region_shape, self.data_field, self.region_field, self.ratio, self.seeds, "ESRI Shapefile", crossValidation=True, random_seed=0) seed0 = fut.getFieldElement(new_region_shape, driverName="ESRI Shapefile", field="seed_0", mode="all", elemType="str") seed1 = fut.getFieldElement(new_region_shape, driverName="ESRI Shapefile", field="seed_1", mode="all", elemType="str") for elem in seed0: self.assertTrue(elem in ["unused", "learn"], msg="flag not in ['unused', 'learn']") for elem in seed1: self.assertTrue(elem in ["unused", "validation"], msg="flag not in ['unused', 'validation']")
def split_vectors_by_regions(path_list: List[str]) -> List[str]: """ split vectors by regions IN: path_list [list(str)]: list of path OUT: list[str] : the list of vector by region """ regions_position = 2 seed_position = 3 output = [] seed_vector_ = fu.sortByFirstElem([ (os.path.split(vec)[-1].split("_")[seed_position], vec) for vec in path_list ]) seed_vector = [seed_vector for seed, seed_vector in seed_vector_] for current_seed in seed_vector: region_vector = [(os.path.split(vec)[-1].split("_")[regions_position], vec) for vec in current_seed] region_vector_sorted_ = fu.sortByFirstElem(region_vector) region_vector_sorted = [ r_vectors for region, r_vectors in region_vector_sorted_ ] for seed_vect_region in region_vector_sorted: output.append(seed_vect_region) return output
def createConfusionMatrix(self, otbmatrix): """Create a multi-level confusion matrix (pandas DataFrame) otbmatrix : csv file an OTB confusionMatrix file Return ------ pandas.core.frame.DataFrame vector or raster QML style file """ mat = fu.confCoordinatesCSV(otbmatrix) d = defaultdict(list) for k, v in mat: d[k].append(v) csv_f = list(d.items()) # get matrix from csv matrix = fu.gen_confusionMatrix(csv_f, self.getCode(self.level)) # MultiIndex Pandas dataframe cmdf = DataFrame(matrix, index=self.HierarchicalNomenclature, \ columns=self.HierarchicalNomenclature) cmdf.sort_index(level=self.level - 1, inplace=True, axis=1) cmdf.sort_index(level=self.level - 1, inplace=True, axis=0) return cmdf
def shape_reference_vector(ref_vector: str, output_name: str) -> str: """ modify reference vector (add field, rename...) Parameters ---------- ref_vector : string output_name : string Return ------ string """ import os from iota2.Common.Utils import run from iota2.Common import FileUtils as fut from iota2.VectorTools.AddField import addField path, _ = os.path.split(ref_vector) tmp = os.path.join(path, output_name + "_TMP") fut.cpShapeFile(ref_vector.replace(".shp", ""), tmp, [".prj", ".shp", ".dbf", ".shx"]) addField(tmp + ".shp", "region", "1", str) addField(tmp + ".shp", "seed_0", "learn", str) cmd = (f"ogr2ogr -dialect 'SQLite' -sql 'select GEOMETRY,seed_0, " f"region, CODE as code from {output_name}_TMP' " f"{path}/{output_name}.shp {tmp}.shp") run(cmd) os.remove(tmp + ".shp") os.remove(tmp + ".shx") os.remove(tmp + ".prj") os.remove(tmp + ".dbf") return path + "/" + output_name + ".shp"
def test_update_flags(self): """ """ from iota2.Sampling.SamplesSelection import update_flags # prepare test input test_vector_name = "T31TCJ_samples_region_1_seed_1_selection.sqlite" test_vector_table = "t31tcj_samples_region_1_seed_0_selection" test_vector = os.path.join(self.test_working_directory, test_vector_name) shutil.copy(self.selection_ref, test_vector) update_flags(test_vector, 2, table_name=test_vector_table) # assert updated_flag = "XXXX" nb_features_origin = len( fut.getFieldElement(self.selection_ref, driverName="SQLite", field="seed_0", mode="all", elemType="str")) features_test = fut.getFieldElement(test_vector, driverName="SQLite", field="seed_0", mode="all", elemType="str") nb_features_test_updated = features_test.count(updated_flag) self.assertTrue(nb_features_origin == nb_features_test_updated, msg="update features failed")
def erodeInter(currentTile, NextTile, intersection, buff, proj): xo, yo = currentTile.getOrigin() xn, yn = NextTile.getOrigin() Extent = getShapeExtent(intersection) if yo == yn and xo != xn: #left priority minX = Extent[0] maxX = Extent[1] - buff minY = Extent[2] maxY = Extent[3] elif yo != yn and xo == xn: #upper priority minX = Extent[0] maxX = Extent[1] minY = Extent[2] + buff maxY = Extent[3] else: return False fu.removeShape(intersection.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) pathFolder = "/".join( intersection.split("/")[0:len(intersection.split("/")) - 1]) createShape(minX, minY, maxX, maxY, pathFolder, intersection.split("/")[-1].replace(".shp", ""), proj) return True
def samples_stats(region_seed_tile: Tuple[str, str, str], iota2_directory: str, data_field: str, working_directory: Optional[Union[str, None]] = None, logger=LOGGER) -> str: """generate samples statistics by tiles Parameters ---------- region_seed_tile: tuple iota2_directory: str iota2 output directory data_field: str data field in region database working_directory: str path to a working directory logger: logging root logger Return ------ str file containing statistics from otbcli_PolygonClassStatistics """ region, seed, tile = region_seed_tile samples_selection_dir = os.path.join(iota2_directory, "samplesSelection") tile_region_dir = os.path.join(iota2_directory, "shapeRegion") w_dir = samples_selection_dir if working_directory: w_dir = working_directory raster_mask = fut.FileSearch_AND(tile_region_dir, True, "region_" + region.split("f")[0] + "_", ".tif", tile)[0] region_vec = fut.FileSearch_AND(samples_selection_dir, True, "_region_" + region, "seed_" + seed, ".shp")[0] logger.info( f"Launch statistics on tile {tile} in region {region} run {seed}") region_tile_stats_name = f"{tile}_region_{region}_seed_{seed}_stats.xml" region_tile_stats = os.path.join(w_dir, region_tile_stats_name) polygon_stats_app = otb.CreatePolygonClassStatisticsApplication({ "in": raster_mask, "mask": raster_mask, "vec": region_vec, "field": data_field, "out": region_tile_stats }) polygon_stats_app.ExecuteAndWriteOutput() if working_directory: shutil.copy(region_tile_stats, samples_selection_dir) return region_tile_stats
def confusion_models_merge_parameters(iota2_dir: str): """ function use to feed confusion_models_merge function Parameter --------- iota2_dir : string path to the iota2 running directory Return ------ list list containing all sub confusion matrix which must be merged. """ ds_sar_opt_conf_dir = os.path.join(iota2_dir, "dataAppVal", "bymodels") csv_seed_pos = 4 csv_model_pos = 2 all_csv = fu.FileSearch_AND(ds_sar_opt_conf_dir, True, "samples", ".csv") # group by models model_group = [] for csv in all_csv: _, csv_name = os.path.split(csv) csv_seed = csv_name.split("_")[csv_seed_pos] csv_model = csv_name.split("_")[csv_model_pos] # csv_mode = "SAR.csv" or "val.csv", use to descriminate models csv_mode = csv_name.split("_")[-1] key_param = (csv_model, csv_seed, csv_mode) model_group.append((key_param, csv)) groups_param = [param for key, param in fu.sortByFirstElem(model_group)] return groups_param
def test_split_vector_by_region(self): """ test : split a vector by the region he belongs to """ from iota2.Sampling.VectorFormatting import split_vector_by_region from iota2.Common.Utils import run from iota2.Tests.UnitTests.Iota2Tests import random_update # define inputs nb_features_origin = len( fut.getFieldElement(self.in_vector, driverName="ESRI shapefile", field="region", mode="all", elemType="str")) nb_features_new_region = 5 test_vector_name = "T31TCJ_Samples.sqlite" test_vector = os.path.join(self.test_working_directory, test_vector_name) cmd = "ogr2ogr -nln output -f SQLite {} {}".format( test_vector, self.in_vector) run(cmd) random_update(test_vector, "output", "seed_0", "learn", nb_features_origin) random_update(test_vector, "output", "region", "2", nb_features_new_region) output_dir = self.test_working_directory region_field = "region" # launch function split_vector_by_region(test_vector, output_dir, region_field, runs=1, driver="SQLite") # assert vector_reg_1 = fut.FileSearch_AND(self.test_working_directory, True, "region_1")[0] vector_reg_2 = fut.FileSearch_AND(self.test_working_directory, True, "region_2")[0] feat_vect_reg_1 = len( fut.getFieldElement(vector_reg_1, driverName="SQLite", field="region", mode="all", elemType="str")) feat_vect_reg_2 = len( fut.getFieldElement(vector_reg_2, driverName="SQLite", field="region", mode="all", elemType="str")) self.assertTrue(nb_features_new_region == feat_vect_reg_2) self.assertTrue(nb_features_origin == feat_vect_reg_1 + feat_vect_reg_2)
def config_model(outputPath, region_field): """ usage : determine which model will class which tile """ #const output = None pos_tile = 0 formatting_vec_dir = os.path.join(outputPath, "formattingVectors") samples = fu.FileSearch_AND(formatting_vec_dir, True, ".shp") #init all_regions = [] for sample in samples: tile_name = os.path.splitext( os.path.basename(sample))[0].split("_")[pos_tile] regions = fu.getFieldElement(sample, driverName="ESRI Shapefile", field=region_field, mode="unique", elemType="str") for region in regions: all_regions.append((region, tile_name)) #{'model_name':[TileName, TileName...],'...':...,...} model_tiles = dict(fu.sortByFirstElem(all_regions)) #add tiles if they are missing by checking in /shapeRegion/ directory shape_region_dir = os.path.join(outputPath, "shapeRegion") shape_region_path = fu.FileSearch_AND(shape_region_dir, True, ".shp") #check if there is actually polygons shape_regions = [ elem for elem in shape_region_path if len( fu.getFieldElement(elem, driverName="ESRI Shapefile", field=region_field, mode="all", elemType="str")) >= 1 ] for shape_region in shape_regions: tile = os.path.splitext( os.path.basename(shape_region))[0].split("_")[-1] region = os.path.splitext( os.path.basename(shape_region))[0].split("_")[-2] for model_name, tiles_model in list(model_tiles.items()): if model_name.split("f")[0] == region and tile not in tiles_model: tiles_model.append(tile) #Construct output file string output = "AllModel:\n[" for model_name, tiles_model in list(model_tiles.items()): output_tmp = "\n\tmodelName:'{}'\n\ttilesList:'{}'".format( model_name, "_".join(tiles_model)) output = output + "\n\t{" + output_tmp + "\n\t}" output += "\n]" return output
def createRasterFootprint(tilePath, commonVecMask, proj=2154): #outpolygonize = pathOut.replace(".shp","_TMP.shp") #cmd = 'gdal_polygonize.py -mask '+tilePath+' '+tilePath+' -f "ESRI Shapefile" '+outpolygonize #run(cmd) fu.keepBiggestArea(tilePath.replace(".tif", ".shp"), commonVecMask) #fu.removeShape(outpolygonize.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) return commonVecMask
def step_inputs(self): """ Return ------ the return could be and iterable or a callable """ from iota2.Common import FileUtils as fut from iota2.Classification.ImageClassifier import autoContext_classification_param from iota2.Common.ServiceConfigFile import iota2_parameters if self.enable_autoContext is False and self.use_scikitlearn is False: parameters = fut.parseClassifCmd( os.path.join(self.output_path, "cmd", "cla", "class.txt")) elif self.enable_autoContext is True: parameters = autoContext_classification_param( self.output_path, self.data_field) elif self.use_scikitlearn: parameters = fut.parseClassifCmd( os.path.join(self.output_path, "cmd", "cla", "class.txt")) running_parameters = iota2_parameters(self.cfg) parameters = [{ "mask": param[1], "model": param[2], "stat": param[3], "out_classif": param[4], "out_confidence": param[5], "out_proba": None, "working_dir": param[6], "tile_name": param[8], "sar_optical_post_fusion": SCF.serviceConfigFile(self.cfg).getParam( 'argTrain', 'dempster_shafer_SAR_Opt_fusion'), "output_path": SCF.serviceConfigFile(self.cfg).getParam( 'chain', 'outputPath'), "sensors_parameters": running_parameters.get_sensors_parameters(tile_name=param[8]), "pixel_type": param[17], "number_of_chunks": self.scikit_tile_split, "targeted_chunk": target_chunk, "ram": param[19] } for param in parameters for target_chunk in range(self.scikit_tile_split)] return parameters
def generate_shape_tile(tiles: List[str], pathWd: str, output_path: str, proj: int) -> None: """generate tile's envelope with priority management Parameters ---------- tiles : list list of tiles envelopes to generate pathOut : str output directory pathWd : str working directory output_path : str iota2 output directory proj : int epsg code of target projection """ pathOut = os.path.join(output_path, "envelope") if not os.path.exists(pathOut): os.mkdir(pathOut) featuresPath = os.path.join(output_path, "features") cMaskName = "MaskCommunSL" for tile in tiles: if not os.path.exists(featuresPath + "/" + tile): os.mkdir(featuresPath + "/" + tile) os.mkdir(featuresPath + "/" + tile + "/tmp") commonDirectory = pathOut + "/commonMasks/" if not os.path.exists(commonDirectory): os.mkdir(commonDirectory) common = [ featuresPath + "/" + Ctile + "/tmp/" + cMaskName + ".tif" for Ctile in tiles ] ObjListTile = [ Tile(currentTile, name) for currentTile, name in zip(common, tiles) ] ObjListTile_sort = sorted(ObjListTile, key=priorityKey) tmpFile = pathOut + "/TMP" if pathWd: tmpFile = pathWd + "/TMP" if not os.path.exists(tmpFile): os.mkdir(tmpFile) genTileEnvPrio(ObjListTile_sort, pathOut, tmpFile, proj) AllPRIO = fu.FileSearch_AND(tmpFile, True, "_PRIO.shp") for prioTile in AllPRIO: tileName = prioTile.split("/")[-1].split("_")[0] fu.cpShapeFile(prioTile.replace(".shp", ""), pathOut + "/" + tileName, [".prj", ".shp", ".dbf", ".shx"]) shutil.rmtree(tmpFile) shutil.rmtree(commonDirectory)
def split_samples(output_path: str, data_field: str, enable_cross_validation: bool, region_threshold: Union[str, float], region_field: str, ratio: float, random_seed: int, runs: int, epsg: Union[str, int], workingDirectory: Optional[str] = None, logger: Optional[Logger] = LOGGER): """ """ from iota2.Common import FileUtils as fut # const regions_pos = -2 if isinstance(epsg, str): epsg = int(epsg.split(":")[-1]) if isinstance(region_threshold, str): region_threshold = float(region_threshold) formatting_vectors_dir = os.path.join(output_path, "formattingVectors") shape_region_dir = os.path.join(output_path, "shapeRegion") vectors = fut.FileSearch_AND(formatting_vectors_dir, True, ".shp") # get All possible regions by parsing shapeFile's name shapes_region = fut.FileSearch_AND(shape_region_dir, True, ".shp") regions = list( set([ os.path.split(shape)[-1].split("_")[regions_pos] for shape in shapes_region ])) # compute region's area areas, regions_tiles, data_to_rm = get_regions_area( vectors, regions, formatting_vectors_dir, workingDirectory, region_field) # get how many sub-regions must be created by too huge regions. regions_split = get_splits_regions(areas, region_threshold) for region_name, area in list(areas.items()): logger.info(f"region : {region_name} , area : {area}") updated_vectors = split(regions_split, regions_tiles, data_field, region_field) # transform sqlites to shape file, according to input data format new_regions_shapes = transform_to_shape(updated_vectors, formatting_vectors_dir) for data in data_to_rm: os.remove(data) data_app_val_dir = os.path.join(output_path, "dataAppVal") update_learning_validation_sets(new_regions_shapes, data_app_val_dir, data_field, region_field, ratio, runs, epsg, enable_cross_validation, random_seed)
def getVectorsChunks(inpath, inbase="dept_"): if not fut.FileSearch_AND(inpath, True, inbase, ".shp", "chk"): listout = fut.FileSearch_AND(inpath, True, inbase, ".shp", "stats") else: listout = fut.FileSearch_AND(inpath, True, inbase, ".shp", "chk") listofchkofzones = fut.sortByFirstElem([ ("_".join(x.split('_')[0:len(x.split('_')) - 1]), x) for x in listout ]) return listofchkofzones
def getListVectToSimplify(path): simplified = [os.path.splitext(x)[0].split('_')[len(os.path.splitext(x)[0].split('_')) - 2] \ for x in fut.FileSearch_AND(path, True, ".shp", "douglas") \ if "hermite" not in x] polygonized = [[x, os.path.splitext(x)[0].split('_')[len(os.path.splitext(x)[0].split('_')) - 1]] \ for x in fut.FileSearch_AND(path, True, "tile_", ".shp") \ if "douglas" not in x and "hermite" not in x] return [ os.path.join(path, x) for x, y in polygonized if y not in simplified ]
def split(regions_split: Dict[str, int], regions_tiles: Dict[str, List[str]], data_field: str, region_field: str) -> List[str]: """ function dedicated to split to huge regions in sub-regions Parameters ---------- regions_split: dict[string, int] regions_tiles: dict[str, List[str]] Return ------ list(str) """ from iota2.Common import FileUtils as fut updated_vectors = [] for region, fold in list(regions_split.items()): vector_paths = regions_tiles[region] for vec in vector_paths: # init dict new regions new_regions_dict = {} for sub_fold in range(fold): # new region's name are define here new_regions_dict["{}f{}".format(region, sub_fold + 1)] = [] # get possible class class_vector = fut.getFieldElement(vec, driverName="SQLite", field=data_field, mode="unique", elemType="str") dic_class = {} # get FID values for all class of current region into # the current tile for c_class in class_vector: dic_class[c_class] = get_fid_values(vec, data_field, region_field, region, c_class) nb_feat = 0 for _, fid_cl in list(dic_class.items()): if fid_cl: fid_folds = fut.splitList(fid_cl, fold) # fill new_regions_dict for i, fid_fold in enumerate(fid_folds): new_regions_dict[f"{region}f{i+1}"] += fid_fold nb_feat += len(fid_cl) update_vector(vec, region_field, new_regions_dict) if vec not in updated_vectors: updated_vectors.append(vec) return updated_vectors
def test_extract_maj_vote_samples(self): """ test the extraction of samples by class according to a ratio """ from iota2.Sampling.VectorFormatting import extract_maj_vote_samples from collections import Counter # define inputs in_vector_name = os.path.basename(self.in_vector) extracted_vector_name = "extracted_samples.sqlite" in_vector = os.path.join(self.test_working_directory, in_vector_name) extracted_vector = os.path.join(self.test_working_directory, extracted_vector_name) fut.cpShapeFile(self.in_vector.replace(".shp", ""), in_vector.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) # launch function dataField = "code" regionField = "region" extraction_ratio = 0.5 extract_maj_vote_samples(in_vector, extracted_vector, extraction_ratio, dataField, regionField) # assert features_origin = fut.getFieldElement(self.in_vector, driverName="ESRI Shapefile", field=dataField, mode="all", elemType="str") by_class_origin = Counter(features_origin) features_in_vector = fut.getFieldElement(in_vector, driverName="ESRI Shapefile", field=dataField, mode="all", elemType="str") by_class_in_vector = Counter(features_in_vector) features_extract_vector = fut.getFieldElement(extracted_vector, driverName="SQLite", field=dataField, mode="all", elemType="str") by_class_extract_vector = Counter(features_extract_vector) buff = [] for class_name, class_count in list(by_class_origin.items()): buff.append(by_class_in_vector[class_name] == extraction_ratio * class_count) self.assertTrue(all(buff), msg="extraction of samples failed")
def test_split_selection(self): """ test dedicated to check if split_sel function works """ from iota2.Sampling.SamplesSelection import split_sel from Iota2Tests import random_update from TestsUtils import rename_table # prepare test input test_vector_name = "samples_region_1_seed_0.sqlite" test_vector_table = "t31tcj_samples_region_1_seed_0_selection" test_vector = os.path.join(self.test_working_directory, test_vector_name) shutil.copy(self.selection_ref, test_vector) # update "nb_feat" features to a new "new_tile_name" tile's name nb_feat = 10 new_tile_name = "T31TDJ" random_update(test_vector, test_vector_table, "tile_o", new_tile_name, nb_feat) rename_table(test_vector, old_table_name=test_vector_table, new_table_name="output") # launch function new_files = split_sel(test_vector, ["T31TCJ", new_tile_name], self.test_working_directory, "EPSG:2154") # assert nb_features_origin = len( fut.getFieldElement(self.selection_ref, driverName="SQLite", field="tile_o", mode="all", elemType="str")) nb_features_t31tcj = len( fut.getFieldElement(new_files[0], driverName="SQLite", field="tile_o", mode="all", elemType="str")) nb_features_t31tdj = len( fut.getFieldElement(new_files[1], driverName="SQLite", field="tile_o", mode="all", elemType="str")) self.assertTrue(nb_features_t31tdj == nb_feat, msg="split samples selection failed") self.assertTrue(nb_features_origin == nb_features_t31tdj + nb_features_t31tcj, msg="split samples selection failed")
def generate_region_shape(envelope_directory: str, output_region_file: str, out_field_name: str, i2_output_path: str, working_directory: str) -> None: """generate regions shape envelope_directory: str directory containing all iota2 tile's envelope output_region_file: str output file out_field_name: str output field containing region i2_output_path: str iota2 output path working_directory: str path to a working directory """ region = [] all_tiles = fu.FileSearch_AND(envelope_directory, False, ".shp") region.append(all_tiles) if not output_region_file: output_region_file = os.path.join(i2_output_path, "MyRegion.shp") p_f = output_region_file.replace(" ", "").split("/") out_name = p_f[-1].split(".")[0] path_mod = "" for i in range(1, len(p_f) - 1): path_mod = path_mod + "/" + p_f[i] CreateModelShapeFromTiles(region, envelope_directory, path_mod, out_name, out_field_name, working_directory)
def test_samples_selection(self): """ test sampling of a shape file (main function of SamplesSelection.py) """ from iota2.Sampling.SamplesSelection import samples_selection from iota2.Common import IOTA2Directory from iota2.Common import ServiceConfigFile as SCF from iota2.Tests.UnitTests.Iota2Tests import compareSQLite from iota2.Common.FileUtils import cpShapeFile from iota2.Common import FileUtils as fut # prepare test input cfg = SCF.serviceConfigFile(self.config_test) cfg.setParam( "chain", "outputPath", os.path.join(self.test_working_directory, "samplesSelTest")) cfg.setParam("chain", "runs", 2) cfg.setParam("argTrain", "sampleSelection", { "sampler": "random", "strategy": "all" }) # create IOTA2 directories IOTA2Directory.generate_directories(os.path.join( self.test_working_directory, "samplesSelTest"), check_inputs=False) shutil.copytree( self.features_ref, os.path.join(self.test_working_directory, "samplesSelTest", "features", "T31TCJ")) shutil.copy( self.in_xml, os.path.join(self.test_working_directory, "samplesSelTest", "samplesSelection", "T31TCJ_region_1_seed_0_stats.xml")) _, in_shape_name = os.path.split(self.in_shape) in_shape_dir = os.path.join(self.test_working_directory, "samplesSelTest", "samplesSelection") in_shape = os.path.join(in_shape_dir, in_shape_name) cpShapeFile(self.in_shape.replace(".shp", ""), in_shape.replace(".shp", ""), extensions=[".prj", ".shp", ".dbf", ".shx"]) #~ # launch function output_path = cfg.getParam("chain", "outputPath") runs = cfg.getParam('chain', 'runs') epsg = cfg.getParam('GlobChain', 'proj') random_seed = cfg.getParam('chain', 'random_seed') data_field = cfg.getParam('chain', 'dataField').lower() parameters = dict(cfg.getParam('argTrain', 'sampleSelection')) masks_name = "MaskCommunSL.tif" samples_selection(in_shape, self.test_working_directory, output_path, runs, epsg, masks_name, parameters, data_field, random_seed) #~ # assert selection_test = fut.FileSearch_AND( os.path.join(self.test_working_directory, "samplesSelTest"), True, os.path.basename(self.selection_ref))[0] same = compareSQLite(self.selection_ref, selection_test, CmpMode='coordinates') self.assertTrue(same, msg="sample selection generation failed")
def test_create_tile_region_masks(self): """ test the generation of the raster mask which define the region in the tile """ from iota2.Sampling.VectorFormatting import create_tile_region_masks from iota2.Common.Utils import run from iota2.Tests.UnitTests.TestsUtils import rasterToArray import numpy as np # define inputs test_vector_name = "T31TCJ.sqlite" test_vector = os.path.join(self.test_working_directory, test_vector_name) cmd = "ogr2ogr -nln t31tcj -f SQLite {} {}".format( test_vector, self.ref_region) run(cmd) # launch function create_tile_region_masks(test_vector, "region", "T31TCJ", self.test_working_directory, "MyRegion", self.ref_img) # assert raster_region = fut.FileSearch_AND(self.test_working_directory, True, "MyRegion", ".tif")[0] raster_region_arr = rasterToArray(raster_region) ref_array = np.ones((50, 50)) self.assertTrue(np.allclose(ref_array, raster_region_arr), msg="problem with the normalization by ref")
def __init__(self, cfg, cfg_resources_file, working_directory=None): # heritage init resources_block_name = "noData" super(fusionsIndecisions, self).__init__(cfg, cfg_resources_file, resources_block_name) # step variables self.working_directory = working_directory self.output_path = SCF.serviceConfigFile(self.cfg).getParam( 'chain', 'outputPath') self.field_region = SCF.serviceConfigFile(self.cfg).getParam( 'chain', 'regionField') self.shape_region = SCF.serviceConfigFile(self.cfg).getParam( 'chain', 'regionPath') self.runs = SCF.serviceConfigFile(self.cfg).getParam('chain', 'runs') self.no_label = SCF.serviceConfigFile(self.cfg).getParam( 'argClassification', 'noLabelManagement') self.features = SCF.serviceConfigFile(self.cfg).getParam( "GlobChain", "features") self.user_feat_path = SCF.serviceConfigFile(self.cfg).getParam( "chain", "userFeatPath") self.pixtype = fut.getOutputPixType( SCF.serviceConfigFile(self.cfg).getParam('chain', 'nomenclaturePath')) self.region_vec = SCF.serviceConfigFile(self.cfg).getParam( 'chain', 'regionPath') self.patterns = SCF.serviceConfigFile(self.cfg).getParam( "userFeat", "patterns") self.sar_opt_fusion = SCF.serviceConfigFile(self.cfg).getParam( 'argTrain', 'dempster_shafer_SAR_Opt_fusion') self.path_to_img = os.path.join(self.output_path, "features")
def test_generate_shape_tile(self): from iota2.Sampling import TileEnvelope as env # Test de création des enveloppes features_path = os.path.join(self.iota2_tests_directory, "features") masks_references = '../../../data/references/features' if os.path.exists(features_path): shutil.rmtree(features_path) shutil.copytree(masks_references, features_path) # Test de création des enveloppes # Launch function env.generate_shape_tile(self.tiles, None, self.iota2_tests_directory, 2154) # For each tile test if the shapefile is ok for i in self.tiles: # generate filename reference_shape_file = (IOTA2_DATATEST + "/references/GenerateShapeTile/" + i + ".shp") shape_file = self.iota2_tests_directory + "/envelope/" + i + ".shp" service_compare_vector_file = fu.serviceCompareVectorFile() # Launch shapefile comparison self.assertTrue( service_compare_vector_file.testSameShapefiles( reference_shape_file, shape_file)) shutil.rmtree(features_path)
def create_dummy_rasters(missing_tiles: List[str], runs: int, output_path: str) -> None: """ Parameters ---------- missing_tiles: list(string) runs: int output_path: string Return ------ None Notes ----- use when mode is 'one_region' but there is no validations / learning samples into a specific tile """ classifications_dir = os.path.join(output_path, "classif") final_dir = os.path.join(output_path, "final", "TMP") for tile in missing_tiles: classif_tile = fu.FileSearch_AND(classifications_dir, True, "Classif_" + str(tile))[0] for seed in range(runs): dummy_raster_name = tile + "_seed_" + str(seed) + "_CompRef.tif" dummy_raster = final_dir + "/" + dummy_raster_name dummy_raster_cmd = (f"gdal_merge.py -ot Byte -n 0 -createonly -o " f"{ dummy_raster} {classif_tile}") run(dummy_raster_cmd)
def test_vector_splits(self): from iota2.Sampling import SplitInSubSets as VS from iota2.Common import FileUtils as fu from iota2.Tests.UnitTests import TestsUtils # We execute the function splitInSubSets() for new_region_shape in self.new_regions_shapes: tile_name = os.path.splitext(os.path.basename(new_region_shape))[0] vectors_to_rm = fu.FileSearch_AND(self.data_app_val_dir, True, tile_name) for vect in vectors_to_rm: os.remove(vect) VS.splitInSubSets(new_region_shape, self.data_field, self.region_field, self.ratio, self.seeds, "ESRI Shapefile", random_seed=0) print(new_region_shape) # We check the output self.assertTrue( TestsUtils.compareVectorFile(self.ref_split_shp, self.out_split_shp, 'coordinates', 'polygon', "ESRI Shapefile"), "Split vector output are different")
def get_values_sorted_by_coordinates( vector: str) -> List[Tuple[float, float]]: """ usage return values sorted by coordinates (x,y) IN vector [string] path to a vector of points OUT values [list of tuple] : [(x,y,[val1,val2]),()...] """ import ogr values = [] driver = ogr.GetDriverByName("SQLite") dataset = driver.Open(vector, 0) lyr = dataset.GetLayer() fields = fut.get_all_fields_in_shape(vector, 'SQLite') for feature in lyr: x_coord = feature.GetGeometryRef().GetX() y_coord = feature.GetGeometryRef().GetY() fields_val = get_field_value(feature, fields) values.append((x_coord, y_coord, fields_val)) values = sorted(values, key=priority) return values
def extract_class(vec_in: str, vec_out: str, target_class: List[str], data_field: str) -> int: """ Extract class IN: vec_in: str vec_out: str target_class:List[str] data_field: str """ from iota2.Common.Utils import run if type(target_class) != type(list()): target_class = target_class.data where = " OR ".join( ["{}={}".format(data_field.lower(), klass) for klass in target_class]) cmd = (f"ogr2ogr -f 'SQLite' -nln output -where '{where}' " f"{vec_out} {vec_in}") run(cmd) return len( fu.getFieldElement(vec_out, driverName="SQLite", field=data_field.lower(), mode="all", elemType="int"))
def get_vectors_to_sample( iota2_formatting_dir: str, ds_fusion_sar_opt: Optional[bool] = False) -> List[Dict[str, str]]: """ get vectors to sample IN : iota2_formatting_dir : str path to shapefiles ds_fusion_sar_opt : bool activate sar mode OUT: List of dictionary containing all shapefiles """ formatting_tiles = fu.FileSearch_AND(iota2_formatting_dir, True, ".shp") # parameters generation tiles_vectors = [{"usually": vector} for vector in formatting_tiles] # parameters dedicated to SAR tiles_vectors_sar = [{"SAR": vector} for vector in formatting_tiles] tiles_vectors_to_sample = tiles_vectors # if we want to have SAR classification and Optical classification belong # we have to double the number of parameters in generateSamples if ds_fusion_sar_opt: tiles_vectors_to_sample = tiles_vectors + tiles_vectors_sar return tiles_vectors_to_sample
def test_iota2_augmentation(self): """Test data augmentation workflow """ from collections import Counter class_augmentation_balance = DataAugmentation.SamplesAugmentationCounter( self.class_count, mode="balance", minNumber=None, byClass=None) DataAugmentation.DoAugmentation(self.vector_test, class_augmentation_balance, strategy="jitter", field="code", excluded_fields=[], Jstdfactor=10, Sneighbors=None, workingDirectory=None) class_count_test = Counter( fut.getFieldElement(self.vector_test, driverName="SQLite", field="code", mode="all", elemType="int")) samples_number = self.class_count[max( self.class_count, key=lambda key: self.class_count[key])] self.assertTrue( all([ samples_number == v for k, v in list(class_count_test.items()) ]))