def test_vector_splits_cross_validation(self):
from iota2.Sampling import SplitInSubSets as VS
from iota2.Common import FileUtils as fut
# We execute the function splitInSubSets()
new_region_shape = self.new_regions_shapes[0]
VS.splitInSubSets(new_region_shape,
self.data_field,
self.region_field,
self.ratio,
self.seeds,
"ESRI Shapefile",
crossValidation=True,
random_seed=0)
seed0 = fut.getFieldElement(new_region_shape,
driverName="ESRI Shapefile",
field="seed_0",
mode="all",
elemType="str")
seed1 = fut.getFieldElement(new_region_shape,
driverName="ESRI Shapefile",
field="seed_1",
mode="all",
elemType="str")
for elem in seed0:
self.assertTrue(elem in ["unused", "learn"],
msg="flag not in ['unused', 'learn']")
for elem in seed1:
self.assertTrue(elem in ["unused", "validation"],
msg="flag not in ['unused', 'validation']")
def split_vectors_by_regions(path_list: List[str]) -> List[str]:
"""
split vectors by regions
IN:
path_list [list(str)]: list of path
OUT:
list[str] : the list of vector by region
"""
regions_position = 2
seed_position = 3
output = []
seed_vector_ = fu.sortByFirstElem([
(os.path.split(vec)[-1].split("_")[seed_position], vec)
for vec in path_list
])
seed_vector = [seed_vector for seed, seed_vector in seed_vector_]
for current_seed in seed_vector:
region_vector = [(os.path.split(vec)[-1].split("_")[regions_position],
vec) for vec in current_seed]
region_vector_sorted_ = fu.sortByFirstElem(region_vector)
region_vector_sorted = [
r_vectors for region, r_vectors in region_vector_sorted_
]
for seed_vect_region in region_vector_sorted:
output.append(seed_vect_region)
return output
def createConfusionMatrix(self, otbmatrix):
"""Create a multi-level confusion matrix (pandas DataFrame)
otbmatrix : csv file
an OTB confusionMatrix file
Return
------
pandas.core.frame.DataFrame
vector or raster QML style file
"""
mat = fu.confCoordinatesCSV(otbmatrix)
d = defaultdict(list)
for k, v in mat:
d[k].append(v)
csv_f = list(d.items())
# get matrix from csv
matrix = fu.gen_confusionMatrix(csv_f, self.getCode(self.level))
# MultiIndex Pandas dataframe
cmdf = DataFrame(matrix, index=self.HierarchicalNomenclature, \
columns=self.HierarchicalNomenclature)
cmdf.sort_index(level=self.level - 1, inplace=True, axis=1)
cmdf.sort_index(level=self.level - 1, inplace=True, axis=0)
return cmdf
def shape_reference_vector(ref_vector: str, output_name: str) -> str:
"""
modify reference vector (add field, rename...)
Parameters
----------
ref_vector : string
output_name : string
Return
------
string
"""
import os
from iota2.Common.Utils import run
from iota2.Common import FileUtils as fut
from iota2.VectorTools.AddField import addField
path, _ = os.path.split(ref_vector)
tmp = os.path.join(path, output_name + "_TMP")
fut.cpShapeFile(ref_vector.replace(".shp", ""), tmp,
[".prj", ".shp", ".dbf", ".shx"])
addField(tmp + ".shp", "region", "1", str)
addField(tmp + ".shp", "seed_0", "learn", str)
cmd = (f"ogr2ogr -dialect 'SQLite' -sql 'select GEOMETRY,seed_0, "
f"region, CODE as code from {output_name}_TMP' "
f"{path}/{output_name}.shp {tmp}.shp")
run(cmd)
os.remove(tmp + ".shp")
os.remove(tmp + ".shx")
os.remove(tmp + ".prj")
os.remove(tmp + ".dbf")
return path + "/" + output_name + ".shp"
def test_update_flags(self):
"""
"""
from iota2.Sampling.SamplesSelection import update_flags
# prepare test input
test_vector_name = "T31TCJ_samples_region_1_seed_1_selection.sqlite"
test_vector_table = "t31tcj_samples_region_1_seed_0_selection"
test_vector = os.path.join(self.test_working_directory,
test_vector_name)
shutil.copy(self.selection_ref, test_vector)
update_flags(test_vector, 2, table_name=test_vector_table)
# assert
updated_flag = "XXXX"
nb_features_origin = len(
fut.getFieldElement(self.selection_ref,
driverName="SQLite",
field="seed_0",
mode="all",
elemType="str"))
features_test = fut.getFieldElement(test_vector,
driverName="SQLite",
field="seed_0",
mode="all",
elemType="str")
nb_features_test_updated = features_test.count(updated_flag)
self.assertTrue(nb_features_origin == nb_features_test_updated,
msg="update features failed")
def erodeInter(currentTile, NextTile, intersection, buff, proj):
xo, yo = currentTile.getOrigin()
xn, yn = NextTile.getOrigin()
Extent = getShapeExtent(intersection)
if yo == yn and xo != xn: #left priority
minX = Extent[0]
maxX = Extent[1] - buff
minY = Extent[2]
maxY = Extent[3]
elif yo != yn and xo == xn: #upper priority
minX = Extent[0]
maxX = Extent[1]
minY = Extent[2] + buff
maxY = Extent[3]
else:
return False
fu.removeShape(intersection.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
pathFolder = "/".join(
intersection.split("/")[0:len(intersection.split("/")) - 1])
createShape(minX, minY, maxX, maxY, pathFolder,
intersection.split("/")[-1].replace(".shp", ""), proj)
return True
def samples_stats(region_seed_tile: Tuple[str, str, str],
iota2_directory: str,
data_field: str,
working_directory: Optional[Union[str, None]] = None,
logger=LOGGER) -> str:
"""generate samples statistics by tiles
Parameters
----------
region_seed_tile: tuple
iota2_directory: str
iota2 output directory
data_field: str
data field in region database
working_directory: str
path to a working directory
logger: logging
root logger
Return
------
str
file containing statistics from otbcli_PolygonClassStatistics
"""
region, seed, tile = region_seed_tile
samples_selection_dir = os.path.join(iota2_directory, "samplesSelection")
tile_region_dir = os.path.join(iota2_directory, "shapeRegion")
w_dir = samples_selection_dir
if working_directory:
w_dir = working_directory
raster_mask = fut.FileSearch_AND(tile_region_dir, True,
"region_" + region.split("f")[0] + "_",
".tif", tile)[0]
region_vec = fut.FileSearch_AND(samples_selection_dir, True,
"_region_" + region, "seed_" + seed,
".shp")[0]
logger.info(
f"Launch statistics on tile {tile} in region {region} run {seed}")
region_tile_stats_name = f"{tile}_region_{region}_seed_{seed}_stats.xml"
region_tile_stats = os.path.join(w_dir, region_tile_stats_name)
polygon_stats_app = otb.CreatePolygonClassStatisticsApplication({
"in":
raster_mask,
"mask":
raster_mask,
"vec":
region_vec,
"field":
data_field,
"out":
region_tile_stats
})
polygon_stats_app.ExecuteAndWriteOutput()
if working_directory:
shutil.copy(region_tile_stats, samples_selection_dir)
return region_tile_stats
def confusion_models_merge_parameters(iota2_dir: str):
"""
function use to feed confusion_models_merge function
Parameter
---------
iota2_dir : string
path to the iota2 running directory
Return
------
list
list containing all sub confusion matrix which must be merged.
"""
ds_sar_opt_conf_dir = os.path.join(iota2_dir, "dataAppVal", "bymodels")
csv_seed_pos = 4
csv_model_pos = 2
all_csv = fu.FileSearch_AND(ds_sar_opt_conf_dir, True, "samples", ".csv")
# group by models
model_group = []
for csv in all_csv:
_, csv_name = os.path.split(csv)
csv_seed = csv_name.split("_")[csv_seed_pos]
csv_model = csv_name.split("_")[csv_model_pos]
# csv_mode = "SAR.csv" or "val.csv", use to descriminate models
csv_mode = csv_name.split("_")[-1]
key_param = (csv_model, csv_seed, csv_mode)
model_group.append((key_param, csv))
groups_param = [param for key, param in fu.sortByFirstElem(model_group)]
return groups_param
def test_split_vector_by_region(self):
"""
test : split a vector by the region he belongs to
"""
from iota2.Sampling.VectorFormatting import split_vector_by_region
from iota2.Common.Utils import run
from iota2.Tests.UnitTests.Iota2Tests import random_update
# define inputs
nb_features_origin = len(
fut.getFieldElement(self.in_vector,
driverName="ESRI shapefile",
field="region",
mode="all",
elemType="str"))
nb_features_new_region = 5
test_vector_name = "T31TCJ_Samples.sqlite"
test_vector = os.path.join(self.test_working_directory,
test_vector_name)
cmd = "ogr2ogr -nln output -f SQLite {} {}".format(
test_vector, self.in_vector)
run(cmd)
random_update(test_vector, "output", "seed_0", "learn",
nb_features_origin)
random_update(test_vector, "output", "region", "2",
nb_features_new_region)
output_dir = self.test_working_directory
region_field = "region"
# launch function
split_vector_by_region(test_vector,
output_dir,
region_field,
runs=1,
driver="SQLite")
# assert
vector_reg_1 = fut.FileSearch_AND(self.test_working_directory, True,
"region_1")[0]
vector_reg_2 = fut.FileSearch_AND(self.test_working_directory, True,
"region_2")[0]
feat_vect_reg_1 = len(
fut.getFieldElement(vector_reg_1,
driverName="SQLite",
field="region",
mode="all",
elemType="str"))
feat_vect_reg_2 = len(
fut.getFieldElement(vector_reg_2,
driverName="SQLite",
field="region",
mode="all",
elemType="str"))
self.assertTrue(nb_features_new_region == feat_vect_reg_2)
self.assertTrue(nb_features_origin == feat_vect_reg_1 +
feat_vect_reg_2)
def config_model(outputPath, region_field):
"""
usage : determine which model will class which tile
"""
#const
output = None
pos_tile = 0
formatting_vec_dir = os.path.join(outputPath, "formattingVectors")
samples = fu.FileSearch_AND(formatting_vec_dir, True, ".shp")
#init
all_regions = []
for sample in samples:
tile_name = os.path.splitext(
os.path.basename(sample))[0].split("_")[pos_tile]
regions = fu.getFieldElement(sample,
driverName="ESRI Shapefile",
field=region_field,
mode="unique",
elemType="str")
for region in regions:
all_regions.append((region, tile_name))
#{'model_name':[TileName, TileName...],'...':...,...}
model_tiles = dict(fu.sortByFirstElem(all_regions))
#add tiles if they are missing by checking in /shapeRegion/ directory
shape_region_dir = os.path.join(outputPath, "shapeRegion")
shape_region_path = fu.FileSearch_AND(shape_region_dir, True, ".shp")
#check if there is actually polygons
shape_regions = [
elem for elem in shape_region_path if len(
fu.getFieldElement(elem,
driverName="ESRI Shapefile",
field=region_field,
mode="all",
elemType="str")) >= 1
]
for shape_region in shape_regions:
tile = os.path.splitext(
os.path.basename(shape_region))[0].split("_")[-1]
region = os.path.splitext(
os.path.basename(shape_region))[0].split("_")[-2]
for model_name, tiles_model in list(model_tiles.items()):
if model_name.split("f")[0] == region and tile not in tiles_model:
tiles_model.append(tile)
#Construct output file string
output = "AllModel:\n["
for model_name, tiles_model in list(model_tiles.items()):
output_tmp = "\n\tmodelName:'{}'\n\ttilesList:'{}'".format(
model_name, "_".join(tiles_model))
output = output + "\n\t{" + output_tmp + "\n\t}"
output += "\n]"
return output
def createRasterFootprint(tilePath, commonVecMask, proj=2154):
#outpolygonize = pathOut.replace(".shp","_TMP.shp")
#cmd = 'gdal_polygonize.py -mask '+tilePath+' '+tilePath+' -f "ESRI Shapefile" '+outpolygonize
#run(cmd)
fu.keepBiggestArea(tilePath.replace(".tif", ".shp"), commonVecMask)
#fu.removeShape(outpolygonize.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"])
return commonVecMask
def step_inputs(self):
"""
Return
------
the return could be and iterable or a callable
"""
from iota2.Common import FileUtils as fut
from iota2.Classification.ImageClassifier import autoContext_classification_param
from iota2.Common.ServiceConfigFile import iota2_parameters
if self.enable_autoContext is False and self.use_scikitlearn is False:
parameters = fut.parseClassifCmd(
os.path.join(self.output_path, "cmd", "cla", "class.txt"))
elif self.enable_autoContext is True:
parameters = autoContext_classification_param(
self.output_path, self.data_field)
elif self.use_scikitlearn:
parameters = fut.parseClassifCmd(
os.path.join(self.output_path, "cmd", "cla", "class.txt"))
running_parameters = iota2_parameters(self.cfg)
parameters = [{
"mask":
param[1],
"model":
param[2],
"stat":
param[3],
"out_classif":
param[4],
"out_confidence":
param[5],
"out_proba":
None,
"working_dir":
param[6],
"tile_name":
param[8],
"sar_optical_post_fusion":
SCF.serviceConfigFile(self.cfg).getParam(
'argTrain', 'dempster_shafer_SAR_Opt_fusion'),
"output_path":
SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath'),
"sensors_parameters":
running_parameters.get_sensors_parameters(tile_name=param[8]),
"pixel_type":
param[17],
"number_of_chunks":
self.scikit_tile_split,
"targeted_chunk":
target_chunk,
"ram":
param[19]
} for param in parameters
for target_chunk in range(self.scikit_tile_split)]
return parameters
def generate_shape_tile(tiles: List[str], pathWd: str, output_path: str,
proj: int) -> None:
"""generate tile's envelope with priority management
Parameters
----------
tiles : list
list of tiles envelopes to generate
pathOut : str
output directory
pathWd : str
working directory
output_path : str
iota2 output directory
proj : int
epsg code of target projection
"""
pathOut = os.path.join(output_path, "envelope")
if not os.path.exists(pathOut):
os.mkdir(pathOut)
featuresPath = os.path.join(output_path, "features")
cMaskName = "MaskCommunSL"
for tile in tiles:
if not os.path.exists(featuresPath + "/" + tile):
os.mkdir(featuresPath + "/" + tile)
os.mkdir(featuresPath + "/" + tile + "/tmp")
commonDirectory = pathOut + "/commonMasks/"
if not os.path.exists(commonDirectory):
os.mkdir(commonDirectory)
common = [
featuresPath + "/" + Ctile + "/tmp/" + cMaskName + ".tif"
for Ctile in tiles
]
ObjListTile = [
Tile(currentTile, name) for currentTile, name in zip(common, tiles)
]
ObjListTile_sort = sorted(ObjListTile, key=priorityKey)
tmpFile = pathOut + "/TMP"
if pathWd:
tmpFile = pathWd + "/TMP"
if not os.path.exists(tmpFile):
os.mkdir(tmpFile)
genTileEnvPrio(ObjListTile_sort, pathOut, tmpFile, proj)
AllPRIO = fu.FileSearch_AND(tmpFile, True, "_PRIO.shp")
for prioTile in AllPRIO:
tileName = prioTile.split("/")[-1].split("_")[0]
fu.cpShapeFile(prioTile.replace(".shp", ""), pathOut + "/" + tileName,
[".prj", ".shp", ".dbf", ".shx"])
shutil.rmtree(tmpFile)
shutil.rmtree(commonDirectory)
def split_samples(output_path: str,
data_field: str,
enable_cross_validation: bool,
region_threshold: Union[str, float],
region_field: str,
ratio: float,
random_seed: int,
runs: int,
epsg: Union[str, int],
workingDirectory: Optional[str] = None,
logger: Optional[Logger] = LOGGER):
"""
"""
from iota2.Common import FileUtils as fut
# const
regions_pos = -2
if isinstance(epsg, str):
epsg = int(epsg.split(":")[-1])
if isinstance(region_threshold, str):
region_threshold = float(region_threshold)
formatting_vectors_dir = os.path.join(output_path, "formattingVectors")
shape_region_dir = os.path.join(output_path, "shapeRegion")
vectors = fut.FileSearch_AND(formatting_vectors_dir, True, ".shp")
# get All possible regions by parsing shapeFile's name
shapes_region = fut.FileSearch_AND(shape_region_dir, True, ".shp")
regions = list(
set([
os.path.split(shape)[-1].split("_")[regions_pos]
for shape in shapes_region
]))
# compute region's area
areas, regions_tiles, data_to_rm = get_regions_area(
vectors, regions, formatting_vectors_dir, workingDirectory,
region_field)
# get how many sub-regions must be created by too huge regions.
regions_split = get_splits_regions(areas, region_threshold)
for region_name, area in list(areas.items()):
logger.info(f"region : {region_name} , area : {area}")
updated_vectors = split(regions_split, regions_tiles, data_field,
region_field)
# transform sqlites to shape file, according to input data format
new_regions_shapes = transform_to_shape(updated_vectors,
formatting_vectors_dir)
for data in data_to_rm:
os.remove(data)
data_app_val_dir = os.path.join(output_path, "dataAppVal")
update_learning_validation_sets(new_regions_shapes, data_app_val_dir,
data_field, region_field, ratio, runs,
epsg, enable_cross_validation, random_seed)
def getVectorsChunks(inpath, inbase="dept_"):
if not fut.FileSearch_AND(inpath, True, inbase, ".shp", "chk"):
listout = fut.FileSearch_AND(inpath, True, inbase, ".shp", "stats")
else:
listout = fut.FileSearch_AND(inpath, True, inbase, ".shp", "chk")
listofchkofzones = fut.sortByFirstElem([
("_".join(x.split('_')[0:len(x.split('_')) - 1]), x) for x in listout
])
return listofchkofzones
def getListVectToSimplify(path):
simplified = [os.path.splitext(x)[0].split('_')[len(os.path.splitext(x)[0].split('_')) - 2] \
for x in fut.FileSearch_AND(path, True, ".shp", "douglas") \
if "hermite" not in x]
polygonized = [[x, os.path.splitext(x)[0].split('_')[len(os.path.splitext(x)[0].split('_')) - 1]] \
for x in fut.FileSearch_AND(path, True, "tile_", ".shp") \
if "douglas" not in x and "hermite" not in x]
return [
os.path.join(path, x) for x, y in polygonized if y not in simplified
]
def split(regions_split: Dict[str, int], regions_tiles: Dict[str, List[str]],
data_field: str, region_field: str) -> List[str]:
"""
function dedicated to split to huge regions in sub-regions
Parameters
----------
regions_split: dict[string, int]
regions_tiles: dict[str, List[str]]
Return
------
list(str)
"""
from iota2.Common import FileUtils as fut
updated_vectors = []
for region, fold in list(regions_split.items()):
vector_paths = regions_tiles[region]
for vec in vector_paths:
# init dict new regions
new_regions_dict = {}
for sub_fold in range(fold):
# new region's name are define here
new_regions_dict["{}f{}".format(region, sub_fold + 1)] = []
# get possible class
class_vector = fut.getFieldElement(vec,
driverName="SQLite",
field=data_field,
mode="unique",
elemType="str")
dic_class = {}
# get FID values for all class of current region into
# the current tile
for c_class in class_vector:
dic_class[c_class] = get_fid_values(vec, data_field,
region_field, region,
c_class)
nb_feat = 0
for _, fid_cl in list(dic_class.items()):
if fid_cl:
fid_folds = fut.splitList(fid_cl, fold)
# fill new_regions_dict
for i, fid_fold in enumerate(fid_folds):
new_regions_dict[f"{region}f{i+1}"] += fid_fold
nb_feat += len(fid_cl)
update_vector(vec, region_field, new_regions_dict)
if vec not in updated_vectors:
updated_vectors.append(vec)
return updated_vectors
def test_extract_maj_vote_samples(self):
"""
test the extraction of samples by class according to a ratio
"""
from iota2.Sampling.VectorFormatting import extract_maj_vote_samples
from collections import Counter
# define inputs
in_vector_name = os.path.basename(self.in_vector)
extracted_vector_name = "extracted_samples.sqlite"
in_vector = os.path.join(self.test_working_directory, in_vector_name)
extracted_vector = os.path.join(self.test_working_directory,
extracted_vector_name)
fut.cpShapeFile(self.in_vector.replace(".shp", ""),
in_vector.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
# launch function
dataField = "code"
regionField = "region"
extraction_ratio = 0.5
extract_maj_vote_samples(in_vector, extracted_vector, extraction_ratio,
dataField, regionField)
# assert
features_origin = fut.getFieldElement(self.in_vector,
driverName="ESRI Shapefile",
field=dataField,
mode="all",
elemType="str")
by_class_origin = Counter(features_origin)
features_in_vector = fut.getFieldElement(in_vector,
driverName="ESRI Shapefile",
field=dataField,
mode="all",
elemType="str")
by_class_in_vector = Counter(features_in_vector)
features_extract_vector = fut.getFieldElement(extracted_vector,
driverName="SQLite",
field=dataField,
mode="all",
elemType="str")
by_class_extract_vector = Counter(features_extract_vector)
buff = []
for class_name, class_count in list(by_class_origin.items()):
buff.append(by_class_in_vector[class_name] == extraction_ratio *
class_count)
self.assertTrue(all(buff), msg="extraction of samples failed")
def test_split_selection(self):
"""
test dedicated to check if split_sel function works
"""
from iota2.Sampling.SamplesSelection import split_sel
from Iota2Tests import random_update
from TestsUtils import rename_table
# prepare test input
test_vector_name = "samples_region_1_seed_0.sqlite"
test_vector_table = "t31tcj_samples_region_1_seed_0_selection"
test_vector = os.path.join(self.test_working_directory,
test_vector_name)
shutil.copy(self.selection_ref, test_vector)
# update "nb_feat" features to a new "new_tile_name" tile's name
nb_feat = 10
new_tile_name = "T31TDJ"
random_update(test_vector, test_vector_table, "tile_o", new_tile_name,
nb_feat)
rename_table(test_vector,
old_table_name=test_vector_table,
new_table_name="output")
# launch function
new_files = split_sel(test_vector, ["T31TCJ", new_tile_name],
self.test_working_directory, "EPSG:2154")
# assert
nb_features_origin = len(
fut.getFieldElement(self.selection_ref,
driverName="SQLite",
field="tile_o",
mode="all",
elemType="str"))
nb_features_t31tcj = len(
fut.getFieldElement(new_files[0],
driverName="SQLite",
field="tile_o",
mode="all",
elemType="str"))
nb_features_t31tdj = len(
fut.getFieldElement(new_files[1],
driverName="SQLite",
field="tile_o",
mode="all",
elemType="str"))
self.assertTrue(nb_features_t31tdj == nb_feat,
msg="split samples selection failed")
self.assertTrue(nb_features_origin == nb_features_t31tdj +
nb_features_t31tcj,
msg="split samples selection failed")
def generate_region_shape(envelope_directory: str, output_region_file: str,
out_field_name: str, i2_output_path: str,
working_directory: str) -> None:
"""generate regions shape
envelope_directory: str
directory containing all iota2 tile's envelope
output_region_file: str
output file
out_field_name: str
output field containing region
i2_output_path: str
iota2 output path
working_directory: str
path to a working directory
"""
region = []
all_tiles = fu.FileSearch_AND(envelope_directory, False, ".shp")
region.append(all_tiles)
if not output_region_file:
output_region_file = os.path.join(i2_output_path, "MyRegion.shp")
p_f = output_region_file.replace(" ", "").split("/")
out_name = p_f[-1].split(".")[0]
path_mod = ""
for i in range(1, len(p_f) - 1):
path_mod = path_mod + "/" + p_f[i]
CreateModelShapeFromTiles(region, envelope_directory, path_mod, out_name,
out_field_name, working_directory)
def test_samples_selection(self):
"""
test sampling of a shape file (main function of SamplesSelection.py)
"""
from iota2.Sampling.SamplesSelection import samples_selection
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
from iota2.Tests.UnitTests.Iota2Tests import compareSQLite
from iota2.Common.FileUtils import cpShapeFile
from iota2.Common import FileUtils as fut
# prepare test input
cfg = SCF.serviceConfigFile(self.config_test)
cfg.setParam(
"chain", "outputPath",
os.path.join(self.test_working_directory, "samplesSelTest"))
cfg.setParam("chain", "runs", 2)
cfg.setParam("argTrain", "sampleSelection", {
"sampler": "random",
"strategy": "all"
})
# create IOTA2 directories
IOTA2Directory.generate_directories(os.path.join(
self.test_working_directory, "samplesSelTest"),
check_inputs=False)
shutil.copytree(
self.features_ref,
os.path.join(self.test_working_directory, "samplesSelTest",
"features", "T31TCJ"))
shutil.copy(
self.in_xml,
os.path.join(self.test_working_directory, "samplesSelTest",
"samplesSelection",
"T31TCJ_region_1_seed_0_stats.xml"))
_, in_shape_name = os.path.split(self.in_shape)
in_shape_dir = os.path.join(self.test_working_directory,
"samplesSelTest", "samplesSelection")
in_shape = os.path.join(in_shape_dir, in_shape_name)
cpShapeFile(self.in_shape.replace(".shp", ""),
in_shape.replace(".shp", ""),
extensions=[".prj", ".shp", ".dbf", ".shx"])
#~ # launch function
output_path = cfg.getParam("chain", "outputPath")
runs = cfg.getParam('chain', 'runs')
epsg = cfg.getParam('GlobChain', 'proj')
random_seed = cfg.getParam('chain', 'random_seed')
data_field = cfg.getParam('chain', 'dataField').lower()
parameters = dict(cfg.getParam('argTrain', 'sampleSelection'))
masks_name = "MaskCommunSL.tif"
samples_selection(in_shape, self.test_working_directory, output_path,
runs, epsg, masks_name, parameters, data_field,
random_seed)
#~ # assert
selection_test = fut.FileSearch_AND(
os.path.join(self.test_working_directory, "samplesSelTest"), True,
os.path.basename(self.selection_ref))[0]
same = compareSQLite(self.selection_ref,
selection_test,
CmpMode='coordinates')
self.assertTrue(same, msg="sample selection generation failed")
def test_create_tile_region_masks(self):
"""
test the generation of the raster mask which define the region
in the tile
"""
from iota2.Sampling.VectorFormatting import create_tile_region_masks
from iota2.Common.Utils import run
from iota2.Tests.UnitTests.TestsUtils import rasterToArray
import numpy as np
# define inputs
test_vector_name = "T31TCJ.sqlite"
test_vector = os.path.join(self.test_working_directory,
test_vector_name)
cmd = "ogr2ogr -nln t31tcj -f SQLite {} {}".format(
test_vector, self.ref_region)
run(cmd)
# launch function
create_tile_region_masks(test_vector, "region", "T31TCJ",
self.test_working_directory, "MyRegion",
self.ref_img)
# assert
raster_region = fut.FileSearch_AND(self.test_working_directory, True,
"MyRegion", ".tif")[0]
raster_region_arr = rasterToArray(raster_region)
ref_array = np.ones((50, 50))
self.assertTrue(np.allclose(ref_array, raster_region_arr),
msg="problem with the normalization by ref")
def __init__(self, cfg, cfg_resources_file, working_directory=None):
# heritage init
resources_block_name = "noData"
super(fusionsIndecisions, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
self.working_directory = working_directory
self.output_path = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
self.field_region = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'regionField')
self.shape_region = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'regionPath')
self.runs = SCF.serviceConfigFile(self.cfg).getParam('chain', 'runs')
self.no_label = SCF.serviceConfigFile(self.cfg).getParam(
'argClassification', 'noLabelManagement')
self.features = SCF.serviceConfigFile(self.cfg).getParam(
"GlobChain", "features")
self.user_feat_path = SCF.serviceConfigFile(self.cfg).getParam(
"chain", "userFeatPath")
self.pixtype = fut.getOutputPixType(
SCF.serviceConfigFile(self.cfg).getParam('chain',
'nomenclaturePath'))
self.region_vec = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'regionPath')
self.patterns = SCF.serviceConfigFile(self.cfg).getParam(
"userFeat", "patterns")
self.sar_opt_fusion = SCF.serviceConfigFile(self.cfg).getParam(
'argTrain', 'dempster_shafer_SAR_Opt_fusion')
self.path_to_img = os.path.join(self.output_path, "features")
def test_generate_shape_tile(self):
from iota2.Sampling import TileEnvelope as env
# Test de création des enveloppes
features_path = os.path.join(self.iota2_tests_directory, "features")
masks_references = '../../../data/references/features'
if os.path.exists(features_path):
shutil.rmtree(features_path)
shutil.copytree(masks_references, features_path)
# Test de création des enveloppes
# Launch function
env.generate_shape_tile(self.tiles, None, self.iota2_tests_directory,
2154)
# For each tile test if the shapefile is ok
for i in self.tiles:
# generate filename
reference_shape_file = (IOTA2_DATATEST +
"/references/GenerateShapeTile/" + i +
".shp")
shape_file = self.iota2_tests_directory + "/envelope/" + i + ".shp"
service_compare_vector_file = fu.serviceCompareVectorFile()
# Launch shapefile comparison
self.assertTrue(
service_compare_vector_file.testSameShapefiles(
reference_shape_file, shape_file))
shutil.rmtree(features_path)
def create_dummy_rasters(missing_tiles: List[str], runs: int,
output_path: str) -> None:
"""
Parameters
----------
missing_tiles: list(string)
runs: int
output_path: string
Return
------
None
Notes
-----
use when mode is 'one_region' but there is no validations / learning
samples into a specific tile
"""
classifications_dir = os.path.join(output_path, "classif")
final_dir = os.path.join(output_path, "final", "TMP")
for tile in missing_tiles:
classif_tile = fu.FileSearch_AND(classifications_dir, True,
"Classif_" + str(tile))[0]
for seed in range(runs):
dummy_raster_name = tile + "_seed_" + str(seed) + "_CompRef.tif"
dummy_raster = final_dir + "/" + dummy_raster_name
dummy_raster_cmd = (f"gdal_merge.py -ot Byte -n 0 -createonly -o "
f"{ dummy_raster} {classif_tile}")
run(dummy_raster_cmd)
def test_vector_splits(self):
from iota2.Sampling import SplitInSubSets as VS
from iota2.Common import FileUtils as fu
from iota2.Tests.UnitTests import TestsUtils
# We execute the function splitInSubSets()
for new_region_shape in self.new_regions_shapes:
tile_name = os.path.splitext(os.path.basename(new_region_shape))[0]
vectors_to_rm = fu.FileSearch_AND(self.data_app_val_dir, True,
tile_name)
for vect in vectors_to_rm:
os.remove(vect)
VS.splitInSubSets(new_region_shape,
self.data_field,
self.region_field,
self.ratio,
self.seeds,
"ESRI Shapefile",
random_seed=0)
print(new_region_shape)
# We check the output
self.assertTrue(
TestsUtils.compareVectorFile(self.ref_split_shp,
self.out_split_shp, 'coordinates',
'polygon', "ESRI Shapefile"),
"Split vector output are different")
def get_values_sorted_by_coordinates(
vector: str) -> List[Tuple[float, float]]:
"""
usage return values sorted by coordinates (x,y)
IN
vector [string] path to a vector of points
OUT
values [list of tuple] : [(x,y,[val1,val2]),()...]
"""
import ogr
values = []
driver = ogr.GetDriverByName("SQLite")
dataset = driver.Open(vector, 0)
lyr = dataset.GetLayer()
fields = fut.get_all_fields_in_shape(vector, 'SQLite')
for feature in lyr:
x_coord = feature.GetGeometryRef().GetX()
y_coord = feature.GetGeometryRef().GetY()
fields_val = get_field_value(feature, fields)
values.append((x_coord, y_coord, fields_val))
values = sorted(values, key=priority)
return values
def extract_class(vec_in: str, vec_out: str, target_class: List[str],
data_field: str) -> int:
"""
Extract class
IN:
vec_in: str
vec_out: str
target_class:List[str]
data_field: str
"""
from iota2.Common.Utils import run
if type(target_class) != type(list()):
target_class = target_class.data
where = " OR ".join(
["{}={}".format(data_field.lower(), klass) for klass in target_class])
cmd = (f"ogr2ogr -f 'SQLite' -nln output -where '{where}' "
f"{vec_out} {vec_in}")
run(cmd)
return len(
fu.getFieldElement(vec_out,
driverName="SQLite",
field=data_field.lower(),
mode="all",
elemType="int"))
def get_vectors_to_sample(
iota2_formatting_dir: str,
ds_fusion_sar_opt: Optional[bool] = False) -> List[Dict[str, str]]:
"""
get vectors to sample
IN :
iota2_formatting_dir : str
path to shapefiles
ds_fusion_sar_opt : bool
activate sar mode
OUT:
List of dictionary containing all shapefiles
"""
formatting_tiles = fu.FileSearch_AND(iota2_formatting_dir, True, ".shp")
# parameters generation
tiles_vectors = [{"usually": vector} for vector in formatting_tiles]
# parameters dedicated to SAR
tiles_vectors_sar = [{"SAR": vector} for vector in formatting_tiles]
tiles_vectors_to_sample = tiles_vectors
# if we want to have SAR classification and Optical classification belong
# we have to double the number of parameters in generateSamples
if ds_fusion_sar_opt:
tiles_vectors_to_sample = tiles_vectors + tiles_vectors_sar
return tiles_vectors_to_sample
def test_iota2_augmentation(self):
"""Test data augmentation workflow
"""
from collections import Counter
class_augmentation_balance = DataAugmentation.SamplesAugmentationCounter(
self.class_count, mode="balance", minNumber=None, byClass=None)
DataAugmentation.DoAugmentation(self.vector_test,
class_augmentation_balance,
strategy="jitter",
field="code",
excluded_fields=[],
Jstdfactor=10,
Sneighbors=None,
workingDirectory=None)
class_count_test = Counter(
fut.getFieldElement(self.vector_test,
driverName="SQLite",
field="code",
mode="all",
elemType="int"))
samples_number = self.class_count[max(
self.class_count, key=lambda key: self.class_count[key])]
self.assertTrue(
all([
samples_number == v for k, v in list(class_count_test.items())
]))
