def samples_stats(region_seed_tile: Tuple[str, str, str],
iota2_directory: str,
data_field: str,
working_directory: Optional[Union[str, None]] = None,
logger=LOGGER) -> str:
"""generate samples statistics by tiles
Parameters
----------
region_seed_tile: tuple
iota2_directory: str
iota2 output directory
data_field: str
data field in region database
working_directory: str
path to a working directory
logger: logging
root logger
Return
------
str
file containing statistics from otbcli_PolygonClassStatistics
"""
region, seed, tile = region_seed_tile
samples_selection_dir = os.path.join(iota2_directory, "samplesSelection")
tile_region_dir = os.path.join(iota2_directory, "shapeRegion")
w_dir = samples_selection_dir
if working_directory:
w_dir = working_directory
raster_mask = fut.FileSearch_AND(tile_region_dir, True,
"region_" + region.split("f")[0] + "_",
".tif", tile)[0]
region_vec = fut.FileSearch_AND(samples_selection_dir, True,
"_region_" + region, "seed_" + seed,
".shp")[0]
logger.info(
f"Launch statistics on tile {tile} in region {region} run {seed}")
region_tile_stats_name = f"{tile}_region_{region}_seed_{seed}_stats.xml"
region_tile_stats = os.path.join(w_dir, region_tile_stats_name)
polygon_stats_app = otb.CreatePolygonClassStatisticsApplication({
"in":
raster_mask,
"mask":
raster_mask,
"vec":
region_vec,
"field":
data_field,
"out":
region_tile_stats
})
polygon_stats_app.ExecuteAndWriteOutput()
if working_directory:
shutil.copy(region_tile_stats, samples_selection_dir)
return region_tile_stats
def test_split_vector_by_region(self):
"""
test : split a vector by the region he belongs to
"""
from iota2.Sampling.VectorFormatting import split_vector_by_region
from iota2.Common.Utils import run
from iota2.Tests.UnitTests.Iota2Tests import random_update
# define inputs
nb_features_origin = len(
fut.getFieldElement(self.in_vector,
driverName="ESRI shapefile",
field="region",
mode="all",
elemType="str"))
nb_features_new_region = 5
test_vector_name = "T31TCJ_Samples.sqlite"
test_vector = os.path.join(self.test_working_directory,
test_vector_name)
cmd = "ogr2ogr -nln output -f SQLite {} {}".format(
test_vector, self.in_vector)
run(cmd)
random_update(test_vector, "output", "seed_0", "learn",
nb_features_origin)
random_update(test_vector, "output", "region", "2",
nb_features_new_region)
output_dir = self.test_working_directory
region_field = "region"
# launch function
split_vector_by_region(test_vector,
output_dir,
region_field,
runs=1,
driver="SQLite")
# assert
vector_reg_1 = fut.FileSearch_AND(self.test_working_directory, True,
"region_1")[0]
vector_reg_2 = fut.FileSearch_AND(self.test_working_directory, True,
"region_2")[0]
feat_vect_reg_1 = len(
fut.getFieldElement(vector_reg_1,
driverName="SQLite",
field="region",
mode="all",
elemType="str"))
feat_vect_reg_2 = len(
fut.getFieldElement(vector_reg_2,
driverName="SQLite",
field="region",
mode="all",
elemType="str"))
self.assertTrue(nb_features_new_region == feat_vect_reg_2)
self.assertTrue(nb_features_origin == feat_vect_reg_1 +
feat_vect_reg_2)
def config_model(outputPath, region_field):
"""
usage : determine which model will class which tile
"""
#const
output = None
pos_tile = 0
formatting_vec_dir = os.path.join(outputPath, "formattingVectors")
samples = fu.FileSearch_AND(formatting_vec_dir, True, ".shp")
#init
all_regions = []
for sample in samples:
tile_name = os.path.splitext(
os.path.basename(sample))[0].split("_")[pos_tile]
regions = fu.getFieldElement(sample,
driverName="ESRI Shapefile",
field=region_field,
mode="unique",
elemType="str")
for region in regions:
all_regions.append((region, tile_name))
#{'model_name':[TileName, TileName...],'...':...,...}
model_tiles = dict(fu.sortByFirstElem(all_regions))
#add tiles if they are missing by checking in /shapeRegion/ directory
shape_region_dir = os.path.join(outputPath, "shapeRegion")
shape_region_path = fu.FileSearch_AND(shape_region_dir, True, ".shp")
#check if there is actually polygons
shape_regions = [
elem for elem in shape_region_path if len(
fu.getFieldElement(elem,
driverName="ESRI Shapefile",
field=region_field,
mode="all",
elemType="str")) >= 1
]
for shape_region in shape_regions:
tile = os.path.splitext(
os.path.basename(shape_region))[0].split("_")[-1]
region = os.path.splitext(
os.path.basename(shape_region))[0].split("_")[-2]
for model_name, tiles_model in list(model_tiles.items()):
if model_name.split("f")[0] == region and tile not in tiles_model:
tiles_model.append(tile)
#Construct output file string
output = "AllModel:\n["
for model_name, tiles_model in list(model_tiles.items()):
output_tmp = "\n\tmodelName:'{}'\n\ttilesList:'{}'".format(
model_name, "_".join(tiles_model))
output = output + "\n\t{" + output_tmp + "\n\t}"
output += "\n]"
return output
def split_samples(output_path: str,
data_field: str,
enable_cross_validation: bool,
region_threshold: Union[str, float],
region_field: str,
ratio: float,
random_seed: int,
runs: int,
epsg: Union[str, int],
workingDirectory: Optional[str] = None,
logger: Optional[Logger] = LOGGER):
"""
"""
from iota2.Common import FileUtils as fut
# const
regions_pos = -2
if isinstance(epsg, str):
epsg = int(epsg.split(":")[-1])
if isinstance(region_threshold, str):
region_threshold = float(region_threshold)
formatting_vectors_dir = os.path.join(output_path, "formattingVectors")
shape_region_dir = os.path.join(output_path, "shapeRegion")
vectors = fut.FileSearch_AND(formatting_vectors_dir, True, ".shp")
# get All possible regions by parsing shapeFile's name
shapes_region = fut.FileSearch_AND(shape_region_dir, True, ".shp")
regions = list(
set([
os.path.split(shape)[-1].split("_")[regions_pos]
for shape in shapes_region
]))
# compute region's area
areas, regions_tiles, data_to_rm = get_regions_area(
vectors, regions, formatting_vectors_dir, workingDirectory,
region_field)
# get how many sub-regions must be created by too huge regions.
regions_split = get_splits_regions(areas, region_threshold)
for region_name, area in list(areas.items()):
logger.info(f"region : {region_name} , area : {area}")
updated_vectors = split(regions_split, regions_tiles, data_field,
region_field)
# transform sqlites to shape file, according to input data format
new_regions_shapes = transform_to_shape(updated_vectors,
formatting_vectors_dir)
for data in data_to_rm:
os.remove(data)
data_app_val_dir = os.path.join(output_path, "dataAppVal")
update_learning_validation_sets(new_regions_shapes, data_app_val_dir,
data_field, region_field, ratio, runs,
epsg, enable_cross_validation, random_seed)
def getVectorsChunks(inpath, inbase="dept_"):
if not fut.FileSearch_AND(inpath, True, inbase, ".shp", "chk"):
listout = fut.FileSearch_AND(inpath, True, inbase, ".shp", "stats")
else:
listout = fut.FileSearch_AND(inpath, True, inbase, ".shp", "chk")
listofchkofzones = fut.sortByFirstElem([
("_".join(x.split('_')[0:len(x.split('_')) - 1]), x) for x in listout
])
return listofchkofzones
def getListVectToSimplify(path):
simplified = [os.path.splitext(x)[0].split('_')[len(os.path.splitext(x)[0].split('_')) - 2] \
for x in fut.FileSearch_AND(path, True, ".shp", "douglas") \
if "hermite" not in x]
polygonized = [[x, os.path.splitext(x)[0].split('_')[len(os.path.splitext(x)[0].split('_')) - 1]] \
for x in fut.FileSearch_AND(path, True, "tile_", ".shp") \
if "douglas" not in x and "hermite" not in x]
return [
os.path.join(path, x) for x, y in polygonized if y not in simplified
]
def generate_region_shape(envelope_directory: str, output_region_file: str,
out_field_name: str, i2_output_path: str,
working_directory: str) -> None:
"""generate regions shape
envelope_directory: str
directory containing all iota2 tile's envelope
output_region_file: str
output file
out_field_name: str
output field containing region
i2_output_path: str
iota2 output path
working_directory: str
path to a working directory
"""
region = []
all_tiles = fu.FileSearch_AND(envelope_directory, False, ".shp")
region.append(all_tiles)
if not output_region_file:
output_region_file = os.path.join(i2_output_path, "MyRegion.shp")
p_f = output_region_file.replace(" ", "").split("/")
out_name = p_f[-1].split(".")[0]
path_mod = ""
for i in range(1, len(p_f) - 1):
path_mod = path_mod + "/" + p_f[i]
CreateModelShapeFromTiles(region, envelope_directory, path_mod, out_name,
out_field_name, working_directory)
def create_dummy_rasters(missing_tiles: List[str], runs: int,
output_path: str) -> None:
"""
Parameters
----------
missing_tiles: list(string)
runs: int
output_path: string
Return
------
None
Notes
-----
use when mode is 'one_region' but there is no validations / learning
samples into a specific tile
"""
classifications_dir = os.path.join(output_path, "classif")
final_dir = os.path.join(output_path, "final", "TMP")
for tile in missing_tiles:
classif_tile = fu.FileSearch_AND(classifications_dir, True,
"Classif_" + str(tile))[0]
for seed in range(runs):
dummy_raster_name = tile + "_seed_" + str(seed) + "_CompRef.tif"
dummy_raster = final_dir + "/" + dummy_raster_name
dummy_raster_cmd = (f"gdal_merge.py -ot Byte -n 0 -createonly -o "
f"{ dummy_raster} {classif_tile}")
run(dummy_raster_cmd)
def get_vectors_to_sample(
iota2_formatting_dir: str,
ds_fusion_sar_opt: Optional[bool] = False) -> List[Dict[str, str]]:
"""
get vectors to sample
IN :
iota2_formatting_dir : str
path to shapefiles
ds_fusion_sar_opt : bool
activate sar mode
OUT:
List of dictionary containing all shapefiles
"""
formatting_tiles = fu.FileSearch_AND(iota2_formatting_dir, True, ".shp")
# parameters generation
tiles_vectors = [{"usually": vector} for vector in formatting_tiles]
# parameters dedicated to SAR
tiles_vectors_sar = [{"SAR": vector} for vector in formatting_tiles]
tiles_vectors_to_sample = tiles_vectors
# if we want to have SAR classification and Optical classification belong
# we have to double the number of parameters in generateSamples
if ds_fusion_sar_opt:
tiles_vectors_to_sample = tiles_vectors + tiles_vectors_sar
return tiles_vectors_to_sample
def test_vector_splits(self):
from iota2.Sampling import SplitInSubSets as VS
from iota2.Common import FileUtils as fu
from iota2.Tests.UnitTests import TestsUtils
# We execute the function splitInSubSets()
for new_region_shape in self.new_regions_shapes:
tile_name = os.path.splitext(os.path.basename(new_region_shape))[0]
vectors_to_rm = fu.FileSearch_AND(self.data_app_val_dir, True,
tile_name)
for vect in vectors_to_rm:
os.remove(vect)
VS.splitInSubSets(new_region_shape,
self.data_field,
self.region_field,
self.ratio,
self.seeds,
"ESRI Shapefile",
random_seed=0)
print(new_region_shape)
# We check the output
self.assertTrue(
TestsUtils.compareVectorFile(self.ref_split_shp,
self.out_split_shp, 'coordinates',
'polygon', "ESRI Shapefile"),
"Split vector output are different")
def test_samples_selection(self):
"""
test sampling of a shape file (main function of SamplesSelection.py)
"""
from iota2.Sampling.SamplesSelection import samples_selection
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
from iota2.Tests.UnitTests.Iota2Tests import compareSQLite
from iota2.Common.FileUtils import cpShapeFile
from iota2.Common import FileUtils as fut
# prepare test input
cfg = SCF.serviceConfigFile(self.config_test)
cfg.setParam(
"chain", "outputPath",
os.path.join(self.test_working_directory, "samplesSelTest"))
cfg.setParam("chain", "runs", 2)
cfg.setParam("argTrain", "sampleSelection", {
"sampler": "random",
"strategy": "all"
})
# create IOTA2 directories
IOTA2Directory.generate_directories(os.path.join(
self.test_working_directory, "samplesSelTest"),
check_inputs=False)
shutil.copytree(
self.features_ref,
os.path.join(self.test_working_directory, "samplesSelTest",
"features", "T31TCJ"))
shutil.copy(
self.in_xml,
os.path.join(self.test_working_directory, "samplesSelTest",
"samplesSelection",
"T31TCJ_region_1_seed_0_stats.xml"))
_, in_shape_name = os.path.split(self.in_shape)
in_shape_dir = os.path.join(self.test_working_directory,
"samplesSelTest", "samplesSelection")
in_shape = os.path.join(in_shape_dir, in_shape_name)
cpShapeFile(self.in_shape.replace(".shp", ""),
in_shape.replace(".shp", ""),
extensions=[".prj", ".shp", ".dbf", ".shx"])
#~ # launch function
output_path = cfg.getParam("chain", "outputPath")
runs = cfg.getParam('chain', 'runs')
epsg = cfg.getParam('GlobChain', 'proj')
random_seed = cfg.getParam('chain', 'random_seed')
data_field = cfg.getParam('chain', 'dataField').lower()
parameters = dict(cfg.getParam('argTrain', 'sampleSelection'))
masks_name = "MaskCommunSL.tif"
samples_selection(in_shape, self.test_working_directory, output_path,
runs, epsg, masks_name, parameters, data_field,
random_seed)
#~ # assert
selection_test = fut.FileSearch_AND(
os.path.join(self.test_working_directory, "samplesSelTest"), True,
os.path.basename(self.selection_ref))[0]
same = compareSQLite(self.selection_ref,
selection_test,
CmpMode='coordinates')
self.assertTrue(same, msg="sample selection generation failed")
def test_create_tile_region_masks(self):
"""
test the generation of the raster mask which define the region
in the tile
"""
from iota2.Sampling.VectorFormatting import create_tile_region_masks
from iota2.Common.Utils import run
from iota2.Tests.UnitTests.TestsUtils import rasterToArray
import numpy as np
# define inputs
test_vector_name = "T31TCJ.sqlite"
test_vector = os.path.join(self.test_working_directory,
test_vector_name)
cmd = "ogr2ogr -nln t31tcj -f SQLite {} {}".format(
test_vector, self.ref_region)
run(cmd)
# launch function
create_tile_region_masks(test_vector, "region", "T31TCJ",
self.test_working_directory, "MyRegion",
self.ref_img)
# assert
raster_region = fut.FileSearch_AND(self.test_working_directory, True,
"MyRegion", ".tif")[0]
raster_region_arr = rasterToArray(raster_region)
ref_array = np.ones((50, 50))
self.assertTrue(np.allclose(ref_array, raster_region_arr),
msg="problem with the normalization by ref")
def confusion_models_merge_parameters(iota2_dir: str):
"""
function use to feed confusion_models_merge function
Parameter
---------
iota2_dir : string
path to the iota2 running directory
Return
------
list
list containing all sub confusion matrix which must be merged.
"""
ds_sar_opt_conf_dir = os.path.join(iota2_dir, "dataAppVal", "bymodels")
csv_seed_pos = 4
csv_model_pos = 2
all_csv = fu.FileSearch_AND(ds_sar_opt_conf_dir, True, "samples", ".csv")
# group by models
model_group = []
for csv in all_csv:
_, csv_name = os.path.split(csv)
csv_seed = csv_name.split("_")[csv_seed_pos]
csv_model = csv_name.split("_")[csv_model_pos]
# csv_mode = "SAR.csv" or "val.csv", use to descriminate models
csv_mode = csv_name.split("_")[-1]
key_param = (csv_model, csv_seed, csv_mode)
model_group.append((key_param, csv))
groups_param = [param for key, param in fu.sortByFirstElem(model_group)]
return groups_param
def step_inputs(self):
"""
Return
------
the return could be and iterable or a callable
"""
from iota2.Common import FileUtils as fut
return fut.FileSearch_AND(os.path.join(self.output_path, "classif"),
True, "_FUSION_")
def generate_shape_tile(tiles: List[str], pathWd: str, output_path: str,
proj: int) -> None:
"""generate tile's envelope with priority management
Parameters
----------
tiles : list
list of tiles envelopes to generate
pathOut : str
output directory
pathWd : str
working directory
output_path : str
iota2 output directory
proj : int
epsg code of target projection
"""
pathOut = os.path.join(output_path, "envelope")
if not os.path.exists(pathOut):
os.mkdir(pathOut)
featuresPath = os.path.join(output_path, "features")
cMaskName = "MaskCommunSL"
for tile in tiles:
if not os.path.exists(featuresPath + "/" + tile):
os.mkdir(featuresPath + "/" + tile)
os.mkdir(featuresPath + "/" + tile + "/tmp")
commonDirectory = pathOut + "/commonMasks/"
if not os.path.exists(commonDirectory):
os.mkdir(commonDirectory)
common = [
featuresPath + "/" + Ctile + "/tmp/" + cMaskName + ".tif"
for Ctile in tiles
]
ObjListTile = [
Tile(currentTile, name) for currentTile, name in zip(common, tiles)
]
ObjListTile_sort = sorted(ObjListTile, key=priorityKey)
tmpFile = pathOut + "/TMP"
if pathWd:
tmpFile = pathWd + "/TMP"
if not os.path.exists(tmpFile):
os.mkdir(tmpFile)
genTileEnvPrio(ObjListTile_sort, pathOut, tmpFile, proj)
AllPRIO = fu.FileSearch_AND(tmpFile, True, "_PRIO.shp")
for prioTile in AllPRIO:
tileName = prioTile.split("/")[-1].split("_")[0]
fu.cpShapeFile(prioTile.replace(".shp", ""), pathOut + "/" + tileName,
[".prj", ".shp", ".dbf", ".shx"])
shutil.rmtree(tmpFile)
shutil.rmtree(commonDirectory)
def getListVectToClip(path, fieldclip, vectorpath):
listtoclip = []
for filetoclip in fut.FileSearch_AND(path, True, ".shp", "hermite"):
listtoclip.append(
(filetoclip,
os.path.basename(filetoclip.replace(fieldclip,
'')).split('_')[2]))
return listtoclip
def step_inputs(self):
"""
Return
------
the return could be and iterable or a callable
"""
from iota2.Common import FileUtils as fut
selected_polygons = fut.FileSearch_AND(
os.path.join(self.output_path, "samplesSelection"), True, ".shp")
if self.enable_cross_validation:
selected_polygons = sorted(selected_polygons,
key=self.sort_by_seed)[:-1]
return selected_polygons
def tile_vectors_to_models(iota2_learning_samples_dir: str) -> List[str]:
"""
use to feed vector_samples_merge function
Parameters
----------
iota2_learning_samples_dir : string
path to "learningSamples" iota² directory
sep_sar_opt : bool
flag use to inform if SAR data has to be computed separately
Return
------
list
list of list of vectors to be merged to form a vector by model
"""
vectors = fu.FileSearch_AND(iota2_learning_samples_dir, True,
"Samples_learn.sqlite")
vectors_sar = fu.FileSearch_AND(iota2_learning_samples_dir, True,
"Samples_SAR_learn.sqlite")
vect_to_model = split_vectors_by_regions(
vectors) + split_vectors_by_regions(vectors_sar)
return vect_to_model
def get_models(formatting_vector_directory: str, region_field: str,
runs: int) -> List[Tuple[str, List[str], int]]:
"""
usage :
describe samples spatial repartition
function use to determine with shapeFile as to be merged in order to
compute statistics thanks to otb_SampleSelection
OUT:
regions_tiles_seed [list] :
example
regions_tiles_seed = [('1', ['T1', 'T2'], 0), ('1', ['T1', T2], 1),
('2', ['T2', 'T3], 0), ('2', ['T2', 'T3], 1)]
mean the region '1' is present in tiles 'T1' and 'T2' in run 0 and 1
and region '2' in 'T2', 'T3' in runs 0 and 1
"""
# the way of getting region could be improve ?
tiles = fut.FileSearch_AND(formatting_vector_directory, True, ".shp")
region_tile = []
all_regions_in_run = []
for tile in tiles:
all_regions = []
tile_name = os.path.splitext(os.path.basename(tile))[0]
r_tmp = fut.getFieldElement(tile,
driverName="ESRI Shapefile",
field=region_field,
mode="unique",
elemType="str")
for r_tile in r_tmp:
if r_tile not in all_regions:
all_regions.append(r_tile)
for region in all_regions:
if region not in all_regions_in_run:
all_regions_in_run.append(region)
region_tile.append((region, tile_name))
region_tile_tmp = dict(fut.sortByFirstElem(region_tile))
region_tile_dic = {}
for region, region_tiles in list(region_tile_tmp.items()):
region_tile_dic[region] = list(set(region_tiles))
all_regions_in_run = sorted(all_regions_in_run)
regions_tiles_seed = [(region, region_tile_dic[region], run)
for run in range(runs)
for region in all_regions_in_run]
return regions_tiles_seed
def update_learning_validation_sets(new_regions_shapes: List[str],
data_app_val_dir: str, data_field: str,
region_field: str, ratio: float,
seeds: int, epsg: str,
enable_cross_validation: bool,
random_seed: int) -> None:
"""
Parameters
----------
new_regions_shapes: list(string)
data_app_val_dir: string
data_field: string
region_field: string
ratio: float
seeds: intersect
epsg: string
enable_cross_validation: bool
random_seed: int
Return
------
"""
from iota2.Sampling.VectorFormatting import split_by_sets
from iota2.Sampling import SplitInSubSets as subset
from iota2.Common import FileUtils as fut
for new_region_shape in new_regions_shapes:
tile_name = os.path.splitext(os.path.basename(new_region_shape))[0]
vectors_to_rm = fut.FileSearch_AND(data_app_val_dir, True, tile_name)
for vect in vectors_to_rm:
os.remove(vect)
# remove seeds fields
subset.splitInSubSets(new_region_shape,
data_field,
region_field,
ratio,
seeds,
"ESRI Shapefile",
crossValidation=enable_cross_validation,
random_seed=random_seed)
split_by_sets(new_region_shape,
seeds,
data_app_val_dir,
epsg,
epsg,
tile_name,
cross_valid=enable_cross_validation)
def getListToPolygonize(path):
if path[len(path) - 1] == "/":
path = path[:-1]
listmos = fut.FileSearch_AND(path, True, ".tif")
listVect = []
for root, dirs, files in os.walk(path):
for filename in files:
if ".shp" in filename and "douglas" not in filename and "hermite" not in filename:
fileToSearch = os.path.join(
root,
os.path.splitext(filename)[0] + ".tif")
if fileToSearch in listmos:
listmos.remove(fileToSearch)
print(listmos)
input("payse")
return listmos
def prepare_annual_features(working_directory,
reference_directory,
pattern,
rename=None):
"""
double all rasters's pixels
rename must be a tuple
"""
import iota2.Common.FileUtils as fut
import shutil
for dirname, dirnames, filenames in os.walk(reference_directory):
# print path to all subdirectories first.
for subdirname in dirnames:
os.mkdir(
os.path.join(dirname, subdirname).replace(
reference_directory,
working_directory).replace(rename[0], rename[1]))
# print path to all filenames.
for filename in filenames:
shutil.copy(
os.path.join(dirname, filename),
os.path.join(dirname, filename).replace(
reference_directory,
working_directory).replace(rename[0], rename[1]))
rasters_path = fut.FileSearch_AND(working_directory, True, pattern)
for raster in rasters_path:
cmd = ('otbcli_BandMathX -il ' + raster + ' -out ' + raster +
' -exp "im1+im1"')
print(cmd)
os.system(cmd)
if rename:
all_content = []
for dirname, dirnames, filenames in os.walk(working_directory):
# print path to all subdirectories first.
for subdirname in dirnames:
all_content.append(os.path.join(dirname, subdirname))
# print path to all filenames.
for filename in filenames:
all_content.append(os.path.join(dirname, filename))
def prepare_selection(sample_sel_directory: str,
tile_name: str,
workingDirectory: Optional[str] = None):
"""
this function is dedicated to merge selection comming from different
models by tiles. It is necessary in order to prepare sampleExtraction
in the step call 'generate samples'
Parameters
----------
sample_sel_directory : string
path to the IOTA² directory containing selections by models
tile_name : string
tile's name
workingDirectory : string
path to a working directory
LOGGER : logging object
root logger
"""
wd = sample_sel_directory
if workingDirectory:
wd = workingDirectory
vectors = fut.FileSearch_AND(sample_sel_directory, True, tile_name,
"selection.sqlite")
merge_selection_name = "{}_selection_merge".format(tile_name)
output_selection_merge = os.path.join(wd, merge_selection_name + ".sqlite")
if not os.path.exists(output_selection_merge):
fut.mergeVectors(merge_selection_name,
wd,
vectors,
ext="sqlite",
out_Tbl_name="output")
if workingDirectory:
shutil.copy(
output_selection_merge,
os.path.join(sample_sel_directory,
merge_selection_name + ".sqlite"))
return os.path.join(sample_sel_directory, merge_selection_name + ".sqlite")
def region_tile(sample_sel_dir: str):
"""
"""
tile_field_name = "tile_o"
region_vectors = fut.FileSearch_AND(sample_sel_dir, True, ".shp")
output = []
region_vectors = sorted(region_vectors)
for region_vector in region_vectors:
tiles = fut.getFieldElement(region_vector,
driverName="ESRI Shapefile",
field=tile_field_name,
mode="unique",
elemType="str")
region_name = os.path.splitext(
os.path.basename(region_vector))[0].split("_")[2]
seed = os.path.splitext(
os.path.basename(region_vector))[0].split("_")[4]
tiles = sorted(tiles)
for tile in tiles:
output.append((region_name, seed, tile))
return output
def get_region_model_in_tile(current_tile: str, current_region: str,
output_path: str, path_wd: str, ref_img: str,
field_region: str, test_mode: bool,
test_path: str, test_output_folder: str) -> str:
"""
usage : rasterize region shape.
IN:
currentTile [string] : tile to compute
currentRegion [string] : current region in tile
output_path [str] : output path
pathWd [string] : working directory
refImg [string] : reference image
testMode [bool] : flag to enable test mode
testPath [string] : path to the vector shape
testOutputFolder [string] : path to the output folder
OUT:
rasterMask [string] : path to the output raster
"""
from iota2.Common.Utils import run
working_directory = os.path.join(output_path, "learningSamples")
if path_wd:
working_directory = path_wd
name_out = f"Mask_region_{current_region}_{current_tile}.tif"
if test_mode:
mask_shp = test_path
working_directory = test_output_folder
else:
mask_shp = fu.FileSearch_AND(output_path + "/shapeRegion/", True,
current_tile,
f"region_{current_region.split('f')[0]}",
".shp")[0]
raster_mask = os.path.join(working_directory, name_out)
cmd_raster = (f"otbcli_Rasterization -in {mask_shp} -mode attribute "
f"-mode.attribute.field {field_region} -im {ref_img} "
f"-out {raster_mask}")
run(cmd_raster)
return raster_mask
def test_iota2_regularisation(self):
"""Test regularization
"""
rules = mr.getMaskRegularisation(self.nomenclature)
for rule in rules:
mr.adaptRegularization(self.wd, self.raster10m,
os.path.join(self.tmp, rule[2]), "1000",
rule, 2)
rasters = fut.FileSearch_AND(self.tmp, True, "mask", ".tif")
mr.mergeRegularization(self.tmp, rasters, 10, self.outfile, "1000")
# test
outtest = testutils.rasterToArray(self.outfile)
outref = testutils.rasterToArray(self.rasterregref)
self.assertTrue(np.array_equal(outtest, outref))
# remove temporary folders
if os.path.exists(self.wd): shutil.rmtree(self.wd, ignore_errors=True)
if os.path.exists(self.out):
shutil.rmtree(self.out, ignore_errors=True)
def gen_raster_ref(vec, output_path, masks_name, working_directory):
"""
generate the reference image needed to sampleSelection application
Parameters
----------
vec : string
path to the shapeFile containing all polygons dedicated to learn
a model.
cfg : ServiceConfigFile object
working_directory : string
Path to a working directory
"""
from iota2.Common.Utils import run
tile_field_name = "tile_o"
# iota2_dir = cfg.getParam('chain', 'outputPath')
features_directory = os.path.join(output_path, "features")
tiles = fut.getFieldElement(vec,
driverName="ESRI Shapefile",
field=tile_field_name,
mode="unique",
elemType="str")
# masks_name = fut.getCommonMaskName(cfg) + ".tif"
rasters_tiles = [
fut.FileSearch_AND(os.path.join(features_directory, tile_name), True,
masks_name)[0] for tile_name in tiles
]
raster_ref_name = "ref_raster_{}.tif".format(
os.path.splitext(os.path.basename(vec))[0])
raster_ref = os.path.join(working_directory, raster_ref_name)
raster_ref_cmd = "gdal_merge.py -ot Byte -n 0 -createonly -o {} {}".format(
raster_ref, " ".join(rasters_tiles))
run(raster_ref_cmd)
return raster_ref, tiles
def get_features_application(
train_shape: str,
working_directory: str,
samples: str,
data_field: str,
output_path: str,
sar_optical_post_fusion: bool,
sensors_parameters: sensors_params_type,
ram: Optional[int] = 128,
mode: Optional[str] = "usually",
only_mask_comm: Optional[bool] = False,
only_sensors_masks: Optional[bool] = False,
logger: Optional[Logger] = LOGGER
) -> Tuple[otb_app_type, List[otb_app_type]]:
"""
usage : compute from a stack of data -> gapFilling -> features computation
-> sampleExtractions
thanks to OTB's applications'
IN:
trainShape [string] : path to a vector shape containing points
workingDirectory [string] : working directory path
samples [string] : output sqlite file
dataField [string] : data's field in trainShape
tile [string] : actual tile to compute. (ex : T31TCJ)
output_path [string] : output_path
onlyMaskComm [bool] : flag to stop the script after common Mask
computation
onlySensorsMasks [bool] : compute only masks
OUT:
sampleExtr [SampleExtraction OTB's object]:
"""
# const
# seed_position = -1
from iota2.Common import GenerateFeatures as genFeatures
from iota2.Sensors.ProcessLauncher import commonMasks
tile = train_shape.split("/")[-1].split(".")[0].split("_")[0]
working_directory_features = os.path.join(working_directory, tile)
c_mask_directory = os.path.join(output_path, "features", tile, "tmp")
if not os.path.exists(working_directory_features):
try:
os.mkdir(working_directory_features)
except OSError:
logger.warning(f"{working_directory_features} allready exists")
(all_features, feat_labels,
dep_features) = genFeatures.generateFeatures(working_directory_features,
tile,
sar_optical_post_fusion,
output_path,
sensors_parameters,
mode=mode)
if only_sensors_masks:
# return AllRefl,AllMask,datesInterp,realDates
return dep_features[1], dep_features[2], dep_features[3], dep_features[
4]
all_features.Execute()
ref = fu.FileSearch_AND(c_mask_directory, True, "MaskCommunSL.tif")
if not ref:
commonMasks(tile, output_path, sensors_parameters)
ref = fu.FileSearch_AND(c_mask_directory, True, "MaskCommunSL.tif")[0]
if only_mask_comm:
return ref
sample_extr = otb.Registry.CreateApplication("SampleExtraction")
sample_extr.SetParameterString("ram", str(0.7 * ram))
sample_extr.SetParameterString("vec", train_shape)
sample_extr.SetParameterInputImage(
"in", all_features.GetParameterOutputImage("out"))
sample_extr.SetParameterString("out", samples)
sample_extr.SetParameterString("outfield", "list")
sample_extr.SetParameterStringList("outfield.list.names", feat_labels)
sample_extr.UpdateParameters()
sample_extr.SetParameterStringList("field", [data_field.lower()])
all_dep = [all_features, dep_features]
return sample_extr, all_dep
def generate_samples_classif_mix(folder_sample: str,
working_directory: str,
train_shape: str,
path_wd: str,
output_path: str,
annual_crop: List[Union[str, int]],
all_class: List[Union[str, int]],
data_field: str,
previous_classif_path: str,
proj: int,
runs: Union[str, int],
enable_cross_validation: bool,
region_field: str,
validity_threshold: int,
target_resolution: int,
sar_optical_post_fusion: bool,
sensors_parameters: sensors_params_type,
folder_features: Optional[str] = None,
ram: Optional[int] = 128,
w_mode: Optional[bool] = False,
test_mode: Optional[bool] = False,
test_shape_region: Optional[str] = None,
sample_sel: Optional[str] = None,
mode: Optional[str] = "usually",
logger: Optional[Logger] = LOGGER):
"""
usage : from one classification, chose randomly annual sample merge
with non annual sample and extract features.
IN:
folderSample [string] : output folder
workingDirectory [string] : computation folder
trainShape [string] : vector shape (polygons) to sample
pathWd [string] : if different from None, enable HPC mode
(copy at ending)
featuresPath [string] : path to all stack
annualCrop [list of string/int] : list containing annual crops
ex : [11,12]
AllClass [list of string/int] : list containing all classes in
vector shape ex : [11,12,51..]
cfg [string] : configuration file class
previousClassifPath [string] : path to the iota2 output
directory which generate previous
classification
dataField [string] : data's field into vector shape
testMode [bool] : enable testMode -> iota2tests.py
testPrevConfig [string] : path to the configuration file which generate
previous classification
testShapeRegion [string] : path to the shapefile representing region in
the tile.
testFeaturePath [string] : path to the stack of data
OUT:
samples [string] : vector shape containing points
"""
from iota2.Sampling.SamplesSelection import prepare_selection
from iota2.Sampling import GenAnnualSamples as genAS
if os.path.exists(
os.path.join(
folder_sample,
train_shape.split("/")[-1].replace(".shp",
"_Samples.sqlite"))):
return None
if enable_cross_validation:
runs = runs - 1
features_path = os.path.join(output_path, "features")
sample_sel_directory = os.path.join(output_path, "samplesSelection")
work_dir = sample_sel_directory
if working_directory:
work_dir = working_directory
data_field = data_field.lower()
current_tile = (os.path.splitext(os.path.basename(train_shape))[0])
if sample_sel:
sample_selection = sample_sel
else:
sample_selection = prepare_selection(sample_sel_directory,
current_tile)
non_annual_shape = os.path.join(
work_dir, "{}_nonAnnual_selection.sqlite".format(current_tile))
annual_shape = os.path.join(
work_dir, "{}_annual_selection.sqlite".format(current_tile))
# garde toutes les classes pérennes
nb_feat_nannu = extract_class(sample_selection, non_annual_shape,
all_class, data_field)
regions = fu.getFieldElement(train_shape,
driverName="ESRI Shapefile",
field=region_field,
mode="unique",
elemType="str")
print(sample_selection)
print(train_shape)
# avoir la répartition des classes anuelles par seed et par region
# -> pouvoir faire annu_repartition[11][R][S]
annu_repartition = get_repartition(sample_selection, annual_crop,
data_field, region_field, regions, runs)
nb_feat_annu = get_number_annual_sample(annu_repartition)
# raster ref (in order to extract ROIs)
ref = fu.FileSearch_AND(os.path.join(features_path, current_tile), True,
"MaskCommunSL.tif")[0]
if nb_feat_nannu > 0:
all_coord = get_points_coord_in_shape(non_annual_shape, "SQLite")
else:
all_coord = [0]
classification_raster = extract_roi(os.path.join(previous_classif_path,
"final",
"Classif_Seed_0.tif"),
current_tile,
path_wd,
output_path,
f"Classif_{current_tile}",
ref,
test_mode,
test_output=folder_sample)
validity_raster = extract_roi(os.path.join(previous_classif_path, "final",
"PixelsValidity.tif"),
current_tile,
path_wd,
output_path,
f"Cloud{current_tile}",
ref,
test_mode,
test_output=folder_sample)
# build regions mask into the tile
masks = [
get_region_model_in_tile(current_tile,
current_region,
output_path,
path_wd,
classification_raster,
region_field,
test_mode,
test_shape_region,
test_output_folder=folder_sample)
for current_region in regions
]
if nb_feat_annu > 0:
annual_points = genAS.genAnnualShapePoints(
all_coord, "SQLite", working_directory, target_resolution,
annual_crop, data_field, current_tile, validity_threshold,
validity_raster, classification_raster, masks, train_shape,
annual_shape, proj, region_field, runs, annu_repartition)
merge_name = train_shape.split("/")[-1].replace(".shp", "_selectionMerge")
sample_selection = os.path.join(working_directory, f"{merge_name}.sqlite")
if (nb_feat_nannu > 0) and (nb_feat_annu > 0 and annual_points):
fu.mergeSQLite(merge_name, working_directory,
[non_annual_shape, annual_shape])
elif (nb_feat_nannu > 0) and not (nb_feat_annu > 0 and annual_points):
# If not annual samples can be added then annual classes are ignored
shutil.copy(non_annual_shape, sample_selection)
elif not (nb_feat_nannu > 0) and (nb_feat_annu > 0 and annual_points):
# If not non annual samples are found then use all annual samples
shutil.copy(annual_shape, sample_selection)
samples = os.path.join(
working_directory,
train_shape.split("/")[-1].replace(".shp", "_Samples.sqlite"))
sample_extr, dep_tmp = get_features_application(
sample_selection, working_directory, samples, data_field, output_path,
sar_optical_post_fusion, sensors_parameters, ram, mode)
# sampleExtr.ExecuteAndWriteOutput()
multi_proc = mp.Process(target=executeApp, args=[sample_extr])
multi_proc.start()
multi_proc.join()
split_vectors = split_vector_by_region(in_vect=samples,
output_dir=working_directory,
region_field=region_field,
runs=int(runs),
driver="SQLite",
proj_in="EPSG:" + str(proj),
proj_out="EPSG:" + str(proj))
if test_mode:
split_vectors = None
if path_wd and os.path.exists(samples):
for sample in split_vectors:
shutil.copy(sample, folder_sample)
if os.path.exists(non_annual_shape):
os.remove(non_annual_shape)
if os.path.exists(annual_shape):
os.remove(annual_shape)
if w_mode:
target_directory = os.path.join(folder_features, current_tile)
if not os.path.exists(target_directory):
try:
os.mkdir(target_directory)
except OSError:
logger.warning(f"{target_directory} allready exists")
try:
os.mkdir(os.path.join(target_directory, "tmp"))
except OSError:
logger.warning(f"{target_directory}/tmp allready exists")
from_dir = os.path.join(working_directory, current_tile, "tmp")
to_dir = os.path.join(target_directory, "tmp")
if os.path.exists(from_dir):
fu.updateDirectory(from_dir, to_dir)
os.remove(samples)
os.remove(classification_raster)
os.remove(validity_raster)
for mask in masks:
os.remove(mask)
return split_vectors
def test_VectorFormatting(self):
"""
test vectorFormatting function
random function is used in Sampling.VectorFormatting.VectorFormatting
we can only check if there is expected number of features with
expected fields and some features values
"""
from iota2.Sampling.VectorFormatting import vector_formatting
from iota2.Common import ServiceConfigFile as SCF
from iota2.Common import IOTA2Directory
from iota2.Common.Utils import run
from iota2.VectorTools.ChangeNameField import changeName
# define inputs
test_output = os.path.join(self.test_working_directory,
"IOTA2_dir_VectorFormatting")
# prepare ground truth
ground_truth = os.path.join(self.test_working_directory,
"groundTruth_test.shp")
cmd = "ogr2ogr -s_srs EPSG:2154 -t_srs EPSG:2154 -dialect 'SQLite' -sql 'select GEOMETRY,code from t31tcj' {} {}".format(
ground_truth, self.in_vector)
run(cmd)
# cfg instance
runs = 2
cfg = SCF.serviceConfigFile(self.config_test)
cfg.setParam('chain', 'outputPath', test_output)
cfg.setParam('chain', 'groundTruth', ground_truth)
cfg.setParam('chain', 'dataField', "code")
cfg.setParam('chain', 'cloud_threshold', 0)
cfg.setParam('chain', 'merge_final_classifications', False)
cfg.setParam('chain', 'runs', runs)
cfg.setParam('GlobChain', 'proj', "EPSG:2154")
cfg.setParam('chain', 'regionPath', self.ref_region)
IOTA2Directory.generate_directories(test_output, check_inputs=False)
# prepare expected function inputs
t31tcj_feat_dir = os.path.join(self.test_working_directory,
"IOTA2_dir_VectorFormatting",
"features", "T31TCJ")
os.mkdir(t31tcj_feat_dir)
# prepare ref img
t31tcj_ref_img = os.path.join(t31tcj_feat_dir, "MaskCommunSL.tif")
shutil.copy(self.ref_img, t31tcj_ref_img)
# prepare envelope
envelope_name = "T31TCJ.shp"
envelope_path = os.path.join(self.test_working_directory,
"IOTA2_dir_VectorFormatting", "envelope",
envelope_name)
fut.cpShapeFile(self.ref_region.replace(".shp", ""),
envelope_path.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
changeName(envelope_path, "region", "FID")
# prepare cloud mask
cloud_name = "CloudThreshold_0.shp"
cloud_path = os.path.join(self.test_working_directory,
"IOTA2_dir_VectorFormatting", "features",
"T31TCJ", cloud_name)
fut.cpShapeFile(self.ref_region.replace(".shp", ""),
cloud_path.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
changeName(cloud_path, "region", "cloud")
# launch function
ratio = cfg.getParam('chain', 'ratio')
random_seed = cfg.getParam('chain', 'random_seed')
enable_cross_validation = cfg.getParam("chain",
"enableCrossValidation")
enable_split_ground_truth = cfg.getParam('chain', 'splitGroundTruth')
fusion_merge_all_validation = cfg.getParam(
'chain', 'fusionOfClassificationAllSamplesValidation')
merge_final_classifications = cfg.getParam(
'chain', 'merge_final_classifications')
merge_final_classifications_ratio = cfg.getParam(
'chain', 'merge_final_classifications_ratio')
region_vec = cfg.getParam('chain', 'regionPath')
epsg = int(cfg.getParam('GlobChain', 'proj').split(":")[-1])
region_field = (cfg.getParam('chain', 'regionField'))
vector_formatting("T31TCJ",
test_output,
ground_truth,
"code",
0,
ratio,
random_seed,
enable_cross_validation,
enable_split_ground_truth,
fusion_merge_all_validation,
runs,
epsg,
region_field,
merge_final_classifications,
merge_final_classifications_ratio,
region_vec,
working_directory=None)
# assert
nb_features_origin = len(
fut.getFieldElement(ground_truth,
driverName="ESRI Shapefile",
field="code",
mode="all",
elemType="str"))
test_vector = fut.FileSearch_AND(
os.path.join(test_output, "formattingVectors"), True,
"T31TCJ.shp")[0]
nb_features_test = len(
fut.getFieldElement(test_vector,
driverName="ESRI Shapefile",
field="code",
mode="all",
elemType="str"))
# check nb features
self.assertTrue(nb_features_origin == nb_features_test,
msg="wrong number of features")
# check fields
origin_fields = fut.get_all_fields_in_shape(ground_truth)
test_fields = fut.get_all_fields_in_shape(test_vector)
new_fields = ['region', 'originfid', 'seed_0', 'seed_1', 'tile_o']
expected_fields = origin_fields + new_fields
self.assertTrue(len(expected_fields) == len(test_fields))
self.assertTrue(all(field in test_fields for field in expected_fields))