def footprint(self, ram=128):
"""
footprint
"""
import os
import glob
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.FileUtils import FileSearch_AND
footprint_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(footprint_dir, raise_exe=False)
footprint_out = os.path.join(footprint_dir, self.footprint_name)
input_dates = [
os.path.join(self.tile_directory, cdir)
for cdir in os.listdir(self.tile_directory)
]
input_dates = self.sort_dates_directories(input_dates)
# get date's footprint
date_edge = []
for date_dir in input_dates:
date_edge.append(
glob.glob(
os.path.join(
date_dir, f"{self.struct_path_masks}"
f"{list(self.masks_rules.keys())[self.border_pos]}"))
[0])
self.generate_raster_ref(date_edge[0])
# seek odd values, then sum it
expr = [
f"(im{i+1}b1/2==rint(im{i+1}b1/2))" for i in range(len(date_edge))
]
expr = f"{'+'.join(expr)}>0?1:0"
masks_rules = CreateBandMathApplication({
"il": date_edge,
"ram": str(ram),
"exp": expr
})
masks_rules.Execute()
app_dep = [masks_rules]
reference_raster = self.ref_image
if self.vhr_path.lower() != "none":
reference_raster = FileSearch_AND(input_dates[0], True,
self.data_type, "COREG",
".TIF")[0]
superimp, _ = CreateSuperimposeApplication({
"inr": reference_raster,
"inm": masks_rules,
"out": footprint_out,
"pixType": "uint8",
"ram": str(ram)
})
return superimp, app_dep
def footprint(self, ram=128):
"""
compute footprint of images
"""
import os
import glob
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.FileUtils import ensure_dir
footprint_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(footprint_dir, raise_exe=False)
footprint_out = os.path.join(footprint_dir, self.footprint_name)
input_dates = [
os.path.join(self.tile_directory, cdir)
for cdir in os.listdir(self.tile_directory)
]
input_dates = self.sort_dates_directories(input_dates)
# get date's footprint
date_edge = []
for date_dir in input_dates:
date_edge.append(
glob.glob(
os.path.join(
date_dir, "{}{}".format(
self.struct_path_masks,
list(self.masks_rules.keys())[self.border_pos])))
[0])
expr = " || ".join("1 - im{}b1".format(i + 1)
for i in range(len(date_edge)))
s2_border = CreateBandMathApplication({
"il": date_edge,
"exp": expr,
"ram": str(ram)
})
s2_border.Execute()
reference_raster = self.ref_image
if self.vhr_path.lower() != "none":
reference_raster = self.get_available_dates()[0]
# superimpose footprint
superimp, _ = CreateSuperimposeApplication({
"inr": reference_raster,
"inm": s2_border,
"out": footprint_out,
"pixType": "uint8",
"ram": str(ram)
})
# needed to travel throught iota2's library
app_dep = [s2_border, _]
return superimp, app_dep
def footprint(self, ram=128):
"""
compute footprint of images
"""
import os
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.FileUtils import FileSearch_AND
footprint_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(footprint_dir, raise_exe=False)
footprint_out = os.path.join(footprint_dir, self.footprint_name)
input_dates = [
os.path.join(self.tile_directory, cdir)
for cdir in os.listdir(self.tile_directory)
]
input_dates = self.sort_dates_directories(input_dates)
all_scl = []
for date_dir in input_dates:
r20m_dir = self.get_date_dir(date_dir, 20)
scl = FileSearch_AND(r20m_dir, True, self.scene_classif)[0]
all_scl.append(scl)
sum_scl = "+".join(
["im{}b1".format(i + 1) for i in range(len(all_scl))])
edge = CreateBandMathApplication({
"il": all_scl,
"exp": "{}==0?0:1".format(sum_scl)
})
edge.Execute()
app_dep = [edge]
# superimpose footprint
reference_raster = self.ref_image
if self.vhr_path.lower() != "none":
reference_raster = self.get_available_dates()[0]
superimp, _ = CreateSuperimposeApplication({
"inr": reference_raster,
"inm": edge,
"out": footprint_out,
"pixType": "uint8",
"ram": str(ram)
})
# needed to travel throught iota2's library
app_dep.append(_)
return superimp, app_dep
def footprint(self, ram=128):
"""
in this case (L3A), we consider the whole tile
"""
import os
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.FileUtils import ensure_dir
footprint_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(footprint_dir, raise_exe=False)
footprint_out = os.path.join(footprint_dir, self.footprint_name)
reference_raster = self.ref_image
if self.vhr_path.lower() != "none":
reference_raster = self.get_available_dates()[0]
s2_l3a_border = CreateBandMathApplication({
"il": reference_raster,
"out": footprint_out,
"exp": "1",
"pixType": "uint8",
"ram": str(ram)
})
# needed to travel throught iota2's library
app_dep = []
return s2_l3a_border, app_dep
def footprint(self, ram=128, data_value=1):
"""get footprint
"""
from gdal import Warp
from osgeo.gdalconst import GDT_Byte
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.FileUtils import getRasterProjectionEPSG
from iota2.Common.FileUtils import getRasterResolution
footprint_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(footprint_dir, raise_exe=False)
footprint_out = os.path.join(footprint_dir, self.footprint_name)
user_feature = FileSearch_AND(self.tile_directory, True,
self.data_type[0])
# tile reference image generation
base_ref = user_feature[0]
LOGGER.info("reference image generation {} from {}".format(
self.ref_image, base_ref))
ensure_dir(os.path.dirname(self.ref_image), raise_exe=False)
base_ref_projection = getRasterProjectionEPSG(base_ref)
base_ref_res_x, _ = getRasterResolution(base_ref)
if not os.path.exists(self.ref_image):
Warp(self.ref_image,
base_ref,
multithread=True,
format="GTiff",
xRes=base_ref_res_x,
yRes=base_ref_res_x,
outputType=GDT_Byte,
srcSRS="EPSG:{}".format(base_ref_projection),
dstSRS="EPSG:{}".format(self.target_proj))
# user features must not contains NODATA
# -> "exp" : 'data_value' mean every data available
footprint = CreateBandMathApplication({
"il": self.ref_image,
"out": footprint_out,
"exp": str(data_value),
"pixType": "uint8",
"ram": str(ram)
})
# needed to travel throught iota2's library
app_dep = []
return footprint, app_dep
def footprint(self, ram=128):
"""get sentinel_1 footprint
"""
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.FileUtils import FileSearch_AND
s1_border_masks = FileSearch_AND(self.output_processing, True,
self.mask_pattern)
sum_mask = "+".join(
["im{}b1".format(i + 1) for i in range(len(s1_border_masks))])
expression = "{}=={}?0:1".format(sum_mask, len(s1_border_masks))
raster_footprint = os.path.join(self.features_dir, "tmp",
self.footprint_name)
footprint_app = CreateBandMathApplication({
"il": s1_border_masks,
"out": raster_footprint,
"exp": expression,
"ram": str(ram)
})
footprint_app_dep = []
return footprint_app, footprint_app_dep
def get_common_sensors_footprint(self, available_ram=128):
"""get common sensor's footprint
provide an otbApplication ready to be 'ExecuteAndWriteOutput'
generating a binary raster which is the intersection of all sensor's
footprint
Parameters
----------
available_ram : int
RAM, usefull to many OTB's applications.
Return
------
tuple
(otbApplication, dependencies)
"""
from iota2.Common.OtbAppBank import CreateBandMathApplication
sensors_footprint = []
all_dep = []
for sensor in self.enabled_sensors:
footprint, _ = sensor.footprint(available_ram)
footprint.Execute()
sensors_footprint.append(footprint)
all_dep.append(_)
all_dep.append(footprint)
expr = "+".join("im{}b1".format(i + 1)
for i in range(len(sensors_footprint)))
expr = "{}=={}?1:0".format(expr, len(sensors_footprint))
common_mask_out = os.path.join(self.common_mask_dir,
self.common_mask_name)
common_mask = CreateBandMathApplication({
"il": sensors_footprint,
"exp": expr,
"out": common_mask_out,
"pixType": "uint8",
"ram": str(available_ram)
})
return common_mask, all_dep
def preprocess_date_masks(self,
date_dir,
out_prepro,
working_dir=None,
ram=128,
logger=LOGGER):
"""
preprocess date mask
"""
import os
import shutil
import glob
import multiprocessing as mp
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.FileUtils import getRasterProjectionEPSG
from iota2.Common.OtbAppBank import executeApp
# TODO : throw Exception if no masks are found
date_mask = []
for mask_name, _ in list(self.masks_rules.items()):
date_mask.append(
glob.glob(
os.path.join(date_dir,
f"{self.struct_path_masks}{mask_name}"))[0])
# manage directories
mask_dir = os.path.dirname(date_mask[0])
logger.debug(f"preprocessing {mask_dir} masks")
mask_name = os.path.basename(date_mask[0]).replace(
list(self.masks_rules.items())[0][0],
"{}.tif".format(self.masks_date_suffix),
)
out_mask = os.path.join(mask_dir, mask_name)
if out_prepro:
out_mask_dir = mask_dir.replace(
os.path.join(self.l8_data, self.tile_name), out_prepro)
ensure_dir(out_mask_dir, raise_exe=False)
out_mask = os.path.join(out_mask_dir, mask_name)
out_mask_processing = out_mask
if working_dir:
out_mask_processing = os.path.join(working_dir, mask_name)
# build binary mask
expr = "+".join([f"im{cpt+1}b1" for cpt in range(len(date_mask))])
expr = f"({expr})==0?0:1"
binary_mask_rule = CreateBandMathApplication({
"il": date_mask,
"exp": expr
})
binary_mask_rule.Execute()
# reproject using reference image
superimp, _ = CreateSuperimposeApplication({
"inr": self.ref_image,
"inm": binary_mask_rule,
"interpolator": "nn",
"out": out_mask_processing,
"pixType": "uint8",
"ram": str(ram)
})
# needed to travel throught iota2's library
app_dep = [binary_mask_rule]
if self.write_dates_stack:
same_proj = False
if os.path.exists(out_mask):
same_proj = int(getRasterProjectionEPSG(out_mask)) == int(
self.target_proj)
if not os.path.exists(out_mask) or same_proj is False:
# ~ superimp.ExecuteAndWriteOutput()
multi_proc = mp.Process(target=executeApp, args=[superimp])
multi_proc.start()
multi_proc.join()
if working_dir:
shutil.copy(out_mask_processing, out_mask)
os.remove(out_mask_processing)
return superimp, app_dep
def get_time_series_masks(self, ram=128):
"""
get time series masks
"""
import os
import glob
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.FileUtils import getRasterProjectionEPSG
time_series_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(time_series_dir, raise_exe=False)
times_series_mask = os.path.join(time_series_dir,
self.time_series_masks_name)
# needed to travel throught iota2's library
app_dep = []
input_dates = [
os.path.join(self.tile_directory, cdir)
for cdir in os.listdir(self.tile_directory)
]
input_dates = self.sort_dates_directories(input_dates)
# get date's data
date_data = []
div_mask_patter = list(self.masks_rules.keys())[self.border_pos]
cloud_mask_patter = list(self.masks_rules.keys())[self.cloud_pos]
sat_mask_patter = list(self.masks_rules.keys())[self.sat_pos]
if self.vhr_path.lower() != "none":
div_mask_patter = div_mask_patter.replace(".TIF", "_COREG.TIF")
cloud_mask_patter = div_mask_patter.replace(".TIF", "_COREG.TIF")
sat_mask_patter = div_mask_patter.replace(".TIF", "_COREG.TIF")
for date_dir in input_dates:
div_mask = glob.glob(
os.path.join(date_dir,
f"{self.struct_path_masks}{div_mask_patter}"))[0]
cloud_mask = glob.glob(
os.path.join(
date_dir,
f"{self.struct_path_masks}{cloud_mask_patter}"))[0]
sat_mask = glob.glob(
os.path.join(date_dir,
f"{self.struct_path_masks}{sat_mask_patter}"))[0]
# im1 = div, im2 = cloud, im3 = sat
div_expr = "(1-(im1b1/2==rint(im1b1/2)))"
cloud_expr = "im2b1"
sat_expr = "im3b1"
# expr = "*".join([div_expr, cloud_expr, sat_expr])
expr = f"({div_expr} + {cloud_expr} + {sat_expr})==0?0:1"
date_binary_mask = CreateBandMathApplication({
"il": [div_mask, cloud_mask, sat_mask],
"exp":
expr
})
date_binary_mask.Execute()
date_data.append(date_binary_mask)
app_dep.append(date_binary_mask)
dates_time_series_mask = CreateConcatenateImagesApplication({
"il":
date_data,
"ram":
str(ram),
"out":
times_series_mask
})
origin_proj = getRasterProjectionEPSG(sat_mask)
if int(origin_proj) != int(self.target_proj):
dates_time_series_mask.Execute()
app_dep.append(dates_time_series_mask)
self.generate_raster_ref(sat_mask)
dates_time_series_mask, _ = CreateSuperimposeApplication({
"inr":
self.ref_image,
"inm":
dates_time_series_mask,
"interpolator":
"nn",
"out":
times_series_mask,
"ram":
str(ram)
})
return dates_time_series_mask, app_dep, len(date_data)
def validity(tile_name,
config_path,
output_path,
maskOut_name,
view_threshold,
workingDirectory=None,
RAM=128):
"""
function dedicated to compute validity raster/vector by tile
Parameters
----------
tile_name [string]
tile's name
config_path [string]
absolute path to the configuration file
maskOut_name [string]
output vector mask's name
view_threshold [int]
threshold
working_directory [string]
absolute path to a working directory
RAM [int]
pipeline's size (Mo)
"""
import os
import shutil
from iota2.Common.ServiceConfigFile import iota2_parameters
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.Utils import run
from iota2.Common.FileUtils import erodeShapeFile
from iota2.Common.FileUtils import removeShape
from iota2.Common.FileUtils import ensure_dir
features_dir = os.path.join(output_path, "features", tile_name)
validity_name = "nbView.tif"
validity_out = os.path.join(features_dir, validity_name)
validity_processing = validity_out
if workingDirectory:
ensure_dir(os.path.join(workingDirectory, tile_name))
validity_processing = os.path.join(workingDirectory, tile_name,
validity_name)
running_parameters = iota2_parameters(config_path)
sensors_parameters = running_parameters.get_sensors_parameters(tile_name)
remote_sensor_container = sensors_container(tile_name, workingDirectory,
output_path,
**sensors_parameters)
sensors_time_series_masks = remote_sensor_container.get_sensors_time_series_masks(
available_ram=RAM)
sensors_masks_size = []
sensors_masks = []
for sensor_name, (time_series_masks, time_series_dep,
nb_bands) in sensors_time_series_masks:
if sensor_name.lower() == "sentinel1":
for _, time_series_masks_app in list(time_series_masks.items()):
time_series_masks_app.Execute()
sensors_masks.append(time_series_masks_app)
else:
time_series_masks.Execute()
sensors_masks.append(time_series_masks)
sensors_masks_size.append(nb_bands)
total_dates = sum(sensors_masks_size)
merge_masks = CreateConcatenateImagesApplication({
"il": sensors_masks,
"ram": str(RAM)
})
merge_masks.Execute()
validity_app = CreateBandMathApplication({
"il":
merge_masks,
"exp":
"{}-({})".format(
total_dates,
"+".join(["im1b{}".format(i + 1) for i in range(total_dates)])),
"ram":
str(0.7 * RAM),
"pixType":
"uint8" if total_dates < 255 else "uint16",
"out":
validity_processing
})
if not os.path.exists(os.path.join(features_dir, validity_name)):
validity_app.ExecuteAndWriteOutput()
if workingDirectory:
shutil.copy(validity_processing,
os.path.join(features_dir, validity_name))
threshold_raster_out = os.path.join(features_dir,
maskOut_name.replace(".shp", ".tif"))
threshold_vector_out_tmp = os.path.join(
features_dir, maskOut_name.replace(".shp", "_TMP.shp"))
threshold_vector_out = os.path.join(features_dir, maskOut_name)
input_threshold = validity_processing if os.path.exists(
validity_processing) else validity_out
threshold_raster = CreateBandMathApplication({
"il":
input_threshold,
"exp":
"im1b1>={}?1:0".format(view_threshold),
"ram":
str(0.7 * RAM),
"pixType":
"uint8",
"out":
threshold_raster_out
})
threshold_raster.ExecuteAndWriteOutput()
cmd_poly = f"gdal_polygonize.py -mask {threshold_raster_out} {threshold_raster_out} -f \"ESRI Shapefile\" {threshold_vector_out_tmp} {os.path.splitext(os.path.basename(threshold_vector_out_tmp))[0]} cloud"
run(cmd_poly)
erodeShapeFile(threshold_vector_out_tmp, threshold_vector_out, 0.1)
os.remove(threshold_raster_out)
removeShape(threshold_vector_out_tmp.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
def generate(self):
"""
"""
import shutil
from iota2.Common.OtbAppBank import CreateImageClassifierApplication
from iota2.Common.OtbAppBank import CreateClassifyAutoContext
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.OtbAppBank import CreateBandMathXApplication
from iota2.Common.FileUtils import ensure_dir
if self.working_directory:
self.classification = os.path.join(
self.working_directory,
os.path.split(self.classification)[-1])
self.confidence = os.path.join(self.working_directory,
os.path.split(self.confidence)[-1])
classifier_options = {
"in": self.features_stack,
"model": self.classifier_model,
"confmap": "{}?&writegeom=false".format(self.confidence),
"ram": str(0.4 * float(self.RAM)),
"pixType": self.pixType,
"out": "{}?&writegeom=false".format(self.classification)
}
if self.auto_context:
tmp_dir = os.path.join(
self.output_directory,
"tmp_model_{}_seed_{}_tile_{}".format(self.model_name,
self.seed, self.tile))
if self.working_directory:
tmp_dir = os.path.join(
self.working_directory,
"tmp_model_{}_seed_{}_tile_{}".format(
self.model_name, self.seed, self.tile))
ensure_dir(tmp_dir)
classifier_options = {
"in": self.features_stack,
"inseg": self.auto_context["tile_segmentation"],
"models": self.classifier_model,
"lablist":
[str(lab) for lab in self.auto_context["labels_list"]],
"confmap": "{}?&writegeom=false".format(self.confidence),
"ram": str(0.4 * float(self.RAM)),
"pixType": self.pixType,
"tmpdir": tmp_dir,
"out": "{}?&writegeom=false".format(self.classification)
}
if self.proba_map_path:
all_class = []
for _, dico_seed in list(self.models_class.items()):
for _, avail_class in list(dico_seed.items()):
all_class += avail_class
all_class = sorted(list(set(all_class)))
nb_class_run = len(all_class)
if self.working_directory:
self.proba_map_path = os.path.join(
self.working_directory,
os.path.split(self.proba_map_path)[-1])
classifier_options["probamap"] = "{}?&writegeom=false".format(
self.proba_map_path)
classifier_options["nbclasses"] = str(nb_class_run)
if self.stats:
classifier_options["imstat"] = self.stats
if self.auto_context:
classifier = CreateClassifyAutoContext(classifier_options)
else:
classifier = CreateImageClassifierApplication(classifier_options)
LOGGER.info("Compute Classification : {}".format(self.classification))
classifier.ExecuteAndWriteOutput()
LOGGER.info("Classification : {} done".format(self.classification))
if self.classif_mask:
mask_filter = CreateBandMathApplication({
"il": [self.classification, self.classif_mask],
"ram":
str(self.RAM),
"pixType":
self.pixType,
"out":
self.classification,
"exp":
"im2b1>=1?im1b1:0"
})
mask_filter.ExecuteAndWriteOutput()
mask_filter = CreateBandMathApplication({
"il": [self.confidence, self.classif_mask],
"ram":
str(self.RAM),
"pixType":
"float",
"out":
self.confidence,
"exp":
"im2b1>=1?im1b1:0"
})
mask_filter.ExecuteAndWriteOutput()
if self.proba_map_path:
expr = "im2b1>=1?im1:{}".format("{" + ";".join(["0"] *
nb_class_run) +
"}")
mask_filter = CreateBandMathXApplication({
"il": [self.proba_map_path, self.classif_mask],
"ram":
str(self.RAM),
"pixType":
"uint16",
"out":
self.proba_map_path,
"exp":
expr
})
mask_filter.ExecuteAndWriteOutput()
if self.proba_map_path:
class_model = self.models_class[self.model_name][int(self.seed)]
if len(class_model) != len(all_class):
LOGGER.info("reordering the probability map : '{}'".format(
self.proba_map_path))
self.reorder_proba_map(self.proba_map_path,
self.proba_map_path, class_model,
all_class)
if self.working_directory:
shutil.copy(
self.classification,
os.path.join(self.output_directory,
os.path.split(self.classification)[-1]))
#~ os.remove(self.classification)
shutil.copy(
self.confidence,
os.path.join(self.output_directory,
os.path.split(self.confidence)[-1]))
#~ os.remove(self.confidence)
if self.proba_map_path:
shutil.copy(
self.proba_map_path,
os.path.join(self.output_directory,
os.path.split(self.proba_map_path)[-1]))
os.remove(self.proba_map_path)
if self.auto_context:
shutil.rmtree(tmp_dir)
def preprocess_date_masks(self,
date_dir,
out_prepro,
working_dir=None,
ram=128,
logger=LOGGER):
"""
preprocess date masks
"""
import shutil
import os
import multiprocessing as mp
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.FileUtils import getRasterProjectionEPSG
from iota2.Common.OtbAppBank import executeApp
# manage directories
date_mask_name = self.build_date_name(date_dir, self.masks_date_suffix)
logger.debug(f"preprocessing {date_dir}")
r10_dir = self.get_date_dir(date_dir, 10)
out_mask = os.path.join(r10_dir, date_mask_name)
if out_prepro:
out_dir = r10_dir.replace(date_dir, out_prepro)
ensure_dir(out_dir, raise_exe=False)
out_mask = os.path.join(out_dir, date_mask_name)
out_mask_processing = out_mask
if working_dir:
out_mask_processing = os.path.join(working_dir, date_mask_name)
r20m_dir = self.get_date_dir(date_dir, 20)
scl = FileSearch_AND(r20m_dir, True, self.scene_classif)[0]
invalid_expr = " or ".join(
["im1b1=={}".format(flag) for flag in self.invalid_flags])
binary_mask = CreateBandMathApplication({
"il":
scl,
"exp":
"{}?1:0".format(invalid_expr),
"pixType":
"uint8"
})
binary_mask.Execute()
app_dep = [binary_mask]
superimp, _ = CreateSuperimposeApplication({
"inr": self.ref_image,
"inm": binary_mask,
"interpolator": "nn",
"out": out_mask_processing,
"pixType": "uint8",
"ram": str(ram)
})
if self.write_dates_stack:
same_proj = False
if os.path.exists(out_mask):
same_proj = int(getRasterProjectionEPSG(out_mask)) == int(
self.target_proj)
if not os.path.exists(out_mask) or same_proj is False:
# superimp.ExecuteAndWriteOutput()
multi_proc = mp.Process(target=executeApp, args=[superimp])
multi_proc.start()
multi_proc.join()
if working_dir:
shutil.copy(out_mask_processing, out_mask)
os.remove(out_mask_processing)
return superimp, app_dep
def preprocess_date_masks(self,
date_dir,
out_prepro,
working_dir=None,
ram=128,
logger=LOGGER):
"""
preprocess date masks
"""
import os
import glob
from gdal import Warp
import shutil
import multiprocessing as mp
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.FileUtils import getRasterProjectionEPSG
from iota2.Common.OtbAppBank import executeApp
# TODO : throw Exception if no masks are found
date_mask = glob.glob(
os.path.join(
date_dir, "{}{}".format(self.struct_path_masks,
self.masks_pattern)))[0]
# manage directories
mask_dir = os.path.dirname(date_mask)
logger.debug(f"preprocessing {mask_dir} masks")
mask_name = os.path.basename(date_mask).replace(
self.masks_pattern, f"{self.suffix_mask}.tif")
out_mask = os.path.join(mask_dir, mask_name)
if out_prepro:
out_mask_dir = mask_dir.replace(
os.path.join(self.s2_l3a_data, self.tile_name), out_prepro)
ensure_dir(out_mask_dir, raise_exe=False)
out_mask = os.path.join(out_mask_dir, mask_name)
out_mask_processing = out_mask
if working_dir:
out_mask_processing = os.path.join(working_dir, mask_name)
# compute mask
if not os.path.exists(out_mask):
mask_exp = "?1:".join(
["im1b1=={}".format(value) for value in self.masks_values])
mask_exp = "{}?1:0".format(mask_exp)
mask_gen = CreateBandMathApplication({
"il": date_mask,
"ram": str(ram),
"exp": mask_exp,
"pixType": "uint8",
"out": out_mask_processing
})
# mask_gen.ExecuteAndWriteOutput()
multi_proc = mp.Process(target=executeApp, args=[mask_gen])
multi_proc.start()
multi_proc.join()
if working_dir:
shutil.copy(out_mask_processing, out_mask)
os.remove(out_mask_processing)
# reproject if needed
mask_projection = getRasterProjectionEPSG(out_mask)
if int(self.target_proj) != int(mask_projection):
logger.info(f"Reprojecting {out_mask}")
Warp(out_mask_processing,
out_mask,
multithread=True,
format="GTiff",
xRes=10,
yRes=10,
srcSRS=f"EPSG:{mask_projection}",
dstSRS=f"EPSG:{self.target_proj}",
options=["INIT_DEST=0"])
if working_dir:
shutil.copy(out_mask_processing, out_mask)
os.remove(out_mask_processing)
logger.info("Reprojection succeed")
logger.info("End preprocessing")
return out_mask
def test_train_and_classify(self):
"""test autoContext training
"""
import re
from iota2.Common import IOTA2Directory
from iota2.Common import ServiceConfigFile as SCF
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.FileUtils import getFieldElement
from iota2.Segmentation import segmentation
from iota2.Learning.trainAutoContext import train_autoContext
from iota2.Classification.ImageClassifier import autoContext_launch_classif
from iota2.Sensors.Sensors_container import sensors_container
from iota2.Common.ServiceConfigFile import iota2_parameters
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Tests.UnitTests.TestsUtils import test_raster_unique_value
# config file
config_path_test = os.path.join(self.test_working_directory,
"Config_TEST.cfg")
test_path = self.generate_cfg_file(self.config_test, config_path_test)
cfg = SCF.serviceConfigFile(config_path_test)
IOTA2Directory.generate_directories(test_path, check_inputs=False)
autocontext_working_dir = os.path.join(self.test_working_directory,
"autoContext_tmp")
slic_working_dir = os.path.join(self.test_working_directory,
"autoContext_tmp")
iota2_dico = SCF.iota2_parameters(
config_path_test).get_sensors_parameters(self.tile_name)
sensors = sensors_container(self.tile_name, None,
self.test_working_directory, **iota2_dico)
sensors.sensors_preprocess()
running_parameters = iota2_parameters(config_path_test)
sensors_parameters = running_parameters.get_sensors_parameters(
"T31TCJ")
segmentation.slicSegmentation(self.tile_name,
test_path,
sensors_parameters,
ram=128,
working_dir=slic_working_dir,
force_spw=1)
slic_seg = FileSearch_AND(
os.path.join(test_path, "features", self.tile_name, "tmp"), True,
"SLIC_{}".format(self.tile_name))[0]
train_auto_data_ref, superpix_data = self.prepare_autoContext_data_ref(
slic_seg, config_path_test)
parameter_dict = {
"model_name": "1",
"seed": "0",
"list_learning_samples": [train_auto_data_ref],
"list_superPixel_samples": [superpix_data],
"list_tiles": ["T31TCJ"],
"list_slic": [slic_seg]
}
# launch tests
# training
e = None
try:
train_autoContext(parameter_dict,
data_field="code",
output_path=test_path,
sensors_parameters=sensors_parameters,
superpix_data_field="superpix",
iterations=3,
RAM=128,
WORKING_DIR=autocontext_working_dir)
except Exception as e:
print(e)
# Asserts training
self.assertTrue(e is None, msg="train_autoContext failed")
models = FileSearch_AND(os.path.join(test_path, "model"), True,
"model_it_", ".rf")
self.assertTrue(len(models) == 4)
# classification
tile_raster = FileSearch_AND(self.fake_data_dir, True,
"BINARY_MASK.tif")[0]
tile_mask = os.path.join(autocontext_working_dir,
"{}_tile_mask.tif".format(self.tile_name))
CreateBandMathApplication({
"il": [tile_raster],
"out": tile_mask,
"exp": "1"
}).ExecuteAndWriteOutput()
labels = getFieldElement(train_auto_data_ref,
driverName="SQLite",
field="code",
mode="unique",
elemType="str")
parameters_dict = {
"model_name":
"1",
"seed_num":
0,
"tile":
self.tile_name,
"tile_segmentation":
slic_seg,
"tile_mask":
tile_mask,
"labels_list":
labels,
"model_list":
sorted(models,
key=lambda x: int(re.findall("\d", os.path.basename(x))[0]))
}
autoContext_launch_classif(parameters_dict,
cfg.getParam("argTrain", "classifier"),
"T31TCJ",
cfg.getParam("argClassification",
"enable_probability_map"),
cfg.getParam("dimRed", "dimRed"),
"code",
False,
False,
iota2_run_dir=test_path,
sar_optical_post_fusion=False,
nomenclature_path=cfg.getParam(
"chain", "nomenclaturePath"),
sensors_parameters=sensors_parameters,
RAM=128,
WORKING_DIR=autocontext_working_dir)
# Asserts classifications
classif = FileSearch_AND(os.path.join(test_path, "classif"), True,
"Classif_T31TCJ_model_1_seed_0.tif")[0]
confidence = FileSearch_AND(os.path.join(test_path, "classif"), True,
"T31TCJ_model_1_confidence_seed_0.tif")[0]
classif_unique_0 = test_raster_unique_value(classif, 0)
confidence_unique_0 = test_raster_unique_value(confidence, 0)
self.assertTrue(
classif_unique_0 == False,
msg=("AutoContext Classifications failed : classification contains"
" only 0 values"))
self.assertTrue(
confidence_unique_0 == False,
msg=("AutoContext Classifications failed : confidence contains "
"only 0 values"))