def footprint(self, ram=128):
"""
footprint
"""
import os
import glob
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.FileUtils import FileSearch_AND
footprint_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(footprint_dir, raise_exe=False)
footprint_out = os.path.join(footprint_dir, self.footprint_name)
input_dates = [
os.path.join(self.tile_directory, cdir)
for cdir in os.listdir(self.tile_directory)
]
input_dates = self.sort_dates_directories(input_dates)
# get date's footprint
date_edge = []
for date_dir in input_dates:
date_edge.append(
glob.glob(
os.path.join(
date_dir, f"{self.struct_path_masks}"
f"{list(self.masks_rules.keys())[self.border_pos]}"))
[0])
self.generate_raster_ref(date_edge[0])
# seek odd values, then sum it
expr = [
f"(im{i+1}b1/2==rint(im{i+1}b1/2))" for i in range(len(date_edge))
]
expr = f"{'+'.join(expr)}>0?1:0"
masks_rules = CreateBandMathApplication({
"il": date_edge,
"ram": str(ram),
"exp": expr
})
masks_rules.Execute()
app_dep = [masks_rules]
reference_raster = self.ref_image
if self.vhr_path.lower() != "none":
reference_raster = FileSearch_AND(input_dates[0], True,
self.data_type, "COREG",
".TIF")[0]
superimp, _ = CreateSuperimposeApplication({
"inr": reference_raster,
"inm": masks_rules,
"out": footprint_out,
"pixType": "uint8",
"ram": str(ram)
})
return superimp, app_dep
def footprint(self, ram=128):
"""
compute footprint of images
"""
import os
import glob
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.FileUtils import ensure_dir
footprint_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(footprint_dir, raise_exe=False)
footprint_out = os.path.join(footprint_dir, self.footprint_name)
input_dates = [
os.path.join(self.tile_directory, cdir)
for cdir in os.listdir(self.tile_directory)
]
input_dates = self.sort_dates_directories(input_dates)
# get date's footprint
date_edge = []
for date_dir in input_dates:
date_edge.append(
glob.glob(
os.path.join(
date_dir, "{}{}".format(
self.struct_path_masks,
list(self.masks_rules.keys())[self.border_pos])))
[0])
expr = " || ".join("1 - im{}b1".format(i + 1)
for i in range(len(date_edge)))
s2_border = CreateBandMathApplication({
"il": date_edge,
"exp": expr,
"ram": str(ram)
})
s2_border.Execute()
reference_raster = self.ref_image
if self.vhr_path.lower() != "none":
reference_raster = self.get_available_dates()[0]
# superimpose footprint
superimp, _ = CreateSuperimposeApplication({
"inr": reference_raster,
"inm": s2_border,
"out": footprint_out,
"pixType": "uint8",
"ram": str(ram)
})
# needed to travel throught iota2's library
app_dep = [s2_border, _]
return superimp, app_dep
def footprint(self, ram=128):
"""
compute footprint of images
"""
import os
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.FileUtils import FileSearch_AND
footprint_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(footprint_dir, raise_exe=False)
footprint_out = os.path.join(footprint_dir, self.footprint_name)
input_dates = [
os.path.join(self.tile_directory, cdir)
for cdir in os.listdir(self.tile_directory)
]
input_dates = self.sort_dates_directories(input_dates)
all_scl = []
for date_dir in input_dates:
r20m_dir = self.get_date_dir(date_dir, 20)
scl = FileSearch_AND(r20m_dir, True, self.scene_classif)[0]
all_scl.append(scl)
sum_scl = "+".join(
["im{}b1".format(i + 1) for i in range(len(all_scl))])
edge = CreateBandMathApplication({
"il": all_scl,
"exp": "{}==0?0:1".format(sum_scl)
})
edge.Execute()
app_dep = [edge]
# superimpose footprint
reference_raster = self.ref_image
if self.vhr_path.lower() != "none":
reference_raster = self.get_available_dates()[0]
superimp, _ = CreateSuperimposeApplication({
"inr": reference_raster,
"inm": edge,
"out": footprint_out,
"pixType": "uint8",
"ram": str(ram)
})
# needed to travel throught iota2's library
app_dep.append(_)
return superimp, app_dep
def get_features(self, ram=128, logger=LOGGER):
"""generate user features. Concatenates all of them
"""
from gdal import Warp
from osgeo.gdalconst import GDT_Byte
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.FileUtils import getRasterProjectionEPSG
from iota2.Common.FileUtils import getRasterResolution
from iota2.Common.FileUtils import getRasterNbands
features_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(features_dir, raise_exe=False)
features_out = os.path.join(features_dir, self.features_names)
user_features_found = []
user_features_bands = []
for pattern in self.data_type:
user_feature = FileSearch_AND(self.tile_directory, True, pattern)
if user_feature:
user_features_bands.append(getRasterNbands(user_feature[0]))
user_features_found.append(user_feature[0])
else:
msg = "WARNING : '{}' not found in {}".format(
pattern, self.tile_directory)
logger.error(msg)
raise Exception(msg)
user_feat_stack = CreateConcatenateImagesApplication({
"il":
user_features_found,
"ram":
str(ram),
"out":
features_out
})
base_ref = user_features_found[0]
base_ref_projection = getRasterProjectionEPSG(base_ref)
if not os.path.exists(self.ref_image):
base_ref_res_x, _ = getRasterResolution(base_ref)
Warp(self.ref_image,
base_ref,
multithread=True,
format="GTiff",
xRes=base_ref_res_x,
yRes=base_ref_res_x,
outputType=GDT_Byte,
srcSRS="EPSG:{}".format(base_ref_projection),
dstSRS="EPSG:{}".format(self.target_proj))
app_dep = []
if int(base_ref_projection) != (self.target_proj):
user_feat_stack.Execute()
app_dep.append(user_feat_stack)
user_feat_stack, _ = CreateSuperimposeApplication({
"inr": self.ref_image,
"inm": user_feat_stack,
"out": features_out,
"ram": str(ram)
})
features_labels = [
"{}_band_{}".format(pattern, band_num)
for pattern, nb_bands in zip(self.data_type, user_features_bands)
for band_num in range(nb_bands)
]
return (user_feat_stack, app_dep), features_labels
def preprocess_date_masks(self,
date_dir,
out_prepro,
working_dir=None,
ram=128,
logger=LOGGER):
"""
preprocess date mask
"""
import os
import shutil
import glob
import multiprocessing as mp
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.FileUtils import getRasterProjectionEPSG
from iota2.Common.OtbAppBank import executeApp
# TODO : throw Exception if no masks are found
date_mask = []
for mask_name, _ in list(self.masks_rules.items()):
date_mask.append(
glob.glob(
os.path.join(date_dir,
f"{self.struct_path_masks}{mask_name}"))[0])
# manage directories
mask_dir = os.path.dirname(date_mask[0])
logger.debug(f"preprocessing {mask_dir} masks")
mask_name = os.path.basename(date_mask[0]).replace(
list(self.masks_rules.items())[0][0],
"{}.tif".format(self.masks_date_suffix),
)
out_mask = os.path.join(mask_dir, mask_name)
if out_prepro:
out_mask_dir = mask_dir.replace(
os.path.join(self.l8_data, self.tile_name), out_prepro)
ensure_dir(out_mask_dir, raise_exe=False)
out_mask = os.path.join(out_mask_dir, mask_name)
out_mask_processing = out_mask
if working_dir:
out_mask_processing = os.path.join(working_dir, mask_name)
# build binary mask
expr = "+".join([f"im{cpt+1}b1" for cpt in range(len(date_mask))])
expr = f"({expr})==0?0:1"
binary_mask_rule = CreateBandMathApplication({
"il": date_mask,
"exp": expr
})
binary_mask_rule.Execute()
# reproject using reference image
superimp, _ = CreateSuperimposeApplication({
"inr": self.ref_image,
"inm": binary_mask_rule,
"interpolator": "nn",
"out": out_mask_processing,
"pixType": "uint8",
"ram": str(ram)
})
# needed to travel throught iota2's library
app_dep = [binary_mask_rule]
if self.write_dates_stack:
same_proj = False
if os.path.exists(out_mask):
same_proj = int(getRasterProjectionEPSG(out_mask)) == int(
self.target_proj)
if not os.path.exists(out_mask) or same_proj is False:
# ~ superimp.ExecuteAndWriteOutput()
multi_proc = mp.Process(target=executeApp, args=[superimp])
multi_proc.start()
multi_proc.join()
if working_dir:
shutil.copy(out_mask_processing, out_mask)
os.remove(out_mask_processing)
return superimp, app_dep
def preprocess_date(self,
date_dir,
out_prepro,
working_dir=None,
ram=128,
logger=LOGGER):
"""
Preprocess each date
"""
import os
import shutil
from gdal import Warp
import multiprocessing as mp
from osgeo.gdalconst import GDT_Byte
from collections import OrderedDict
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.FileUtils import getRasterProjectionEPSG
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.OtbAppBank import executeApp
# manage directories
date_stack_name = self.build_stack_date_name(date_dir)
logger.debug(f"preprocessing {date_dir}")
out_stack = os.path.join(date_dir, date_stack_name)
if out_prepro:
_, date_dir_name = os.path.split(date_dir)
out_dir = os.path.join(out_prepro, date_dir_name)
if not os.path.exists(out_dir):
try:
os.mkdir(out_dir)
except OSError:
logger.warning(f"{out_dir} already exists")
out_stack = os.path.join(out_dir, date_stack_name)
out_stack_processing = out_stack
if working_dir:
out_stack_processing = os.path.join(working_dir, date_stack_name)
# get bands
date_bands = [
FileSearch_AND(date_dir, True,
"{}_{}.tif".format(self.data_type, bands_name))[0]
for bands_name in self.stack_band_position
]
# tile reference image generation
base_ref = date_bands[0]
ensure_dir(os.path.dirname(self.ref_image), raise_exe=False)
base_ref_projection = getRasterProjectionEPSG(base_ref)
if not os.path.exists(self.ref_image):
logger.info(
f"reference image generation {self.ref_image} from {base_ref}")
Warp(self.ref_image,
base_ref,
multithread=True,
format="GTiff",
xRes=self.native_res,
yRes=self.native_res,
outputType=GDT_Byte,
srcSRS="EPSG:{}".format(base_ref_projection),
dstSRS="EPSG:{}".format(self.target_proj))
# reproject / resample
bands_proj = OrderedDict()
all_reproj = []
for band, band_name in zip(date_bands, self.stack_band_position):
superimp, _ = CreateSuperimposeApplication({
"inr": self.ref_image,
"inm": band,
"ram": str(ram)
})
bands_proj[band_name] = superimp
all_reproj.append(superimp)
if self.write_dates_stack:
for reproj in all_reproj:
reproj.Execute()
date_stack = CreateConcatenateImagesApplication({
"il":
all_reproj,
"ram":
str(ram),
"pixType":
"int16",
"out":
out_stack_processing
})
same_proj = False
if os.path.exists(out_stack):
same_proj = int(getRasterProjectionEPSG(out_stack)) == int(
self.target_proj)
if not os.path.exists(out_stack) or same_proj is False:
# ~ date_stack.ExecuteAndWriteOutput()
multi_proc = mp.Process(target=executeApp, args=[date_stack])
multi_proc.start()
multi_proc.join()
if working_dir:
shutil.copy(out_stack_processing, out_stack)
os.remove(out_stack_processing)
return bands_proj if self.write_dates_stack is False else out_stack
def get_time_series_masks(self, ram=128):
"""
get time series masks
"""
import os
import glob
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.FileUtils import getRasterProjectionEPSG
time_series_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(time_series_dir, raise_exe=False)
times_series_mask = os.path.join(time_series_dir,
self.time_series_masks_name)
# needed to travel throught iota2's library
app_dep = []
input_dates = [
os.path.join(self.tile_directory, cdir)
for cdir in os.listdir(self.tile_directory)
]
input_dates = self.sort_dates_directories(input_dates)
# get date's data
date_data = []
div_mask_patter = list(self.masks_rules.keys())[self.border_pos]
cloud_mask_patter = list(self.masks_rules.keys())[self.cloud_pos]
sat_mask_patter = list(self.masks_rules.keys())[self.sat_pos]
if self.vhr_path.lower() != "none":
div_mask_patter = div_mask_patter.replace(".TIF", "_COREG.TIF")
cloud_mask_patter = div_mask_patter.replace(".TIF", "_COREG.TIF")
sat_mask_patter = div_mask_patter.replace(".TIF", "_COREG.TIF")
for date_dir in input_dates:
div_mask = glob.glob(
os.path.join(date_dir,
f"{self.struct_path_masks}{div_mask_patter}"))[0]
cloud_mask = glob.glob(
os.path.join(
date_dir,
f"{self.struct_path_masks}{cloud_mask_patter}"))[0]
sat_mask = glob.glob(
os.path.join(date_dir,
f"{self.struct_path_masks}{sat_mask_patter}"))[0]
# im1 = div, im2 = cloud, im3 = sat
div_expr = "(1-(im1b1/2==rint(im1b1/2)))"
cloud_expr = "im2b1"
sat_expr = "im3b1"
# expr = "*".join([div_expr, cloud_expr, sat_expr])
expr = f"({div_expr} + {cloud_expr} + {sat_expr})==0?0:1"
date_binary_mask = CreateBandMathApplication({
"il": [div_mask, cloud_mask, sat_mask],
"exp":
expr
})
date_binary_mask.Execute()
date_data.append(date_binary_mask)
app_dep.append(date_binary_mask)
dates_time_series_mask = CreateConcatenateImagesApplication({
"il":
date_data,
"ram":
str(ram),
"out":
times_series_mask
})
origin_proj = getRasterProjectionEPSG(sat_mask)
if int(origin_proj) != int(self.target_proj):
dates_time_series_mask.Execute()
app_dep.append(dates_time_series_mask)
self.generate_raster_ref(sat_mask)
dates_time_series_mask, _ = CreateSuperimposeApplication({
"inr":
self.ref_image,
"inm":
dates_time_series_mask,
"interpolator":
"nn",
"out":
times_series_mask,
"ram":
str(ram)
})
return dates_time_series_mask, app_dep, len(date_data)
def get_time_series(self, ram=128):
"""
TODO : be able of using a date interval
Return
------
list
[(otb_Application, some otb's objects), time_series_labels]
Functions dealing with otb's application instance has to
returns every objects in the pipeline
"""
import os
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.FileUtils import getRasterProjectionEPSG
from iota2.Common.FileUtils import FileSearch_AND
# needed to travel throught iota2's library
app_dep = []
input_dates = [
os.path.join(self.tile_directory, cdir)
for cdir in os.listdir(self.tile_directory)
]
input_dates = self.sort_dates_directories(input_dates)
# get date's data
date_data = []
for date_dir in input_dates:
l5_old_date = FileSearch_AND(date_dir, True, self.data_type,
".TIF")[0]
if self.vhr_path.lower() != "none":
l5_old_date = FileSearch_AND(date_dir, True, self.data_type,
"COREG", ".TIF")[0]
date_data.append(l5_old_date)
time_series_dir = os.path.join(self.features_dir, "tmp")
ensure_dir(time_series_dir, raise_exe=False)
times_series_raster = os.path.join(time_series_dir,
self.time_series_name)
dates_time_series = CreateConcatenateImagesApplication({
"il": date_data,
"out": times_series_raster,
"ram": str(ram)
})
_, dates_in = self.write_dates_file()
# build labels
features_labels = [
f"{self.__class__.name}_{band_name}_{date}" for date in dates_in
for band_name in self.stack_band_position
]
# if not all bands must be used
if self.extracted_bands:
app_dep.append(dates_time_series)
(dates_time_series,
features_labels) = self.extract_bands_time_series(
dates_time_series, dates_in, len(self.stack_band_position),
self.extracted_bands, ram)
origin_proj = getRasterProjectionEPSG(date_data[0])
if int(origin_proj) != int(self.target_proj):
dates_time_series.Execute()
app_dep.append(dates_time_series)
self.generate_raster_ref(date_data[0])
dates_time_series, _ = CreateSuperimposeApplication({
"inr": self.ref_image,
"inm": self.masks_rules,
"out": times_series_raster,
"ram": str(ram)
})
return (dates_time_series, app_dep), features_labels
def preprocess_date_masks(self,
date_dir,
out_prepro,
working_dir=None,
ram=128,
logger=LOGGER):
"""
preprocess date masks
"""
import shutil
import os
import multiprocessing as mp
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.OtbAppBank import CreateBandMathApplication
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.FileUtils import getRasterProjectionEPSG
from iota2.Common.OtbAppBank import executeApp
# manage directories
date_mask_name = self.build_date_name(date_dir, self.masks_date_suffix)
logger.debug(f"preprocessing {date_dir}")
r10_dir = self.get_date_dir(date_dir, 10)
out_mask = os.path.join(r10_dir, date_mask_name)
if out_prepro:
out_dir = r10_dir.replace(date_dir, out_prepro)
ensure_dir(out_dir, raise_exe=False)
out_mask = os.path.join(out_dir, date_mask_name)
out_mask_processing = out_mask
if working_dir:
out_mask_processing = os.path.join(working_dir, date_mask_name)
r20m_dir = self.get_date_dir(date_dir, 20)
scl = FileSearch_AND(r20m_dir, True, self.scene_classif)[0]
invalid_expr = " or ".join(
["im1b1=={}".format(flag) for flag in self.invalid_flags])
binary_mask = CreateBandMathApplication({
"il":
scl,
"exp":
"{}?1:0".format(invalid_expr),
"pixType":
"uint8"
})
binary_mask.Execute()
app_dep = [binary_mask]
superimp, _ = CreateSuperimposeApplication({
"inr": self.ref_image,
"inm": binary_mask,
"interpolator": "nn",
"out": out_mask_processing,
"pixType": "uint8",
"ram": str(ram)
})
if self.write_dates_stack:
same_proj = False
if os.path.exists(out_mask):
same_proj = int(getRasterProjectionEPSG(out_mask)) == int(
self.target_proj)
if not os.path.exists(out_mask) or same_proj is False:
# superimp.ExecuteAndWriteOutput()
multi_proc = mp.Process(target=executeApp, args=[superimp])
multi_proc.start()
multi_proc.join()
if working_dir:
shutil.copy(out_mask_processing, out_mask)
os.remove(out_mask_processing)
return superimp, app_dep
def preprocess_date(self,
date_dir,
out_prepro,
working_dir=None,
ram=128,
logger=LOGGER):
"""
preprocess date
"""
import os
import shutil
from collections import OrderedDict
from gdal import Warp
from osgeo.gdalconst import GDT_Byte
import multiprocessing as mp
from iota2.Common.FileUtils import ensure_dir
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.FileUtils import getRasterProjectionEPSG
from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication
from iota2.Common.OtbAppBank import CreateSuperimposeApplication
from iota2.Common.OtbAppBank import executeApp
# manage directories
date_stack_name = self.build_date_name(date_dir, self.suffix)
logger.debug(f"preprocessing {date_dir}")
r10_dir = self.get_date_dir(date_dir, 10)
out_stack = os.path.join(r10_dir, date_stack_name)
if out_prepro:
out_dir = r10_dir.replace(date_dir, out_prepro)
ensure_dir(out_dir, raise_exe=False)
out_stack = os.path.join(out_dir, date_stack_name)
out_stack_processing = out_stack
if working_dir:
out_stack_processing = os.path.join(working_dir, date_stack_name)
# get bands
date_bands = []
for band in self.stack_band_position:
if band in ["B02", "B03", "B04", "B08"]:
date_bands.append(
FileSearch_AND(date_dir, True,
"{}_".format(self.tile_name),
"{}_10m.jp2".format(band))[0])
elif band in ["B05", "B06", "B07", "B8A", "B11", "B12"]:
date_bands.append(
FileSearch_AND(date_dir, True,
"{}_".format(self.tile_name),
"{}_20m.jp2".format(band))[0])
# tile reference image generation
base_ref = date_bands[0]
logger.info(f"reference image generation {self.ref_image}"
f" from {base_ref}")
ensure_dir(os.path.dirname(self.ref_image), raise_exe=False)
base_ref_projection = getRasterProjectionEPSG(base_ref)
if not os.path.exists(self.ref_image):
Warp(self.ref_image,
base_ref,
multithread=True,
format="GTiff",
xRes=10,
yRes=10,
outputType=GDT_Byte,
srcSRS="EPSG:{}".format(base_ref_projection),
dstSRS="EPSG:{}".format(self.target_proj))
# reproject / resample
bands_proj = OrderedDict()
all_reproj = []
for band, band_name in zip(date_bands, self.stack_band_position):
superimp, _ = CreateSuperimposeApplication({
"inr": self.ref_image,
"inm": band,
"ram": str(ram)
})
bands_proj[band_name] = superimp
all_reproj.append(superimp)
if self.write_dates_stack:
for reproj in all_reproj:
reproj.Execute()
date_stack = CreateConcatenateImagesApplication({
"il":
all_reproj,
"ram":
str(ram),
"pixType":
"int16",
"out":
out_stack_processing
})
same_proj = False
if os.path.exists(out_stack):
same_proj = int(getRasterProjectionEPSG(out_stack)) == int(
self.target_proj)
if not os.path.exists(out_stack) or same_proj is False:
# date_stack.ExecuteAndWriteOutput()
multi_proc = mp.Process(target=executeApp, args=[date_stack])
multi_proc.start()
multi_proc.join()
if working_dir:
shutil.copy(out_stack_processing, out_stack)
os.remove(out_stack_processing)
return bands_proj if self.write_dates_stack is False else out_stack