def footprint(self, ram=128): """ footprint """ import os import glob from iota2.Common.OtbAppBank import CreateSuperimposeApplication from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.FileUtils import ensure_dir from iota2.Common.FileUtils import FileSearch_AND footprint_dir = os.path.join(self.features_dir, "tmp") ensure_dir(footprint_dir, raise_exe=False) footprint_out = os.path.join(footprint_dir, self.footprint_name) input_dates = [ os.path.join(self.tile_directory, cdir) for cdir in os.listdir(self.tile_directory) ] input_dates = self.sort_dates_directories(input_dates) # get date's footprint date_edge = [] for date_dir in input_dates: date_edge.append( glob.glob( os.path.join( date_dir, f"{self.struct_path_masks}" f"{list(self.masks_rules.keys())[self.border_pos]}")) [0]) self.generate_raster_ref(date_edge[0]) # seek odd values, then sum it expr = [ f"(im{i+1}b1/2==rint(im{i+1}b1/2))" for i in range(len(date_edge)) ] expr = f"{'+'.join(expr)}>0?1:0" masks_rules = CreateBandMathApplication({ "il": date_edge, "ram": str(ram), "exp": expr }) masks_rules.Execute() app_dep = [masks_rules] reference_raster = self.ref_image if self.vhr_path.lower() != "none": reference_raster = FileSearch_AND(input_dates[0], True, self.data_type, "COREG", ".TIF")[0] superimp, _ = CreateSuperimposeApplication({ "inr": reference_raster, "inm": masks_rules, "out": footprint_out, "pixType": "uint8", "ram": str(ram) }) return superimp, app_dep
def footprint(self, ram=128): """ compute footprint of images """ import os import glob from iota2.Common.OtbAppBank import CreateSuperimposeApplication from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.FileUtils import ensure_dir footprint_dir = os.path.join(self.features_dir, "tmp") ensure_dir(footprint_dir, raise_exe=False) footprint_out = os.path.join(footprint_dir, self.footprint_name) input_dates = [ os.path.join(self.tile_directory, cdir) for cdir in os.listdir(self.tile_directory) ] input_dates = self.sort_dates_directories(input_dates) # get date's footprint date_edge = [] for date_dir in input_dates: date_edge.append( glob.glob( os.path.join( date_dir, "{}{}".format( self.struct_path_masks, list(self.masks_rules.keys())[self.border_pos]))) [0]) expr = " || ".join("1 - im{}b1".format(i + 1) for i in range(len(date_edge))) s2_border = CreateBandMathApplication({ "il": date_edge, "exp": expr, "ram": str(ram) }) s2_border.Execute() reference_raster = self.ref_image if self.vhr_path.lower() != "none": reference_raster = self.get_available_dates()[0] # superimpose footprint superimp, _ = CreateSuperimposeApplication({ "inr": reference_raster, "inm": s2_border, "out": footprint_out, "pixType": "uint8", "ram": str(ram) }) # needed to travel throught iota2's library app_dep = [s2_border, _] return superimp, app_dep
def footprint(self, ram=128): """ compute footprint of images """ import os from iota2.Common.OtbAppBank import CreateSuperimposeApplication from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.FileUtils import ensure_dir from iota2.Common.FileUtils import FileSearch_AND footprint_dir = os.path.join(self.features_dir, "tmp") ensure_dir(footprint_dir, raise_exe=False) footprint_out = os.path.join(footprint_dir, self.footprint_name) input_dates = [ os.path.join(self.tile_directory, cdir) for cdir in os.listdir(self.tile_directory) ] input_dates = self.sort_dates_directories(input_dates) all_scl = [] for date_dir in input_dates: r20m_dir = self.get_date_dir(date_dir, 20) scl = FileSearch_AND(r20m_dir, True, self.scene_classif)[0] all_scl.append(scl) sum_scl = "+".join( ["im{}b1".format(i + 1) for i in range(len(all_scl))]) edge = CreateBandMathApplication({ "il": all_scl, "exp": "{}==0?0:1".format(sum_scl) }) edge.Execute() app_dep = [edge] # superimpose footprint reference_raster = self.ref_image if self.vhr_path.lower() != "none": reference_raster = self.get_available_dates()[0] superimp, _ = CreateSuperimposeApplication({ "inr": reference_raster, "inm": edge, "out": footprint_out, "pixType": "uint8", "ram": str(ram) }) # needed to travel throught iota2's library app_dep.append(_) return superimp, app_dep
def footprint(self, ram=128): """ in this case (L3A), we consider the whole tile """ import os from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.FileUtils import ensure_dir footprint_dir = os.path.join(self.features_dir, "tmp") ensure_dir(footprint_dir, raise_exe=False) footprint_out = os.path.join(footprint_dir, self.footprint_name) reference_raster = self.ref_image if self.vhr_path.lower() != "none": reference_raster = self.get_available_dates()[0] s2_l3a_border = CreateBandMathApplication({ "il": reference_raster, "out": footprint_out, "exp": "1", "pixType": "uint8", "ram": str(ram) }) # needed to travel throught iota2's library app_dep = [] return s2_l3a_border, app_dep
def footprint(self, ram=128, data_value=1): """get footprint """ from gdal import Warp from osgeo.gdalconst import GDT_Byte from iota2.Common.FileUtils import FileSearch_AND from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.FileUtils import ensure_dir from iota2.Common.FileUtils import getRasterProjectionEPSG from iota2.Common.FileUtils import getRasterResolution footprint_dir = os.path.join(self.features_dir, "tmp") ensure_dir(footprint_dir, raise_exe=False) footprint_out = os.path.join(footprint_dir, self.footprint_name) user_feature = FileSearch_AND(self.tile_directory, True, self.data_type[0]) # tile reference image generation base_ref = user_feature[0] LOGGER.info("reference image generation {} from {}".format( self.ref_image, base_ref)) ensure_dir(os.path.dirname(self.ref_image), raise_exe=False) base_ref_projection = getRasterProjectionEPSG(base_ref) base_ref_res_x, _ = getRasterResolution(base_ref) if not os.path.exists(self.ref_image): Warp(self.ref_image, base_ref, multithread=True, format="GTiff", xRes=base_ref_res_x, yRes=base_ref_res_x, outputType=GDT_Byte, srcSRS="EPSG:{}".format(base_ref_projection), dstSRS="EPSG:{}".format(self.target_proj)) # user features must not contains NODATA # -> "exp" : 'data_value' mean every data available footprint = CreateBandMathApplication({ "il": self.ref_image, "out": footprint_out, "exp": str(data_value), "pixType": "uint8", "ram": str(ram) }) # needed to travel throught iota2's library app_dep = [] return footprint, app_dep
def footprint(self, ram=128): """get sentinel_1 footprint """ from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.FileUtils import FileSearch_AND s1_border_masks = FileSearch_AND(self.output_processing, True, self.mask_pattern) sum_mask = "+".join( ["im{}b1".format(i + 1) for i in range(len(s1_border_masks))]) expression = "{}=={}?0:1".format(sum_mask, len(s1_border_masks)) raster_footprint = os.path.join(self.features_dir, "tmp", self.footprint_name) footprint_app = CreateBandMathApplication({ "il": s1_border_masks, "out": raster_footprint, "exp": expression, "ram": str(ram) }) footprint_app_dep = [] return footprint_app, footprint_app_dep
def get_common_sensors_footprint(self, available_ram=128): """get common sensor's footprint provide an otbApplication ready to be 'ExecuteAndWriteOutput' generating a binary raster which is the intersection of all sensor's footprint Parameters ---------- available_ram : int RAM, usefull to many OTB's applications. Return ------ tuple (otbApplication, dependencies) """ from iota2.Common.OtbAppBank import CreateBandMathApplication sensors_footprint = [] all_dep = [] for sensor in self.enabled_sensors: footprint, _ = sensor.footprint(available_ram) footprint.Execute() sensors_footprint.append(footprint) all_dep.append(_) all_dep.append(footprint) expr = "+".join("im{}b1".format(i + 1) for i in range(len(sensors_footprint))) expr = "{}=={}?1:0".format(expr, len(sensors_footprint)) common_mask_out = os.path.join(self.common_mask_dir, self.common_mask_name) common_mask = CreateBandMathApplication({ "il": sensors_footprint, "exp": expr, "out": common_mask_out, "pixType": "uint8", "ram": str(available_ram) }) return common_mask, all_dep
def preprocess_date_masks(self, date_dir, out_prepro, working_dir=None, ram=128, logger=LOGGER): """ preprocess date mask """ import os import shutil import glob import multiprocessing as mp from iota2.Common.FileUtils import ensure_dir from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.OtbAppBank import CreateSuperimposeApplication from iota2.Common.FileUtils import getRasterProjectionEPSG from iota2.Common.OtbAppBank import executeApp # TODO : throw Exception if no masks are found date_mask = [] for mask_name, _ in list(self.masks_rules.items()): date_mask.append( glob.glob( os.path.join(date_dir, f"{self.struct_path_masks}{mask_name}"))[0]) # manage directories mask_dir = os.path.dirname(date_mask[0]) logger.debug(f"preprocessing {mask_dir} masks") mask_name = os.path.basename(date_mask[0]).replace( list(self.masks_rules.items())[0][0], "{}.tif".format(self.masks_date_suffix), ) out_mask = os.path.join(mask_dir, mask_name) if out_prepro: out_mask_dir = mask_dir.replace( os.path.join(self.l8_data, self.tile_name), out_prepro) ensure_dir(out_mask_dir, raise_exe=False) out_mask = os.path.join(out_mask_dir, mask_name) out_mask_processing = out_mask if working_dir: out_mask_processing = os.path.join(working_dir, mask_name) # build binary mask expr = "+".join([f"im{cpt+1}b1" for cpt in range(len(date_mask))]) expr = f"({expr})==0?0:1" binary_mask_rule = CreateBandMathApplication({ "il": date_mask, "exp": expr }) binary_mask_rule.Execute() # reproject using reference image superimp, _ = CreateSuperimposeApplication({ "inr": self.ref_image, "inm": binary_mask_rule, "interpolator": "nn", "out": out_mask_processing, "pixType": "uint8", "ram": str(ram) }) # needed to travel throught iota2's library app_dep = [binary_mask_rule] if self.write_dates_stack: same_proj = False if os.path.exists(out_mask): same_proj = int(getRasterProjectionEPSG(out_mask)) == int( self.target_proj) if not os.path.exists(out_mask) or same_proj is False: # ~ superimp.ExecuteAndWriteOutput() multi_proc = mp.Process(target=executeApp, args=[superimp]) multi_proc.start() multi_proc.join() if working_dir: shutil.copy(out_mask_processing, out_mask) os.remove(out_mask_processing) return superimp, app_dep
def get_time_series_masks(self, ram=128): """ get time series masks """ import os import glob from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication from iota2.Common.OtbAppBank import CreateSuperimposeApplication from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.FileUtils import ensure_dir from iota2.Common.FileUtils import getRasterProjectionEPSG time_series_dir = os.path.join(self.features_dir, "tmp") ensure_dir(time_series_dir, raise_exe=False) times_series_mask = os.path.join(time_series_dir, self.time_series_masks_name) # needed to travel throught iota2's library app_dep = [] input_dates = [ os.path.join(self.tile_directory, cdir) for cdir in os.listdir(self.tile_directory) ] input_dates = self.sort_dates_directories(input_dates) # get date's data date_data = [] div_mask_patter = list(self.masks_rules.keys())[self.border_pos] cloud_mask_patter = list(self.masks_rules.keys())[self.cloud_pos] sat_mask_patter = list(self.masks_rules.keys())[self.sat_pos] if self.vhr_path.lower() != "none": div_mask_patter = div_mask_patter.replace(".TIF", "_COREG.TIF") cloud_mask_patter = div_mask_patter.replace(".TIF", "_COREG.TIF") sat_mask_patter = div_mask_patter.replace(".TIF", "_COREG.TIF") for date_dir in input_dates: div_mask = glob.glob( os.path.join(date_dir, f"{self.struct_path_masks}{div_mask_patter}"))[0] cloud_mask = glob.glob( os.path.join( date_dir, f"{self.struct_path_masks}{cloud_mask_patter}"))[0] sat_mask = glob.glob( os.path.join(date_dir, f"{self.struct_path_masks}{sat_mask_patter}"))[0] # im1 = div, im2 = cloud, im3 = sat div_expr = "(1-(im1b1/2==rint(im1b1/2)))" cloud_expr = "im2b1" sat_expr = "im3b1" # expr = "*".join([div_expr, cloud_expr, sat_expr]) expr = f"({div_expr} + {cloud_expr} + {sat_expr})==0?0:1" date_binary_mask = CreateBandMathApplication({ "il": [div_mask, cloud_mask, sat_mask], "exp": expr }) date_binary_mask.Execute() date_data.append(date_binary_mask) app_dep.append(date_binary_mask) dates_time_series_mask = CreateConcatenateImagesApplication({ "il": date_data, "ram": str(ram), "out": times_series_mask }) origin_proj = getRasterProjectionEPSG(sat_mask) if int(origin_proj) != int(self.target_proj): dates_time_series_mask.Execute() app_dep.append(dates_time_series_mask) self.generate_raster_ref(sat_mask) dates_time_series_mask, _ = CreateSuperimposeApplication({ "inr": self.ref_image, "inm": dates_time_series_mask, "interpolator": "nn", "out": times_series_mask, "ram": str(ram) }) return dates_time_series_mask, app_dep, len(date_data)
def validity(tile_name, config_path, output_path, maskOut_name, view_threshold, workingDirectory=None, RAM=128): """ function dedicated to compute validity raster/vector by tile Parameters ---------- tile_name [string] tile's name config_path [string] absolute path to the configuration file maskOut_name [string] output vector mask's name view_threshold [int] threshold working_directory [string] absolute path to a working directory RAM [int] pipeline's size (Mo) """ import os import shutil from iota2.Common.ServiceConfigFile import iota2_parameters from iota2.Sensors.Sensors_container import sensors_container from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.Utils import run from iota2.Common.FileUtils import erodeShapeFile from iota2.Common.FileUtils import removeShape from iota2.Common.FileUtils import ensure_dir features_dir = os.path.join(output_path, "features", tile_name) validity_name = "nbView.tif" validity_out = os.path.join(features_dir, validity_name) validity_processing = validity_out if workingDirectory: ensure_dir(os.path.join(workingDirectory, tile_name)) validity_processing = os.path.join(workingDirectory, tile_name, validity_name) running_parameters = iota2_parameters(config_path) sensors_parameters = running_parameters.get_sensors_parameters(tile_name) remote_sensor_container = sensors_container(tile_name, workingDirectory, output_path, **sensors_parameters) sensors_time_series_masks = remote_sensor_container.get_sensors_time_series_masks( available_ram=RAM) sensors_masks_size = [] sensors_masks = [] for sensor_name, (time_series_masks, time_series_dep, nb_bands) in sensors_time_series_masks: if sensor_name.lower() == "sentinel1": for _, time_series_masks_app in list(time_series_masks.items()): time_series_masks_app.Execute() sensors_masks.append(time_series_masks_app) else: time_series_masks.Execute() sensors_masks.append(time_series_masks) sensors_masks_size.append(nb_bands) total_dates = sum(sensors_masks_size) merge_masks = CreateConcatenateImagesApplication({ "il": sensors_masks, "ram": str(RAM) }) merge_masks.Execute() validity_app = CreateBandMathApplication({ "il": merge_masks, "exp": "{}-({})".format( total_dates, "+".join(["im1b{}".format(i + 1) for i in range(total_dates)])), "ram": str(0.7 * RAM), "pixType": "uint8" if total_dates < 255 else "uint16", "out": validity_processing }) if not os.path.exists(os.path.join(features_dir, validity_name)): validity_app.ExecuteAndWriteOutput() if workingDirectory: shutil.copy(validity_processing, os.path.join(features_dir, validity_name)) threshold_raster_out = os.path.join(features_dir, maskOut_name.replace(".shp", ".tif")) threshold_vector_out_tmp = os.path.join( features_dir, maskOut_name.replace(".shp", "_TMP.shp")) threshold_vector_out = os.path.join(features_dir, maskOut_name) input_threshold = validity_processing if os.path.exists( validity_processing) else validity_out threshold_raster = CreateBandMathApplication({ "il": input_threshold, "exp": "im1b1>={}?1:0".format(view_threshold), "ram": str(0.7 * RAM), "pixType": "uint8", "out": threshold_raster_out }) threshold_raster.ExecuteAndWriteOutput() cmd_poly = f"gdal_polygonize.py -mask {threshold_raster_out} {threshold_raster_out} -f \"ESRI Shapefile\" {threshold_vector_out_tmp} {os.path.splitext(os.path.basename(threshold_vector_out_tmp))[0]} cloud" run(cmd_poly) erodeShapeFile(threshold_vector_out_tmp, threshold_vector_out, 0.1) os.remove(threshold_raster_out) removeShape(threshold_vector_out_tmp.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"])
def generate(self): """ """ import shutil from iota2.Common.OtbAppBank import CreateImageClassifierApplication from iota2.Common.OtbAppBank import CreateClassifyAutoContext from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.OtbAppBank import CreateBandMathXApplication from iota2.Common.FileUtils import ensure_dir if self.working_directory: self.classification = os.path.join( self.working_directory, os.path.split(self.classification)[-1]) self.confidence = os.path.join(self.working_directory, os.path.split(self.confidence)[-1]) classifier_options = { "in": self.features_stack, "model": self.classifier_model, "confmap": "{}?&writegeom=false".format(self.confidence), "ram": str(0.4 * float(self.RAM)), "pixType": self.pixType, "out": "{}?&writegeom=false".format(self.classification) } if self.auto_context: tmp_dir = os.path.join( self.output_directory, "tmp_model_{}_seed_{}_tile_{}".format(self.model_name, self.seed, self.tile)) if self.working_directory: tmp_dir = os.path.join( self.working_directory, "tmp_model_{}_seed_{}_tile_{}".format( self.model_name, self.seed, self.tile)) ensure_dir(tmp_dir) classifier_options = { "in": self.features_stack, "inseg": self.auto_context["tile_segmentation"], "models": self.classifier_model, "lablist": [str(lab) for lab in self.auto_context["labels_list"]], "confmap": "{}?&writegeom=false".format(self.confidence), "ram": str(0.4 * float(self.RAM)), "pixType": self.pixType, "tmpdir": tmp_dir, "out": "{}?&writegeom=false".format(self.classification) } if self.proba_map_path: all_class = [] for _, dico_seed in list(self.models_class.items()): for _, avail_class in list(dico_seed.items()): all_class += avail_class all_class = sorted(list(set(all_class))) nb_class_run = len(all_class) if self.working_directory: self.proba_map_path = os.path.join( self.working_directory, os.path.split(self.proba_map_path)[-1]) classifier_options["probamap"] = "{}?&writegeom=false".format( self.proba_map_path) classifier_options["nbclasses"] = str(nb_class_run) if self.stats: classifier_options["imstat"] = self.stats if self.auto_context: classifier = CreateClassifyAutoContext(classifier_options) else: classifier = CreateImageClassifierApplication(classifier_options) LOGGER.info("Compute Classification : {}".format(self.classification)) classifier.ExecuteAndWriteOutput() LOGGER.info("Classification : {} done".format(self.classification)) if self.classif_mask: mask_filter = CreateBandMathApplication({ "il": [self.classification, self.classif_mask], "ram": str(self.RAM), "pixType": self.pixType, "out": self.classification, "exp": "im2b1>=1?im1b1:0" }) mask_filter.ExecuteAndWriteOutput() mask_filter = CreateBandMathApplication({ "il": [self.confidence, self.classif_mask], "ram": str(self.RAM), "pixType": "float", "out": self.confidence, "exp": "im2b1>=1?im1b1:0" }) mask_filter.ExecuteAndWriteOutput() if self.proba_map_path: expr = "im2b1>=1?im1:{}".format("{" + ";".join(["0"] * nb_class_run) + "}") mask_filter = CreateBandMathXApplication({ "il": [self.proba_map_path, self.classif_mask], "ram": str(self.RAM), "pixType": "uint16", "out": self.proba_map_path, "exp": expr }) mask_filter.ExecuteAndWriteOutput() if self.proba_map_path: class_model = self.models_class[self.model_name][int(self.seed)] if len(class_model) != len(all_class): LOGGER.info("reordering the probability map : '{}'".format( self.proba_map_path)) self.reorder_proba_map(self.proba_map_path, self.proba_map_path, class_model, all_class) if self.working_directory: shutil.copy( self.classification, os.path.join(self.output_directory, os.path.split(self.classification)[-1])) #~ os.remove(self.classification) shutil.copy( self.confidence, os.path.join(self.output_directory, os.path.split(self.confidence)[-1])) #~ os.remove(self.confidence) if self.proba_map_path: shutil.copy( self.proba_map_path, os.path.join(self.output_directory, os.path.split(self.proba_map_path)[-1])) os.remove(self.proba_map_path) if self.auto_context: shutil.rmtree(tmp_dir)
def preprocess_date_masks(self, date_dir, out_prepro, working_dir=None, ram=128, logger=LOGGER): """ preprocess date masks """ import shutil import os import multiprocessing as mp from iota2.Common.FileUtils import FileSearch_AND from iota2.Common.FileUtils import ensure_dir from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.OtbAppBank import CreateSuperimposeApplication from iota2.Common.FileUtils import getRasterProjectionEPSG from iota2.Common.OtbAppBank import executeApp # manage directories date_mask_name = self.build_date_name(date_dir, self.masks_date_suffix) logger.debug(f"preprocessing {date_dir}") r10_dir = self.get_date_dir(date_dir, 10) out_mask = os.path.join(r10_dir, date_mask_name) if out_prepro: out_dir = r10_dir.replace(date_dir, out_prepro) ensure_dir(out_dir, raise_exe=False) out_mask = os.path.join(out_dir, date_mask_name) out_mask_processing = out_mask if working_dir: out_mask_processing = os.path.join(working_dir, date_mask_name) r20m_dir = self.get_date_dir(date_dir, 20) scl = FileSearch_AND(r20m_dir, True, self.scene_classif)[0] invalid_expr = " or ".join( ["im1b1=={}".format(flag) for flag in self.invalid_flags]) binary_mask = CreateBandMathApplication({ "il": scl, "exp": "{}?1:0".format(invalid_expr), "pixType": "uint8" }) binary_mask.Execute() app_dep = [binary_mask] superimp, _ = CreateSuperimposeApplication({ "inr": self.ref_image, "inm": binary_mask, "interpolator": "nn", "out": out_mask_processing, "pixType": "uint8", "ram": str(ram) }) if self.write_dates_stack: same_proj = False if os.path.exists(out_mask): same_proj = int(getRasterProjectionEPSG(out_mask)) == int( self.target_proj) if not os.path.exists(out_mask) or same_proj is False: # superimp.ExecuteAndWriteOutput() multi_proc = mp.Process(target=executeApp, args=[superimp]) multi_proc.start() multi_proc.join() if working_dir: shutil.copy(out_mask_processing, out_mask) os.remove(out_mask_processing) return superimp, app_dep
def preprocess_date_masks(self, date_dir, out_prepro, working_dir=None, ram=128, logger=LOGGER): """ preprocess date masks """ import os import glob from gdal import Warp import shutil import multiprocessing as mp from iota2.Common.FileUtils import ensure_dir from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.FileUtils import getRasterProjectionEPSG from iota2.Common.OtbAppBank import executeApp # TODO : throw Exception if no masks are found date_mask = glob.glob( os.path.join( date_dir, "{}{}".format(self.struct_path_masks, self.masks_pattern)))[0] # manage directories mask_dir = os.path.dirname(date_mask) logger.debug(f"preprocessing {mask_dir} masks") mask_name = os.path.basename(date_mask).replace( self.masks_pattern, f"{self.suffix_mask}.tif") out_mask = os.path.join(mask_dir, mask_name) if out_prepro: out_mask_dir = mask_dir.replace( os.path.join(self.s2_l3a_data, self.tile_name), out_prepro) ensure_dir(out_mask_dir, raise_exe=False) out_mask = os.path.join(out_mask_dir, mask_name) out_mask_processing = out_mask if working_dir: out_mask_processing = os.path.join(working_dir, mask_name) # compute mask if not os.path.exists(out_mask): mask_exp = "?1:".join( ["im1b1=={}".format(value) for value in self.masks_values]) mask_exp = "{}?1:0".format(mask_exp) mask_gen = CreateBandMathApplication({ "il": date_mask, "ram": str(ram), "exp": mask_exp, "pixType": "uint8", "out": out_mask_processing }) # mask_gen.ExecuteAndWriteOutput() multi_proc = mp.Process(target=executeApp, args=[mask_gen]) multi_proc.start() multi_proc.join() if working_dir: shutil.copy(out_mask_processing, out_mask) os.remove(out_mask_processing) # reproject if needed mask_projection = getRasterProjectionEPSG(out_mask) if int(self.target_proj) != int(mask_projection): logger.info(f"Reprojecting {out_mask}") Warp(out_mask_processing, out_mask, multithread=True, format="GTiff", xRes=10, yRes=10, srcSRS=f"EPSG:{mask_projection}", dstSRS=f"EPSG:{self.target_proj}", options=["INIT_DEST=0"]) if working_dir: shutil.copy(out_mask_processing, out_mask) os.remove(out_mask_processing) logger.info("Reprojection succeed") logger.info("End preprocessing") return out_mask
def test_train_and_classify(self): """test autoContext training """ import re from iota2.Common import IOTA2Directory from iota2.Common import ServiceConfigFile as SCF from iota2.Common.FileUtils import FileSearch_AND from iota2.Common.FileUtils import getFieldElement from iota2.Segmentation import segmentation from iota2.Learning.trainAutoContext import train_autoContext from iota2.Classification.ImageClassifier import autoContext_launch_classif from iota2.Sensors.Sensors_container import sensors_container from iota2.Common.ServiceConfigFile import iota2_parameters from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Tests.UnitTests.TestsUtils import test_raster_unique_value # config file config_path_test = os.path.join(self.test_working_directory, "Config_TEST.cfg") test_path = self.generate_cfg_file(self.config_test, config_path_test) cfg = SCF.serviceConfigFile(config_path_test) IOTA2Directory.generate_directories(test_path, check_inputs=False) autocontext_working_dir = os.path.join(self.test_working_directory, "autoContext_tmp") slic_working_dir = os.path.join(self.test_working_directory, "autoContext_tmp") iota2_dico = SCF.iota2_parameters( config_path_test).get_sensors_parameters(self.tile_name) sensors = sensors_container(self.tile_name, None, self.test_working_directory, **iota2_dico) sensors.sensors_preprocess() running_parameters = iota2_parameters(config_path_test) sensors_parameters = running_parameters.get_sensors_parameters( "T31TCJ") segmentation.slicSegmentation(self.tile_name, test_path, sensors_parameters, ram=128, working_dir=slic_working_dir, force_spw=1) slic_seg = FileSearch_AND( os.path.join(test_path, "features", self.tile_name, "tmp"), True, "SLIC_{}".format(self.tile_name))[0] train_auto_data_ref, superpix_data = self.prepare_autoContext_data_ref( slic_seg, config_path_test) parameter_dict = { "model_name": "1", "seed": "0", "list_learning_samples": [train_auto_data_ref], "list_superPixel_samples": [superpix_data], "list_tiles": ["T31TCJ"], "list_slic": [slic_seg] } # launch tests # training e = None try: train_autoContext(parameter_dict, data_field="code", output_path=test_path, sensors_parameters=sensors_parameters, superpix_data_field="superpix", iterations=3, RAM=128, WORKING_DIR=autocontext_working_dir) except Exception as e: print(e) # Asserts training self.assertTrue(e is None, msg="train_autoContext failed") models = FileSearch_AND(os.path.join(test_path, "model"), True, "model_it_", ".rf") self.assertTrue(len(models) == 4) # classification tile_raster = FileSearch_AND(self.fake_data_dir, True, "BINARY_MASK.tif")[0] tile_mask = os.path.join(autocontext_working_dir, "{}_tile_mask.tif".format(self.tile_name)) CreateBandMathApplication({ "il": [tile_raster], "out": tile_mask, "exp": "1" }).ExecuteAndWriteOutput() labels = getFieldElement(train_auto_data_ref, driverName="SQLite", field="code", mode="unique", elemType="str") parameters_dict = { "model_name": "1", "seed_num": 0, "tile": self.tile_name, "tile_segmentation": slic_seg, "tile_mask": tile_mask, "labels_list": labels, "model_list": sorted(models, key=lambda x: int(re.findall("\d", os.path.basename(x))[0])) } autoContext_launch_classif(parameters_dict, cfg.getParam("argTrain", "classifier"), "T31TCJ", cfg.getParam("argClassification", "enable_probability_map"), cfg.getParam("dimRed", "dimRed"), "code", False, False, iota2_run_dir=test_path, sar_optical_post_fusion=False, nomenclature_path=cfg.getParam( "chain", "nomenclaturePath"), sensors_parameters=sensors_parameters, RAM=128, WORKING_DIR=autocontext_working_dir) # Asserts classifications classif = FileSearch_AND(os.path.join(test_path, "classif"), True, "Classif_T31TCJ_model_1_seed_0.tif")[0] confidence = FileSearch_AND(os.path.join(test_path, "classif"), True, "T31TCJ_model_1_confidence_seed_0.tif")[0] classif_unique_0 = test_raster_unique_value(classif, 0) confidence_unique_0 = test_raster_unique_value(confidence, 0) self.assertTrue( classif_unique_0 == False, msg=("AutoContext Classifications failed : classification contains" " only 0 values")) self.assertTrue( confidence_unique_0 == False, msg=("AutoContext Classifications failed : confidence contains " "only 0 values"))