def test_config_dictionaries(self): """check iota2_parameters class """ from iota2.Common.ServiceConfigFile import iota2_parameters sensors_parameters = iota2_parameters( self.config_test).get_sensors_parameters("T31TCJ") self.assertTrue(len(sensors_parameters.keys()) == 3)
def preprocess(tile_name, config_path, output_path, working_directory=None, RAM=128): """ preprocessing input rasters data by tile Parameters ---------- tile_name [string] tile's name config_path [string] absolute path to the configuration file output_path : str iota2 output path working_directory [string] absolute path to a working directory RAM [int] pipeline's size (Mo) """ from iota2.Sensors.Sensors_container import sensors_container from iota2.Common.ServiceConfigFile import iota2_parameters running_parameters = iota2_parameters(config_path) sensors_parameters = running_parameters.get_sensors_parameters(tile_name) remote_sensor_container = sensors_container(tile_name, working_directory, output_path, **sensors_parameters) remote_sensor_container.sensors_preprocess(available_ram=RAM)
def step_inputs(self): """ Return ------ the return could be and iterable or a callable """ from iota2.Common import FileUtils as fut from iota2.Classification.ImageClassifier import autoContext_classification_param from iota2.Common.ServiceConfigFile import iota2_parameters if self.enable_autoContext is False and self.use_scikitlearn is False: parameters = fut.parseClassifCmd( os.path.join(self.output_path, "cmd", "cla", "class.txt")) elif self.enable_autoContext is True: parameters = autoContext_classification_param( self.output_path, self.data_field) elif self.use_scikitlearn: parameters = fut.parseClassifCmd( os.path.join(self.output_path, "cmd", "cla", "class.txt")) running_parameters = iota2_parameters(self.cfg) parameters = [{ "mask": param[1], "model": param[2], "stat": param[3], "out_classif": param[4], "out_confidence": param[5], "out_proba": None, "working_dir": param[6], "tile_name": param[8], "sar_optical_post_fusion": SCF.serviceConfigFile(self.cfg).getParam( 'argTrain', 'dempster_shafer_SAR_Opt_fusion'), "output_path": SCF.serviceConfigFile(self.cfg).getParam( 'chain', 'outputPath'), "sensors_parameters": running_parameters.get_sensors_parameters(tile_name=param[8]), "pixel_type": param[17], "number_of_chunks": self.scikit_tile_split, "targeted_chunk": target_chunk, "ram": param[19] } for param in parameters for target_chunk in range(self.scikit_tile_split)] return parameters
def step_execute(self): """ Return ------ lambda the function to execute as a lambda function. The returned object must be a lambda function. """ from iota2.MPI import launch_tasks as tLauncher from iota2.Learning import TrainSkLearn from iota2.Learning.TrainSkLearn import cast_config_cv_parameters from iota2.Learning.trainAutoContext import train_autoContext from iota2.Common.ServiceConfigFile import iota2_parameters if self.use_scikitlearn is True and self.enable_autoContext is False: model_parameters = SCF.serviceConfigFile( self.cfg).getSection("scikit_models_parameters") sk_model = model_parameters['model_type'] del model_parameters['model_type'] del model_parameters['standardization'] del model_parameters['cross_validation_grouped'] del model_parameters['cross_validation_folds'] del model_parameters['cross_validation_parameters'] cv_params = SCF.serviceConfigFile(self.cfg).getParam( "scikit_models_parameters", "cross_validation_parameters") cv_params = cast_config_cv_parameters(cv_params) step_function = lambda x: TrainSkLearn.sk_learn( x["learning_file"], x["feat_labels"], x[ "model_path"], self.data_field, sk_model, SCF.serviceConfigFile(self.cfg).getParam( "scikit_models_parameters", "standardization"), cv_params, SCF.serviceConfigFile(self.cfg).getParam( "scikit_models_parameters", "cross_validation_grouped"), SCF.serviceConfigFile(self.cfg).getParam( "scikit_models_parameters", "cross_validation_folds"), self .available_ram, **model_parameters) elif self.enable_autoContext is True: running_parameters = iota2_parameters(self.cfg) step_function = lambda x: train_autoContext( x, SCF.serviceConfigFile(self.cfg).getParam("chain", "dataField" ), self.output_path, running_parameters.get_sensors_parameters(self.tiles[ 0]), self.superpix_data_field, self.autoContext_iterations, self.RAM, self.workingDirectory) else: bashLauncherFunction = tLauncher.launchBashCmd step_function = lambda x: bashLauncherFunction(x) return step_function
def step_execute(self): """ Return ------ lambda the function to execute as a lambda function. The returned object must be a lambda function. """ from iota2.Classification.ImageClassifier import autoContext_launch_classif from iota2.Classification import ImageClassifier as imageClassifier from iota2.Common.ServiceConfigFile import iota2_parameters from iota2.Classification import skClassifier from iota2.MPI import launch_tasks as tLauncher if self.enable_autoContext is False and self.use_scikitlearn is False: launch_py_cmd = tLauncher.launchPythonCmd step_function = lambda x: launch_py_cmd( imageClassifier.launchClassification, *x) elif self.enable_autoContext is True and self.use_scikitlearn is False: running_parameters = iota2_parameters(self.cfg) step_function = lambda x: autoContext_launch_classif( x, SCF.serviceConfigFile(self.cfg).getParam( 'argTrain', 'classifier'), x["tile"], SCF.serviceConfigFile(self.cfg).getParam( 'argClassification', 'enable_probability_map'), SCF.serviceConfigFile(self.cfg).getParam('dimRed', 'dimRed'), SCF.serviceConfigFile(self.cfg).getParam('chain', 'dataField'), SCF.serviceConfigFile(self.cfg).getParam( 'GlobChain', 'writeOutputs'), SCF.serviceConfigFile(self.cfg).getParam( 'dimRed', 'reductionMode'), SCF.serviceConfigFile(self.cfg).getParam( 'chain', 'outputPath'), SCF.serviceConfigFile(self.cfg).getParam( 'argTrain', 'dempster_shafer_SAR_Opt_fusion'), SCF.serviceConfigFile(self.cfg).getParam( 'chain', 'nomenclaturePath'), running_parameters.get_sensors_parameters(x["tile"]), self.RAM, self.workingDirectory) elif self.enable_autoContext is False and self.use_scikitlearn is True: step_function = lambda x: skClassifier.predict(**x) return step_function
def step_execute(self): """ Return ------ lambda the function to execute as a lambda function. The returned object must be a lambda function. """ from iota2.Segmentation import segmentation from iota2.Common.ServiceConfigFile import iota2_parameters running_parameters = iota2_parameters(self.cfg) step_function = lambda x: segmentation.slicSegmentation( x, SCF.serviceConfigFile(self.cfg).getParam('chain', 'outputPath'), running_parameters.get_sensors_parameters(x), self.RAM, self. workingDirectory) return step_function
def step_execute(self): """ Return ------ lambda the function to execute as a lambda function. The returned object must be a lambda function. """ from iota2.Sensors.Sensors_container import sensors_container from iota2.Common.ServiceConfigFile import iota2_parameters check_inputs = SCF.serviceConfigFile(self.cfg).getParam( 'chain', 'check_inputs') tiles = SCF.serviceConfigFile(self.cfg).getParam('chain', 'listTile').split(" ") output_path = SCF.serviceConfigFile(self.cfg).getParam( 'chain', 'outputPath') running_parameters = iota2_parameters( SCF.serviceConfigFile(self.cfg).pathConf) sensors_parameters = running_parameters.get_sensors_parameters( tiles[0]) sensor_tile_container = sensors_container(tiles[0], None, output_path, **sensors_parameters) sensor_path = sensor_tile_container.get_enabled_sensors_path()[0] step_function = lambda x: IOTA2_dir.generate_directories( x, check_inputs, SCF.serviceConfigFile(self.cfg).getParam( 'chain', 'merge_final_classifications'), SCF.serviceConfigFile(self.cfg).getParam('chain', 'groundTruth'), SCF.serviceConfigFile(self.cfg).getParam('chain', 'regionPath'), SCF.serviceConfigFile(self.cfg).getParam('chain', 'dataField'), SCF.serviceConfigFile(self.cfg).getParam('chain', 'regionField'), int( SCF.serviceConfigFile(self.cfg).getParam('GlobChain', 'proj'). split(":")[-1]), sensor_path, tiles) return step_function
def validity(tile_name, config_path, output_path, maskOut_name, view_threshold, workingDirectory=None, RAM=128): """ function dedicated to compute validity raster/vector by tile Parameters ---------- tile_name [string] tile's name config_path [string] absolute path to the configuration file maskOut_name [string] output vector mask's name view_threshold [int] threshold working_directory [string] absolute path to a working directory RAM [int] pipeline's size (Mo) """ import os import shutil from iota2.Common.ServiceConfigFile import iota2_parameters from iota2.Sensors.Sensors_container import sensors_container from iota2.Common.OtbAppBank import CreateConcatenateImagesApplication from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Common.Utils import run from iota2.Common.FileUtils import erodeShapeFile from iota2.Common.FileUtils import removeShape from iota2.Common.FileUtils import ensure_dir features_dir = os.path.join(output_path, "features", tile_name) validity_name = "nbView.tif" validity_out = os.path.join(features_dir, validity_name) validity_processing = validity_out if workingDirectory: ensure_dir(os.path.join(workingDirectory, tile_name)) validity_processing = os.path.join(workingDirectory, tile_name, validity_name) running_parameters = iota2_parameters(config_path) sensors_parameters = running_parameters.get_sensors_parameters(tile_name) remote_sensor_container = sensors_container(tile_name, workingDirectory, output_path, **sensors_parameters) sensors_time_series_masks = remote_sensor_container.get_sensors_time_series_masks( available_ram=RAM) sensors_masks_size = [] sensors_masks = [] for sensor_name, (time_series_masks, time_series_dep, nb_bands) in sensors_time_series_masks: if sensor_name.lower() == "sentinel1": for _, time_series_masks_app in list(time_series_masks.items()): time_series_masks_app.Execute() sensors_masks.append(time_series_masks_app) else: time_series_masks.Execute() sensors_masks.append(time_series_masks) sensors_masks_size.append(nb_bands) total_dates = sum(sensors_masks_size) merge_masks = CreateConcatenateImagesApplication({ "il": sensors_masks, "ram": str(RAM) }) merge_masks.Execute() validity_app = CreateBandMathApplication({ "il": merge_masks, "exp": "{}-({})".format( total_dates, "+".join(["im1b{}".format(i + 1) for i in range(total_dates)])), "ram": str(0.7 * RAM), "pixType": "uint8" if total_dates < 255 else "uint16", "out": validity_processing }) if not os.path.exists(os.path.join(features_dir, validity_name)): validity_app.ExecuteAndWriteOutput() if workingDirectory: shutil.copy(validity_processing, os.path.join(features_dir, validity_name)) threshold_raster_out = os.path.join(features_dir, maskOut_name.replace(".shp", ".tif")) threshold_vector_out_tmp = os.path.join( features_dir, maskOut_name.replace(".shp", "_TMP.shp")) threshold_vector_out = os.path.join(features_dir, maskOut_name) input_threshold = validity_processing if os.path.exists( validity_processing) else validity_out threshold_raster = CreateBandMathApplication({ "il": input_threshold, "exp": "im1b1>={}?1:0".format(view_threshold), "ram": str(0.7 * RAM), "pixType": "uint8", "out": threshold_raster_out }) threshold_raster.ExecuteAndWriteOutput() cmd_poly = f"gdal_polygonize.py -mask {threshold_raster_out} {threshold_raster_out} -f \"ESRI Shapefile\" {threshold_vector_out_tmp} {os.path.splitext(os.path.basename(threshold_vector_out_tmp))[0]} cloud" run(cmd_poly) erodeShapeFile(threshold_vector_out_tmp, threshold_vector_out, 0.1) os.remove(threshold_raster_out) removeShape(threshold_vector_out_tmp.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"])
PARSER = argparse.ArgumentParser(description=DESCRIPTION) PARSER.add_argument("-config", dest="config", help="configuration file path", default=None, required=True) PARSER.add_argument("-output", dest="output", help="output raster", default=None, required=True) PARSER.add_argument("-tile", dest="tile_name", help="tile's name", default=None, required=True) PARSER.add_argument("-working_dir", dest="working_dir", help="tile's name", default=None, required=True) ARGS = PARSER.parse_args() # load configuration file CFG = SCF.serviceConfigFile(ARGS.pathConf) PARAMS = iota2_parameters(ARGS.pathConf) SEN_PARAM = PARAMS.get_sensors_parameters(ARGS.tile) OUTPUT_PATH = CFG.getParam("chain", "outputPath") compute_features(OUTPUT_PATH, SEN_PARAM, ARGS.output, ARGS.tile_name, ARGS.working_dir)
def test_train_and_classify(self): """test autoContext training """ import re from iota2.Common import IOTA2Directory from iota2.Common import ServiceConfigFile as SCF from iota2.Common.FileUtils import FileSearch_AND from iota2.Common.FileUtils import getFieldElement from iota2.Segmentation import segmentation from iota2.Learning.trainAutoContext import train_autoContext from iota2.Classification.ImageClassifier import autoContext_launch_classif from iota2.Sensors.Sensors_container import sensors_container from iota2.Common.ServiceConfigFile import iota2_parameters from iota2.Common.OtbAppBank import CreateBandMathApplication from iota2.Tests.UnitTests.TestsUtils import test_raster_unique_value # config file config_path_test = os.path.join(self.test_working_directory, "Config_TEST.cfg") test_path = self.generate_cfg_file(self.config_test, config_path_test) cfg = SCF.serviceConfigFile(config_path_test) IOTA2Directory.generate_directories(test_path, check_inputs=False) autocontext_working_dir = os.path.join(self.test_working_directory, "autoContext_tmp") slic_working_dir = os.path.join(self.test_working_directory, "autoContext_tmp") iota2_dico = SCF.iota2_parameters( config_path_test).get_sensors_parameters(self.tile_name) sensors = sensors_container(self.tile_name, None, self.test_working_directory, **iota2_dico) sensors.sensors_preprocess() running_parameters = iota2_parameters(config_path_test) sensors_parameters = running_parameters.get_sensors_parameters( "T31TCJ") segmentation.slicSegmentation(self.tile_name, test_path, sensors_parameters, ram=128, working_dir=slic_working_dir, force_spw=1) slic_seg = FileSearch_AND( os.path.join(test_path, "features", self.tile_name, "tmp"), True, "SLIC_{}".format(self.tile_name))[0] train_auto_data_ref, superpix_data = self.prepare_autoContext_data_ref( slic_seg, config_path_test) parameter_dict = { "model_name": "1", "seed": "0", "list_learning_samples": [train_auto_data_ref], "list_superPixel_samples": [superpix_data], "list_tiles": ["T31TCJ"], "list_slic": [slic_seg] } # launch tests # training e = None try: train_autoContext(parameter_dict, data_field="code", output_path=test_path, sensors_parameters=sensors_parameters, superpix_data_field="superpix", iterations=3, RAM=128, WORKING_DIR=autocontext_working_dir) except Exception as e: print(e) # Asserts training self.assertTrue(e is None, msg="train_autoContext failed") models = FileSearch_AND(os.path.join(test_path, "model"), True, "model_it_", ".rf") self.assertTrue(len(models) == 4) # classification tile_raster = FileSearch_AND(self.fake_data_dir, True, "BINARY_MASK.tif")[0] tile_mask = os.path.join(autocontext_working_dir, "{}_tile_mask.tif".format(self.tile_name)) CreateBandMathApplication({ "il": [tile_raster], "out": tile_mask, "exp": "1" }).ExecuteAndWriteOutput() labels = getFieldElement(train_auto_data_ref, driverName="SQLite", field="code", mode="unique", elemType="str") parameters_dict = { "model_name": "1", "seed_num": 0, "tile": self.tile_name, "tile_segmentation": slic_seg, "tile_mask": tile_mask, "labels_list": labels, "model_list": sorted(models, key=lambda x: int(re.findall("\d", os.path.basename(x))[0])) } autoContext_launch_classif(parameters_dict, cfg.getParam("argTrain", "classifier"), "T31TCJ", cfg.getParam("argClassification", "enable_probability_map"), cfg.getParam("dimRed", "dimRed"), "code", False, False, iota2_run_dir=test_path, sar_optical_post_fusion=False, nomenclature_path=cfg.getParam( "chain", "nomenclaturePath"), sensors_parameters=sensors_parameters, RAM=128, WORKING_DIR=autocontext_working_dir) # Asserts classifications classif = FileSearch_AND(os.path.join(test_path, "classif"), True, "Classif_T31TCJ_model_1_seed_0.tif")[0] confidence = FileSearch_AND(os.path.join(test_path, "classif"), True, "T31TCJ_model_1_confidence_seed_0.tif")[0] classif_unique_0 = test_raster_unique_value(classif, 0) confidence_unique_0 = test_raster_unique_value(confidence, 0) self.assertTrue( classif_unique_0 == False, msg=("AutoContext Classifications failed : classification contains" " only 0 values")) self.assertTrue( confidence_unique_0 == False, msg=("AutoContext Classifications failed : confidence contains " "only 0 values"))
def prepare_autoContext_data_ref(self, slic_seg, config_path_test): """ """ from iota2.Common.ServiceConfigFile import iota2_parameters from iota2.VectorTools.AddField import addField from iota2.Sampling.SuperPixelsSelection import merge_ref_super_pix from iota2.Sampling.SplitSamples import split_superpixels_and_reference from iota2.Common.FileUtils import FileSearch_AND from iota2.Common.OtbAppBank import CreateSampleSelectionApplication from iota2.Common.OtbAppBank import CreatePolygonClassStatisticsApplication from iota2.Common.OtbAppBank import CreateSampleExtractionApplication from iota2.Common.GenerateFeatures import generateFeatures from iota2.Common import ServiceConfigFile as SCF raster_ref = FileSearch_AND(self.fake_data_dir, True, ".tif")[0] CreatePolygonClassStatisticsApplication({ "in": raster_ref, "vec": self.ref_data, "field": "CODE", "out": os.path.join(self.test_working_directory, "stats.xml") }).ExecuteAndWriteOutput() ref_sampled = os.path.join( self.test_working_directory, "T31TCJ_samples_region_1_seed_0_selection.sqlite") CreateSampleSelectionApplication({ "in": raster_ref, "vec": self.ref_data, "field": "CODE", "strategy": "all", "instats": os.path.join(self.test_working_directory, "stats.xml"), "out": ref_sampled }).ExecuteAndWriteOutput() cfg = SCF.serviceConfigFile(config_path_test) params = iota2_parameters(config_path_test) sensors_parameters = params.get_sensors_parameters("T31TCJ") features, feat_labels, dep = generateFeatures( None, "T31TCJ", sar_optical_post_fusion=False, output_path=cfg.getParam("chain", "outputPath"), sensors_parameters=sensors_parameters) extraction = CreateSampleExtractionApplication({ "in": features, "vec": ref_sampled, "field": "code", "outfield.list.names": feat_labels, "outfield": "list" }) extraction.ExecuteAndWriteOutput() addField(ref_sampled, "newregion", valueField="1", valueType=str, driver_name="SQLite") os.remove(os.path.join(self.test_working_directory, "stats.xml")) merge_ref_super_pix( { "selection_samples": ref_sampled, "SLIC": slic_seg }, "code", "superpix", "is_superpix", "newregion") learning_vector = os.path.join( self.test_working_directory, "T31TCJ_region_1_seed0_Samples_learn.sqlite") shutil.move(ref_sampled, learning_vector) ref, superpix = split_superpixels_and_reference( learning_vector, superpix_column="is_superpix") return learning_vector, superpix