def __init__(self,
cfg,
cfg_resources_file,
umc,
nomenclature=None,
stepname=None,
workingDirectory=None):
# heritage init
resources_block_name = "regularisation"
super(Regularization, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
if stepname:
self.step_name = stepname
self.RAM = 1024.0 * get_RAM(self.resources["ram"])
self.CPU = self.resources["cpu"]
self.workingDirectory = workingDirectory
self.outputPath = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
self.rastclass = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'classification')
self.seed = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'seed')
self.umc = umc
self.outtmpdir = os.path.join(self.outputPath, 'final',
'simplification', 'tmp')
self.nomenclature = nomenclature
def __init__(self, cfg, cfg_resources_file, workingDirectory=None):
# heritage init
resources_block_name = "samplesManagement"
super(copySamples, self).__init__(cfg, cfg_resources_file, resources_block_name)
# step variables
self.workingDirectory = workingDirectory
self.output_path = SCF.serviceConfigFile(self.cfg).getParam('chain', 'outputPath')
self.dataField = SCF.serviceConfigFile(self.cfg).getParam('chain', 'dataField')
self.sampleManagement = SCF.serviceConfigFile(self.cfg).getParam('argTrain', 'sampleManagement')
def __init__(self, cfg, cfg_resources_file, workingDirectory=None):
# heritage init
resources_block_name = "crownsearch"
super(crownSearch, self).__init__(cfg, cfg_resources_file, resources_block_name)
# step variables
self.RAM = 1024.0 * get_RAM(self.resources["ram"])
self.CPU = self.resources["cpu"]
self.workingDirectory = workingDirectory
self.outputPath = SCF.serviceConfigFile(self.cfg).getParam('chain', 'outputPath')
self.gridsize = SCF.serviceConfigFile(self.cfg).getParam('Simplification', 'gridsize')
def __init__(self, cfg, cfg_resources_file, workingDirectory=None):
# heritage init
resources_block_name = "SAROptConfusionMatrixFusion"
super(confusionSAROptMerge, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
self.workingDirectory = workingDirectory
self.output_path = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
self.data_field = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'dataField')
def __init__(self, cfg, cfg_resources_file, workingDirectory=None):
# heritage init
resources_block_name = "SAROptFusion"
super(SAROptFusion, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
self.workingDirectory = workingDirectory
self.output_path = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
self.enable_proba_map = SCF.serviceConfigFile(self.cfg).getParam(
'argClassification', 'enable_probability_map')
def __init__(self, cfg, cfg_resources_file, workingDirectory=None):
# heritage init
resources_block_name = "prodVectors"
super(prodVectors, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
self.RAM = 1024.0 * get_RAM(self.resources["ram"])
self.workingDirectory = workingDirectory
self.outputPath = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
self.nomenclature = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'nomenclature')
def __init__(self, cfg, cfg_resources_file, workingDirectory=None):
# heritage init
resources_block_name = "clump"
super(Clump, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
self.RAM = 1024.0 * get_RAM(self.resources["ram"])
self.workingDirectory = workingDirectory
self.outputPath = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
self.lib64bit = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'lib64bit')
def __init__(self, cfg, cfg_resources_file, workingDirectory=None):
# heritage init
resources_block_name = "grid"
super(Grid, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
self.RAM = 1024.0 * get_RAM(self.resources["ram"])
self.workingDirectory = workingDirectory
self.outputPath = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
self.gridsize = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'gridsize')
self.epsg = int(
SCF.serviceConfigFile(self.cfg).getParam('GlobChain',
'proj').split(":")[-1])
def __init__(self, cfg, cfg_resources_file, workingDirectory=None):
# heritage init
resources_block_name = "samplesAugmentation"
super(genSyntheticSamples, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
self.workingDirectory = workingDirectory
self.output_path = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
self.ground_truth = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'groundTruth')
self.data_field = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'dataField')
self.sample_augmentation = SCF.serviceConfigFile(self.cfg).getParam(
'argTrain', 'sampleAugmentation')
def __init__(self, cfg, cfg_resources_file, working_directory=None):
# heritage init
resources_block_name = "mergeClassifications"
super(ScikitClassificationsMerge,
self).__init__(cfg, cfg_resources_file, resources_block_name)
# step variables
self.working_directory = working_directory
self.output_path = ServiceConfigFile.serviceConfigFile(
self.cfg).getParam('chain', 'outputPath')
self.epsg_code = int(
ServiceConfigFile.serviceConfigFile(self.cfg).getParam(
'GlobChain', 'proj').split(":")[-1])
self.use_scikitlearn = ServiceConfigFile.serviceConfigFile(
self.cfg).getParam('scikit_models_parameters',
'model_type') is not None
def __init__(self,
cfg: str,
cfg_resources_file: str,
workingDirectory: Optional[str] = None) -> None:
"""set up the step
"""
# heritage init
resources_block_name = "superPixSplit"
super(superPixSplit, self).__init__(cfg, cfg_resources_file, resources_block_name)
# step variables
self.workingDirectory = workingDirectory
self.execution_dir = SCF.serviceConfigFile(self.cfg).getParam('chain', 'outputPath')
self.is_superPix_field = "is_super_pix"
self.region_field = SCF.serviceConfigFile(self.cfg).getParam('chain', 'regionField')
self.ram = 1024.0 * get_RAM(self.resources["ram"])
def __init__(self, cfg, cfg_resources_file, workingDirectory=None):
# heritage init
resources_block_name = "confusionMatrix"
super(confusionGeneration, self).__init__(cfg, cfg_resources_file, resources_block_name)
# step variables
self.workingDirectory = workingDirectory
self.output_path = SCF.serviceConfigFile(self.cfg).getParam('chain', 'outputPath')
def __init__(self, cfg, cfg_resources_file, workingDirectory=None):
# heritage init
resources_block_name = None
super(Sentinel1PreProcess, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
self.workingDirectory = workingDirectory
self.Sentinel1 = SCF.serviceConfigFile(cfg).getParam('chain', 'S1Path')
def step_inputs(self):
"""
Return
------
the return could be and iterable or a callable
"""
from Common import ServiceConfigFile as SCF
tiles = SCF.serviceConfigFile(self.cfg).getParam('chain',
'listTile').split(" ")
return tiles
def __init__(self, cfg, cfg_resources_file, workdir=None):
# heritage init
resources_block_name = "clipvectors"
super(clipVectors, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
self.RAM = 1024.0 * get_RAM(self.resources["ram"])
self.workdir = workdir
self.outputPath = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
self.grasslib = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'grasslib')
self.clipfile = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'clipfile')
self.clipfield = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'clipfield')
self.clipvalue = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'clipvalue')
self.outprefix = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'outprefix')
self.outmos = os.path.join(self.outputPath, 'final', 'simplification',
'mosaic')
self.outfilevect = os.path.join(self.outputPath, 'final',
'simplification', 'vectors')
self.grid = os.path.join(self.outputPath, 'final', 'simplification',
'grid.shp')
def __init__(self,
cfg: str,
cfg_resources_file: str,
workingDirectory: Optional[str] = None) -> None:
"""set up the step
"""
from Common.FileUtils import FileSearch_AND
# heritage init
resources_block_name = "superPixPos"
super(superPixPos, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
self.workingDirectory = workingDirectory
self.execution_dir = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
self.superPix_field = "superpix"
self.superPix_belong_field = "is_super_pix"
self.region_field = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'regionField')
self.data_field = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'dataField')
self.ram = 1024.0 * get_RAM(self.resources["ram"])
self.sampling_labels_from_raster = {}
if (SCF.serviceConfigFile(self.cfg).getParam('argTrain', 'cropMix')
and SCF.serviceConfigFile(self.cfg).getParam(
'argTrain', 'samplesClassifMix')):
source_dir = SCF.serviceConfigFile(self.cfg).getParam(
'argTrain', 'annualClassesExtractionSource')
annual_labels = SCF.serviceConfigFile(self.cfg).getParam(
'argTrain', 'annualCrop')
classification_raster = os.path.join(source_dir, "final",
"Classif_Seed_0.tif")
validity_raster = os.path.join(source_dir, "final",
"PixelsValidity.tif")
val_thresh = SCF.serviceConfigFile(self.cfg).getParam(
'argTrain', 'validityThreshold')
region_mask_dir = os.path.join(self.execution_dir, "shapeRegion")
self.sampling_labels_from_raster = {
"annual_labels": annual_labels,
"classification_raster": classification_raster,
"validity_raster": validity_raster,
"region_mask_directory": region_mask_dir,
"val_threshold": val_thresh
}
def setUpClass(self):
#class variables
self.iota2_directory = os.environ.get('IOTA2DIR')
self.iota2_tests_directory = os.path.join(self.iota2_directory, "data",
"test_features_labels")
self.test_working_directory = None
self.test_working_directory_tmp = None
config_path = os.path.join(self.iota2_directory, "config",
"Config_4Tuiles_Multi_FUS_Confidence.cfg")
self.config = SCF.serviceConfigFile(config_path)
if os.path.exists(self.iota2_tests_directory):
shutil.rmtree(self.iota2_tests_directory, ignore_errors=False)
os.mkdir(self.iota2_tests_directory)
def __init__(self, cfg, cfg_resources_file, stepname=None, output=None, umc=None, resample=None, water=None, workingDirectory=None):
# heritage init
resources_block_name = "mergeregularisation"
super(mergeRegularization, self).__init__(cfg, cfg_resources_file, resources_block_name)
# step variables
if stepname:
self.step_name = stepname
self.RAM = 1024.0 * get_RAM(self.resources["ram"])
self.CPU = self.resources["cpu"]
self.workingDirectory = workingDirectory
self.outputPath = SCF.serviceConfigFile(self.cfg).getParam('chain', 'outputPath')
self.resample = resample
self.water = water
self.umc = umc
self.output = output
self.tmpdir = os.path.join(self.outputPath, 'final', 'simplification', 'tmp')
def iota2_ressources(
iota2_ressources_description="iota2_HPC_ressources_request.cfg"):
"""
usage :
IN :
iota2_ressources_description [string] : path to a configuration file which
describe step's HPC request.
OUT :
iota2_HPC_requests [dic of Ressources Object] : dictionnary containing all
ressources request
"""
iota2_ressources_description = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
iota2_ressources_description)
cfg = SCF.serviceConfigFile(iota2_ressources_description,
iota_config=False)
available_steps = cfg.getAvailableSections()
iota2_HPC_requests = {}
for step in available_steps:
try:
process_max = cfg.getParam(step, 'process_max')
except:
process_max = -1
try:
process_min = cfg.getParam(step, 'process_min')
except:
process_min = 1
iota2_HPC_requests[step] = Ressources(
name=cfg.getParam(step, 'name'),
nb_cpu=cfg.getParam(step, 'nb_cpu'),
ram=cfg.getParam(step, 'ram'),
walltime=cfg.getParam(step, 'walltime'),
process_min=process_min,
process_max=process_max)
return iota2_HPC_requests
def __init__(self, cfg, cfg_resources_file, workingDirectory=None):
# heritage init
resources_block_name = "vectorisation"
super(simplification, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
self.RAM = 1024.0 * get_RAM(self.resources["ram"])
self.workingDirectory = workingDirectory
self.outputPath = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
self.mmu = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'mmu')
self.douglas = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'douglas')
self.hermite = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'hermite')
self.angle = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'angle')
self.grasslib = SCF.serviceConfigFile(self.cfg).getParam(
'Simplification', 'grasslib')
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Computes a time series of features")
parser.add_argument("-wd",
dest="pathWd",
help="path to the working directory",
default=None,
required=False)
parser.add_argument("-tile",
dest="tile",
help="tile to be processed",
required=True)
parser.add_argument("-conf",
dest="pathConf",
help="path to the configuration file (mandatory)",
required=True)
parser.add_argument("-writeFeatures",
type=str2bool,
dest="writeFeatures",
Shelp="path to the working directory",
default=False,
required=False)
args = parser.parse_args()
# load configuration file
cfg = SCF.serviceConfigFile(args.pathConf)
generateFeatures(args.pathWd, args.tile, cfg, args.writeFeatures)
def readNomenclatureFile(self, nomen, typefile='csv'):
"""Read a csv nomenclature file
Example of csv structure for 2 nested levels (l1 = level 1, l2 = level 2) :
classname_l1, code_l1, colour_l1, alias_l1, classname_l2, code_l2, colour_l2, alias_l2
Urbain, 100, #b106b1, Urbain, Urbain dense, 1, #b186c1, Urbaindens
Example of config structure for 2 nested levels (l1 = level 1, l2 = level 2) :
>>> Classes:
>>> {
>>> Level1:
>>> {
>>> "Urbain":
>>> {
>>> code:100
>>> alias:'Urbain'
>>> color:'#b106b1'
>>> }
>>> ...
>>> }
>>> Level2:
>>> {
>>> "Urbain dense":
>>> {
>>> code:1
>>> ...
>>> parent:100
>>> }
>>> ...
>>> }
>>> }
Example of config structure for 2 nested levels (l1 = level 1, l2 = level 2) :
>>> {'Level1': {'Urbain': {'color': '#b106b1', 'alias': 'Urbain', 'code': 100}, ...},
'Level2': {'Zones indus. et comm.': {'color': ..., 'parent' : 100}, ...}}
alias : maximum 10 caracters
color : HEX or RGB
Parameters
----------
nomen: csv or config file or dict object
nomenclature description
Return
------
list of tuples
raw nomenclature
"""
if isinstance(nomen, dict):
try:
tabnomenc = convertDictToList(nomen)
except IOError:
raise Exception(
"Nomenclature reading failed, nomenclature is not properly built"
)
elif typefile == 'cfg':
from Common import ServiceConfigFile as SCF
try:
cfg = SCF.serviceConfigFile(nomen, False).cfg
dictnomenc = {}
for level in cfg.Classes:
cls = {}
for classe in cfg.Classes[level]:
cls[classe] = cfg.Classes[level][classe]
dictnomenc[level] = cls
tabnomenc = convertDictToList(dictnomenc)
except IOError:
raise Exception(
"Nomenclature reading failed, nomenclature is not properly built"
)
elif typefile == 'csv' or isinstance(nomen, list):
if isinstance(nomen, list):
tabnomenc = nomen
else:
try:
fileclasses = codecs.open(nomen, "r", "utf-8")
tabnomenc = [
tuple(line.rstrip('\n').split(','))
for line in fileclasses.readlines()
]
except IOError:
raise Exception(
"Nomenclature reading failed, nomenclature is not properly built"
)
else:
raise Exception("The type of nomenclature file is not handled")
return tabnomenc
errors = check_errors_JA(log_dir=log_err,
task_name=steps[step_num].step_name)
if errors:
print("ERROR in step '" + steps[step_num].step_name + "'")
print(errors)
return errors
current_step += 1
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="This function allows you launch the chain according to a configuration file")
parser.add_argument("-config", dest="config", help="path to IOTA2 configuration file",
required=True)
parser.add_argument("-config_ressources", dest="config_ressources",
help="path to IOTA2 ressources configuration file", required=False)
parser.add_argument("-mode", dest="parallel_mode",
help="parallel jobs strategy",
required=False,
default="MPI",
choices=["MPI", "JobArray"])
args = parser.parse_args()
cfg = SCF.serviceConfigFile(args.config)
try:
launchChain(cfg, args.config_ressources, args.parallel_mode)
except sErr.i2Error as e:
print(e)
except Exception as e:
print(e)
raise
def __init__(self, cfg, cfg_resources_file, workingDirectory=None):
# heritage init
resources_block_name = "merge_final_classifications"
super(mergeSeedClassifications, self).__init__(cfg, cfg_resources_file,
resources_block_name)
# step variables
self.workingDirectory = workingDirectory
self.output_path = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'outputPath')
self.data_field = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'dataField')
self.nomenclature = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'nomenclaturePath')
self.color_table = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'colorTable')
self.runs = SCF.serviceConfigFile(self.cfg).getParam('chain', 'runs')
self.undecidedlabel = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'merge_final_classifications_undecidedlabel')
self.dempstershafer_mob = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'dempstershafer_mob')
self.keep_runs_results = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'keep_runs_results')
self.enableCrossValidation = SCF.serviceConfigFile(self.cfg).getParam(
'chain', 'enableCrossValidation')
self.ground_truth = SCF.serviceConfigFile(cfg).getParam(
'chain', 'groundTruth')
self.fusionClaAllSamplesVal = SCF.serviceConfigFile(cfg).getParam(
'chain', 'fusionOfClassificationAllSamplesValidation')
self.merge_final_classifications_method = SCF.serviceConfigFile(
cfg).getParam('chain', 'merge_final_classifications_method')
def launch_coregister(tile, cfg, workingDirectory, launch_mask=True):
""" register an image / a time series on a reference image
Parameters
----------
tile : string
tile id
cfg : serviceConfig obj
configuration object for parameters
workingDirectory : string
path to the working directory
launch_mask : bool
boolean to launch common mask
Note
------
This function use the OTB's application **PointMatchCoregistrationModel**,**OrthoRectification** and **SuperImpose**
more documentation for
`OrthoRectification <https://www.orfeo-toolbox.org/Applications/OrthoRectification.html>`_
and
`SuperImpose <https://www.orfeo-toolbox.org/Applications/Superimpose.html>`_
"""
from Sensors.ProcessLauncher import commonMasks
from Common import ServiceConfigFile as SCF
logger.info("Source Raster Registration")
if not isinstance(cfg, SCF.serviceConfigFile):
cfg = SCF.serviceConfigFile(cfg)
ipathL5 = cfg.getParam('chain', 'L5Path_old')
if ipathL5 != "None" and os.path.exists(os.path.join(ipathL5, tile)):
datadir = os.path.join(ipathL5, tile)
datatype = 'L5'
pattern = "ORTHO_SURF_CORR_PENTE*.TIF"
ipathL8 = cfg.getParam('chain', 'L8Path_old')
if ipathL8 != "None" and os.path.exists(os.path.join(ipathL8, tile)):
datadir = os.path.join(ipathL8, tile)
datatype = 'L8'
pattern = "ORTHO_SURF_CORR_PENTE*.TIF"
ipathS2 = cfg.getParam('chain', 'S2Path')
if ipathS2 != "None" and os.path.exists(os.path.join(ipathS2, tile)):
datadir = os.path.join(ipathS2, tile)
datatype = 'S2'
pattern = "*STACK.tif"
ipathS2_S2C = cfg.getParam('chain', 'S2_S2C_Path')
if ipathS2_S2C != "None" and os.path.exists(os.path.join(
ipathS2_S2C, tile)):
datadir = os.path.join(ipathS2_S2C, tile)
datatype = 'S2_S2C'
pattern = "*STACK.tif"
if cfg.getParam('coregistration', 'pattern') != "None":
pattern = cfg.getParam('coregistration', 'pattern')
inref = os.path.join(cfg.getParam('coregistration', 'VHRPath'))
datesSrc = cfg.getParam('coregistration', 'dateSrc')
if datesSrc != "None":
tiles = cfg.getParam('chain', 'listTile').split(" ")
tile_ind = tiles.index(tile)
dateSrc = datesSrc.split(" ")[tile_ind]
if dateSrc == "None":
dateVHR = cfg.getParam('coregistration', 'dateVHR')
if dateVHR == 'None':
logger.warning(
"No dateVHR in configuration file, please fill dateVHR value"
)
else:
dateSrc = fitnessDateScore(dateVHR, datadir, datatype)
else:
dateVHR = cfg.getParam('coregistration', 'dateVHR')
if dateVHR == 'None':
logger.warning(
"No dateVHR in configuration file, please fill dateVHR value")
else:
dateSrc = fitnessDateScore(dateVHR, datadir, datatype)
insrc = glob.glob(os.path.join(datadir, '*' + str(dateSrc) + '*',
pattern))[0]
bandsrc = cfg.getParam('coregistration', 'bandSrc')
bandref = cfg.getParam('coregistration', 'bandRef')
resample = cfg.getParam('coregistration', 'resample')
step = cfg.getParam('coregistration', 'step')
minstep = cfg.getParam('coregistration', 'minstep')
minsiftpoints = cfg.getParam('coregistration', 'minsiftpoints')
iterate = cfg.getParam('coregistration', 'iterate')
prec = cfg.getParam('coregistration', 'prec')
mode = int(cfg.getParam('coregistration', 'mode'))
if workingDirectory != None:
workingDirectory = os.path.join(workingDirectory, tile)
coregister(insrc, inref, bandsrc, bandref, resample, step, minstep,
minsiftpoints, iterate, prec, mode, datadir, pattern, datatype,
False, workingDirectory)
if launch_mask:
commonMasks(tile, cfg.pathConf)
def get_learning_samples(
learning_samples_dir: str,
config_path: str) -> List[Dict[str, Union[str, List[str]]]]:
"""get sorted learning samples files from samples directory
Parameters
----------
learning_samples_dir : str
path to iota2 output directory containing learning data base
config_path : str
path to iota2 configuration file
Return
------
list
ist of dictionaries
example :
get_learning_samples() = [{"learning_file": path to the learning file data base,
"feat_labels": feature's labels,
"model_path": output model path
},
...
]
"""
from Common import ServiceConfigFile
from iota2.Common.FileUtils import FileSearch_AND
from iota2.Common.FileUtils import getVectorFeatures
sar_suffix = "SAR"
ground_truth = ServiceConfigFile.serviceConfigFile(config_path).getParam(
"chain", "groundTruth")
region_field = ServiceConfigFile.serviceConfigFile(config_path).getParam(
"chain", "regionField")
sar_opt_post_fusion = ServiceConfigFile.serviceConfigFile(
config_path).getParam("argTrain", "dempster_shafer_SAR_Opt_fusion")
iota2_outputs = ServiceConfigFile.serviceConfigFile(config_path).getParam(
"chain", "outputPath")
iota2_models_dir = os.path.join(iota2_outputs, "model")
parameters = []
seed_pos = 3
model_pos = 2
learning_files = FileSearch_AND(learning_samples_dir, True,
"Samples_region_", "_learn.sqlite")
if sar_opt_post_fusion:
learning_files_sar = FileSearch_AND(
learning_samples_dir, True, "Samples_region_",
"_learn_{}.sqlite".format(sar_suffix))
learning_files += learning_files_sar
learning_files_sorted = []
output_model_files_sorted = []
features_labels_sorted = []
for learning_file in learning_files:
seed = os.path.basename(learning_file).split("_")[seed_pos].replace(
"seed", "")
model = os.path.basename(learning_file).split("_")[model_pos]
learning_files_sorted.append((seed, model, learning_file))
learning_files_sorted = sorted(learning_files_sorted,
key=operator.itemgetter(0, 1))
for _, _, learning_file in learning_files_sorted:
seed = os.path.basename(learning_file).split("_")[seed_pos].replace(
"seed", "")
model = os.path.basename(learning_file).split("_")[model_pos]
model_name = "model_{}_seed_{}.txt".format(model, seed)
if "{}.sqlite".format(sar_suffix) in learning_file:
model_name = model_name.replace(".txt",
"_{}.txt".format(sar_suffix))
model_path = os.path.join(iota2_models_dir, model_name)
output_model_files_sorted.append(model_path)
features_labels_sorted.append(
getVectorFeatures(ground_truth, region_field, learning_file))
parameters = [{
"learning_file": learning_file,
"feat_labels": features_labels,
"model_path": model_path
} for (seed, model, learning_file), model_path, features_labels in zip(
learning_files_sorted, output_model_files_sorted,
features_labels_sorted)]
return parameters
def GenerateDirectories(cfg):
"""
generate IOTA2 output directories
"""
if not isinstance(cfg, SCF.serviceConfigFile):
cfg = SCF.serviceConfigFile(cfg)
root = cfg.getParam('chain', 'outputPath')
rm_PathTEST = cfg.getParam("chain", "remove_outputPath")
start_step = cfg.getParam("chain", "firstStep")
if os.path.exists(
root) and root != "/" and rm_PathTEST and start_step == "init":
shutil.rmtree(root, ignore_errors=False)
os.mkdir(root)
if os.path.exists(root + "/logs"):
shutil.rmtree(root + "/logs")
os.mkdir(root + "/logs")
if os.path.exists(root + "/samplesSelection"):
shutil.rmtree(root + "/samplesSelection")
os.mkdir(root + "/samplesSelection")
if os.path.exists(root + "/model"):
shutil.rmtree(root + "/model")
os.mkdir(root + "/model")
if os.path.exists(root + "/formattingVectors"):
shutil.rmtree(root + "/formattingVectors")
os.mkdir(root + "/formattingVectors")
if os.path.exists(root + "/config_model"):
shutil.rmtree(root + "/config_model")
os.mkdir(root + "/config_model")
if os.path.exists(root + "/envelope"):
shutil.rmtree(root + "/envelope")
os.mkdir(root + "/envelope")
if os.path.exists(root + "/classif"):
shutil.rmtree(root + "/classif")
os.mkdir(root + "/classif")
if os.path.exists(root + "/shapeRegion"):
shutil.rmtree(root + "/shapeRegion")
os.mkdir(root + "/shapeRegion")
if os.path.exists(root + "/final"):
shutil.rmtree(root + "/final")
os.mkdir(root + "/final")
os.mkdir(root + "/final/simplification")
os.mkdir(root + "/final/simplification/tiles")
os.mkdir(root + "/final/simplification/vectors")
os.mkdir(root + "/final/simplification/tmp")
if os.path.exists(root + "/features"):
shutil.rmtree(root + "/features")
os.mkdir(root + "/features")
if os.path.exists(root + "/dataRegion"):
shutil.rmtree(root + "/dataRegion")
os.mkdir(root + "/dataRegion")
if os.path.exists(root + "/learningSamples"):
shutil.rmtree(root + "/learningSamples")
os.mkdir(root + "/learningSamples")
if os.path.exists(root + "/dataAppVal"):
shutil.rmtree(root + "/dataAppVal")
os.mkdir(root + "/dataAppVal")
if os.path.exists(root + "/stats"):
shutil.rmtree(root + "/stats")
os.mkdir(root + "/stats")
if os.path.exists(root + "/cmd"):
shutil.rmtree(root + "/cmd")
os.mkdir(root + "/cmd")
os.mkdir(root + "/cmd/stats")
os.mkdir(root + "/cmd/train")
os.mkdir(root + "/cmd/cla")
os.mkdir(root + "/cmd/confusion")
os.mkdir(root + "/cmd/features")
os.mkdir(root + "/cmd/fusion")
os.mkdir(root + "/cmd/splitShape")
merge_final_classifications = cfg.getParam('chain',
'merge_final_classifications')
if merge_final_classifications:
if os.path.exists(root + "/final/merge_final_classifications"):
shutil.rmtree(root + "/final/merge_final_classifications")
os.mkdir(root + "/final/merge_final_classifications")