def applyKmeansMasks(image_input, mask_samples_macro_input_list, image_samples_merged_output, proposal_table_output, micro_samples_images_output_list, centroids_files_output_list, macroclass_sampling_list, macroclass_labels_list, no_data_value, path_time_log, kmeans_param_maximum_iterations=200, kmeans_param_training_set_size_weight=1, kmeans_param_minimum_training_set_size=-1, rate_clean_micro_class=0.0, rand_otb=0, ram_otb=0, number_of_actives_pixels_threshold=200, extension_raster=".tif", save_results_intermediate=False, overwrite=True):
# Mise à jour du Log
starting_event = "applyKmeansMasks() : Kmeans and mask starting : "
timeLine(path_time_log,starting_event)
print(endC)
print(cyan + "applyKmeansMasks() : " + bold + green + "## START : SUBSAMPLING OF " + str(macroclass_labels_list) + endC)
print(endC)
if debug >= 2:
print(cyan + "applyKmeansMasks() : variables dans la fonction" + endC)
print(cyan + "applyKmeansMasks() : " + endC + "image_input : " + str(image_input) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "image_samples_merged_output : " + str(image_samples_merged_output) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "proposal_table_output : " + str(proposal_table_output) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "mask_samples_macro_input_list : " + str(mask_samples_macro_input_list) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "micro_samples_images_output_list : " + str(micro_samples_images_output_list) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "centroids_files_output_list : " + str(centroids_files_output_list) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "macroclass_sampling_list : " + str(macroclass_sampling_list) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "macroclass_labels_list : " + str(macroclass_labels_list) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "kmeans_param_maximum_iterations : " + str(kmeans_param_maximum_iterations) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "kmeans_param_training_set_size_weight : " + str(kmeans_param_training_set_size_weight) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "kmeans_param_minimum_training_set_size : " + str(kmeans_param_minimum_training_set_size) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "rate_clean_micro_class : " + str(rate_clean_micro_class))
print(cyan + "applyKmeansMasks() : " + endC + "no_data_value : " + str(no_data_value) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "rand_otb : " + str(rand_otb) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "ram_otb : " + str(ram_otb) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "number_of_actives_pixels_threshold : " + str(number_of_actives_pixels_threshold))
print(cyan + "applyKmeansMasks() : " + endC + "extension_raster : " + str(extension_raster) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "save_results_intermediate : " + str(save_results_intermediate) + endC)
print(cyan + "applyKmeansMasks() : " + endC + "overwrite : " + str(overwrite) + endC)
# constantes
HEADER_TABLEAU_MODIF = "MICROCLASSE;TRAITEMENT\n"
CODAGE_16B = "uint16"
CODAGE_8B = "uint8"
EXT_XML = ".xml"
SUFFIX_MASK_CLEAN = "_clean"
SUFFIX_SAMPLE_MICRO = "_sample_micro"
SUFFIX_STATISTICS = "_statistics"
SUFFIX_CENTROID = "_centroid"
SUFFIX_MASK_TEMP = "_tmp"
# Creation des fichiers temporaires de sortie si ils ne sont pas spécifier
#-------------------------------------------------------------------------
length_mask = len(mask_samples_macro_input_list)
images_mask_cleaned_list = []
temporary_files_list = []
micro_samples_images_list = []
centroids_files_list = []
repertory_output_tmp_list = []
if image_samples_merged_output != "" :
repertory_base_output = os.path.dirname(image_samples_merged_output)
filename = os.path.splitext(os.path.basename(image_samples_merged_output))[0]
else :
repertory_base_output = os.path.dirname(micro_samples_images_output_list[0])
filename = os.path.splitext(os.path.basename(micro_samples_images_output_list[0]))[0]
file_statistic_points = repertory_base_output + os.sep + filename + SUFFIX_STATISTICS + EXT_XML
for macroclass_id in range(length_mask):
repertory_output = repertory_base_output + os.sep + str(macroclass_labels_list[macroclass_id])
if not os.path.isdir(repertory_output):
os.makedirs(repertory_output)
repertory_output_tmp_list.append(repertory_output)
samples_image_input = mask_samples_macro_input_list[macroclass_id]
filename = os.path.splitext(os.path.basename(samples_image_input))[0]
image_mask_cleaned = repertory_output + os.sep + filename + SUFFIX_MASK_CLEAN + extension_raster
images_mask_cleaned_list.append(image_mask_cleaned)
image_tmp = repertory_output + os.sep + filename + SUFFIX_MASK_TEMP + extension_raster
temporary_files_list.append(image_tmp)
if micro_samples_images_output_list == [] :
micro_samples_image = repertory_output + os.sep + filename + SUFFIX_SAMPLE_MICRO + extension_raster
else :
micro_samples_image = micro_samples_images_output_list[macroclass_id]
micro_samples_images_list.append(micro_samples_image)
if centroids_files_output_list == [] :
centroids_file = repertory_output + os.sep + filename + SUFFIX_CENTROID + extension_raster
else :
centroids_file = centroids_files_output_list[macroclass_id]
centroids_files_list.append(centroids_file)
# Nettoyage des pixels superposés sur plusieurs images
#-----------------------------------------------------
if length_mask > 1:
image_name = os.path.splitext(os.path.basename(image_input))[0]
deletePixelsSuperpositionMasks(mask_samples_macro_input_list, images_mask_cleaned_list, image_name, CODAGE_8B)
else:
images_mask_cleaned_list = mask_samples_macro_input_list
# Execution du kmeans pour chaque macroclasse
#--------------------------------------------
# Initialisation de la liste pour le multi-threading
thread_list = []
for macroclass_id in range(length_mask):
mask_sample_input = images_mask_cleaned_list[macroclass_id]
micro_samples_image = micro_samples_images_list[macroclass_id]
image_tmp = temporary_files_list[macroclass_id]
centroids_file = centroids_files_list[macroclass_id]
check = os.path.isfile(micro_samples_image)
if check and not overwrite : # Si un fichier de sortie avec le même nom existe déjà, et si l'option ecrasement est à false, alors passe à la classification suivante
print(cyan + "applyKmeansMasks() : " + bold + yellow + "Computing kmeans from %s with %s already done : no actualisation" % (image_input, mask_sample_input) + endC)
else: # Si non, on applique un kmeans
if check :
removeFile(micro_samples_image) # Suppression de l'éventuel fichier existant
print(cyan + "applyKmeansMasks() : " + bold + green + "Computing kmeans from %s with %s ; output image is %s" %(image_input, mask_sample_input,micro_samples_image) + endC)
# Obtention du nombre de microclasses
number_of_classes = macroclass_sampling_list[macroclass_id] # Nombre de microclasses
label = macroclass_labels_list[macroclass_id] # Label de la macroclasse Ex : 11000
# Gestion du multi threading pour l'appel du calcul du kmeans
thread = threading.Thread(target=computeKmeans, args=(image_input, mask_sample_input, image_tmp, micro_samples_image, centroids_file, label, number_of_classes, macroclass_id, number_of_actives_pixels_threshold, kmeans_param_minimum_training_set_size, kmeans_param_maximum_iterations, length_mask, no_data_value, rand_otb, int(ram_otb/length_mask), CODAGE_8B, CODAGE_16B, save_results_intermediate, overwrite))
thread.start()
thread_list.append(thread)
# Start Kmeans all macro classes
try:
for thread in thread_list:
thread.join()
except:
print(cyan + "applyKmeansMasks() : " + bold + red + "applyKmeansMasks() : " + endC + "Erreur lors du calcul du kmeans : impossible de demarrer le thread" + endC, file=sys.stderr)
# Fusion des echantillons micro
#------------------------------
if image_samples_merged_output != "" :
mergeListRaster(micro_samples_images_list, image_samples_merged_output, CODAGE_16B)
updateReferenceProjection(image_input, image_samples_merged_output)
# Creation de la table de proposition et le fichier statistique
#--------------------------------------------------------------
if proposal_table_output != "" :
suppress_micro_class_list = []
info_micoclass_nbpoints_dico = {}
nb_points_total = 0
nb_points_medium = 0
# Liste des identifants des micro classes disponibles
id_micro_list = identifyPixelValues(image_samples_merged_output)
if 0 in id_micro_list :
id_micro_list.remove(0)
nb_micr_class = len(id_micro_list)
# Pour toutes les micro classes
for id_micro in id_micro_list :
nb_pixels = countPixelsOfValue(image_samples_merged_output, id_micro)
info_micoclass_nbpoints_dico[id_micro] = nb_pixels
nb_points_total += nb_pixels
# Valeur moyenne de nombre de points
if nb_micr_class != 0 :
nb_points_medium = int(nb_points_total / nb_micr_class)
nb_points_min = int((nb_points_medium * rate_clean_micro_class) / 100)
# Identifier les micro classes trop petites
if debug >= 4:
print("rate_clean_micro_class = " + str(rate_clean_micro_class))
print("nb_points_medium = " + str(nb_points_medium))
print("nb_points_min = " + str(nb_points_min))
# Preparation du fichier statistique
writeTextFile(file_statistic_points, '<?xml version="1.0" ?>\n')
appendTextFileCR(file_statistic_points, '<GeneralStatistics>')
appendTextFileCR(file_statistic_points, ' <Statistic name="pointsPerClassRaw">')
for micro_class_id in info_micoclass_nbpoints_dico :
nb_points = info_micoclass_nbpoints_dico[micro_class_id]
if debug >= 4:
print("micro_class_id = " + str(micro_class_id) + ", nb_points = " + str(nb_points))
appendTextFileCR(file_statistic_points, ' <StatisticPoints class="%d" value="%d" />' %(micro_class_id, nb_points))
if nb_points < nb_points_min :
# Micro_class à proposer en effacement
suppress_micro_class_list.append(micro_class_id)
# Fin du fichier statistique
appendTextFileCR(file_statistic_points, ' </Statistic>')
appendTextFileCR(file_statistic_points, '</GeneralStatistics>')
# Test si ecrassement de la table précédemment créée
check = os.path.isfile(proposal_table_output)
if check and not overwrite :
print(cyan + "applyKmeansMasks() : " + bold + yellow + "Modifier table already exists." + '\n' + endC)
else:
# Tenter de supprimer le fichier
try:
removeFile(proposal_table_output)
except Exception:
pass # Ignore l'exception levee si le fichier n'existe pas (et ne peut donc pas être supprime)
# lister les micro classes à supprimer
text_output = HEADER_TABLEAU_MODIF
for micro_class_del in suppress_micro_class_list:
text_output += "%d;-1\n" %(micro_class_del)
# Ecriture du fichier proposition de réaffectation
writeTextFile(proposal_table_output, text_output)
# Suppresions fichiers intermediaires inutiles
#---------------------------------------------
if not save_results_intermediate:
for macroclass_id in range(length_mask):
if (os.path.isfile(temporary_files_list[macroclass_id])) :
removeFile(temporary_files_list[macroclass_id])
if (length_mask > 1) and (os.path.isfile(images_mask_cleaned_list[macroclass_id])) :
removeFile(images_mask_cleaned_list[macroclass_id])
if (micro_samples_images_output_list == []) and (os.path.isfile(micro_samples_images_list[macroclass_id])) :
removeFile(micro_samples_images_list[macroclass_id])
if (centroids_files_output_list == []) and (os.path.isfile(centroids_files_list[macroclass_id])) :
removeFile(centroids_files_list[macroclass_id])
if os.path.isdir(repertory_output_tmp_list[macroclass_id]) :
removeDir(repertory_output_tmp_list[macroclass_id])
print(cyan + "applyKmeansMasks() : " + bold + green + "## END : KMEANS CLASSIFICATION" + endC)
print(endC)
# Mise à jour du Log
ending_event = "applyKmeansMasks() : Kmeans and mask ending : "
timeLine(path_time_log,ending_event)
return
def createTableModifier(centroids_input_files_list, indicators_input_file, matrix_input_file, validation_input_vector, label_macro_list, table_output_file, path_time_log, rate_area_min, threshold_delete_perf, threshold_alert_perf, format_vector='ESRI Shapefile', save_results_intermediate=False, overwrite=True) :
# Mise à jour du Log
starting_event = "createTableModifier() : Create table modifier starting : "
timeLine(path_time_log,starting_event)
print(cyan + "createTableModifier() : " + bold + green + "START ...\n" + endC)
# Affichage des parametres
if debug >= 3:
print(cyan + "createTableModifier() : " + endC + "centroids_input_files_list: "+ str(centroids_input_files_list))
print(cyan + "createTableModifier() : " + endC + "indicators_input_file: "+ str(indicators_input_file))
print(cyan + "createTableModifier() : " + endC + "matrix_input_file: "+ str(matrix_input_file))
print(cyan + "createTableModifier() : " + endC + "validation_input_vector: "+ str(validation_input_vector))
print(cyan + "createTableModifier() : " + endC + "label_macro_list: "+ str(label_macro_list))
print(cyan + "createTableModifier() : " + endC + "table_output_file: "+ str(table_output_file))
print(cyan + "createTableModifier() : " + endC + "path_time_log: " + str(path_time_log))
print(cyan + "createTableModifier() : " + endC + "rate_area_min: " + str(rate_area_min))
print(cyan + "createTableModifier() : " + endC + "threshold_delete_perf: " + str(threshold_delete_perf))
print(cyan + "createTableModifier() : " + endC + "threshold_alert_perf: " + str(threshold_alert_perf))
print(cyan + "createTableModifier() : " + endC + "format_vector: " + str(format_vector))
print(cyan + "createTableModifier() : " + endC + "save_results_intermediate: "+ str(save_results_intermediate))
print(cyan + "createTableModifier() : " + endC + "overwrite: "+ str(overwrite))
# Test si ecrassement de la table précédemment créée
check = os.path.isfile(table_output_file)
if check and not overwrite :
print(cyan + "createTableModifier() : " + bold + yellow + "Modifier table already exists." + '\n' + endC)
else:
# Tenter de supprimer le fichier
try:
removeFile(table_output_file)
except Exception:
# Ignore l'exception levee si le fichier n'existe pas (et ne peut donc pas être supprime)
pass
# lecture des fichiers centroides
microclass_centroides_list = readCentroidsFiles(centroids_input_files_list)
# lecture du fichier indicateurs de qualité
indicator_macro_dico,indicator_general_dico = readQualityIndicatorsFile(indicators_input_file)
# lecture de la matrice de confusion
matrix,class_ref_list,class_pro_list = readConfusionMatrix(matrix_input_file)
# correction de la matrice de confision pour identifier les micro classes manquantes
missed_micro_list = []
if class_ref_list != class_pro_list:
matrix, missed_micro_list = correctMatrix(class_ref_list, class_pro_list, matrix)
# creer la liste de microclasse en de 'int'
class_labels_list =[]
for class_elem in class_pro_list:
class_labels_list.append(int(class_elem))
if debug >= 3:
print(cyan + "createTableModifier() : " + endC + "indicator_macro_dico : " + str(indicator_macro_dico) + "\n")
print(cyan + "createTableModifier() : " + endC + "indicator_general_dico : " + str(indicator_general_dico) + "\n")
print(cyan + "createTableModifier() : " + endC + "microclass_centroides_list = " + str(microclass_centroides_list))
print(cyan + "createTableModifier() : " + endC + "missed_micro_list : " + str(missed_micro_list))
print(cyan + "createTableModifier() : " + endC + "class_ref_list : " + str(class_ref_list))
print(cyan + "createTableModifier() : " + endC + "class_pro_list : " + str(class_pro_list))
print(cyan + "createTableModifier() : " + endC + "class_labels_list : " + str(class_labels_list))
# indentifier les microclasses suspectes
s_suspect_microclass1,s_suspect_microclass2 = findSuspiciousMicroClass(label_macro_list, microclass_centroides_list)
# detecter les micro classes suspectes
suspect_micro_list,s_performance_list = detectSuspiciousMicroclass(indicator_macro_dico, class_labels_list, threshold_alert_perf)
if debug >= 3:
print(cyan + "createTableModifier() : " + endC + "s_suspect_microclass1 = " + str(s_suspect_microclass1))
print(cyan + "createTableModifier() : " + endC + "s_suspect_microclass2 = " + str(s_suspect_microclass2))
print(cyan + "createTableModifier() : " + endC + "suspect_micro_list = " + str(suspect_micro_list))
print(cyan + "createTableModifier() : " + endC + "s_performance_list = " + str(s_performance_list))
# creation de la liste de proposition de réaffectation des micro classes
proposal_text = proposeReallocationMicroClass(validation_input_vector, s_suspect_microclass1, s_suspect_microclass2, s_performance_list, missed_micro_list, suspect_micro_list, table_output_file, label_macro_list, rate_area_min, threshold_delete_perf, threshold_alert_perf, format_vector)
# ecriture du fichier proposition de réaffectation
writeTextFile(table_output_file, proposal_text)
print(cyan + "createTableModifier() : " + bold + green + "END\n" + endC)
# Mise à jour du Log
ending_event = "createTableModifier() : Create table modifier ending : "
timeLine(path_time_log,ending_event)
return
def distanceTDCPointLine(input_points_shp,
input_tdc_shp,
output_dir,
input_sea_points,
evolution_column_name,
path_time_log,
server_postgis="localhost",
user_postgis="postgres",
password_postgis="postgres",
database_postgis="db_buffer_tdc",
schema_name="directionevolution",
port_number=5432,
epsg=2154,
project_encoding="UTF-8",
format_vector="ESRI Shapefile",
save_results_intermediate=True,
overwrite=True):
# Mise à jour du Log
starting_event = "distanceTDCPointLine() : Distance TDC Point Line starting : "
timeLine(path_time_log, starting_event)
if debug >= 3:
print(bold + green +
"Variables dans distanceTDCPointLine - Variables générales" +
endC)
print(cyan + "distanceTDCPointLine() : " + endC +
"input_points_shp : " + str(input_points_shp) + endC)
print(cyan + "distanceTDCPointLine() : " + endC + "input_tdc_shp : " +
str(input_tdc_shp) + endC)
print(cyan + "distanceTDCPointLine() : " + endC + "output_dir : " +
str(output_dir) + endC)
print(cyan + "distanceTDCPointLine() : " + endC +
"input_sea_points : " + str(input_sea_points) + endC)
print(cyan + "distanceTDCPointLine() : " + endC +
"evolution_column_name : " + str(evolution_column_name) + endC)
print(cyan + "distanceTDCPointLine() : " + endC + "path_time_log : " +
str(path_time_log) + endC)
print(cyan + "distanceTDCPointLine() : " + endC + "server_postgis : " +
str(server_postgis) + endC)
print(cyan + "distanceTDCPointLine() : " + endC + "user_postgis : " +
str(user_postgis) + endC)
print(cyan + "distanceTDCPointLine() : " + endC +
"password_postgis : " + str(password_postgis) + endC)
print(cyan + "distanceTDCPointLine() : " + endC +
"database_postgis : " + str(database_postgis) + endC)
print(cyan + "distanceTDCPointLine() : " + endC + "schema_name : " +
str(schema_name) + endC)
print(cyan + "distanceTDCPointLine() : " + endC + "port_number : " +
str(port_number) + endC)
print(cyan + "distanceTDCPointLine() : " + endC + "epsg : " +
str(epsg) + endC)
print(cyan + "distanceTDCPointLine() : " + endC +
"project_encoding : " + str(project_encoding) + endC)
print(cyan + "distanceTDCPointLine() : " + endC + "format_vector : " +
str(format_vector) + endC)
print(cyan + "distanceTDCPointLine() : " + endC +
"save_results_inter : " + str(save_results_intermediate) + endC)
print(cyan + "distanceTDCPointLine() : " + endC + "overwrite : " +
str(overwrite) + endC)
print(bold + green + "## START : DistanceTDCPointLine" + endC)
# Initialisation des constantes
EXT_TEXT = ".txt"
# Configuration du format vecteur
driver = ogr.GetDriverByName(format_vector)
# Création de la référence spatiale
srs = osr.SpatialReference()
srs.ImportFromEPSG(epsg)
# Constitution des noms
ext_vect = os.path.splitext(os.path.split(input_points_shp)[1])[1]
input_points_shp_name = os.path.splitext(
os.path.split(input_points_shp)[1])[0]
input_tdc_shp_name = os.path.splitext(os.path.split(input_tdc_shp)[1])[0]
vector_output_points = output_dir + os.sep + "distance_" + input_points_shp_name + "_" + input_tdc_shp_name + ext_vect
output_text_file = output_dir + os.sep + "distance_sens_evol_" + input_points_shp_name + "_" + input_tdc_shp_name + EXT_TEXT
vector_output_points_sens = output_dir + os.sep + "distance_sens_evol_" + input_points_shp_name + "_" + input_tdc_shp_name + ext_vect
## Gestion des fichiers
# Création du répertoire de sortie s'il n'existe pas déjà
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# Suppression de la couche en sortie si elle existe déjà
if os.path.exists(vector_output_points):
driver.DeleteDataSource(vector_output_points)
# Suppression de la couche en sortie si elle existe déjà
if os.path.exists(vector_output_points_sens):
driver.DeleteDataSource(vector_output_points_sens)
## Ouverture des fichiers existants
# Ouverture de la couche points
data_source_points_ref = driver.Open(input_points_shp, 0)
input_layer_pts = data_source_points_ref.GetLayer()
input_layer_pts_defn = input_layer_pts.GetLayerDefn()
# Ouverture de la couche TDC
data_source_tdc = driver.Open(input_tdc_shp, 0)
input_layer_tdc = data_source_tdc.GetLayer(0)
## Création des fichiers de sortie
# Création de la couche en sortie
output_data_source = driver.CreateDataSource(vector_output_points)
output_layer = output_data_source.CreateLayer(input_points_shp_name,
srs,
geom_type=ogr.wkbPoint)
output_layer_defn = output_layer.GetLayerDefn()
# Ajouter les champs du fichier d'entrée au shapefile de sortie
for i in range(0, input_layer_pts_defn.GetFieldCount()):
field_defn = input_layer_pts_defn.GetFieldDefn(i)
output_layer.CreateField(field_defn)
# Ajout du champ "distance" à la couche en sortie
dist_field = ogr.FieldDefn("distance", ogr.OFTReal)
output_layer.CreateField(dist_field)
liste_all_distances = []
liste_min_distances = []
# Ajout des valeurs à la couche de sortie
for i in range(0, input_layer_pts.GetFeatureCount()):
# Récupération de l'entité de la couche en entrée
input_feature = input_layer_pts.GetFeature(i)
# Calcul de la distance du point courant à la ligne la plus proche
geom_point = input_feature.GetGeometryRef()
input_layer_tdc.ResetReading()
liste_distances = []
for ligne in input_layer_tdc:
geom_ligne = ligne.GetGeometryRef()
dist = geom_point.Distance(geom_ligne)
liste_distances.append(dist)
liste_all_distances.append(dist)
dist_min = min(liste_distances)
liste_min_distances.append(dist_min)
# Création de l'entité en sortie
output_feature = ogr.Feature(output_layer_defn)
# Ajout des champs de la couche d'entrée à la couche de sortie
if debug >= 4:
print(cyan + "distanceTDCPointLine : " + endC + green + bold +
"POINT " + str(i) + endC)
for i in range(0, output_layer_defn.GetFieldCount() - 1):
if debug >= 4:
print(cyan + "distanceTDCPointLine : " + endC +
str(output_layer_defn.GetFieldDefn(i).GetNameRef()) +
" : " + str(input_feature.GetField(i)))
output_feature.SetField(
output_layer_defn.GetFieldDefn(i).GetNameRef(),
input_feature.GetField(i))
output_feature.SetField("distance", dist_min)
if debug >= 4:
print(cyan + "distanceTDCPointLine : " + endC +
"Distance à la ligne la plus proche : " + str(dist_min) +
"\n")
# Géométrie de la couche de sortie
input_geom = input_feature.GetGeometryRef()
output_feature.SetGeometry(input_geom)
# Ajout de la nouvelle entité à la couche de sortie
output_layer.CreateFeature(output_feature)
## Calcul du sens de l'évolution
tdc_reference_buffers_sens = evolvingDirectionTDC(
input_tdc_shp, input_sea_points, output_dir,
int(ceil(max(liste_min_distances))), path_time_log, server_postgis,
user_postgis, password_postgis, database_postgis, schema_name,
port_number, epsg, project_encoding, format_vector,
save_results_intermediate, overwrite)
data_source_tdc_ref_buffers_sens = driver.Open(tdc_reference_buffers_sens,
1)
layer_tdc_ref_buffers_sens = data_source_tdc_ref_buffers_sens.GetLayer(0)
# Création de l'intersection avec les buffers unilatéraux pour avoir le sens d'évolution
data_source_intersection_sens = driver.CreateDataSource(
vector_output_points_sens)
intersection_sens_layer = data_source_intersection_sens.CreateLayer(
vector_output_points_sens, srs, geom_type=ogr.wkbPoint)
intersection_sens_layer_defn = intersection_sens_layer.GetLayerDefn()
intersection_sens = layer_tdc_ref_buffers_sens.Intersection(
output_layer, intersection_sens_layer)
# Ajout champ evolution, multiplication du buffer multiple dans lequel il se trouve avec sens évolution
evolution_field = ogr.FieldDefn("evolution", ogr.OFTReal)
intersection_sens_layer.CreateField(evolution_field)
for i in range(0, intersection_sens_layer.GetFeatureCount()):
feature = intersection_sens_layer.GetFeature(i)
distance = feature.GetField("distance")
evol_direction = feature.GetField("num_side")
feature.SetField(evolution_column_name, distance * evol_direction)
intersection_sens_layer.SetFeature(feature)
feature.Destroy()
liste_all_evolutions = []
## Création du fichier texte en sortie
writeTextFile(output_text_file, "")
# Initialisation des colonnes du fichier texte
for i in range(intersection_sens_layer_defn.GetFieldCount() - 1):
appendTextFile(
output_text_file,
intersection_sens_layer_defn.GetFieldDefn(i).GetName() + "\t \t")
appendTextFile(output_text_file, str(evolution_column_name) + " (m)\n")
for i in range(0, intersection_sens_layer.GetFeatureCount()):
# Récupération de l'entité de la couche en entrée
feature = intersection_sens_layer.GetFeature(i)
for j in range(0, intersection_sens_layer_defn.GetFieldCount() - 1):
appendTextFile(output_text_file,
str(feature.GetField(j)) + "\t \t")
appendTextFile(output_text_file,
str(feature.GetField(evolution_column_name)) + "\n")
liste_all_evolutions.append(feature.GetField(evolution_column_name))
appendTextFile(
output_text_file, "\nDistance min à la ligne (m) : " +
str(min(liste_all_evolutions)) + "\n")
appendTextFile(
output_text_file, "Distance max à la ligne (m) : " +
str(max(liste_all_evolutions)) + "\n")
appendTextFile(
output_text_file, "Moyenne des distances à la ligne (m) : " +
str(np.mean(liste_all_evolutions)) + "\n")
appendTextFile(
output_text_file, "Ecart-type des distances à la ligne (m) : " +
str(np.std(liste_all_evolutions)) + "\n")
if debug >= 3:
print(cyan + "distanceTDCPointLine() : " + endC + bold + green +
"L'évolution a bien été calculée dans le fichier " + endC +
bold + green + vector_output_points_sens + " dans la colonne '" +
evolution_column_name +
"' et la distance en valeur absolue dans la colonne 'distance'" +
endC)
print(cyan + "distanceTDCPointLine() : " + endC + bold + green +
"Un résumé est fait dans le fichier texte : " +
output_text_file + endC)
# Mise à jour du Log
ending_event = "distanceTDCPointLine() : Distance TDC Point Line ending : "
timeLine(path_time_log, ending_event)
return
def classRasterSubSampling(satellite_image_input, classified_image_input, image_output, table_reallocation, sub_sampling_number, no_data_value, path_time_log, rand_otb=0, ram_otb=0, number_of_actives_pixels_threshold=8000, extension_raster=".tif", save_results_intermediate=False, overwrite=True) :
# Mise à jour du Log
starting_event = "classRasterSubSampling() : Micro class subsampling on classification image starting : "
timeLine(path_time_log,starting_event)
if debug >= 3:
print(cyan + "classRasterSubSampling() : " + endC + "satellite_image_input : " + str(satellite_image_input) + endC)
print(cyan + "classRasterSubSampling() : " + endC + "classified_image_input : " + str(classified_image_input) + endC)
print(cyan + "classRasterSubSampling() : " + endC + "image_output : " + str(image_output) + endC)
print(cyan + "classRasterSubSampling() : " + endC + "table_reallocation : " + str(table_reallocation) + endC)
print(cyan + "classRasterSubSampling() : " + endC + "sub_sampling_number : " + str(sub_sampling_number) + endC)
print(cyan + "classRasterSubSampling() : " + endC + "no_data_value : " + str(no_data_value) + endC)
print(cyan + "classRasterSubSampling() : " + endC + "path_time_log : " + str(path_time_log) + endC)
print(cyan + "classRasterSubSampling() : " + endC + "rand_otb : " + str(rand_otb) + endC)
print(cyan + "classRasterSubSampling() : " + endC + "ram_otb : " + str(ram_otb) + endC)
print(cyan + "classRasterSubSampling() : " + endC + "number_of_actives_pixels_threshold : " + str(number_of_actives_pixels_threshold) + endC)
print(cyan + "classRasterSubSampling() : " + endC + "extension_raster : " + str(extension_raster) + endC)
print(cyan + "classRasterSubSampling() : " + endC + "save_results_intermediate : " + str(save_results_intermediate) + endC)
print(cyan + "classRasterSubSampling() : " + endC + "overwrite : " + str(overwrite) + endC)
# Constantes
CODAGE = "uint16"
CODAGE_8B = "uint8"
TEMP = "TempSubSampling_"
MASK_SUF = "_Mask"
SUB_SAMPLE_SUF = "_SubSampled"
CENTROID_SUF = "_Centroids"
TEMP_OUT = "_temp_out"
EXTENSION_TXT = ".txt"
# Contenu de la nouvelle table
text_new_table = ""
# CREATION DES NOMS DE CHEMINS UTILES
name = os.path.splitext(os.path.basename(image_output))[0]
input_classified_image_path = os.path.dirname(classified_image_input) # Ex : D2_Par_Zone/Paysage_01/Corr_2/Resultats/Temp/
temp_sub_sampling_path = input_classified_image_path + os.sep + TEMP + name + os.sep # Dossier contenant les fichiers temporaires de cette brique. Ex : D2_Par_Zone/Paysage_01/Corr_2/Resultats/Temp/Temp_Sub_Sampling/
input_classified_image_complete_name = os.path.basename(classified_image_input) # Ex : Paysage_01_raw.tif
input_classified_image_name = os.path.splitext(input_classified_image_complete_name)[0] # Ex : Paysage_01_raw
input_classified_image_extend = os.path.splitext(input_classified_image_complete_name)[1] # Ex : .tif
image_output_temp = os.path.splitext(image_output)[0] + TEMP_OUT + extension_raster # Ex : D2_Par_Zone/Paysage_01/Corr_2/Resultats/Temp/Temp_Sub_Sampling/Paysage_01_raw_temp.tif
# Création de temp_sub_sampling_path s'il n'existe pas
if not os.path.isdir(os.path.dirname(temp_sub_sampling_path)) :
os.makedirs(os.path.dirname(temp_sub_sampling_path))
print(cyan + "classRasterSubSampling() : " + bold + green + "START ...\n" + endC)
# Lecture du fichier table de proposition
supp_class_list, reaff_class_list, macro_reaff_class_list, sub_sampling_class_list, sub_sampling_number_list = readReallocationTable(table_reallocation, sub_sampling_number) # Fonction de Lib_text
info_table_list = readTextFileBySeparator(table_reallocation, "\n")
# Recherche de la liste des micro classes contenu dans le fichier de classification d'entrée
class_values_list = identifyPixelValues(classified_image_input)
# Supression dans la table des lignes correspondant aux actions "-2"
for ligne_table in info_table_list:
if not "-2" in ligne_table[0]:
text_new_table += str(ligne_table[0]) + "\n"
if debug >= 3:
print("supp_class_list : " + str(supp_class_list))
print("reaff_class_list : " + str(reaff_class_list))
print("macro_reaff_class_list : " + str(macro_reaff_class_list))
print("sub_sampling_class_list : " + str(sub_sampling_class_list))
print("sub_sampling_number_list : " + str(sub_sampling_number_list))
# Dans cettre brique, on ne s'intéresse qu'à la partie sous echantillonage
# Gestion du cas de suppression
if len(supp_class_list) > 0:
print(cyan + "classRasterSubSampling() : " + bold + yellow + "ATTENTION : Les classes ne sont pas supprimees pour le fichier classification format raster." + '\n' + endC)
# Gestion du cas de réaffectation
if len(reaff_class_list) > 0:
print(cyan + "classRasterSubSampling() : " + bold + yellow + "ATTENTION : la brique SpecificSubSampling ne traite pas les reaffectation. A l'issue de cette brique, verifier la table de reallocation et executer la brique de reallocation." + '\n' + endC)
if len(sub_sampling_class_list) > 0 :
if debug >= 3:
print(cyan + "classRasterSubSampling() : " + bold + green + "DEBUT DU SOUS ECHANTILLONAGE DES CLASSES %s " %(sub_sampling_class_list) + endC)
# Parcours des classes à sous échantilloner
processing_pass_first = False
for idx_class in range(len(sub_sampling_class_list)) :
# INITIALISATION DU TRAITEMENT DE LA CLASSE
# Classe à sous échantilloner. Ex : 21008
class_to_sub_sample = sub_sampling_class_list[idx_class]
if idx_class == 0 or not processing_pass_first :
# Image à reclassifier : classified_image_input au premier tour
image_to_sub_sample = classified_image_input
else :
# Image à reclassifier : la sortie de la boucle précédente ensuite
image_to_sub_sample = image_output
# determiner le label disponible de la classe
base_subclass_label = int(class_to_sub_sample/100)*100
subclass_label = base_subclass_label
for class_value in class_values_list:
if (class_value > subclass_label) and (class_value < base_subclass_label + 100) :
subclass_label = class_value
subclass_label += 1
# subclass_label = int(class_to_sub_sample/100)*100 + 20 + class_to_sub_sample%20 * 5
# Label de départ des sous classes. Formule proposée : 3 premiers chiffres de class_to_sub_sample puis ajout de 20 + 5 * class_to_sub_sample modulo 20. Ex : 21000 -> 21020, 21001-> 21025, 21002-> 21030 etc...
# Part du principe qu'il y a moins de 20 micro classes et que chacune est sous échantillonnée au maximum en 5 sous parties. Si ce n'est pas le cas : A ADAPTER
number_of_sub_samples = sub_sampling_number_list[idx_class] # Nombre de sous classes demandées pour le sous échantillonage de class_to_sub_sample. Ex : 4
class_mask_raster = temp_sub_sampling_path + input_classified_image_name + "_" + str(class_to_sub_sample) + MASK_SUF + input_classified_image_extend # Ex : D2_Par_Zone/Paysage_01/Corr_2/Resultats/Temp/Temp_Sub_Sampling/Paysage_01_raw_21008_Mask.tif
class_subsampled_raster = temp_sub_sampling_path + input_classified_image_name + "_" + str(class_to_sub_sample) + SUB_SAMPLE_SUF + input_classified_image_extend # Ex : D2_Par_Zone/Paysage_01/Corr_2/Resultats/Temp/Temp_Sub_Sampling/Paysage_01_raw_21008_SubSampled.tif
centroid_file = temp_sub_sampling_path + input_classified_image_name + "_" + str(class_to_sub_sample) + CENTROID_SUF + EXTENSION_TXT # Ex : D2_Par_Zone/Paysage_01/Corr_2/Resultats/Temp/Temp_Sub_Sampling/Paysage_01_raw_21008_Centroid.txt
if debug >= 5:
print(cyan + "classRasterSubSampling() : " + endC + "class_to_sub_sample :" , class_to_sub_sample)
print(cyan + "classRasterSubSampling() : " + endC + "subclass_label :" , subclass_label)
print(cyan + "classRasterSubSampling() : " + endC + "number_of_sub_samples :" , number_of_sub_samples)
print(cyan + "classRasterSubSampling() : " + endC + "class_mask_raster :" , class_mask_raster)
print(cyan + "classRasterSubSampling() : " + endC + "class_subsampled_raster :" , class_subsampled_raster)
print(cyan + "classRasterSubSampling() : " + endC + "centroid_file :" , centroid_file)
if debug >= 3:
print(cyan + "classRasterSubSampling() : " + bold + green + "CLASSE %s/%s : SOUS ECHANTILLONAGE DE %s EN %s CLASSES " %(idx_class+1, len(sub_sampling_class_list), class_to_sub_sample, number_of_sub_samples) + endC)
# ETAPE 1/5 : EXTRACTION DU MASQUE BINAIRE DES PIXELS CORRESPONDANT A LA CLASSE
expression_masque = "\"im1b1 == %s? 1 : 0\"" %(class_to_sub_sample)
command = "otbcli_BandMath -il %s -out %s %s -exp %s" %(classified_image_input, class_mask_raster, CODAGE_8B, expression_masque)
if debug >=2:
print("\n" + cyan + "classRasterSubSampling() : " + bold + green + "CLASSE %s/%s - ETAPE 1/5 : Debut de l extraction du masque binaire de la classe %s" %(idx_class+1, len(sub_sampling_class_list),class_to_sub_sample) + endC)
print(command)
os.system(command)
if debug >=2:
print(cyan + "classRasterSubSampling() : " + bold + green + "CLASSE %s/%s - ETAPE 1/5 : Fin de l extraction du masque binaire de la classe %s, disponible ici : %s" %(idx_class+1, len(sub_sampling_class_list),class_to_sub_sample, class_mask_raster) + endC)
# TEST POUR SAVOIR SI ON EST EN CAPACITE D'EFFECTUER LE KMEANS
number_of_actives_pixels = countPixelsOfValue(class_mask_raster, 1) # Comptage du nombre de pixels disponibles pour effectuer le kmeans
if number_of_actives_pixels > (number_of_sub_samples * number_of_actives_pixels_threshold) : # Cas où il y a plus de pixels disponibles pour effectuer le kmeans que le seuil
# ETAPE 2/5 : CLASSIFICATION NON SUPERVISEE DES PIXELS CORRESPONDANT A LA CLASSE
if debug >= 3:
print("\n" + cyan + "classRasterSubSampling() : " + bold + green + "CLASSE %s/%s - ETAPE 2/5 : Il y a assez de pixels pour faire le sous echantillonage : %s sur %s requis au minimum " %(idx_class+1, len(sub_sampling_class_list), number_of_actives_pixels, int(number_of_sub_samples) * number_of_actives_pixels_threshold) + endC)
if debug >=2:
print("\n" + cyan + "classRasterSubSampling() : " + bold + green + "CLASSE %s/%s - ETAPE 2/5 : Debut du sous echantillonage par classification non supervisee en %s classes " %(idx_class+1, len(sub_sampling_class_list), number_of_sub_samples) + endC)
# appel du kmeans
input_mask_list = []
input_mask_list.append(class_mask_raster)
output_masked_image_list = []
output_masked_image_list.append(class_subsampled_raster)
output_centroids_files_list = []
output_centroids_files_list.append(centroid_file)
macroclass_sampling_list = []
macroclass_sampling_list.append(number_of_sub_samples)
macroclass_labels_list = []
macroclass_labels_list.append(subclass_label)
applyKmeansMasks(satellite_image_input, input_mask_list, "", "", output_masked_image_list, output_centroids_files_list, macroclass_sampling_list, macroclass_labels_list, no_data_value, path_time_log, 200, 1, -1, 0.0, rand_otb, ram_otb, number_of_actives_pixels_threshold, extension_raster, save_results_intermediate, overwrite)
if debug >=2:
print(cyan + "classRasterSubSampling() : " + bold + green + "CLASSE %s/%s - ETAPE 2/5 : Fin du sous echantillonage par classification non supervisee en %s classes, disponible ici %s : " %(idx_class+1, len(sub_sampling_class_list), number_of_sub_samples, class_subsampled_raster) + endC)
# ETAPE 3/5 : INTEGRATION DES NOUVELLES SOUS CLASSES DANS LA TABLE DE REALLOCATION
# Ouveture du fichier table de proposition pour re-ecriture
for i in range(number_of_sub_samples):
class_values_list.append(subclass_label + i)
text_new_table += str(subclass_label + i) + ";" + str(subclass_label + i) + "; METTRE A JOUR MANUELLEMENT (origine : " + str(class_to_sub_sample) + ")" + "\n"
# ETAPE 4/5 : APPLICATION DU SOUS ECHANTILLONAGE AU RESULTAT DE CLASSIFICATION
expression_application_sous_echantillonage = "\"im1b1 == %s? im2b1 : im1b1\"" %(class_to_sub_sample)
command = "otbcli_BandMath -il %s %s -out %s %s -exp %s" %(image_to_sub_sample, class_subsampled_raster, image_output_temp, CODAGE, expression_application_sous_echantillonage)
if debug >=2:
print("\n" + cyan + "classRasterSubSampling() : " + bold + green + "CLASSE %s/%s - ETAPE 4/5 : Debut de l application du sous echantillonage present dans %s sur %s" %(idx_class+1, len(sub_sampling_class_list), class_subsampled_raster, classified_image_input) + endC)
print(command)
os.system(command)
if debug >=2:
print(cyan + "classRasterSubSampling() : " + bold + green + "CLASSE %s/%s - ETAPE 4/5 : Fin de l application du sous echantillonage present dans %s sur %s, sortie disponible ici : %s" %(idx_class+1, len(sub_sampling_class_list), class_subsampled_raster, classified_image_input, image_output_temp) + endC)
# ETAPE 5/5 : GESTION DES RENOMMAGES ET SUPPRESSIONS
if debug >=2:
print("\n" + cyan + "classRasterSubSampling() : " + bold + green + "CLASSE %s/%s - ETAPE 5/5 : Debut du renommage et suppression des dossiers intermediaires" %(idx_class+1, len(sub_sampling_class_list)) + endC)
if debug >=3 :
print("\n" + green + "classified image input: %s" %(classified_image_input) + endC)
print("\n" + green + "image to sub sample: %s" %(image_to_sub_sample) + endC)
print("\n" + green + "image temp : %s" %(image_output_temp) + endC)
print("\n" + green + "image output : %s" %(image_output) + endC)
# Si l'image d'entrée et l'image de sorte sont le même fichier on efface le fichier d'entrée pour le re-creer avec le fichier re-travaillé
if image_output == classified_image_input and os.path.isfile(classified_image_input) :
removeFile(classified_image_input)
os.rename(image_output_temp,image_output)
processing_pass_first = True
# SUPPRESSION DES FICHIERS TEMPORAIRES
if not save_results_intermediate :
if os.path.isfile(class_mask_raster) :
removeFile(class_mask_raster)
if os.path.isfile(class_subsampled_raster) :
removeFile(class_subsampled_raster)
if os.path.isfile(centroid_file) :
removeFile(centroid_file)
if debug >=2:
print(cyan + "classRasterSubSampling() : " + bold + green + "CLASSE %s/%s - ETAPE 5/5 : Fin du renommage et suppression des dossiers intermediaires" %(idx_class+1, len(sub_sampling_class_list)) + endC)
else: # Cas où il n'y a pas assez de pixels pour effectuer le kmeans
if debug >=2:
print("\n" + cyan + "classRasterSubSampling() : " + bold + yellow + "CLASSE %s/%s - ETAPE 2/5 : Nombre insuffisant de pixels disponibles pour appliquer le kmeans : %s sur %s requis au minimum " %(idx_class+1, len(sub_sampling_class_list), number_of_actives_pixels, int(number_of_sub_samples) * number_of_actives_pixels_threshold) + endC)
print(cyan + "classRasterSubSampling() : " + bold + yellow + "CLASSE %s/%s - ETAPE 2/5 : SOUS ECHANTILLONAGE NON APPLIQUE A LA CLASSE %s" %(idx_class+1, len(sub_sampling_class_list), class_to_sub_sample) + endC + "\n")
# MISE A JOUR DU FICHIER image_to_sub_sample
if idx_class == 0:
processing_pass_first = False
# MISE A JOUR DE LA TABLE DE REALLOCATION
text_new_table += str(class_to_sub_sample) + ";" + str(class_to_sub_sample) + ";CLASSE TROP PETITE POUR SOUS ECHANTILLONAGE" + "\n"
# SUPPRESSION DU MASQUE
if not save_results_intermediate and os.path.isfile(class_mask_raster) :
removeFile(class_mask_raster)
else:
shutil.copy2(classified_image_input, image_output) # Copie du raster d'entree si pas de sous-echantillonnage
# Ecriture de la nouvelle table dans le fichier
writeTextFile(table_reallocation, text_new_table)
# SUPPRESSION DU DOSSIER ET DES FICHIERS TEMPORAIRES
if not save_results_intermediate and os.path.isdir(os.path.dirname(temp_sub_sampling_path)) :
shutil.rmtree(os.path.dirname(temp_sub_sampling_path))
print(cyan + "classRasterSubSampling() : " + bold + green + "END\n" + endC)
# Mise à jour du Log
ending_event = "classRasterSubSampling() : Micro class subsampling on classification image ending : "
timeLine(path_time_log,ending_event)
return
def executeCommand(ip_serveur,
port,
id_command,
command_to_execute,
type_execution,
error_management,
base_name_shell_command,
ip_remote="",
login="",
password=""):
EXT_SHELL = '.sh'
EXT_ERR = '.err'
EXT_LOG = '.log'
new_state = ''
# Preparation du fichier d'execution en background-local ou background-remote
if type_execution == TAG_ACTION_TO_MAKE_BG or type_execution == TAG_ACTION_TO_MAKE_RE:
# Pour les executions a faire en background ou en remote preparation des fichiers .sh et .err
shell_command = base_name_shell_command + str(id_command) + EXT_SHELL
error_file = base_name_shell_command + str(id_command) + EXT_ERR
log_file = base_name_shell_command + str(id_command) + EXT_LOG
# Creation du fichier shell
error_management_option = ""
if not error_management:
error_management_option = " -nem "
command_to_execute = command_to_execute.replace('\n', '')
if six.PY2:
cmd_tmp = command_to_execute + " 1> " + log_file.encode(
"utf-8") + " 2> " + error_file.encode("utf-8") + "\n"
else:
cmd_tmp = command_to_execute + " 1> " + log_file + " 2> " + error_file + "\n"
writeTextFile(shell_command, cmd_tmp)
appendTextFileCR(
shell_command, FUNCTION_PYTHON + "ReplyEndCommand -ip_serveur " +
str(ip_serveur) + " -port " + str(port) + " -id_command " +
str(id_command) + error_management_option + " -err " + error_file)
appendTextFileCR(shell_command, "rm " + shell_command)
os.chmod(shell_command, stat.S_IRWXU)
# Selon le type d'execution
while switch(type_execution):
if case(TAG_ACTION_TO_MAKE_NOW):
# Execution en direct (local)
exitCode = subprocess.call(command_to_execute, shell=True)
new_state = TAG_STATE_END
if exitCode != 0: # Si la commande command_to_execute a eu un probleme
new_state = TAG_STATE_ERROR
print(cyan + "executeCommand : " + endC + bold + red +
"ERREUR EXECUTION DE LA COMMANDE : " +
str(command_to_execute) + endC,
file=sys.stderr)
break
if case(TAG_ACTION_TO_MAKE_BG):
# Execution en back ground (local)
process = subprocess.Popen(shell_command,
shell=True,
stderr=subprocess.STDOUT)
time.sleep(0.1)
if process == None:
new_state = TAG_STATE_ERROR
print(cyan + "executeCommand : " + endC + bold + red +
"ERREUR EXECUTION DE LA COMMANDE EN BACKGROUND : " +
str(command_to_execute) + endC,
file=sys.stderr)
else:
print(cyan + "executeCommand : " + endC +
" background pid = " + str(process.pid))
break
if case(TAG_ACTION_TO_MAKE_RE):
# Test si la machine Remote est accesible
if ping(ip_remote):
# Execution en remote execution
try:
s = pxssh.pxssh()
s.login(ip_remote, login, password)
time.sleep(0.5)
s.sendline(shell_command + '&')
time.sleep(0.01)
s.logout()
except pxssh.ExceptionPxssh as e:
new_state = TAG_STATE_ERROR
print(
cyan + "executeCommand : " + endC + bold + red +
"ERREUR EXECUTION DE LA COMMANDE EN REMOTE (login failed) : "
+ str(command_to_execute) + endC,
file=sys.stderr)
print(e, file=sys.stderr)
else:
new_state = TAG_STATE_ERROR
print(
cyan + "executeCommand : " + endC + bold + red +
"ERREUR EXECUTION DE LA COMMANDE EN REMOTE (Computeur : " +
ip_remote + " non disponible) : " +
str(command_to_execute) + endC,
file=sys.stderr)
break
break # Sortie du while
return new_state
def main(gui=False):
print(endC)
print(bold + green + "#######################################################################################################" + endC)
print(bold + green + "# DEBUT DE L'EXECUTION GENERALE DE LA CHAINE #" + endC)
print(bold + green + "#######################################################################################################" + endC)
print(endC)
# Définition des différents paramètres du parser
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, prog="TaskSequencer", description="\
Info : Run a complete processing chain consting of a task list. \n\
Objectif : exécuter une chaine de traitement complète composée d'une luste de tache. \n\
Example : python TaskSequencer.py Settings.xml")
parser.add_argument('setting_file',nargs="+",type=str, help="Settings file contain configuration of chain execution")
args = displayIHM(gui, parser)
# Récupération du fichier de setting d'entrée
if args.setting_file != None:
setting_file_list = args.setting_file
print(bold + green + "TaskSequencer : Variables dans le parser" + endC)
print(cyan + "TaskSequencer : " + endC + "setting_file_list : " + str(setting_file_list) + endC)
# EXECUTION DE LA CHAINE
command_doc = ''
global id_command
id_command = ID_COMMAND_INIT
settings_struct_dico = {}
correspondence_task_process_dico = {}
for setting_file in setting_file_list:
# nom du settting
name_setting = os.path.splitext(os.path.basename(setting_file))[0]
# Parser le fichier xml de settings
settings_struct_dico[name_setting] = xmlSettingsParser(setting_file)
# Ecriture des commandes
command_doc,id_command = commandsWritingManagement(name_setting, settings_struct_dico, correspondence_task_process_dico, setting_file, id_command, command_doc)
# Correction du fichier command_doc pour traiter les dépendances croisées nom encore traitées
command_txt = readTextFile(command_doc)
for correspondence_task in correspondence_task_process_dico :
command_txt = command_txt.replace(correspondence_task, correspondence_task_process_dico[correspondence_task])
writeTextFile(command_doc, command_txt)
# Definition du niveau de debug pour toute la chaine
debug = settings_struct_dico[list(settings_struct_dico)[0]].general.processing.debug
# Lancement de la supervison du fichier commande
setEndDisplay(False)
threadSupervision = supervisionCommands(command_doc, debug)
threadSupervision.start()
# Execution des commandes
link = settings_struct_dico[list(settings_struct_dico)[0]].general.processing.link
port = settings_struct_dico[list(settings_struct_dico)[0]].general.processing.port
if settings_struct_dico[list(settings_struct_dico)[0]].general.processing.running :
executeCommands(command_doc, debug, link, port)
else :
time.sleep(5)
# Arret de la supervision
time.sleep(1)
setEndDisplay(True)
time.sleep(1)
terminateThread(threadSupervision)
def selectSamples(image_input_list, sample_image_input, vector_output, table_statistics_output, sampler_strategy, select_ratio_floor, ratio_per_class_dico, name_column, no_data_value, path_time_log, rand_seed=0, ram_otb=0, epsg=2154, format_vector='ESRI Shapefile', extension_vector=".shp", save_results_intermediate=False, overwrite=True) :
# Mise à jour du Log
starting_event = "selectSamples() : Select points in raster mask macro input starting : "
timeLine(path_time_log, starting_event)
if debug >= 3:
print(cyan + "selectSamples() : " + endC + "image_input_list : " + str(image_input_list) + endC)
print(cyan + "selectSamples() : " + endC + "sample_image_input : " + str(sample_image_input) + endC)
print(cyan + "selectSamples() : " + endC + "vector_output : " + str(vector_output) + endC)
print(cyan + "selectSamples() : " + endC + "table_statistics_output : " + str(table_statistics_output) + endC)
print(cyan + "selectSamples() : " + endC + "sampler_strategy : " + str(sampler_strategy) + endC)
print(cyan + "selectSamples() : " + endC + "select_ratio_floor : " + str(select_ratio_floor) + endC)
print(cyan + "selectSamples() : " + endC + "ratio_per_class_dico : " + str(ratio_per_class_dico) + endC)
print(cyan + "selectSamples() : " + endC + "name_column : " + str(name_column) + endC)
print(cyan + "selectSamples() : " + endC + "no_data_value : " + str(no_data_value) + endC)
print(cyan + "selectSamples() : " + endC + "path_time_log : " + str(path_time_log) + endC)
print(cyan + "selectSamples() : " + endC + "rand_seed : " + str(rand_seed) + endC)
print(cyan + "selectSamples() : " + endC + "ram_otb : " + str(ram_otb) + endC)
print(cyan + "selectSamples() : " + endC + "epsg : " + str(epsg) + endC)
print(cyan + "selectSamples() : " + endC + "format_vector : " + str(format_vector) + endC)
print(cyan + "selectSamples() : " + endC + "extension_vector : " + str(extension_vector) + endC)
print(cyan + "selectSamples() : " + endC + "save_results_intermediate : " + str(save_results_intermediate) + endC)
print(cyan + "selectSamples() : " + endC + "overwrite : " + str(overwrite) + endC)
# Constantes
EXT_XML = ".xml"
SUFFIX_SAMPLE = "_sample"
SUFFIX_STATISTICS = "_statistics"
SUFFIX_POINTS = "_points"
SUFFIX_VALUE = "_value"
BAND_NAME = "band_"
COLUMN_CLASS = "class"
COLUMN_ORIGINFID = "originfid"
NB_POINTS = "nb_points"
AVERAGE = "average"
STANDARD_DEVIATION = "st_dev"
print(cyan + "selectSamples() : " + bold + green + "DEBUT DE LA SELECTION DE POINTS" + endC)
# Definition variables et chemins
repertory_output = os.path.dirname(vector_output)
filename = os.path.splitext(os.path.basename(vector_output))[0]
sample_points_output = repertory_output + os.sep + filename + SUFFIX_SAMPLE + extension_vector
file_statistic_points = repertory_output + os.sep + filename + SUFFIX_STATISTICS + SUFFIX_POINTS + EXT_XML
if debug >= 3:
print(cyan + "selectSamples() : " + endC + "file_statistic_points : " + str(file_statistic_points) + endC)
# 0. EXISTENCE DU FICHIER DE SORTIE
#----------------------------------
# Si le fichier vecteur points de sortie existe deja et que overwrite n'est pas activé
check = os.path.isfile(vector_output)
if check and not overwrite:
print(bold + yellow + "Samples points already done for file %s and will not be calculated again." %(vector_output) + endC)
else: # Si non ou si la vérification est désactivée : creation du fichier d'échantillons points
# Suppression de l'éventuel fichier existant
if check:
try:
removeVectorFile(vector_output)
except Exception:
pass # Si le fichier ne peut pas être supprimé, on suppose qu'il n'existe pas et on passe à la suite
if os.path.isfile(table_statistics_output) :
try:
removeFile(table_statistics_output)
except Exception:
pass # Si le fichier ne peut pas être supprimé, on suppose qu'il n'existe pas et on passe à la suite
# 1. STATISTIQUE SUR L'IMAGE DES ECHANTILLONS RASTEUR
#----------------------------------------------------
if debug >= 3:
print(cyan + "selectSamples() : " + bold + green + "Start statistique sur l'image des echantillons rasteur..." + endC)
id_micro_list = identifyPixelValues(sample_image_input)
if 0 in id_micro_list :
id_micro_list.remove(0)
min_micro_class_nb_points = -1
min_micro_class_label = 0
infoStructPointSource_dico = {}
writeTextFile(file_statistic_points, '<?xml version="1.0" ?>\n')
appendTextFileCR(file_statistic_points, '<GeneralStatistics>')
appendTextFileCR(file_statistic_points, ' <Statistic name="pointsPerClassRaw">')
if debug >= 2:
print("Nombre de points par micro classe :" + endC)
for id_micro in id_micro_list :
nb_pixels = countPixelsOfValue(sample_image_input, id_micro)
if debug >= 2:
print("MicroClass : " + str(id_micro) + ", nb_points = " + str(nb_pixels))
appendTextFileCR(file_statistic_points, ' <StatisticPoints class="%d" value="%d" />' %(id_micro, nb_pixels))
if min_micro_class_nb_points == -1 or min_micro_class_nb_points > nb_pixels :
min_micro_class_nb_points = nb_pixels
min_micro_class_label = id_micro
infoStructPointSource_dico[id_micro] = StructInfoMicoClass()
infoStructPointSource_dico[id_micro].label_class = id_micro
infoStructPointSource_dico[id_micro].nb_points = nb_pixels
infoStructPointSource_dico[id_micro].info_points_list = []
del nb_pixels
if debug >= 2:
print("MicroClass min points find : " + str(min_micro_class_label) + ", nb_points = " + str(min_micro_class_nb_points))
appendTextFileCR(file_statistic_points, ' </Statistic>')
pending_event = cyan + "selectSamples() : " + bold + green + "End statistique sur l'image des echantillons rasteur. " + endC
if debug >= 3:
print(pending_event)
timeLine(path_time_log,pending_event)
# 2. CHARGEMENT DE L'IMAGE DES ECHANTILLONS
#------------------------------------------
if debug >= 3:
print(cyan + "selectSamples() : " + bold + green + "Start chargement de l'image des echantillons..." + endC)
# Information image
cols, rows, bands = getGeometryImage(sample_image_input)
xmin, xmax, ymin, ymax = getEmpriseImage(sample_image_input)
pixel_width, pixel_height = getPixelWidthXYImage(sample_image_input)
projection_input = getProjectionImage(sample_image_input)
if projection_input == None or projection_input == 0 :
projection_input = epsg
else :
projection_input = int(projection_input)
pixel_width = abs(pixel_width)
pixel_height = abs(pixel_height)
# Lecture des données
raw_data = getRawDataImage(sample_image_input)
if debug >= 3:
print("projection = " + str(projection_input))
print("cols = " + str(cols))
print("rows = " + str(rows))
# Creation d'une structure dico contenent tous les points différents de zéro
progress = 0
pass_prog = False
for y_row in range(rows) :
for x_col in range(cols) :
value_class = raw_data[y_row][x_col]
if value_class != 0 :
infoStructPointSource_dico[value_class].info_points_list.append(x_col + (y_row * cols))
# Barre de progression
if debug >= 4:
if ((float(y_row) / rows) * 100.0 > progress) and not pass_prog :
progress += 1
pass_prog = True
print("Progression => " + str(progress) + "%")
if ((float(y_row) / rows) * 100.0 > progress + 1) :
pass_prog = False
del raw_data
pending_event = cyan + "selectSamples() : " + bold + green + "End chargement de l'image des echantillons. " + endC
if debug >= 3:
print(pending_event)
timeLine(path_time_log,pending_event)
# 3. SELECTION DES POINTS D'ECHANTILLON
#--------------------------------------
if debug >= 3:
print(cyan + "selectSamples() : " + bold + green + "Start selection des points d'echantillon..." + endC)
appendTextFileCR(file_statistic_points, ' <Statistic name="pointsPerClassSelect">')
# Rendre deterministe la fonction aléatoire de random.sample
if rand_seed > 0:
random.seed( rand_seed )
# Pour toute les micro classes
for id_micro in id_micro_list :
# Selon la stategie de selection
nb_points_ratio = 0
while switch(sampler_strategy.lower()):
if case('all'):
# Le mode de selection 'all' est choisi
nb_points_ratio = infoStructPointSource_dico[id_micro].nb_points
infoStructPointSource_dico[id_micro].sample_points_list = range(nb_points_ratio)
break
if case('percent'):
# Le mode de selection 'percent' est choisi
id_macro_class = int(math.floor(id_micro / 100) * 100)
select_ratio_class = ratio_per_class_dico[id_macro_class]
nb_points_ratio = int(infoStructPointSource_dico[id_micro].nb_points * select_ratio_class / 100)
infoStructPointSource_dico[id_micro].sample_points_list = random.sample(range(infoStructPointSource_dico[id_micro].nb_points), nb_points_ratio)
break
if case('mixte'):
# Le mode de selection 'mixte' est choisi
nb_points_ratio = int(infoStructPointSource_dico[id_micro].nb_points * select_ratio_floor / 100)
if id_micro == min_micro_class_label :
# La plus petite micro classe est concervée intégralement
infoStructPointSource_dico[id_micro].sample_points_list = range(infoStructPointSource_dico[id_micro].nb_points)
nb_points_ratio = min_micro_class_nb_points
elif nb_points_ratio <= min_micro_class_nb_points :
# Les micro classes dont le ratio de selection est inferieur au nombre de points de la plus petite classe sont égement conservées intégralement
infoStructPointSource_dico[id_micro].sample_points_list = random.sample(range(infoStructPointSource_dico[id_micro].nb_points), min_micro_class_nb_points)
nb_points_ratio = min_micro_class_nb_points
else :
# Pour toutes les autres micro classes tirage aleatoire d'un nombre de points correspondant au ratio
infoStructPointSource_dico[id_micro].sample_points_list = random.sample(range(infoStructPointSource_dico[id_micro].nb_points), nb_points_ratio)
break
break
if debug >= 2:
print("MicroClass = " + str(id_micro) + ", nb_points_ratio " + str(nb_points_ratio))
appendTextFileCR(file_statistic_points, ' <StatisticPoints class="%d" value="%d" />' %(id_micro, nb_points_ratio))
appendTextFileCR(file_statistic_points, ' </Statistic>')
appendTextFileCR(file_statistic_points, '</GeneralStatistics>')
pending_event = cyan + "selectSamples() : " + bold + green + "End selection des points d'echantillon. " + endC
if debug >= 3:
print(pending_event)
timeLine(path_time_log,pending_event)
# 4. PREPARATION DES POINTS D'ECHANTILLON
#----------------------------------------
if debug >= 3:
print(cyan + "selectSamples() : " + bold + green + "Start preparation des points d'echantillon..." + endC)
# Création du dico de points
points_random_value_dico = {}
index_dico_point = 0
for micro_class in infoStructPointSource_dico :
micro_class_struct = infoStructPointSource_dico[micro_class]
label_class = micro_class_struct.label_class
point_attr_dico = {name_column:int(label_class), COLUMN_CLASS:int(label_class), COLUMN_ORIGINFID:0}
for id_point in micro_class_struct.sample_points_list:
# Recuperer les valeurs des coordonnees des points
coor_x = float(xmin + (int(micro_class_struct.info_points_list[id_point] % cols) * pixel_width)) + (pixel_width / 2.0)
coor_y = float(ymax - (int(micro_class_struct.info_points_list[id_point] / cols) * pixel_height)) - (pixel_height / 2.0)
points_random_value_dico[index_dico_point] = [[coor_x, coor_y], point_attr_dico]
del coor_x
del coor_y
index_dico_point += 1
del point_attr_dico
del infoStructPointSource_dico
pending_event = cyan + "selectSamples() : " + bold + green + "End preparation des points d'echantillon. " + endC
if debug >=3:
print(pending_event)
timeLine(path_time_log,pending_event)
# 5. CREATION DU FICHIER SHAPE DE POINTS D'ECHANTILLON
#-----------------------------------------------------
if debug >= 3:
print(cyan + "selectSamples() : " + bold + green + "Start creation du fichier shape de points d'echantillon..." + endC)
# Définir les attibuts du fichier résultat
attribute_dico = {name_column:ogr.OFTInteger, COLUMN_CLASS:ogr.OFTInteger, COLUMN_ORIGINFID:ogr.OFTInteger}
# Creation du fichier shape
createPointsFromCoordList(attribute_dico, points_random_value_dico, sample_points_output, projection_input, format_vector)
del attribute_dico
del points_random_value_dico
pending_event = cyan + "selectSamples() : " + bold + green + "End creation du fichier shape de points d'echantillon. " + endC
if debug >=3:
print(pending_event)
timeLine(path_time_log,pending_event)
# 6. EXTRACTION DES POINTS D'ECHANTILLONS
#-----------------------------------------
if debug >= 3:
print(cyan + "selectSamples() : " + bold + green + "Start extraction des points d'echantillon dans l'image..." + endC)
# Cas ou l'on a une seule image
if len(image_input_list) == 1:
# Extract sample
image_input = image_input_list[0]
command = "otbcli_SampleExtraction -in %s -vec %s -outfield prefix -outfield.prefix.name %s -out %s -field %s" %(image_input, sample_points_output, BAND_NAME, vector_output, name_column)
if ram_otb > 0:
command += " -ram %d" %(ram_otb)
if debug >= 3:
print(command)
exitCode = os.system(command)
if exitCode != 0:
raise NameError(cyan + "selectSamples() : " + bold + red + "An error occured during otbcli_SampleExtraction command. See error message above." + endC)
# Cas de plusieurs imagettes
else :
# Le repertoire de sortie
repertory_output = os.path.dirname(vector_output)
# Initialisation de la liste pour le multi-threading et la liste de l'ensemble des echantions locaux
thread_list = []
vector_local_output_list = []
# Obtenir l'emprise des images d'entrées pour redecouper le vecteur d'echantillon d'apprentissage pour chaque image
for image_input in image_input_list :
# Definition des fichiers sur emprise local
file_name = os.path.splitext(os.path.basename(image_input))[0]
emprise_local_sample = repertory_output + os.sep + file_name + SUFFIX_SAMPLE + extension_vector
vector_sample_local_output = repertory_output + os.sep + file_name + SUFFIX_VALUE + extension_vector
vector_local_output_list.append(vector_sample_local_output)
# Gestion sans thread...
#SampleLocalExtraction(image_input, sample_points_output, emprise_local_sample, vector_sample_local_output, name_column, BAND_NAME, ram_otb, format_vector, extension_vector, save_results_intermediate)
# Gestion du multi threading
thread = threading.Thread(target=SampleLocalExtraction, args=(image_input, sample_points_output, emprise_local_sample, vector_sample_local_output, name_column, BAND_NAME, ram_otb, format_vector, extension_vector, save_results_intermediate))
thread.start()
thread_list.append(thread)
# Extraction des echantions points des images
try:
for thread in thread_list:
thread.join()
except:
print(cyan + "selectSamples() : " + bold + red + "Erreur lors de l'éextaction des valeurs d'echantion : impossible de demarrer le thread" + endC, file=sys.stderr)
# Fusion des multi vecteurs de points contenant les valeurs des bandes de l'image
fusionVectors(vector_local_output_list, vector_output, format_vector)
# Clean des vecteurs point sample local file
for vector_sample_local_output in vector_local_output_list :
removeVectorFile(vector_sample_local_output)
if debug >= 3:
print(cyan + "selectSamples() : " + bold + green + "End extraction des points d'echantillon dans l'image." + endC)
# 7. CALCUL DES STATISTIQUES SUR LES VALEURS DES POINTS D'ECHANTILLONS SELECTIONNEES
#-----------------------------------------------------------------------------------
if debug >= 3:
print(cyan + "selectSamples() : " + bold + green + "Start calcul des statistiques sur les valeurs des points d'echantillons selectionnees..." + endC)
# Si le calcul des statistiques est demandé presence du fichier stat
if table_statistics_output != "":
# On récupère la liste de données
pending_event = cyan + "selectSamples() : " + bold + green + "Encours calcul des statistiques part1... " + endC
if debug >=4:
print(pending_event)
timeLine(path_time_log,pending_event)
attribute_name_dico = {}
name_field_value_list = []
names_attribut_list = getAttributeNameList(vector_output, format_vector)
if debug >=4:
print("names_attribut_list = " + str(names_attribut_list))
attribute_name_dico[name_column] = ogr.OFTInteger
for name_attribut in names_attribut_list :
if BAND_NAME in name_attribut :
attribute_name_dico[name_attribut] = ogr.OFTReal
name_field_value_list.append(name_attribut)
name_field_value_list.sort()
res_values_dico = getAttributeValues(vector_output, None, None, attribute_name_dico, format_vector)
del attribute_name_dico
# Trie des données par identifiant micro classes
pending_event = cyan + "selectSamples() : " + bold + green + "Encours calcul des statistiques part2... " + endC
if debug >=4:
print(pending_event)
timeLine(path_time_log,pending_event)
data_value_by_micro_class_dico = {}
stat_by_micro_class_dico = {}
# Initilisation du dico complexe
for id_micro in id_micro_list :
data_value_by_micro_class_dico[id_micro] = {}
stat_by_micro_class_dico[id_micro] = {}
for name_field_value in res_values_dico :
if name_field_value != name_column :
data_value_by_micro_class_dico[id_micro][name_field_value] = []
stat_by_micro_class_dico[id_micro][name_field_value] = {}
stat_by_micro_class_dico[id_micro][name_field_value][AVERAGE] = 0.0
stat_by_micro_class_dico[id_micro][name_field_value][STANDARD_DEVIATION] = 0.0
# Trie des valeurs
pending_event = cyan + "selectSamples() : " + bold + green + "Encours calcul des statistiques part3... " + endC
if debug >=4:
print(pending_event)
timeLine(path_time_log,pending_event)
for index in range(len(res_values_dico[name_column])) :
id_micro = res_values_dico[name_column][index]
for name_field_value in name_field_value_list :
data_value_by_micro_class_dico[id_micro][name_field_value].append(res_values_dico[name_field_value][index])
del res_values_dico
# Calcul des statistiques
pending_event = cyan + "selectSamples() : " + bold + green + "Encours calcul des statistiques part4... " + endC
if debug >=4:
print(pending_event)
timeLine(path_time_log,pending_event)
for id_micro in id_micro_list :
for name_field_value in name_field_value_list :
try :
stat_by_micro_class_dico[id_micro][name_field_value][AVERAGE] = average(data_value_by_micro_class_dico[id_micro][name_field_value])
except:
stat_by_micro_class_dico[id_micro][name_field_value][AVERAGE] = 0
try :
stat_by_micro_class_dico[id_micro][name_field_value][STANDARD_DEVIATION] = standardDeviation(data_value_by_micro_class_dico[id_micro][name_field_value])
except:
stat_by_micro_class_dico[id_micro][name_field_value][STANDARD_DEVIATION] = 0
try :
stat_by_micro_class_dico[id_micro][name_field_value][NB_POINTS] = len(data_value_by_micro_class_dico[id_micro][name_field_value])
except:
stat_by_micro_class_dico[id_micro][name_field_value][NB_POINTS] = 0
del data_value_by_micro_class_dico
# Creation du fichier statistique .csv
pending_event = cyan + "selectSamples() : " + bold + green + "Encours calcul des statistiques part5... " + endC
if debug >= 4:
print(pending_event)
timeLine(path_time_log,pending_event)
text_csv = " Micro classes ; Champs couche image ; Nombre de points ; Moyenne ; Ecart type \n"
writeTextFile(table_statistics_output, text_csv)
for id_micro in id_micro_list :
for name_field_value in name_field_value_list :
# Ecriture du fichier
text_csv = " %d " %(id_micro)
text_csv += " ; %s" %(name_field_value)
text_csv += " ; %d" %(stat_by_micro_class_dico[id_micro][name_field_value][NB_POINTS])
text_csv += " ; %f" %(stat_by_micro_class_dico[id_micro][name_field_value][AVERAGE])
text_csv += " ; %f" %(stat_by_micro_class_dico[id_micro][name_field_value][STANDARD_DEVIATION])
appendTextFileCR(table_statistics_output, text_csv)
del name_field_value_list
else :
if debug >=3:
print(cyan + "selectSamples() : " + bold + green + "Pas de calcul des statistiques sur les valeurs des points demander!!!." + endC)
del id_micro_list
pending_event = cyan + "selectSamples() : " + bold + green + "End calcul des statistiques sur les valeurs des points d'echantillons selectionnees. " + endC
if debug >= 3:
print(pending_event)
timeLine(path_time_log,pending_event)
# 8. SUPRESSION DES FICHIERS INTERMEDIAIRES
#------------------------------------------
if not save_results_intermediate:
if os.path.isfile(sample_points_output) :
removeVectorFile(sample_points_output)
print(cyan + "selectSamples() : " + bold + green + "FIN DE LA SELECTION DE POINTS" + endC)
# Mise à jour du Log
ending_event = "selectSamples() : Select points in raster mask macro input ending : "
timeLine(path_time_log,ending_event)
return