def genAnnualShapePoints(coord,
gdalDriver,
workingDirectory,
rasterResolution,
classToKeep,
dataField,
tile,
validityThreshold,
validityRaster,
classificationRaster,
masks,
inlearningShape,
outlearningShape,
epsg,
region_field_name,
runs,
annu_repartition,
logger=logger):
#Const
region_pos = 2 #in mask name if splited by '_'
learn_flag = "learn"
undetermined_flag = "XXXX"
tile_pos = 0
currentTile = os.path.splitext(os.path.basename(inlearningShape))[0]
classifName = currentTile + "_Classif.tif"
#check HPC mode
try:
PathWd = os.environ['TMPDIR']
except:
PathWd = None
projection = int(epsg)
vector_regions = []
add = 0
for current_seed in range(runs):
for currentMask in masks:
currentRegion = os.path.split(currentMask)[-1].split(
"_")[region_pos]
vector_region = os.path.join(
workingDirectory, "Annual_" + currentTile + "_region_" +
currentRegion + "_seed_" + str(current_seed) + ".sqlite")
vector_regions.append(vector_region)
rasterRdy = workingDirectory + "/" + classifName.replace(
".tif", "_RDY_" + str(currentRegion) + "_seed_" +
str(current_seed) + ".tif")
mapReg = otb.Registry.CreateApplication(
"ClassificationMapRegularization")
mapReg.SetParameterString("io.in", classificationRaster)
mapReg.SetParameterString("ip.undecidedlabel", "0")
mapReg.Execute()
useless = otb.Registry.CreateApplication("BandMath")
useless.SetParameterString("exp", "im1b1")
useless.SetParameterStringList("il", [validityRaster])
useless.SetParameterString("ram", "10000")
useless.Execute()
uselessMask = otb.Registry.CreateApplication("BandMath")
uselessMask.SetParameterString("exp", "im1b1")
uselessMask.SetParameterStringList("il", [currentMask])
uselessMask.SetParameterString("ram", "10000")
uselessMask.Execute()
valid = otb.Registry.CreateApplication("BandMath")
valid.SetParameterString(
"exp", "im1b1>" + str(validityThreshold) + "?im2b1:0")
valid.AddImageToParameterInputImageList(
"il", useless.GetParameterOutputImage("out"))
valid.AddImageToParameterInputImageList(
"il", mapReg.GetParameterOutputImage("io.out"))
valid.SetParameterString("ram", "10000")
valid.Execute()
rdy = otb.Registry.CreateApplication("BandMath")
rdy.SetParameterString("exp", "im1b1*(im2b1>=1?1:0)")
rdy.AddImageToParameterInputImageList(
"il", valid.GetParameterOutputImage("out"))
rdy.AddImageToParameterInputImageList(
"il", uselessMask.GetParameterOutputImage("out"))
rdy.SetParameterString(
"out", rasterRdy +
"?&streaming:type=stripped&streaming:sizemode=nbsplits&streaming:sizevalue=10"
)
rdy.SetParameterOutputImagePixelType("out",
otb.ImagePixelType_uint8)
rdy.ExecuteAndWriteOutput()
rasterArray = raster2array(rasterRdy)
rasterFile = gdal.Open(rasterRdy)
x_origin, y_origin = rasterFile.GetGeoTransform(
)[0], rasterFile.GetGeoTransform()[3]
sizeX, sizeY = rasterFile.GetGeoTransform(
)[1], rasterFile.GetGeoTransform()[5]
driver = ogr.GetDriverByName(gdalDriver)
if os.path.exists(vector_region):
driver.DeleteDataSource(vector_region)
data_source = driver.CreateDataSource(vector_region)
srs = osr.SpatialReference()
srs.ImportFromEPSG(projection)
layerName = "output" #layerName
layerOUT = data_source.CreateLayer(layerName, srs, ogr.wkbPoint)
add_origin_fields(inlearningShape, layerOUT, region_field_name,
runs)
for currentVal in classToKeep.data:
try:
nbSamples = annu_repartition[str(
currentVal)][currentRegion][current_seed]
except:
logger.info(
"class : {} does not exists in {} at seed {} in region {}"
.format(currentVal, inlearningShape, current_seed,
currentRegion))
continue
Y, X = np.where(rasterArray == int(currentVal))
XYcoordinates = []
for y, x in zip(Y, X):
X_c, Y_c = pixCoordinates(x, y, x_origin, y_origin, sizeX,
sizeY)
XYcoordinates.append((X_c, Y_c))
if nbSamples > len(XYcoordinates):
nbSamples = len(XYcoordinates)
for Xc, Yc in random.sample(
XYcoordinates,
nbSamples): #"0" for nbSamples allready manage ?
if coord and (Xc, Yc) not in coord:
feature = ogr.Feature(layerOUT.GetLayerDefn())
feature.SetField(dataField, int(currentVal))
wkt = "POINT(%f %f)" % (Xc, Yc)
point = ogr.CreateGeometryFromWkt(wkt)
feature.SetGeometry(point)
layerOUT.CreateFeature(feature)
feature.Destroy()
add += 1
data_source.Destroy()
os.remove(rasterRdy)
layerOUT = None
#Add region column and value
addField(vector_region,
region_field_name,
str(currentRegion),
valueType=str,
driver_name="SQLite")
#Add seed columns and value
for run in range(runs):
if run == current_seed:
addField(vector_region,
"seed_" + str(run),
learn_flag,
valueType=str,
driver_name="SQLite")
else:
addField(vector_region,
"seed_" + str(run),
undetermined_flag,
valueType=str,
driver_name="SQLite")
outlearningShape_name = os.path.splitext(
os.path.split(outlearningShape)[-1])[0]
outlearningShape_dir = os.path.split(outlearningShape)[0]
fu.mergeSQLite(outlearningShape_name, outlearningShape_dir, vector_regions)
for vec in vector_regions:
os.remove(vec)
if add == 0:
return False
else:
return True
def mergeFinalClassifications(iota2_dir,
dataField,
nom_path,
colorFile,
runs=1,
pixType='uint8',
method="majorityvoting",
undecidedlabel=255,
dempstershafer_mob="precision",
keep_runs_results=True,
enableCrossValidation=False,
validationShape=None,
workingDirectory=None,
logger=logger):
"""function use to merge classifications by majorityvoting or dempstershafer's method and evaluate it.
get all classifications Classif_Seed_*.tif in the /final directory and fusion them
under the raster call Classifications_fusion.tif. Then compute statistics using the
results_utils library
Parameters
----------
iota2_dir : string
path to the iota2's output path
dataField : string
data's field name
nom_path : string
path to the nomenclature file
colorFile : string
path to the color file description
runs : int
number of iota2 runs (random learning splits)
pixType : string
output pixel format (available in OTB)
method : string
fusion's method (majorityvoting/dempstershafer)
undecidedlabel : int
label for label for un-decisions
dempstershafer_mob : string
mass of belief measurement (precision/recall/accuracy/kappa)
keep_runs_results : bool
flag to inform if seeds results could be overwritten
enableCrossValidation : bool
flag to inform if cross validation is enable
validationShape : string
path to a shape dedicated to validate fusion of classifications
workingDirectory : string
path to a working directory
See Also
--------
results_utils.gen_confusion_matrix_fig
results_utils.stats_report
"""
import shutil
from Common import OtbAppBank as otbApp
from Validation import ResultsUtils as ru
from Common import CreateIndexedColorImage as color
fusion_name = "Classifications_fusion.tif"
new_results_seed_file = "RESULTS_seeds.txt"
fusion_vec_name = "fusion_validation" #without extension
confusion_matrix_name = "fusionConfusion.png"
if not method in ["majorityvoting", "dempstershafer"]:
err_msg = "the fusion method must be 'majorityvoting' or 'dempstershafer'"
logger.error(err_msg)
raise Exception(err_msg)
if not dempstershafer_mob in ["precision", "recall", "accuracy", "kappa"]:
err_msg = "the dempstershafer MoB must be 'precision' or 'recall' or 'accuracy' or 'kappa'"
logger.error(err_msg)
raise Exception(err_msg)
iota2_dir_final = os.path.join(iota2_dir, "final")
wd = iota2_dir_final
wd_merge = os.path.join(iota2_dir_final, "merge_final_classifications")
if workingDirectory:
wd = workingDirectory
wd_merge = workingDirectory
final_classifications = [
fut.FileSearch_AND(iota2_dir_final, True,
"Classif_Seed_{}.tif".format(run))[0]
for run in range(runs)
]
fusion_path = os.path.join(wd, fusion_name)
fusion_options = compute_fusion_options(iota2_dir_final,
final_classifications, method,
undecidedlabel, dempstershafer_mob,
pixType, fusion_path)
logger.debug("fusion options:")
logger.debug(fusion_options)
fusion_app = otbApp.CreateFusionOfClassificationsApplication(
fusion_options)
logger.debug("START fusion of final classifications")
fusion_app.ExecuteAndWriteOutput()
logger.debug("END fusion of final classifications")
fusion_color_index = color.CreateIndexedColorImage(
fusion_path,
colorFile,
co_option=["COMPRESS=LZW"],
output_pix_type=gdal.GDT_Byte
if pixType == "uint8" else gdal.GDT_UInt16)
confusion_matrix = os.path.join(iota2_dir_final,
"merge_final_classifications",
"confusion_mat_maj_vote.csv")
if enableCrossValidation is False:
vector_val = fut.FileSearch_AND(
os.path.join(iota2_dir_final, "merge_final_classifications"), True,
"_majvote.sqlite")
else:
vector_val = fut.FileSearch_AND(os.path.join(iota2_dir, "dataAppVal"),
True, "val.sqlite")
if validationShape:
validation_vector = validationShape
else:
fut.mergeSQLite(fusion_vec_name, wd_merge, vector_val)
validation_vector = os.path.join(wd_merge, fusion_vec_name + ".sqlite")
confusion = otbApp.CreateComputeConfusionMatrixApplication({
"in":
fusion_path,
"out":
confusion_matrix,
"ref":
"vector",
"ref.vector.nodata":
"0",
"ref.vector.in":
validation_vector,
"ref.vector.field":
dataField.lower(),
"nodatalabel":
"0",
"ram":
"5000"
})
confusion.ExecuteAndWriteOutput()
maj_vote_conf_mat = os.path.join(iota2_dir_final, confusion_matrix_name)
ru.gen_confusion_matrix_fig(csv_in=confusion_matrix,
out_png=maj_vote_conf_mat,
nomenclature_path=nom_path,
undecidedlabel=undecidedlabel,
dpi=900)
if keep_runs_results:
seed_results = fut.FileSearch_AND(iota2_dir_final, True,
"RESULTS.txt")[0]
shutil.copy(seed_results,
os.path.join(iota2_dir_final, new_results_seed_file))
maj_vote_report = os.path.join(iota2_dir_final, "RESULTS.txt")
ru.stats_report(csv_in=[confusion_matrix],
nomenclature_path=nom_path,
out_report=maj_vote_report,
undecidedlabel=undecidedlabel)
if workingDirectory:
shutil.copy(fusion_path, iota2_dir_final)
shutil.copy(fusion_color_index, iota2_dir_final)
os.remove(fusion_path)
def DoAugmentation(samples,
class_augmentation,
strategy,
field,
excluded_fields=[],
Jstdfactor=None,
Sneighbors=None,
workingDirectory=None,
logger=logger):
"""perform data augmentation according to input parameters
Parameters
----------
samples : string
path to the set of samples to augment (OGR geometries must be 'POINT')
class_augmentation : dict
number of new samples to compute by class
strategy : string
which method to use in order to perform data augmentation (replicate/jitter/smote)
field : string
data's field
excluded_fields : list
do not consider these fields to perform data augmentation
Jstdfactor : float
Factor for dividing the standard deviation of each feature
Sneighbors : int
Number of nearest neighbors (smote's method)
workingDirectory : string
path to a working directory
Note
----
This function use the OTB's application **SampleAugmentation**,
more documentation
`here <http://www.orfeo-toolbox.org/Applications/SampleAugmentation.html>`_
"""
from Common import OtbAppBank
samples_dir_o, samples_name = os.path.split(samples)
samples_dir = samples_dir_o
if workingDirectory:
samples_dir = workingDirectory
shutil.copy(samples, samples_dir)
samples = os.path.join(samples_dir, samples_name)
augmented_files = []
for class_name, class_samples_augmentation in list(
class_augmentation.items()):
logger.info(
"{} samples of class {} will be generated by data augmentation ({} method) in {}"
.format(class_samples_augmentation, class_name, strategy, samples))
sample_name_augmented = "_".join([
os.path.splitext(samples_name)[0],
"aug_class_{}.sqlite".format(class_name)
])
output_sample_augmented = os.path.join(samples_dir,
sample_name_augmented)
parameters = {
"in": samples,
"field": field,
"out": output_sample_augmented,
"label": class_name,
"strategy": strategy,
"samples": class_samples_augmentation
}
if excluded_fields:
parameters["exclude"] = excluded_fields
if strategy.lower() == "jitter":
parameters["strategy.jitter.stdfactor"] = Jstdfactor
elif strategy.lower() == "smote":
parameters["strategy.smote.neighbors"] = Sneighbors
augmentation_application = OtbAppBank.CreateSampleAugmentationApplication(
parameters)
augmentation_application.ExecuteAndWriteOutput()
logger.debug("{} samples of class {} were added in {}".format(
class_samples_augmentation, class_name, samples))
augmented_files.append(output_sample_augmented)
outputVector = os.path.join(
samples_dir,
"_".join([os.path.splitext(samples_name)[0], "augmented.sqlite"]))
fut.mergeSQLite("_".join([os.path.splitext(samples_name)[0], "augmented"]),
samples_dir, [samples] + augmented_files)
logger.info("Every data augmentation done in {}".format(samples))
shutil.move(outputVector, os.path.join(samples_dir_o, samples_name))
#clean-up
for augmented_file in augmented_files:
os.remove(augmented_file)
