def generateStatModel(pathShapes, pathToTiles, pathToStats, pathToCmdStats,
pathWd, pathConf):
AllCmd = []
modTiles = GM.getModel(pathShapes)
cfg = Config(pathConf)
Stack_ind = fu.getFeatStackName(pathConf)
classifier = Config(file(pathConf)).argTrain.classifier
for mod, Tiles in modTiles:
allpath = ""
for tile in Tiles:
pathToFeat = pathToTiles + "/" + tile + "/Final/" + Stack_ind
allpath = allpath + " " + pathToFeat + " "
# hpc case
#if pathWd != None:
# pathToStats = "$TMPDIR"
if classifier == "svm":
cmd = "otbcli_ComputeImagesStatistics -il " + allpath + " -out " + pathToStats + "/Model_" + str(
mod) + ".xml"
else:
cmd = "echo 'random forest does not need stats'"
AllCmd.append(cmd)
fu.writeCmds(pathToCmdStats + "/stats.txt", AllCmd)
return AllCmd
def outStats(config, tile, sample, workingDirectory):
Testpath = Config(file(config)).chain.outputPath
Nruns = int(Config(file(config)).chain.runs)
featuresPath = Config(file(config)).chain.featuresPath
stackName = fu.getFeatStackName(config)
statsName = ["ValidNOK", "ValidOK", "AppNOK", "AppOK"]
"""
1 valid NOK
2 valid OK
3 app NOK
4 app OK
"""
confStep = 1
confMin = 1
confMax = 100
cloudAllTile = Testpath + "/final/PixelsValidity.tif"
src_ds = gdal.Open(cloudAllTile)
if src_ds is None:
print 'Unable to open %s' % cloudAllTile
sys.exit(1)
srcband = src_ds.GetRasterBand(1).ReadAsArray()
maxView = np.amax(srcband)
Cloud = raster2array(Testpath + "/final/TMP/" + tile + "_Cloud_StatsOK.tif")
for seed in range(Nruns):
Classif = raster2array(Testpath + "/final/TMP/" + tile + "_seed_" + str(seed) + ".tif")
confidence = raster2array(Testpath + "/final/TMP/" + tile + "_GlobalConfidence_seed_" + str(seed) + ".tif")
difference = raster2array(Testpath + "/final/TMP/" + tile + "_seed_" + str(seed) + "_CompRef.tif")
diffHisto = getDiffHisto(confMin, confMax, confStep, confidence, difference)
statsTile = Testpath + "/final/TMP/" + tile + "_stats_seed_" + str(seed) + ".cfg"
stats = open(statsTile, "a")
stats.write("AllDiffStats:'" + ",".join(statsName) + "'\n")
stats.close()
for i in range(len(statsName)):
hist, bin_edges = histo(diffHisto[i + 1], bins=np.arange(confMin, confMax + 1, confStep))
hist_str = " ".join([str(currentVal) for currentVal in hist])
bin_edges_str = " ".join([str(currentVal) for currentVal in bin_edges])
genStatsDiff(statsTile, statsName[i], hist_str, bin_edges_str)
histNView, binsNview = histo(Cloud, bins=np.arange(0, maxView + 1, 1))
hist_str = " ".join([str(currentVal) for currentVal in histNView])
bin_edges_str = " ".join([str(currentVal) for currentVal in binsNview])
genStatsDiff(statsTile, "TileValidity", hist_str, bin_edges_str)
def generateSamples_classifMix(folderSample, workingDirectory, trainShape,
pathWd, featuresPath, samplesOptions,
annualCrop, AllClass, dataField, pathConf,
configPrevClassif):
currentTile, bindingPy = trainShape.split("/")[-1].split("_")[0], Config(
file(pathConf)).GlobChain.bindingPython
targetResolution, validityThreshold = Config(
file(pathConf)).chain.spatialResolution, Config(
file(pathConf)).argTrain.validityThreshold
previousClassifPath, projOut = Config(
file(configPrevClassif)).chain.outputPath, Config(
file(configPrevClassif)).GlobChain.proj
projOut = int(projOut.split(":")[-1])
stack = "/Final/" + fu.getFeatStackName(pathConf)
userFeatPath = Config(file(pathConf)).chain.userFeatPath
if userFeatPath == "None": userFeatPath = None
featImg = featuresPath + "/" + currentTile + "/" + stack
if bindingPy == "True":
featImg = fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/",
True, "ST_MASK")[0]
nameNonAnnual = trainShape.split("/")[-1].replace(".shp", "_NonAnnu.shp")
nonAnnualShape = workingDirectory + "/" + nameNonAnnual
nameAnnual = trainShape.split("/")[-1].replace(".shp", "_Annu.shp")
AnnualShape = workingDirectory + "/" + nameAnnual
nonAnnualCropFind = filterShpByClass(dataField, nonAnnualShape, AllClass,
trainShape)
annualCropFind = filterShpByClass(dataField, AnnualShape, annualCrop,
trainShape)
gdalDriver = "SQLite"
SampleSel_NA = workingDirectory + "/" + nameNonAnnual.replace(
".shp", "_SampleSel_NA.sqlite")
stats_NA = workingDirectory + "/" + nameNonAnnual.replace(
".shp", "_STATS.xml")
if nonAnnualCropFind:
cmd = "otbcli_PolygonClassStatistics -in " + featImg + " -vec " + nonAnnualShape + " -field " + dataField + " -out " + stats_NA
print cmd
os.system(cmd)
verifPolyStats(stats_NA)
cmd = "otbcli_SampleSelection -in " + featImg + " -vec " + nonAnnualShape + " -field " + dataField + " -instats " + stats_NA + " -out " + SampleSel_NA + " " + samplesOptions
print cmd
os.system(cmd)
allCoord = getPointsCoordInShape(SampleSel_NA, gdalDriver)
else:
allCoord = [0]
nameAnnual = trainShape.split("/")[-1].replace(".shp", "_Annu.sqlite")
annualShape = workingDirectory + "/" + nameAnnual
validityRaster = fu.FileSearch_AND(previousClassifPath + "/final/TMP",
True, currentTile, "Cloud.tif")[0]
classificationRaster = fu.FileSearch_AND(
previousClassifPath + "/final/TMP", True,
currentTile + "_seed_0.tif")[0]
maskFolder = previousClassifPath + "/classif/MASK"
if annualCropFind:
genAS.genAnnualShapePoints(allCoord, gdalDriver, workingDirectory,
targetResolution, annualCrop, dataField,
currentTile, validityThreshold,
validityRaster, classificationRaster,
maskFolder, trainShape, annualShape)
MergeName = trainShape.split("/")[-1].replace(".shp", "_selectionMerge")
sampleSelection = workingDirectory + "/" + MergeName + ".sqlite"
if nonAnnualCropFind and annualCropFind:
createSamplePoint(SampleSel_NA, annualShape, dataField,
sampleSelection, projOut)
elif nonAnnualCropFind and not annualCropFind:
shutil.copy(SampleSel_NA, sampleSelection)
elif not nonAnnualCropFind and annualCropFind:
shutil.copy(annualShape, sampleSelection)
samples = workingDirectory + "/" + trainShape.split("/")[-1].replace(
".shp", "_Samples.sqlite")
if bindingPy == "False":
folderSample + "/" + trainShape.split("/")[-1].replace(
".shp", "_Samples.sqlite")
if not os.path.exists(
folderSample + "/" +
trainShape.split("/")[-1].replace(".shp", "_Samples.sqlite")):
cmd = "otbcli_SampleExtraction -in " + featImg + " -vec " + sampleSelection + " -field " + dataField + " -out " + samples
print cmd
os.system(cmd)
else:
AllRefl = sorted(
fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True,
"REFL.tif"))
AllMask = sorted(
fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True,
"MASK.tif"))
datesInterp = sorted(
fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True,
"DatesInterp"))
realDates = sorted(
fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True,
"imagesDate"))
print AllRefl
print AllMask
print datesInterp
print realDates
# gapFill + feat
features = []
concatSensors = otb.Registry.CreateApplication("ConcatenateImages")
for refl, mask, datesInterp, realDates in zip(AllRefl, AllMask,
datesInterp, realDates):
gapFill = otb.Registry.CreateApplication(
"ImageTimeSeriesGapFilling")
nbDate = fu.getNbDateInTile(realDates)
nbReflBands = fu.getRasterNbands(refl)
comp = int(nbReflBands) / int(nbDate)
print datesInterp
if not isinstance(comp, int):
raise Exception("unvalid component by date (not integer) : " +
comp)
gapFill.SetParameterString("in", refl)
gapFill.SetParameterString("mask", mask)
gapFill.SetParameterString("comp", str(comp))
gapFill.SetParameterString("it", "linear")
gapFill.SetParameterString("id", realDates)
gapFill.SetParameterString("od", datesInterp)
gapFill.Execute()
concatSensors.AddImageToParameterInputImageList(
"il", gapFill.GetParameterOutputImage("out"))
features.append(gapFill)
# sensors Concatenation + sampleExtraction
sampleExtr = otb.Registry.CreateApplication("SampleExtraction")
sampleExtr.SetParameterString("ram", "128")
sampleExtr.SetParameterString("vec", sampleSelection)
sampleExtr.SetParameterString("field", dataField)
sampleExtr.SetParameterString("out", samples)
# if len(AllRefl) > 1:
# concatSensors.Execute()
# sampleExtr.SetParameterInputImage("in",concatSensors.GetParameterOutputImage("out"))
# else:
# sampleExtr.SetParameterInputImage("in",features[0].GetParameterOutputImage("out"))
# sampleExtr.ExecuteAndWriteOutput()
if len(AllRefl) > 1:
concatSensors.Execute()
allFeatures = concatSensors.GetParameterOutputImage("out")
else:
allFeatures = features[0].GetParameterOutputImage("out")
if userFeatPath:
print "Add user features"
userFeat_arbo = Config(file(pathConf)).userFeat.arbo
userFeat_pattern = (Config(
file(pathConf)).userFeat.patterns).split(",")
concatFeatures = otb.Registry.CreateApplication(
"ConcatenateImages")
userFeatures = fu.getUserFeatInTile(userFeatPath, currentTile,
userFeat_arbo,
userFeat_pattern)
concatFeatures.SetParameterStringList("il", userFeatures)
concatFeatures.Execute()
concatAllFeatures = otb.Registry.CreateApplication(
"ConcatenateImages")
concatAllFeatures.AddImageToParameterInputImageList(
"il", allFeatures)
concatAllFeatures.AddImageToParameterInputImageList(
"il", concatFeatures.GetParameterOutputImage("out"))
concatAllFeatures.Execute()
allFeatures = concatAllFeatures.GetParameterOutputImage("out")
sampleExtr.SetParameterInputImage("in", allFeatures)
sampleExtr.ExecuteAndWriteOutput()
if pathWd:
shutil.copy(
samples, folderSample + "/" +
trainShape.split("/")[-1].replace(".shp", "_Samples.sqlite"))
os.remove(SampleSel_NA)
os.remove(sampleSelection)
os.remove(stats_NA)
def generateSamples_cropMix(folderSample, workingDirectory, trainShape, pathWd,
featuresPath, samplesOptions, prevFeatures,
annualCrop, AllClass, dataField, pathConf):
currentTile = trainShape.split("/")[-1].split("_")[0]
bindingPy = Config(file(pathConf)).GlobChain.bindingPython
samplesClassifMix = Config(file(pathConf)).argTrain.samplesClassifMix
userFeatPath = Config(file(pathConf)).chain.userFeatPath
if userFeatPath == "None": userFeatPath = None
stack = "/Final/" + fu.getFeatStackName(pathConf)
NA_img = featuresPath + "/" + currentTile + "/" + stack
A_img = prevFeatures + "/" + currentTile + "/" + stack
if bindingPy == "True":
NA_img = fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/",
True, "ST_MASK")[0]
A_img = fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/",
True, "ST_MASK")[0]
# Step 1 : filter trainShape in order to keep non-annual class
nameNonAnnual = trainShape.split("/")[-1].replace(".shp", "_NonAnnu.shp")
nonAnnualShape = workingDirectory + "/" + nameNonAnnual
filterShpByClass(dataField, nonAnnualShape, AllClass, trainShape)
# Step 2 : filter trainShape in order to keep annual class
nameAnnual = trainShape.split("/")[-1].replace(".shp", "_Annu.shp")
annualShape = workingDirectory + "/" + nameAnnual
annualCropFind = filterShpByClass(dataField, annualShape, annualCrop,
trainShape)
# Step 3 : nonAnnual stats
stats_NA = workingDirectory + "/" + nameNonAnnual.replace(
".shp", "_STATS.xml")
cmd = "otbcli_PolygonClassStatistics -in " + NA_img + " -vec " + nonAnnualShape + " -field " + dataField + " -out " + stats_NA
print cmd
os.system(cmd)
verifPolyStats(stats_NA)
# Step 4 : Annual stats
stats_A = workingDirectory + "/" + nameAnnual.replace(".shp", "_STATS.xml")
cmd = "otbcli_PolygonClassStatistics -in " + A_img + " -vec " + annualShape + " -field " + dataField + " -out " + stats_A
if annualCropFind:
print cmd
os.system(cmd)
verifPolyStats(stats_A)
# Step 5 : Sample Selection NonAnnual
SampleSel_NA = workingDirectory + "/" + nameNonAnnual.replace(
".shp", "_SampleSel_NA.sqlite")
cmd = "otbcli_SampleSelection -in " + NA_img + " -vec " + nonAnnualShape + " -field " + dataField + " -instats " + stats_NA + " -out " + SampleSel_NA + " " + samplesOptions
print cmd
os.system(cmd)
# Step 6 : Sample Selection Annual
SampleSel_A = workingDirectory + "/" + nameAnnual.replace(
".shp", "_SampleSel_A.sqlite")
cmd = "otbcli_SampleSelection -in " + A_img + " -vec " + annualShape + " -field " + dataField + " -instats " + stats_A + " -out " + SampleSel_A + " " + samplesOptions
if annualCropFind:
print cmd
os.system(cmd)
SampleExtr_NA = workingDirectory + "/" + nameNonAnnual.replace(
".shp", "_SampleExtr_NA.sqlite")
SampleExtr_A = workingDirectory + "/" + nameAnnual.replace(
".shp", "_SampleExtr_A.sqlite")
if bindingPy == "False":
# Step 7 : Sample extraction NonAnnual
cmd = "otbcli_SampleExtraction -in " + NA_img + " -vec " + SampleSel_NA + " -field " + dataField + " -out " + SampleExtr_NA
print cmd
os.system(cmd)
# Step 8 : Sample extraction Annual
cmd = "otbcli_SampleExtraction -in " + A_img + " -vec " + SampleSel_A + " -field " + dataField + " -out " + SampleExtr_A
if annualCropFind:
print cmd
os.system(cmd)
else:
# Step 7 : Sample extraction NonAnnual
concatSensors = otb.Registry.CreateApplication("ConcatenateImages")
AllRefl = sorted(
fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True,
"REFL.tif"))
AllMask = sorted(
fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True,
"MASK.tif"))
datesInterp = sorted(
fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True,
"DatesInterp"))
realDates = sorted(
fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True,
"imagesDate"))
features = []
for refl, mask, datesInterp, realDates in zip(AllRefl, AllMask,
datesInterp, realDates):
gapFill = otb.Registry.CreateApplication(
"ImageTimeSeriesGapFilling")
nbDate = fu.getNbDateInTile(realDates)
nbReflBands = fu.getRasterNbands(refl)
comp = int(nbReflBands) / int(nbDate)
if not isinstance(comp, int):
raise Exception("unvalid component by date (not integer) : " +
comp)
gapFill.SetParameterString("in", refl)
gapFill.SetParameterString("mask", mask)
gapFill.SetParameterString("comp", str(comp))
gapFill.SetParameterString("it", "linear")
gapFill.SetParameterString("id", realDates)
gapFill.SetParameterString("od", datesInterp)
# gapFill.SetParameterString("ram","1024")
gapFill.Execute()
concatSensors.AddImageToParameterInputImageList(
"il", gapFill.GetParameterOutputImage("out"))
features.append(gapFill)
sampleExtr = otb.Registry.CreateApplication("SampleExtraction")
sampleExtr.SetParameterString("ram", "128")
sampleExtr.SetParameterString("vec", SampleSel_NA)
sampleExtr.SetParameterString("field", dataField)
sampleExtr.SetParameterString("out", SampleExtr_NA)
# if len(AllRefl) > 1:
# concatSensors.Execute()
# sampleExtr.SetParameterInputImage("in",concatSensors.GetParameterOutputImage("out"))
# else:
# sampleExtr.SetParameterInputImage("in",features[0].GetParameterOutputImage("out"))
# sampleExtr.ExecuteAndWriteOutput()
if len(AllRefl) > 1:
concatSensors.Execute()
allFeatures = concatSensors.GetParameterOutputImage("out")
else:
allFeatures = features[0].GetParameterOutputImage("out")
if userFeatPath:
print "Add user features"
userFeat_arbo = Config(file(pathConf)).userFeat.arbo
userFeat_pattern = (Config(
file(pathConf)).userFeat.patterns).split(",")
concatFeatures = otb.Registry.CreateApplication(
"ConcatenateImages")
userFeatures = fu.getUserFeatInTile(userFeatPath, currentTile,
userFeat_arbo,
userFeat_pattern)
concatFeatures.SetParameterStringList("il", userFeatures)
concatFeatures.Execute()
concatAllFeatures = otb.Registry.CreateApplication(
"ConcatenateImages")
concatAllFeatures.AddImageToParameterInputImageList(
"il", allFeatures)
concatAllFeatures.AddImageToParameterInputImageList(
"il", concatFeatures.GetParameterOutputImage("out"))
concatAllFeatures.Execute()
allFeatures = concatAllFeatures.GetParameterOutputImage("out")
sampleExtr.SetParameterInputImage("in", allFeatures)
sampleExtr.ExecuteAndWriteOutput()
# Step 8 : Sample extraction Annual
concatSensors = otb.Registry.CreateApplication("ConcatenateImages")
AllRefl = sorted(
fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True,
"REFL.tif"))
AllMask = sorted(
fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True,
"MASK.tif"))
datesInterp = sorted(
fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True,
"DatesInterp"))
realDates = sorted(
fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True,
"imagesDate"))
features = []
for refl, mask, datesInterp, realDates in zip(AllRefl, AllMask,
datesInterp, realDates):
gapFill = otb.Registry.CreateApplication(
"ImageTimeSeriesGapFilling")
nbDate = fu.getNbDateInTile(realDates)
nbReflBands = fu.getRasterNbands(refl)
comp = int(nbReflBands) / int(nbDate)
if not isinstance(comp, int):
raise Exception("unvalid component by date (not integer) : " +
comp)
gapFill.SetParameterString("in", refl)
gapFill.SetParameterString("mask", mask)
gapFill.SetParameterString("comp", str(comp))
gapFill.SetParameterString("it", "linear")
gapFill.SetParameterString("id", realDates)
gapFill.SetParameterString("od", datesInterp)
# gapFill.SetParameterString("ram","1024")
gapFill.Execute()
concatSensors.AddImageToParameterInputImageList(
"il", gapFill.GetParameterOutputImage("out"))
features.append(gapFill)
sampleExtr = otb.Registry.CreateApplication("SampleExtraction")
sampleExtr.SetParameterString("ram", "128")
sampleExtr.SetParameterString("vec", SampleSel_A)
sampleExtr.SetParameterString("field", dataField)
sampleExtr.SetParameterString("out", SampleExtr_A)
if len(AllRefl) > 1:
concatSensors.Execute()
allFeatures = concatSensors.GetParameterOutputImage("out")
else:
allFeatures = features[0].GetParameterOutputImage("out")
if userFeatPath:
print "Add user features"
userFeat_arbo = Config(file(pathConf)).userFeat.arbo
userFeat_pattern = (Config(
file(pathConf)).userFeat.patterns).split(",")
concatFeatures = otb.Registry.CreateApplication(
"ConcatenateImages")
userFeatures = fu.getUserFeatInTile(userFeatPath, currentTile,
userFeat_arbo,
userFeat_pattern)
concatFeatures.SetParameterStringList("il", userFeatures)
concatFeatures.Execute()
concatAllFeatures = otb.Registry.CreateApplication(
"ConcatenateImages")
concatAllFeatures.AddImageToParameterInputImageList(
"il", allFeatures)
concatAllFeatures.AddImageToParameterInputImageList(
"il", concatFeatures.GetParameterOutputImage("out"))
concatAllFeatures.Execute()
allFeatures = concatAllFeatures.GetParameterOutputImage("out")
sampleExtr.SetParameterInputImage("in", allFeatures)
if annualCropFind: sampleExtr.ExecuteAndWriteOutput()
# Step 9 : Merge
MergeName = trainShape.split("/")[-1].replace(".shp", "_Samples")
listToMerge = [SampleExtr_NA]
if annualCropFind:
# listToMerge = [SampleExtr_A,SampleExtr_NA]
listToMerge = [SampleExtr_NA, SampleExtr_A]
fu.mergeSQLite(MergeName, workingDirectory, listToMerge)
samples = workingDirectory + "/" + trainShape.split("/")[-1].replace(
".shp", "_Samples.sqlite")
os.remove(stats_NA)
os.remove(SampleSel_NA)
os.remove(SampleExtr_NA)
fu.removeShape(nonAnnualShape.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
if annualCropFind:
os.remove(stats_A)
os.remove(SampleSel_A)
os.remove(SampleExtr_A)
fu.removeShape(annualShape.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
if pathWd:
shutil.copy(
samples, folderSample + "/" +
trainShape.split("/")[-1].replace(".shp", "_Samples.sqlite"))
def generateSamples_simple(folderSample, workingDirectory, trainShape, pathWd,
featuresPath, samplesOptions, pathConf, dataField):
bindingPython = Config(file(pathConf)).GlobChain.bindingPython
dataField = Config(file(pathConf)).chain.dataField
outputPath = Config(file(pathConf)).chain.outputPath
userFeatPath = Config(file(pathConf)).chain.userFeatPath
if userFeatPath == "None": userFeatPath = None
tmpFolder = outputPath + "/TMPFOLDER"
if not os.path.exists(tmpFolder): os.mkdir(tmpFolder)
# Sensors
S2 = Sensors.Sentinel_2("", Opath(tmpFolder), pathConf, "")
L8 = Sensors.Landsat8("", Opath(tmpFolder), pathConf, "")
L5 = Sensors.Landsat5("", Opath(tmpFolder), pathConf, "")
# shutil.rmtree(tmpFolder, ignore_errors=True)
SensorsList = [S2, L8, L5]
stats = workingDirectory + "/" + trainShape.split("/")[-1].replace(
".shp", "_stats.xml")
tile = trainShape.split("/")[-1].split("_")[0]
stack = fu.getFeatStackName(pathConf)
feat = featuresPath + "/" + tile + "/Final/" + stack
if bindingPython == "True":
feat = fu.FileSearch_AND(featuresPath + "/" + tile + "/tmp/", True,
"ST_MASK")[0]
os.environ["ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS"] = "1"
cmd = "otbcli_PolygonClassStatistics -in " + feat + " -vec " + trainShape + " -out " + stats + " -field " + dataField
print cmd
os.system(cmd)
verifPolyStats(stats)
sampleSelection = workingDirectory + "/" + trainShape.split(
"/")[-1].replace(".shp", "_SampleSel.sqlite")
cmd = "otbcli_SampleSelection -out " + sampleSelection + " " + samplesOptions + " -field " + dataField + " -in " + feat + " -vec " + trainShape + " -instats " + stats
print cmd
os.system(cmd)
# if pathWd:shutil.copy(sampleSelection,folderSample)
os.environ["ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS"] = "5"
samples = workingDirectory + "/" + trainShape.split("/")[-1].replace(
".shp", "_Samples.sqlite")
if bindingPython == "True":
sampleExtr = otb.Registry.CreateApplication("SampleExtraction")
sampleExtr.SetParameterString("vec", sampleSelection)
sampleExtr.SetParameterString("field", dataField)
sampleExtr.SetParameterString("out", samples)
AllRefl = sorted(
fu.FileSearch_AND(featuresPath + "/" + tile + "/tmp/", True,
"REFL.tif"))
AllMask = sorted(
fu.FileSearch_AND(featuresPath + "/" + tile + "/tmp/", True,
"MASK.tif"))
datesInterp = sorted(
fu.FileSearch_AND(featuresPath + "/" + tile + "/tmp/", True,
"DatesInterp"))
realDates = sorted(
fu.FileSearch_AND(featuresPath + "/" + tile + "/tmp/", True,
"imagesDate"))
print AllRefl
print AllMask
print datesInterp
print realDates
# gapFill + feat
features = []
concatSensors = otb.Registry.CreateApplication("ConcatenateImages")
for refl, mask, datesInterp, realDates in zip(AllRefl, AllMask,
datesInterp, realDates):
gapFill = otb.Registry.CreateApplication(
"ImageTimeSeriesGapFilling")
nbDate = fu.getNbDateInTile(realDates)
nbReflBands = fu.getRasterNbands(refl)
comp = int(nbReflBands) / int(nbDate)
print datesInterp
if not isinstance(comp, int):
raise Exception("unvalid component by date (not integer) : " +
comp)
gapFill.SetParameterString("in", refl)
gapFill.SetParameterString("mask", mask)
gapFill.SetParameterString("comp", str(comp))
gapFill.SetParameterString("it", "linear")
gapFill.SetParameterString("id", realDates)
gapFill.SetParameterString("od", datesInterp)
gapFill.Execute()
# gapFill.SetParameterString("out","/ptmp/vincenta/tmp/TestGapFill.tif")
# gapFill.ExecuteAndWriteOutput()
# pause = raw_input("Pause1")
# featExtr = otb.Registry.CreateApplication("iota2FeatureExtraction")
# featExtr.SetParameterInputImage("in",gapFill.GetParameterOutputImage("out"))
# featExtr.SetParameterString("comp",str(comp))
# for currentSensor in SensorsList:
# if currentSensor.name in refl:
# red = str(currentSensor.bands["BANDS"]["red"])
# nir = str(currentSensor.bands["BANDS"]["NIR"])
# swir = str(currentSensor.bands["BANDS"]["SWIR"])
# featExtr.SetParameterString("red",red)
# featExtr.SetParameterString("nir",nir)
# featExtr.SetParameterString("swir",swir)
# featExtr.SetParameterString("ram","256")
# featExtr.Execute()
# features.append(featExtr)
concatSensors.AddImageToParameterInputImageList(
"il", gapFill.GetParameterOutputImage("out"))
features.append(gapFill)
# sensors Concatenation + sampleExtraction
sampleExtr = otb.Registry.CreateApplication("SampleExtraction")
sampleExtr.SetParameterString("ram", "1024")
sampleExtr.SetParameterString("vec", sampleSelection)
sampleExtr.SetParameterString("field", dataField)
sampleExtr.SetParameterString("out", samples)
if len(AllRefl) > 1:
concatSensors.Execute()
allFeatures = concatSensors.GetParameterOutputImage("out")
else:
allFeatures = features[0].GetParameterOutputImage("out")
if userFeatPath:
print "Add user features"
userFeat_arbo = Config(file(pathConf)).userFeat.arbo
userFeat_pattern = (Config(
file(pathConf)).userFeat.patterns).split(",")
concatFeatures = otb.Registry.CreateApplication(
"ConcatenateImages")
userFeatures = fu.getUserFeatInTile(userFeatPath, tile,
userFeat_arbo,
userFeat_pattern)
concatFeatures.SetParameterStringList("il", userFeatures)
concatFeatures.Execute()
concatAllFeatures = otb.Registry.CreateApplication(
"ConcatenateImages")
concatAllFeatures.AddImageToParameterInputImageList(
"il", allFeatures)
concatAllFeatures.AddImageToParameterInputImageList(
"il", concatFeatures.GetParameterOutputImage("out"))
concatAllFeatures.Execute()
allFeatures = concatAllFeatures.GetParameterOutputImage("out")
sampleExtr.SetParameterInputImage("in", allFeatures)
sampleExtr.ExecuteAndWriteOutput()
# cmd = "otbcli_SampleExtraction -field "+dataField+" -out "+samples+" -vec "+sampleSelection+" -in /ptmp/vincenta/tmp/TestGapFill.tif"
# print cmd
# pause = raw_input("Pause")
# os.system(cmd)
else:
cmd = "otbcli_SampleExtraction -field " + dataField + " -out " + samples + " -vec " + sampleSelection + " -in " + feat
print cmd
os.system(cmd)
if pathWd:
shutil.copy(
samples, folderSample + "/" +
trainShape.split("/")[-1].replace(".shp", "_Samples.sqlite"))
os.remove(sampleSelection)
os.remove(stats)
def launchClassification(model, pathConf, stat, pathToRT, pathToImg,
pathToRegion, fieldRegion, N, pathToCmdClassif,
pathOut, pathWd):
f = file(pathConf)
cfg = Config(f)
classif = cfg.argTrain.classifier
mode = cfg.chain.mode
outputPath = cfg.chain.outputPath
classifMode = cfg.argClassification.classifMode
regionMode = cfg.chain.mode
pixType = cfg.argClassification.pixType
bindingPy = cfg.GlobChain.bindingPython
Stack_ind = fu.getFeatStackName(pathConf)
AllCmd = []
allTiles_s = cfg.chain.listTile
allTiles = allTiles_s.split(" ")
maskFiles = pathOut + "/MASK"
if not os.path.exists(maskFiles):
os.system("mkdir " + maskFiles)
shpRName = pathToRegion.split("/")[-1].replace(".shp", "")
AllModel = fu.FileSearch_AND(model, True, "model", ".txt")
AllModel = [f for f in AllModel if os.path.splitext(f)[1] == '.txt']
for path in AllModel:
model = path.split("/")[-1].split("_")[1]
tiles = fu.getListTileFromModel(
model, outputPath + "/config_model/configModel.cfg")
model_Mask = model
if re.search('model_.*f.*_', path.split("/")[-1]):
model_Mask = path.split("/")[-1].split("_")[1].split("f")[0]
seed = path.split("/")[-1].split("_")[-1].replace(".txt", "")
tilesToEvaluate = tiles
if ("fusion" in classifMode
and regionMode != "outside") or (regionMode == "one_region"):
tilesToEvaluate = allTiles
# construction du string de sortie
for tile in tilesToEvaluate:
pathToFeat = pathToImg + "/" + tile + "/Final/" + Stack_ind
if bindingPy == "True":
pathToFeat = fu.FileSearch_AND(
pathToImg + "/" + tile + "/tmp/", True, ".tif")[0]
maskSHP = pathToRT + "/" + shpRName + "_region_" + model_Mask + "_" + tile + ".shp"
maskTif = shpRName + "_region_" + model_Mask + "_" + tile + ".tif"
CmdConfidenceMap = ""
confidenceMap = ""
if "fusion" in classifMode:
if mode != "outside":
tmp = pathOut.split("/")
if pathOut[-1] == "/":
del tmp[-1]
tmp[-1] = "envelope"
pathToEnvelope = "/".join(tmp)
maskSHP = pathToEnvelope + "/" + tile + ".shp"
confidenceMap = tile + "_model_" + model + "_confidence_seed_" + seed + ".tif"
CmdConfidenceMap = " -confmap " + pathOut + "/" + confidenceMap
if not os.path.exists(maskFiles + "/" + maskTif):
pathToMaskCommun = pathToImg + "/" + tile + "/tmp/MaskCommunSL.shp"
# cas cluster
if pathWd != None:
pathToMaskCommun = pathToImg + "/" + tile + "/MaskCommunSL.shp"
maskFiles = pathWd
nameOut = fu.ClipVectorData(maskSHP, pathToMaskCommun,
maskFiles,
maskTif.replace(".tif", ""))
cmdRaster = "otbcli_Rasterization -in " + nameOut + " -mode attribute -mode.attribute.field " + fieldRegion + " -im " + pathToFeat + " -out " + maskFiles + "/" + maskTif
if "fusion" in classifMode:
cmdRaster = "otbcli_Rasterization -in " + nameOut + " -mode binary -mode.binary.foreground 1 -im " + pathToFeat + " -out " + maskFiles + "/" + maskTif
print cmdRaster
os.system(cmdRaster)
if pathWd != None:
os.system("cp " + pathWd + "/" + maskTif + " " + pathOut +
"/MASK")
out = pathOut + "/Classif_" + tile + "_model_" + model + "_seed_" + seed + ".tif"
# hpc case
if pathWd != None:
out = "$TMPDIR/Classif_" + tile + "_model_" + model + "_seed_" + seed + ".tif"
CmdConfidenceMap = " -confmap $TMPDIR/" + confidenceMap
appli = "otbcli_ImageClassifier "
pixType_cmd = pixType
if bindingPy == "True":
appli = "python bPy_ImageClassifier.py -conf " + pathConf + " "
pixType_cmd = " -pixType " + pixType
cmd = appli + " -in " + pathToFeat + " -model " + path + " -mask " + pathOut + "/MASK/" + maskTif + " -out " + out + " " + pixType_cmd + " -ram 128 " + CmdConfidenceMap
# Ajout des stats lors de la phase de classification
if ("svm" in classif):
cmd = cmd + " -imstat " + stat + "/Model_" + str(
model) + ".xml"
AllCmd.append(cmd)
fu.writeCmds(pathToCmdClassif + "/class.txt", AllCmd)
return AllCmd
landsat5.setDatesVoulues(datesVoulues)
list_Sensor.append(landsat5)
if not ("None" in args.ipathS2):
PreProcessS2(args.config, args.ipathS2, args.opath) # resample if needed
Sentinel2 = Sentinel_2(args.ipathS2, opath, fconf, workRes)
datesVoulues = CreateFichierDatesReg(args.dateB_S2, args.dateE_S2, args.gapS2, opath.opathT, Sentinel2.name)
Sentinel2.setDatesVoulues(datesVoulues)
list_Sensor.append(Sentinel2)
imRef = list_Sensor[0].imRef
sensorRef = list_Sensor[0].name
StackName = fu.getFeatStackName(args.config)
Stack = args.opath + "/Final/" + StackName
allTiles = (Config(file(args.config)).chain.listTile).split()
userFeatPath = Config(file(args.config)).chain.userFeatPath
if userFeatPath == "None": userFeatPath = None
if userFeatPath:
userFeat_arbo = Config(file(args.config)).userFeat.arbo
userFeat_pattern = (Config(file(args.config)).userFeat.patterns).split(",")
tile = fu.findCurrentTileInString(Stack, allTiles)
allUserFeatures = " ".join(fu.getUserFeatInTile(userFeatPath, tile, userFeat_arbo, userFeat_pattern))
if not os.path.exists(Stack):
# Step 1 Creation des masques de bords
Step = 1
if log.dico[Step]:
for sensor in list_Sensor:
def launchTraining(pathShapes, pathConf, pathToTiles, dataField, stat, N,
pathToCmdTrain, out, pathWd, pathlog):
"""
OUT : les commandes pour l'app
"""
cmd_out = []
f = file(pathConf)
cfg = Config(f)
classif = cfg.argTrain.classifier
options = cfg.argTrain.options
outputPath = cfg.chain.outputPath
samplesMode = Config(file(pathConf)).argTrain.shapeMode
dataField = Config(file(pathConf)).chain.dataField
binding = Config(file(pathConf)).GlobChain.bindingPython
posModel = -3 # model's position, if training shape is split by "_"
Stack_ind = fu.getFeatStackName(pathConf)
pathToModelConfig = outputPath + "/config_model/configModel.cfg"
configModel = open(pathToModelConfig, "w")
configModel.write("AllModel:\n[\n")
configModel.close()
for seed in range(N):
pathAppVal = fu.FileSearch_AND(pathShapes, True, "seed" + str(seed),
".shp", "learn")
sort = [(path.split("/")[-1].split("_")[posModel], path)
for path in pathAppVal]
sort = fu.sortByFirstElem(sort)
# get tiles by model
names = []
for r, paths in sort:
tmp = ""
for i in range(len(paths)):
if i < len(paths) - 1:
tmp = tmp + paths[i].split("/")[-1].split("_")[0] + "_"
else:
tmp = tmp + paths[i].split("/")[-1].split("_")[0]
names.append(tmp)
cpt = 0
for r, paths in sort:
writeConfigName(r, names[cpt], pathToModelConfig)
cpt += 1
if samplesMode == "points":
pathAppVal = fu.FileSearch_AND(outputPath + "/learningSamples",
True, "seed" + str(seed), ".sqlite",
"learn")
sort = [(path.split("/")[-1].split("_")[posModel], path)
for path in pathAppVal]
for r, paths in sort:
print r
if samplesMode != "points":
cmd = buildTrainCmd_poly(r, paths, pathToTiles, Stack_ind,
classif, options, dataField, out,
seed, stat, pathlog)
else:
if binding == "True" and classif == "svm":
outStats = outputPath + "/stats/Model_" + r + ".xml"
if os.path.exists(outStats):
os.remove(outStats)
writeStatsFromSample(paths, outStats)
cmd = buildTrainCmd_points(r, paths, classif, options,
dataField, out, seed, stat, pathlog)
cmd_out.append(cmd)
configModel = open(pathToModelConfig, "a")
configModel.write("\n]\n")
configModel.close()
fu.writeCmds(pathToCmdTrain + "/train.txt", cmd_out)
return cmd_out
def ClassificationShaping(pathClassif, pathEnvelope, pathImg, fieldEnv, N,
pathOut, pathWd, pathConf, colorpath):
f = file(pathConf)
cfg = Config(f)
Stack_ind = fu.getFeatStackName(pathConf)
if pathWd == None:
TMP = pathOut + "/TMP"
if not os.path.exists(pathOut + "/TMP"):
os.mkdir(TMP)
else:
TMP = pathWd
if not os.path.exists(pathOut + "/TMP"):
os.mkdir(pathOut + "/TMP")
classifMode, pathTest, proj = cfg.argClassification.classifMode, cfg.chain.outputPath, \
cfg.GlobChain.proj.split(":")[-1]
AllTile, mode, pixType = cfg.chain.listTile.split(
" "), cfg.chain.mode, cfg.argClassification.pixType
featuresPath, outputStatistics, spatialResolution = cfg.chain.featuresPath, cfg.chain.outputStatistics, cfg.chain.spatialResolution
allTMPFolder = fu.fileSearchRegEx(pathTest + "/TMPFOLDER*")
if allTMPFolder:
for tmpFolder in allTMPFolder:
shutil.rmtree(tmpFolder)
genGlobalConfidence(AllTile, pathTest, N, mode, classifMode, pathWd,
pathConf)
if mode == "outside" and classifMode == "fusion":
old_classif = fu.fileSearchRegEx(
pathTest + "/classif/Classif_*_model_*f*_seed_*.tif")
for rm in old_classif:
print rm
os.remove(rm)
# os.system("mv "+rm+" "+pathTest+"/final/TMP/")
classification = []
confidence = []
cloud = []
for seed in range(N):
classification.append([])
confidence.append([])
cloud.append([])
sort = []
if classifMode == "separate" or mode == "outside":
AllClassifSeed = fu.FileSearch_AND(pathClassif, True, ".tif",
"Classif", "seed_" + str(seed))
ind = 1
elif classifMode == "fusion":
AllClassifSeed = fu.FileSearch_AND(
pathClassif, True, "_FUSION_NODATA_seed" + str(seed) + ".tif")
ind = 0
for tile in AllClassifSeed:
sort.append((tile.split("/")[-1].split("_")[ind], tile))
sort = fu.sortByFirstElem(sort)
for tile, paths in sort:
exp = ""
allCl = ""
allCl_rm = []
for i in range(len(paths)):
allCl = allCl + paths[i] + " "
allCl_rm.append(paths[i])
if i < len(paths) - 1:
exp = exp + "im" + str(i + 1) + "b1 + "
else:
exp = exp + "im" + str(i + 1) + "b1"
path_Cl_final = TMP + "/" + tile + "_seed_" + str(seed) + ".tif"
classification[seed].append(path_Cl_final)
cmd = 'otbcli_BandMath -il ' + allCl + '-out ' + path_Cl_final + ' ' + pixType + ' -exp "' + exp + '"'
print cmd
os.system(cmd)
for currentTileClassif in allCl_rm:
os.remove(currentTileClassif)
tileConfidence = pathOut + "/TMP/" + tile + "_GlobalConfidence_seed_" + str(
seed) + ".tif"
confidence[seed].append(tileConfidence)
cloudTile = fu.FileSearch_AND(featuresPath + "/" + tile, True,
"nbView.tif")[0]
ClassifTile = TMP + "/" + tile + "_seed_" + str(seed) + ".tif"
cloudTilePriority = pathTest + "/final/TMP/" + tile + "_Cloud.tif"
cloudTilePriority_tmp = TMP + "/" + tile + "_Cloud.tif"
cloudTilePriority_StatsOK = pathTest + "/final/TMP/" + tile + "_Cloud_StatsOK.tif"
cloudTilePriority_tmp_StatsOK = TMP + "/" + tile + "_Cloud_StatsOK.tif"
cloud[seed].append(cloudTilePriority)
if not os.path.exists(cloudTilePriority):
cmd_cloud = 'otbcli_BandMath -il ' + cloudTile + ' ' + ClassifTile + ' -out ' + cloudTilePriority_tmp + ' int16 -exp "im2b1>0?im1b1:0"'
print cmd_cloud
os.system(cmd_cloud)
if outputStatistics == "True":
cmd_cloud = 'otbcli_BandMath -il ' + cloudTile + ' ' + ClassifTile + ' -out ' + cloudTilePriority_tmp_StatsOK + ' int16 -exp "im2b1>0?im1b1:-1"'
print cmd_cloud
os.system(cmd_cloud)
if pathWd:
shutil.copy(cloudTilePriority_tmp_StatsOK,
cloudTilePriority_StatsOK)
os.remove(cloudTilePriority_tmp_StatsOK)
if pathWd:
shutil.copy(cloudTilePriority_tmp, cloudTilePriority)
os.remove(cloudTilePriority_tmp)
if pathWd != None:
os.system("cp -a " + TMP + "/* " + pathOut + "/TMP")
for seed in range(N):
assembleFolder = pathTest + "/final"
if pathWd: assembleFolder = pathWd
fu.assembleTile_Merge(
classification[seed], spatialResolution,
assembleFolder + "/Classif_Seed_" + str(seed) + ".tif")
if pathWd:
shutil.copy(pathWd + "/Classif_Seed_" + str(seed) + ".tif",
pathTest + "/final")
fu.assembleTile_Merge(
confidence[seed], spatialResolution,
assembleFolder + "/Confidence_Seed_" + str(seed) + ".tif")
if pathWd:
shutil.copy(pathWd + "/Confidence_Seed_" + str(seed) + ".tif",
pathTest + "/final")
color.CreateIndexedColorImage(
pathTest + "/final/Classif_Seed_" + str(seed) + ".tif", colorpath,
pixType)
fu.assembleTile_Merge(cloud[0], spatialResolution,
assembleFolder + "/PixelsValidity.tif")
if pathWd: shutil.copy(pathWd + "/PixelsValidity.tif", pathTest + "/final")