def erodeInter(currentTile, NextTile, intersection, buff, proj):
xo, yo = currentTile.getOrigin()
xn, yn = NextTile.getOrigin()
Extent = getShapeExtent(intersection)
if yo == yn and xo != xn: # left priority
minX = Extent[0]
maxX = Extent[1] - buff
minY = Extent[2]
maxY = Extent[3]
elif yo != yn and xo == xn: # upper priority
minX = Extent[0]
maxX = Extent[1]
minY = Extent[2] + buff
maxY = Extent[3]
else:
return False
fu.removeShape(intersection.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
pathFolder = "/".join(
intersection.split("/")[0:len(intersection.split("/")) - 1])
createShape(minX, minY, maxX, maxY, pathFolder,
intersection.split("/")[-1].replace(".shp", ""), proj)
return True
def genNbView(TilePath, maskOut, nbview, workingDirectory=None):
"""
"""
nameNbView = "nbView.tif"
wd = TilePath
if workingDirectory: wd = workingDirectory
tmp1 = wd + "/" + nameNbView
if not os.path.exists(TilePath + "/" + nameNbView):
# build stack
MaskStack = "AllSensorMask.tif"
maskList = fu.FileSearch_AND(TilePath, True, "_ST_MASK.tif")
maskList = " ".join(maskList)
# cmd = "gdalbuildvrt "+TilePath+"/"+MaskStack+" "+maskList
cmd = "otbcli_ConcatenateImages -il " + maskList + " -out " + TilePath + "/" + MaskStack + " int16"
print cmd
os.system(cmd)
exp = buildExpression_cloud(TilePath + "/" + MaskStack)
tmp2 = maskOut.replace(".shp", "_tmp_2.tif").replace(TilePath, wd)
cmd = 'otbcli_BandMath -il ' + TilePath + "/" + MaskStack + ' -out ' + tmp1 + ' uint16 -exp "' + exp + '"'
print cmd
os.system(cmd)
cmd = 'otbcli_BandMath -il ' + tmp1 + ' -out ' + tmp2 + ' -exp "im1b1>=' + str(
nbview) + '?1:0"'
print cmd
os.system(cmd)
maskOut_tmp = maskOut.replace(".shp", "_tmp.shp").replace(TilePath, wd)
cmd = "gdal_polygonize.py -mask " + tmp2 + " " + tmp2 + " -f \"ESRI Shapefile\" " + maskOut_tmp
print cmd
os.system(cmd)
fu.erodeShapeFile(maskOut_tmp, wd + "/" + maskOut.split("/")[-1], 0.1)
os.remove(tmp2)
fu.removeShape(maskOut_tmp.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
if workingDirectory:
shutil.copy(tmp1, TilePath)
fu.cpShapeFile(wd + "/" +
maskOut.split("/")[-1].replace(".shp", ""),
TilePath, [".prj", ".shp", ".dbf", ".shx"],
spe=True)
def split_All_shape(shape, folds, pathConf, pathWd):
f = file(pathConf)
cfg = Config(f)
regionField = cfg.chain.regionField
outputpath = cfg.chain.outputPath
dataField = cfg.chain.dataField
workingDirectory = outputpath + "/dataAppVal"
if pathWd != None:
workingDirectory = pathWd
createdShape = SplitShape(shape, dataField, folds, workingDirectory,
shape.split("/")[-1])
fu.removeShape(shape.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"])
if pathWd != None:
for NewShape in createdShape:
fu.cpShapeFile(NewShape.replace(".shp", ""),
outputpath + "/dataAppVal",
[".prj", ".shp", ".dbf", ".shx"],
spe=True)
def generateSamples_cropMix(folderSample, workingDirectory, trainShape, pathWd,
featuresPath, samplesOptions, prevFeatures,
annualCrop, AllClass, dataField, pathConf):
currentTile = trainShape.split("/")[-1].split("_")[0]
bindingPy = Config(file(pathConf)).GlobChain.bindingPython
samplesClassifMix = Config(file(pathConf)).argTrain.samplesClassifMix
userFeatPath = Config(file(pathConf)).chain.userFeatPath
if userFeatPath == "None": userFeatPath = None
stack = "/Final/" + fu.getFeatStackName(pathConf)
NA_img = featuresPath + "/" + currentTile + "/" + stack
A_img = prevFeatures + "/" + currentTile + "/" + stack
if bindingPy == "True":
NA_img = fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/",
True, "ST_MASK")[0]
A_img = fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/",
True, "ST_MASK")[0]
# Step 1 : filter trainShape in order to keep non-annual class
nameNonAnnual = trainShape.split("/")[-1].replace(".shp", "_NonAnnu.shp")
nonAnnualShape = workingDirectory + "/" + nameNonAnnual
filterShpByClass(dataField, nonAnnualShape, AllClass, trainShape)
# Step 2 : filter trainShape in order to keep annual class
nameAnnual = trainShape.split("/")[-1].replace(".shp", "_Annu.shp")
annualShape = workingDirectory + "/" + nameAnnual
annualCropFind = filterShpByClass(dataField, annualShape, annualCrop,
trainShape)
# Step 3 : nonAnnual stats
stats_NA = workingDirectory + "/" + nameNonAnnual.replace(
".shp", "_STATS.xml")
cmd = "otbcli_PolygonClassStatistics -in " + NA_img + " -vec " + nonAnnualShape + " -field " + dataField + " -out " + stats_NA
print cmd
os.system(cmd)
verifPolyStats(stats_NA)
# Step 4 : Annual stats
stats_A = workingDirectory + "/" + nameAnnual.replace(".shp", "_STATS.xml")
cmd = "otbcli_PolygonClassStatistics -in " + A_img + " -vec " + annualShape + " -field " + dataField + " -out " + stats_A
if annualCropFind:
print cmd
os.system(cmd)
verifPolyStats(stats_A)
# Step 5 : Sample Selection NonAnnual
SampleSel_NA = workingDirectory + "/" + nameNonAnnual.replace(
".shp", "_SampleSel_NA.sqlite")
cmd = "otbcli_SampleSelection -in " + NA_img + " -vec " + nonAnnualShape + " -field " + dataField + " -instats " + stats_NA + " -out " + SampleSel_NA + " " + samplesOptions
print cmd
os.system(cmd)
# Step 6 : Sample Selection Annual
SampleSel_A = workingDirectory + "/" + nameAnnual.replace(
".shp", "_SampleSel_A.sqlite")
cmd = "otbcli_SampleSelection -in " + A_img + " -vec " + annualShape + " -field " + dataField + " -instats " + stats_A + " -out " + SampleSel_A + " " + samplesOptions
if annualCropFind:
print cmd
os.system(cmd)
SampleExtr_NA = workingDirectory + "/" + nameNonAnnual.replace(
".shp", "_SampleExtr_NA.sqlite")
SampleExtr_A = workingDirectory + "/" + nameAnnual.replace(
".shp", "_SampleExtr_A.sqlite")
if bindingPy == "False":
# Step 7 : Sample extraction NonAnnual
cmd = "otbcli_SampleExtraction -in " + NA_img + " -vec " + SampleSel_NA + " -field " + dataField + " -out " + SampleExtr_NA
print cmd
os.system(cmd)
# Step 8 : Sample extraction Annual
cmd = "otbcli_SampleExtraction -in " + A_img + " -vec " + SampleSel_A + " -field " + dataField + " -out " + SampleExtr_A
if annualCropFind:
print cmd
os.system(cmd)
else:
# Step 7 : Sample extraction NonAnnual
concatSensors = otb.Registry.CreateApplication("ConcatenateImages")
AllRefl = sorted(
fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True,
"REFL.tif"))
AllMask = sorted(
fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True,
"MASK.tif"))
datesInterp = sorted(
fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True,
"DatesInterp"))
realDates = sorted(
fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True,
"imagesDate"))
features = []
for refl, mask, datesInterp, realDates in zip(AllRefl, AllMask,
datesInterp, realDates):
gapFill = otb.Registry.CreateApplication(
"ImageTimeSeriesGapFilling")
nbDate = fu.getNbDateInTile(realDates)
nbReflBands = fu.getRasterNbands(refl)
comp = int(nbReflBands) / int(nbDate)
if not isinstance(comp, int):
raise Exception("unvalid component by date (not integer) : " +
comp)
gapFill.SetParameterString("in", refl)
gapFill.SetParameterString("mask", mask)
gapFill.SetParameterString("comp", str(comp))
gapFill.SetParameterString("it", "linear")
gapFill.SetParameterString("id", realDates)
gapFill.SetParameterString("od", datesInterp)
# gapFill.SetParameterString("ram","1024")
gapFill.Execute()
concatSensors.AddImageToParameterInputImageList(
"il", gapFill.GetParameterOutputImage("out"))
features.append(gapFill)
sampleExtr = otb.Registry.CreateApplication("SampleExtraction")
sampleExtr.SetParameterString("ram", "128")
sampleExtr.SetParameterString("vec", SampleSel_NA)
sampleExtr.SetParameterString("field", dataField)
sampleExtr.SetParameterString("out", SampleExtr_NA)
# if len(AllRefl) > 1:
# concatSensors.Execute()
# sampleExtr.SetParameterInputImage("in",concatSensors.GetParameterOutputImage("out"))
# else:
# sampleExtr.SetParameterInputImage("in",features[0].GetParameterOutputImage("out"))
# sampleExtr.ExecuteAndWriteOutput()
if len(AllRefl) > 1:
concatSensors.Execute()
allFeatures = concatSensors.GetParameterOutputImage("out")
else:
allFeatures = features[0].GetParameterOutputImage("out")
if userFeatPath:
print "Add user features"
userFeat_arbo = Config(file(pathConf)).userFeat.arbo
userFeat_pattern = (Config(
file(pathConf)).userFeat.patterns).split(",")
concatFeatures = otb.Registry.CreateApplication(
"ConcatenateImages")
userFeatures = fu.getUserFeatInTile(userFeatPath, currentTile,
userFeat_arbo,
userFeat_pattern)
concatFeatures.SetParameterStringList("il", userFeatures)
concatFeatures.Execute()
concatAllFeatures = otb.Registry.CreateApplication(
"ConcatenateImages")
concatAllFeatures.AddImageToParameterInputImageList(
"il", allFeatures)
concatAllFeatures.AddImageToParameterInputImageList(
"il", concatFeatures.GetParameterOutputImage("out"))
concatAllFeatures.Execute()
allFeatures = concatAllFeatures.GetParameterOutputImage("out")
sampleExtr.SetParameterInputImage("in", allFeatures)
sampleExtr.ExecuteAndWriteOutput()
# Step 8 : Sample extraction Annual
concatSensors = otb.Registry.CreateApplication("ConcatenateImages")
AllRefl = sorted(
fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True,
"REFL.tif"))
AllMask = sorted(
fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True,
"MASK.tif"))
datesInterp = sorted(
fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True,
"DatesInterp"))
realDates = sorted(
fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True,
"imagesDate"))
features = []
for refl, mask, datesInterp, realDates in zip(AllRefl, AllMask,
datesInterp, realDates):
gapFill = otb.Registry.CreateApplication(
"ImageTimeSeriesGapFilling")
nbDate = fu.getNbDateInTile(realDates)
nbReflBands = fu.getRasterNbands(refl)
comp = int(nbReflBands) / int(nbDate)
if not isinstance(comp, int):
raise Exception("unvalid component by date (not integer) : " +
comp)
gapFill.SetParameterString("in", refl)
gapFill.SetParameterString("mask", mask)
gapFill.SetParameterString("comp", str(comp))
gapFill.SetParameterString("it", "linear")
gapFill.SetParameterString("id", realDates)
gapFill.SetParameterString("od", datesInterp)
# gapFill.SetParameterString("ram","1024")
gapFill.Execute()
concatSensors.AddImageToParameterInputImageList(
"il", gapFill.GetParameterOutputImage("out"))
features.append(gapFill)
sampleExtr = otb.Registry.CreateApplication("SampleExtraction")
sampleExtr.SetParameterString("ram", "128")
sampleExtr.SetParameterString("vec", SampleSel_A)
sampleExtr.SetParameterString("field", dataField)
sampleExtr.SetParameterString("out", SampleExtr_A)
if len(AllRefl) > 1:
concatSensors.Execute()
allFeatures = concatSensors.GetParameterOutputImage("out")
else:
allFeatures = features[0].GetParameterOutputImage("out")
if userFeatPath:
print "Add user features"
userFeat_arbo = Config(file(pathConf)).userFeat.arbo
userFeat_pattern = (Config(
file(pathConf)).userFeat.patterns).split(",")
concatFeatures = otb.Registry.CreateApplication(
"ConcatenateImages")
userFeatures = fu.getUserFeatInTile(userFeatPath, currentTile,
userFeat_arbo,
userFeat_pattern)
concatFeatures.SetParameterStringList("il", userFeatures)
concatFeatures.Execute()
concatAllFeatures = otb.Registry.CreateApplication(
"ConcatenateImages")
concatAllFeatures.AddImageToParameterInputImageList(
"il", allFeatures)
concatAllFeatures.AddImageToParameterInputImageList(
"il", concatFeatures.GetParameterOutputImage("out"))
concatAllFeatures.Execute()
allFeatures = concatAllFeatures.GetParameterOutputImage("out")
sampleExtr.SetParameterInputImage("in", allFeatures)
if annualCropFind: sampleExtr.ExecuteAndWriteOutput()
# Step 9 : Merge
MergeName = trainShape.split("/")[-1].replace(".shp", "_Samples")
listToMerge = [SampleExtr_NA]
if annualCropFind:
# listToMerge = [SampleExtr_A,SampleExtr_NA]
listToMerge = [SampleExtr_NA, SampleExtr_A]
fu.mergeSQLite(MergeName, workingDirectory, listToMerge)
samples = workingDirectory + "/" + trainShape.split("/")[-1].replace(
".shp", "_Samples.sqlite")
os.remove(stats_NA)
os.remove(SampleSel_NA)
os.remove(SampleExtr_NA)
fu.removeShape(nonAnnualShape.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
if annualCropFind:
os.remove(stats_A)
os.remove(SampleSel_A)
os.remove(SampleExtr_A)
fu.removeShape(annualShape.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
if pathWd:
shutil.copy(
samples, folderSample + "/" +
trainShape.split("/")[-1].replace(".shp", "_Samples.sqlite"))
def genTileEnvPrio(ObjListTile, out, tmpFile, proj):
buff = 600 # offset in order to manage nodata in image's border
ObjListTile.reverse()
listSHP = [
createRasterFootprint(c_ObjListTile.getPath(),
tmpFile + "/" + c_ObjListTile.getName() + ".shp")
for c_ObjListTile in ObjListTile
]
for env, currentTile in zip(listSHP, ObjListTile):
currentTile.setEnvelope(env)
currentTile.setPriorityEnv(env.replace(".shp", "_PRIO.shp"))
fu.cpShapeFile(env.replace(".shp", ""),
env.replace(".shp", "") + "_PRIO",
[".prj", ".shp", ".dbf", ".shx"])
for i in range(len(ObjListTile)):
currentTileEnv = ObjListTile[i].getEnvelope()
for j in range(1 + i, len(ObjListTile)):
NextTileEnv = ObjListTile[j].getEnvelope()
if IsIntersect(currentTileEnv, NextTileEnv):
InterName = ObjListTile[i].getName(
) + "_inter_" + ObjListTile[j].getName()
intersection = fu.ClipVectorData(ObjListTile[i].getEnvelope(),
ObjListTile[j].getEnvelope(),
tmpFile, InterName)
notDiag = erodeInter(ObjListTile[i], ObjListTile[j],
intersection, buff, proj)
if notDiag:
tmpName = ObjListTile[i].getName() + "_TMP"
subtractShape(ObjListTile[i].getPriorityEnv(),
intersection, tmpFile, tmpName, proj)
fu.removeShape(
ObjListTile[i].getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.cpShapeFile(
tmpFile + "/" + tmpName.replace(".shp", ""),
ObjListTile[i].getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(tmpFile + "/" + tmpName.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
ObjListTile.reverse()
for i in range(len(ObjListTile)):
currentTileEnv = ObjListTile[i].getEnvelope()
for j in range(1 + i, len(ObjListTile)):
NextTileEnv = ObjListTile[j].getEnvelope()
if IsIntersect(currentTileEnv, NextTileEnv):
if diag(ObjListTile[i], ObjListTile[j]):
InterName = ObjListTile[i].getName(
) + "_inter_" + ObjListTile[j].getName()
intersection = fu.ClipVectorData(
ObjListTile[i].getEnvelope(),
ObjListTile[j].getEnvelope(), tmpFile, InterName)
erodeDiag(ObjListTile[i], ObjListTile[j], intersection,
buff, tmpFile, proj)
else:
tmpName = ObjListTile[i].getName() + "_TMP"
subtractShape(ObjListTile[i].getPriorityEnv(),
ObjListTile[j].getPriorityEnv(), tmpFile,
tmpName, proj)
fu.removeShape(
ObjListTile[i].getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.cpShapeFile(
tmpFile + "/" + tmpName.replace(".shp", ""),
ObjListTile[i].getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(tmpFile + "/" + tmpName.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
def erodeDiag(currentTile, NextTile, intersection, buff, TMP, proj):
xo, yo = currentTile.getOrigin() # tuile la plus prio
xn, yn = NextTile.getOrigin()
Extent = getShapeExtent(intersection) # [minX, maxX, minY, maxY]
if yo > yn and xo > xn:
minX = Extent[1] - buff
maxX = Extent[1]
minY = Extent[2]
maxY = Extent[3]
fu.removeShape(intersection.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
pathFolder = "/".join(
intersection.split("/")[0:len(intersection.split("/")) - 1])
createShape(minX, minY, maxX, maxY, pathFolder,
intersection.split("/")[-1].replace(".shp", ""), proj)
tmpName = NextTile.getName() + "_TMP"
subtractShape(NextTile.getPriorityEnv(), intersection, TMP, tmpName,
proj)
fu.removeShape(NextTile.getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.cpShapeFile(TMP + "/" + tmpName.replace(".shp", ""),
NextTile.getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(TMP + "/" + tmpName.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
tmpName = currentTile.getName() + "_TMP"
subtractShape(currentTile.getPriorityEnv(), NextTile.getPriorityEnv(),
TMP, tmpName, proj)
fu.removeShape(currentTile.getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.cpShapeFile(TMP + "/" + tmpName.replace(".shp", ""),
currentTile.getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(TMP + "/" + tmpName.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
if yo > yn and xo < xn:
tmpName = NextTile.getName() + "_TMP"
subtractShape(NextTile.getPriorityEnv(), currentTile.getPriorityEnv(),
TMP, tmpName, proj)
fu.removeShape(NextTile.getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.cpShapeFile(TMP + "/" + tmpName.replace(".shp", ""),
NextTile.getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(TMP + "/" + tmpName.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
def ExtractData(pathToClip, shapeData, pathOut, pathFeat, pathConf, pathWd):
"""
Clip the shapeFile pathToClip with the shapeFile shapeData and store it in pathOut
"""
f = file(pathConf)
cfg = Config(f)
cloud_threshold = cfg.chain.cloud_threshold
featuresPath = cfg.chain.featuresPath
currentTile = pathToClip.split("_")[-1].split(".")[0]
driver = ogr.GetDriverByName('ESRI Shapefile')
dataSource = driver.Open(pathToClip,
0) # 0 means read-only. 1 means writeable.
# Check to see if shapefile is found.
if dataSource is None:
print 'Could not open %s' % (pathToClip)
else:
layer = dataSource.GetLayer()
featureCount = layer.GetFeatureCount()
if featureCount != 0:
tmpdir = ""
pathName = pathWd
command = "cp "
suffix = " " + pathOut
if pathWd == None:
tmpdir = "/tmp"
pathName = pathOut
command = "rm "
suffix = ""
CloudMask = featuresPath + "/" + currentTile + "/CloudThreshold_" + cloud_threshold + ".shp"
NbView.genNbView(featuresPath + "/" + currentTile, CloudMask,
cloud_threshold, pathWd)
path_tmp = fu.ClipVectorData(
shapeData,
pathFeat + "/" + currentTile + tmpdir + "/MaskCommunSL.shp",
pathName)
path_tmp2 = fu.ClipVectorData(path_tmp, pathToClip, pathName)
path = fu.ClipVectorData(path_tmp2, CloudMask, pathName)
if fu.multiSearch(path):
NoMulti = path.replace(".shp", "_NoMulti.shp")
fu.multiPolyToPoly(path, NoMulti)
fu.removeShape(path.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.cpShapeFile(NoMulti.replace(".shp", ""),
path.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(NoMulti.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
if pathWd != None:
fu.cpShapeFile(
path.replace(".shp", ""),
pathOut + "/" + path.split("/")[-1].replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
else:
fu.removeShape(path_tmp.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(path_tmp2.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])