def GenerateShapeTile(tiles, pathTiles, pathOut, pathWd, pathConf):
f = file(pathConf)
cfg = Config(f)
proj = int(cfg.GlobChain.proj.split(":")[-1])
executionMode = cfg.chain.executionMode
# tilesPath = [pathTiles+"/"+tile+"/Final/"+fu.getFeatStackName(pathConf) for tile in tiles]
MaskCommunPath = ""
if executionMode == "sequential":
MaskCommunPath = "/tmp/"
tilesPath = [
pathTiles + "/" + tile + MaskCommunPath + "/MaskCommunSL.tif"
for tile in tiles
]
ObjListTile = [
Tile(currentTile, name) for currentTile, name in zip(tilesPath, tiles)
]
ObjListTile_sort = sorted(ObjListTile, key=priorityKey)
tmpFile = pathOut + "/TMP"
if pathWd:
tmpFile = pathWd + "/TMP"
if not os.path.exists(tmpFile):
os.mkdir(tmpFile)
genTileEnvPrio(ObjListTile_sort, pathOut, tmpFile, proj)
AllPRIO = fu.FileSearch_AND(tmpFile, True, "_PRIO.shp")
for prioTile in AllPRIO:
tileName = prioTile.split("/")[-1].split("_")[0]
fu.cpShapeFile(prioTile.replace(".shp", ""), pathOut + "/" + tileName,
[".prj", ".shp", ".dbf", ".shx"])
shutil.rmtree(tmpFile)
def genNbView(TilePath, maskOut, nbview, workingDirectory=None):
"""
"""
nameNbView = "nbView.tif"
wd = TilePath
if workingDirectory: wd = workingDirectory
tmp1 = wd + "/" + nameNbView
if not os.path.exists(TilePath + "/" + nameNbView):
# build stack
MaskStack = "AllSensorMask.tif"
maskList = fu.FileSearch_AND(TilePath, True, "_ST_MASK.tif")
maskList = " ".join(maskList)
# cmd = "gdalbuildvrt "+TilePath+"/"+MaskStack+" "+maskList
cmd = "otbcli_ConcatenateImages -il " + maskList + " -out " + TilePath + "/" + MaskStack + " int16"
print cmd
os.system(cmd)
exp = buildExpression_cloud(TilePath + "/" + MaskStack)
tmp2 = maskOut.replace(".shp", "_tmp_2.tif").replace(TilePath, wd)
cmd = 'otbcli_BandMath -il ' + TilePath + "/" + MaskStack + ' -out ' + tmp1 + ' uint16 -exp "' + exp + '"'
print cmd
os.system(cmd)
cmd = 'otbcli_BandMath -il ' + tmp1 + ' -out ' + tmp2 + ' -exp "im1b1>=' + str(
nbview) + '?1:0"'
print cmd
os.system(cmd)
maskOut_tmp = maskOut.replace(".shp", "_tmp.shp").replace(TilePath, wd)
cmd = "gdal_polygonize.py -mask " + tmp2 + " " + tmp2 + " -f \"ESRI Shapefile\" " + maskOut_tmp
print cmd
os.system(cmd)
fu.erodeShapeFile(maskOut_tmp, wd + "/" + maskOut.split("/")[-1], 0.1)
os.remove(tmp2)
fu.removeShape(maskOut_tmp.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
if workingDirectory:
shutil.copy(tmp1, TilePath)
fu.cpShapeFile(wd + "/" +
maskOut.split("/")[-1].replace(".shp", ""),
TilePath, [".prj", ".shp", ".dbf", ".shx"],
spe=True)
def split_All_shape(shape, folds, pathConf, pathWd):
f = file(pathConf)
cfg = Config(f)
regionField = cfg.chain.regionField
outputpath = cfg.chain.outputPath
dataField = cfg.chain.dataField
workingDirectory = outputpath + "/dataAppVal"
if pathWd != None:
workingDirectory = pathWd
createdShape = SplitShape(shape, dataField, folds, workingDirectory,
shape.split("/")[-1])
fu.removeShape(shape.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"])
if pathWd != None:
for NewShape in createdShape:
fu.cpShapeFile(NewShape.replace(".shp", ""),
outputpath + "/dataAppVal",
[".prj", ".shp", ".dbf", ".shx"],
spe=True)
def erodeDiag(currentTile, NextTile, intersection, buff, TMP, proj):
xo, yo = currentTile.getOrigin() # tuile la plus prio
xn, yn = NextTile.getOrigin()
Extent = getShapeExtent(intersection) # [minX, maxX, minY, maxY]
if yo > yn and xo > xn:
minX = Extent[1] - buff
maxX = Extent[1]
minY = Extent[2]
maxY = Extent[3]
fu.removeShape(intersection.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
pathFolder = "/".join(
intersection.split("/")[0:len(intersection.split("/")) - 1])
createShape(minX, minY, maxX, maxY, pathFolder,
intersection.split("/")[-1].replace(".shp", ""), proj)
tmpName = NextTile.getName() + "_TMP"
subtractShape(NextTile.getPriorityEnv(), intersection, TMP, tmpName,
proj)
fu.removeShape(NextTile.getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.cpShapeFile(TMP + "/" + tmpName.replace(".shp", ""),
NextTile.getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(TMP + "/" + tmpName.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
tmpName = currentTile.getName() + "_TMP"
subtractShape(currentTile.getPriorityEnv(), NextTile.getPriorityEnv(),
TMP, tmpName, proj)
fu.removeShape(currentTile.getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.cpShapeFile(TMP + "/" + tmpName.replace(".shp", ""),
currentTile.getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(TMP + "/" + tmpName.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
if yo > yn and xo < xn:
tmpName = NextTile.getName() + "_TMP"
subtractShape(NextTile.getPriorityEnv(), currentTile.getPriorityEnv(),
TMP, tmpName, proj)
fu.removeShape(NextTile.getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.cpShapeFile(TMP + "/" + tmpName.replace(".shp", ""),
NextTile.getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(TMP + "/" + tmpName.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
def genTileEnvPrio(ObjListTile, out, tmpFile, proj):
buff = 600 # offset in order to manage nodata in image's border
ObjListTile.reverse()
listSHP = [
createRasterFootprint(c_ObjListTile.getPath(),
tmpFile + "/" + c_ObjListTile.getName() + ".shp")
for c_ObjListTile in ObjListTile
]
for env, currentTile in zip(listSHP, ObjListTile):
currentTile.setEnvelope(env)
currentTile.setPriorityEnv(env.replace(".shp", "_PRIO.shp"))
fu.cpShapeFile(env.replace(".shp", ""),
env.replace(".shp", "") + "_PRIO",
[".prj", ".shp", ".dbf", ".shx"])
for i in range(len(ObjListTile)):
currentTileEnv = ObjListTile[i].getEnvelope()
for j in range(1 + i, len(ObjListTile)):
NextTileEnv = ObjListTile[j].getEnvelope()
if IsIntersect(currentTileEnv, NextTileEnv):
InterName = ObjListTile[i].getName(
) + "_inter_" + ObjListTile[j].getName()
intersection = fu.ClipVectorData(ObjListTile[i].getEnvelope(),
ObjListTile[j].getEnvelope(),
tmpFile, InterName)
notDiag = erodeInter(ObjListTile[i], ObjListTile[j],
intersection, buff, proj)
if notDiag:
tmpName = ObjListTile[i].getName() + "_TMP"
subtractShape(ObjListTile[i].getPriorityEnv(),
intersection, tmpFile, tmpName, proj)
fu.removeShape(
ObjListTile[i].getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.cpShapeFile(
tmpFile + "/" + tmpName.replace(".shp", ""),
ObjListTile[i].getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(tmpFile + "/" + tmpName.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
ObjListTile.reverse()
for i in range(len(ObjListTile)):
currentTileEnv = ObjListTile[i].getEnvelope()
for j in range(1 + i, len(ObjListTile)):
NextTileEnv = ObjListTile[j].getEnvelope()
if IsIntersect(currentTileEnv, NextTileEnv):
if diag(ObjListTile[i], ObjListTile[j]):
InterName = ObjListTile[i].getName(
) + "_inter_" + ObjListTile[j].getName()
intersection = fu.ClipVectorData(
ObjListTile[i].getEnvelope(),
ObjListTile[j].getEnvelope(), tmpFile, InterName)
erodeDiag(ObjListTile[i], ObjListTile[j], intersection,
buff, tmpFile, proj)
else:
tmpName = ObjListTile[i].getName() + "_TMP"
subtractShape(ObjListTile[i].getPriorityEnv(),
ObjListTile[j].getPriorityEnv(), tmpFile,
tmpName, proj)
fu.removeShape(
ObjListTile[i].getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.cpShapeFile(
tmpFile + "/" + tmpName.replace(".shp", ""),
ObjListTile[i].getPriorityEnv().replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(tmpFile + "/" + tmpName.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
def ExtractData(pathToClip, shapeData, pathOut, pathFeat, pathConf, pathWd):
"""
Clip the shapeFile pathToClip with the shapeFile shapeData and store it in pathOut
"""
f = file(pathConf)
cfg = Config(f)
cloud_threshold = cfg.chain.cloud_threshold
featuresPath = cfg.chain.featuresPath
currentTile = pathToClip.split("_")[-1].split(".")[0]
driver = ogr.GetDriverByName('ESRI Shapefile')
dataSource = driver.Open(pathToClip,
0) # 0 means read-only. 1 means writeable.
# Check to see if shapefile is found.
if dataSource is None:
print 'Could not open %s' % (pathToClip)
else:
layer = dataSource.GetLayer()
featureCount = layer.GetFeatureCount()
if featureCount != 0:
tmpdir = ""
pathName = pathWd
command = "cp "
suffix = " " + pathOut
if pathWd == None:
tmpdir = "/tmp"
pathName = pathOut
command = "rm "
suffix = ""
CloudMask = featuresPath + "/" + currentTile + "/CloudThreshold_" + cloud_threshold + ".shp"
NbView.genNbView(featuresPath + "/" + currentTile, CloudMask,
cloud_threshold, pathWd)
path_tmp = fu.ClipVectorData(
shapeData,
pathFeat + "/" + currentTile + tmpdir + "/MaskCommunSL.shp",
pathName)
path_tmp2 = fu.ClipVectorData(path_tmp, pathToClip, pathName)
path = fu.ClipVectorData(path_tmp2, CloudMask, pathName)
if fu.multiSearch(path):
NoMulti = path.replace(".shp", "_NoMulti.shp")
fu.multiPolyToPoly(path, NoMulti)
fu.removeShape(path.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.cpShapeFile(NoMulti.replace(".shp", ""),
path.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(NoMulti.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
if pathWd != None:
fu.cpShapeFile(
path.replace(".shp", ""),
pathOut + "/" + path.split("/")[-1].replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
else:
fu.removeShape(path_tmp.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
fu.removeShape(path_tmp2.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
sensor.serieTemp = args.wOut + "/tmp/" + os.path.split(
sensor.serieTemp)[1]
if binding == 'False':
for sensor in list_Sensor:
DP.Gapfilling(sensor.serieTemp, sensor.serieTempMask,
sensor.serieTempGap, sensor.nbBands, 0,
sensor.fdates, datesVoulues, args.wOut)
for sensor in list_Sensor:
if not os.path.exists(args.wOut + "/tmp"):
os.mkdir(args.wOut + "/tmp")
shutil.copy(sensor.fdates, args.wOut + "/tmp")
shutil.copy(sensor.DatesVoulues, args.wOut + "/tmp")
shutil.copy(args.opath + "/tmp/MaskCommunSL.tif", args.wOut)
fu.cpShapeFile(args.opath + "/tmp/MaskCommunSL",
args.wOut, [".prj", ".shp", ".dbf", ".shx"],
spe=True)
Step = log.update(Step)
if binding == 'False':
if batchProcessing == 'False':
"""
if os.path.exists(args.wOut+"/tmp"):
os.system("rm -rf "+args.opath+"/tmp/*")
os.system("rm -r "+args.opath+"/Final")
for sensor in list_Sensor:
os.system("cp "+args.wOut+"/tmp/"+sensor.serieTempGap+" "+args.opath+"/tmp")
os.system("cp "+args.wOut+"/tmp/DatesInterpReg"+str(sensor.name)+".txt "+args.opath+"/tmp")
os.system("cp -R "+args.wOut+"/Final "+args.opath)
os.system("rm -r "+args.wOut+"/tmp")
def RandomInSitu(vectorFile, field, nbdraws, opath, name, AllFields, ratio,
pathWd):
"""
"""
AllPath = []
crop = 0
classes = []
shapefile = vectorFile
field = field
dicoprop = {}
allFID = []
nbtirage = nbdraws
nameshp = shapefile.split('.')
namefile = nameshp[0].split('/')
driver = ogr.GetDriverByName("ESRI Shapefile")
dataSource = driver.Open(shapefile, 0)
layer = dataSource.GetLayer()
# Count the total features of cropland
if crop == 1:
layer.SetAttributeFilter("CROP =1")
count = float(layer.GetFeatureCount())
# print count
# Find the number of polygons by class
for feature in layer:
pid = feature.GetFID()
allFID.append(pid)
cl = feature.GetField(field)
if cl not in classes:
classes.append(cl)
AllTrain = []
AllValid = []
for tirage in range(0, nbtirage):
listallid, listValid = get_randomPoly(dataSource, field, classes,
ratio)
ch = ""
listFid = []
for fid in listallid:
listFid.append("FID=" + str(fid))
resultA = []
for e in listFid:
resultA.append(e)
resultA.append(' OR ')
resultA.pop()
chA = ''.join(resultA)
layer.SetAttributeFilter(chA)
learningShape = opath + "/" + name + "_seed" + str(
tirage) + "_learn.shp"
if pathWd == None:
outShapefile = opath + "/" + name + "_seed" + str(
tirage) + "_learn.shp"
fu.CreateNewLayer(layer, outShapefile, AllFields)
else:
outShapefile = pathWd + "/" + name + "_seed" + str(
tirage) + "_learn.shp"
fu.CreateNewLayer(layer, outShapefile, AllFields)
fu.cpShapeFile(
outShapefile.replace(".shp", ""),
opath + "/" + name + "_seed" + str(tirage) + "_learn",
[".prj", ".shp", ".dbf", ".shx"])
for i in allFID:
if i not in listallid:
listValid.append(i)
chV = ""
listFidV = []
for fid in listValid:
listFidV.append("FID=" + str(fid))
resultV = []
for e in listFidV:
resultV.append(e)
resultV.append(' OR ')
resultV.pop()
chV = ''.join(resultV)
layer.SetAttributeFilter(chV)
validationShape = opath + "/" + name + "_seed" + str(
tirage) + "_val.shp"
if pathWd == None:
outShapefile2 = opath + "/" + name + "_seed" + str(
tirage) + "_val.shp"
fu.CreateNewLayer(layer, outShapefile2, AllFields)
else:
outShapefile2 = pathWd + "/" + name + "_seed" + str(
tirage) + "_val.shp"
fu.CreateNewLayer(layer, outShapefile2, AllFields)
fu.cpShapeFile(outShapefile2.replace(".shp", ""),
opath + "/" + name + "_seed" + str(tirage) + "_val",
[".prj", ".shp", ".dbf", ".shx"])
AllTrain.append(learningShape)
AllValid.append(validationShape)
return AllTrain, AllValid