def erodeInter(currentTile, NextTile, intersection, buff, proj): xo, yo = currentTile.getOrigin() xn, yn = NextTile.getOrigin() Extent = getShapeExtent(intersection) if yo == yn and xo != xn: # left priority minX = Extent[0] maxX = Extent[1] - buff minY = Extent[2] maxY = Extent[3] elif yo != yn and xo == xn: # upper priority minX = Extent[0] maxX = Extent[1] minY = Extent[2] + buff maxY = Extent[3] else: return False fu.removeShape(intersection.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) pathFolder = "/".join( intersection.split("/")[0:len(intersection.split("/")) - 1]) createShape(minX, minY, maxX, maxY, pathFolder, intersection.split("/")[-1].replace(".shp", ""), proj) return True
def genNbView(TilePath, maskOut, nbview, workingDirectory=None): """ """ nameNbView = "nbView.tif" wd = TilePath if workingDirectory: wd = workingDirectory tmp1 = wd + "/" + nameNbView if not os.path.exists(TilePath + "/" + nameNbView): # build stack MaskStack = "AllSensorMask.tif" maskList = fu.FileSearch_AND(TilePath, True, "_ST_MASK.tif") maskList = " ".join(maskList) # cmd = "gdalbuildvrt "+TilePath+"/"+MaskStack+" "+maskList cmd = "otbcli_ConcatenateImages -il " + maskList + " -out " + TilePath + "/" + MaskStack + " int16" print cmd os.system(cmd) exp = buildExpression_cloud(TilePath + "/" + MaskStack) tmp2 = maskOut.replace(".shp", "_tmp_2.tif").replace(TilePath, wd) cmd = 'otbcli_BandMath -il ' + TilePath + "/" + MaskStack + ' -out ' + tmp1 + ' uint16 -exp "' + exp + '"' print cmd os.system(cmd) cmd = 'otbcli_BandMath -il ' + tmp1 + ' -out ' + tmp2 + ' -exp "im1b1>=' + str( nbview) + '?1:0"' print cmd os.system(cmd) maskOut_tmp = maskOut.replace(".shp", "_tmp.shp").replace(TilePath, wd) cmd = "gdal_polygonize.py -mask " + tmp2 + " " + tmp2 + " -f \"ESRI Shapefile\" " + maskOut_tmp print cmd os.system(cmd) fu.erodeShapeFile(maskOut_tmp, wd + "/" + maskOut.split("/")[-1], 0.1) os.remove(tmp2) fu.removeShape(maskOut_tmp.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) if workingDirectory: shutil.copy(tmp1, TilePath) fu.cpShapeFile(wd + "/" + maskOut.split("/")[-1].replace(".shp", ""), TilePath, [".prj", ".shp", ".dbf", ".shx"], spe=True)
def split_All_shape(shape, folds, pathConf, pathWd): f = file(pathConf) cfg = Config(f) regionField = cfg.chain.regionField outputpath = cfg.chain.outputPath dataField = cfg.chain.dataField workingDirectory = outputpath + "/dataAppVal" if pathWd != None: workingDirectory = pathWd createdShape = SplitShape(shape, dataField, folds, workingDirectory, shape.split("/")[-1]) fu.removeShape(shape.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) if pathWd != None: for NewShape in createdShape: fu.cpShapeFile(NewShape.replace(".shp", ""), outputpath + "/dataAppVal", [".prj", ".shp", ".dbf", ".shx"], spe=True)
def generateSamples_cropMix(folderSample, workingDirectory, trainShape, pathWd, featuresPath, samplesOptions, prevFeatures, annualCrop, AllClass, dataField, pathConf): currentTile = trainShape.split("/")[-1].split("_")[0] bindingPy = Config(file(pathConf)).GlobChain.bindingPython samplesClassifMix = Config(file(pathConf)).argTrain.samplesClassifMix userFeatPath = Config(file(pathConf)).chain.userFeatPath if userFeatPath == "None": userFeatPath = None stack = "/Final/" + fu.getFeatStackName(pathConf) NA_img = featuresPath + "/" + currentTile + "/" + stack A_img = prevFeatures + "/" + currentTile + "/" + stack if bindingPy == "True": NA_img = fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True, "ST_MASK")[0] A_img = fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True, "ST_MASK")[0] # Step 1 : filter trainShape in order to keep non-annual class nameNonAnnual = trainShape.split("/")[-1].replace(".shp", "_NonAnnu.shp") nonAnnualShape = workingDirectory + "/" + nameNonAnnual filterShpByClass(dataField, nonAnnualShape, AllClass, trainShape) # Step 2 : filter trainShape in order to keep annual class nameAnnual = trainShape.split("/")[-1].replace(".shp", "_Annu.shp") annualShape = workingDirectory + "/" + nameAnnual annualCropFind = filterShpByClass(dataField, annualShape, annualCrop, trainShape) # Step 3 : nonAnnual stats stats_NA = workingDirectory + "/" + nameNonAnnual.replace( ".shp", "_STATS.xml") cmd = "otbcli_PolygonClassStatistics -in " + NA_img + " -vec " + nonAnnualShape + " -field " + dataField + " -out " + stats_NA print cmd os.system(cmd) verifPolyStats(stats_NA) # Step 4 : Annual stats stats_A = workingDirectory + "/" + nameAnnual.replace(".shp", "_STATS.xml") cmd = "otbcli_PolygonClassStatistics -in " + A_img + " -vec " + annualShape + " -field " + dataField + " -out " + stats_A if annualCropFind: print cmd os.system(cmd) verifPolyStats(stats_A) # Step 5 : Sample Selection NonAnnual SampleSel_NA = workingDirectory + "/" + nameNonAnnual.replace( ".shp", "_SampleSel_NA.sqlite") cmd = "otbcli_SampleSelection -in " + NA_img + " -vec " + nonAnnualShape + " -field " + dataField + " -instats " + stats_NA + " -out " + SampleSel_NA + " " + samplesOptions print cmd os.system(cmd) # Step 6 : Sample Selection Annual SampleSel_A = workingDirectory + "/" + nameAnnual.replace( ".shp", "_SampleSel_A.sqlite") cmd = "otbcli_SampleSelection -in " + A_img + " -vec " + annualShape + " -field " + dataField + " -instats " + stats_A + " -out " + SampleSel_A + " " + samplesOptions if annualCropFind: print cmd os.system(cmd) SampleExtr_NA = workingDirectory + "/" + nameNonAnnual.replace( ".shp", "_SampleExtr_NA.sqlite") SampleExtr_A = workingDirectory + "/" + nameAnnual.replace( ".shp", "_SampleExtr_A.sqlite") if bindingPy == "False": # Step 7 : Sample extraction NonAnnual cmd = "otbcli_SampleExtraction -in " + NA_img + " -vec " + SampleSel_NA + " -field " + dataField + " -out " + SampleExtr_NA print cmd os.system(cmd) # Step 8 : Sample extraction Annual cmd = "otbcli_SampleExtraction -in " + A_img + " -vec " + SampleSel_A + " -field " + dataField + " -out " + SampleExtr_A if annualCropFind: print cmd os.system(cmd) else: # Step 7 : Sample extraction NonAnnual concatSensors = otb.Registry.CreateApplication("ConcatenateImages") AllRefl = sorted( fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True, "REFL.tif")) AllMask = sorted( fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True, "MASK.tif")) datesInterp = sorted( fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True, "DatesInterp")) realDates = sorted( fu.FileSearch_AND(featuresPath + "/" + currentTile + "/tmp/", True, "imagesDate")) features = [] for refl, mask, datesInterp, realDates in zip(AllRefl, AllMask, datesInterp, realDates): gapFill = otb.Registry.CreateApplication( "ImageTimeSeriesGapFilling") nbDate = fu.getNbDateInTile(realDates) nbReflBands = fu.getRasterNbands(refl) comp = int(nbReflBands) / int(nbDate) if not isinstance(comp, int): raise Exception("unvalid component by date (not integer) : " + comp) gapFill.SetParameterString("in", refl) gapFill.SetParameterString("mask", mask) gapFill.SetParameterString("comp", str(comp)) gapFill.SetParameterString("it", "linear") gapFill.SetParameterString("id", realDates) gapFill.SetParameterString("od", datesInterp) # gapFill.SetParameterString("ram","1024") gapFill.Execute() concatSensors.AddImageToParameterInputImageList( "il", gapFill.GetParameterOutputImage("out")) features.append(gapFill) sampleExtr = otb.Registry.CreateApplication("SampleExtraction") sampleExtr.SetParameterString("ram", "128") sampleExtr.SetParameterString("vec", SampleSel_NA) sampleExtr.SetParameterString("field", dataField) sampleExtr.SetParameterString("out", SampleExtr_NA) # if len(AllRefl) > 1: # concatSensors.Execute() # sampleExtr.SetParameterInputImage("in",concatSensors.GetParameterOutputImage("out")) # else: # sampleExtr.SetParameterInputImage("in",features[0].GetParameterOutputImage("out")) # sampleExtr.ExecuteAndWriteOutput() if len(AllRefl) > 1: concatSensors.Execute() allFeatures = concatSensors.GetParameterOutputImage("out") else: allFeatures = features[0].GetParameterOutputImage("out") if userFeatPath: print "Add user features" userFeat_arbo = Config(file(pathConf)).userFeat.arbo userFeat_pattern = (Config( file(pathConf)).userFeat.patterns).split(",") concatFeatures = otb.Registry.CreateApplication( "ConcatenateImages") userFeatures = fu.getUserFeatInTile(userFeatPath, currentTile, userFeat_arbo, userFeat_pattern) concatFeatures.SetParameterStringList("il", userFeatures) concatFeatures.Execute() concatAllFeatures = otb.Registry.CreateApplication( "ConcatenateImages") concatAllFeatures.AddImageToParameterInputImageList( "il", allFeatures) concatAllFeatures.AddImageToParameterInputImageList( "il", concatFeatures.GetParameterOutputImage("out")) concatAllFeatures.Execute() allFeatures = concatAllFeatures.GetParameterOutputImage("out") sampleExtr.SetParameterInputImage("in", allFeatures) sampleExtr.ExecuteAndWriteOutput() # Step 8 : Sample extraction Annual concatSensors = otb.Registry.CreateApplication("ConcatenateImages") AllRefl = sorted( fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True, "REFL.tif")) AllMask = sorted( fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True, "MASK.tif")) datesInterp = sorted( fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True, "DatesInterp")) realDates = sorted( fu.FileSearch_AND(prevFeatures + "/" + currentTile + "/tmp/", True, "imagesDate")) features = [] for refl, mask, datesInterp, realDates in zip(AllRefl, AllMask, datesInterp, realDates): gapFill = otb.Registry.CreateApplication( "ImageTimeSeriesGapFilling") nbDate = fu.getNbDateInTile(realDates) nbReflBands = fu.getRasterNbands(refl) comp = int(nbReflBands) / int(nbDate) if not isinstance(comp, int): raise Exception("unvalid component by date (not integer) : " + comp) gapFill.SetParameterString("in", refl) gapFill.SetParameterString("mask", mask) gapFill.SetParameterString("comp", str(comp)) gapFill.SetParameterString("it", "linear") gapFill.SetParameterString("id", realDates) gapFill.SetParameterString("od", datesInterp) # gapFill.SetParameterString("ram","1024") gapFill.Execute() concatSensors.AddImageToParameterInputImageList( "il", gapFill.GetParameterOutputImage("out")) features.append(gapFill) sampleExtr = otb.Registry.CreateApplication("SampleExtraction") sampleExtr.SetParameterString("ram", "128") sampleExtr.SetParameterString("vec", SampleSel_A) sampleExtr.SetParameterString("field", dataField) sampleExtr.SetParameterString("out", SampleExtr_A) if len(AllRefl) > 1: concatSensors.Execute() allFeatures = concatSensors.GetParameterOutputImage("out") else: allFeatures = features[0].GetParameterOutputImage("out") if userFeatPath: print "Add user features" userFeat_arbo = Config(file(pathConf)).userFeat.arbo userFeat_pattern = (Config( file(pathConf)).userFeat.patterns).split(",") concatFeatures = otb.Registry.CreateApplication( "ConcatenateImages") userFeatures = fu.getUserFeatInTile(userFeatPath, currentTile, userFeat_arbo, userFeat_pattern) concatFeatures.SetParameterStringList("il", userFeatures) concatFeatures.Execute() concatAllFeatures = otb.Registry.CreateApplication( "ConcatenateImages") concatAllFeatures.AddImageToParameterInputImageList( "il", allFeatures) concatAllFeatures.AddImageToParameterInputImageList( "il", concatFeatures.GetParameterOutputImage("out")) concatAllFeatures.Execute() allFeatures = concatAllFeatures.GetParameterOutputImage("out") sampleExtr.SetParameterInputImage("in", allFeatures) if annualCropFind: sampleExtr.ExecuteAndWriteOutput() # Step 9 : Merge MergeName = trainShape.split("/")[-1].replace(".shp", "_Samples") listToMerge = [SampleExtr_NA] if annualCropFind: # listToMerge = [SampleExtr_A,SampleExtr_NA] listToMerge = [SampleExtr_NA, SampleExtr_A] fu.mergeSQLite(MergeName, workingDirectory, listToMerge) samples = workingDirectory + "/" + trainShape.split("/")[-1].replace( ".shp", "_Samples.sqlite") os.remove(stats_NA) os.remove(SampleSel_NA) os.remove(SampleExtr_NA) fu.removeShape(nonAnnualShape.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) if annualCropFind: os.remove(stats_A) os.remove(SampleSel_A) os.remove(SampleExtr_A) fu.removeShape(annualShape.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) if pathWd: shutil.copy( samples, folderSample + "/" + trainShape.split("/")[-1].replace(".shp", "_Samples.sqlite"))
def genTileEnvPrio(ObjListTile, out, tmpFile, proj): buff = 600 # offset in order to manage nodata in image's border ObjListTile.reverse() listSHP = [ createRasterFootprint(c_ObjListTile.getPath(), tmpFile + "/" + c_ObjListTile.getName() + ".shp") for c_ObjListTile in ObjListTile ] for env, currentTile in zip(listSHP, ObjListTile): currentTile.setEnvelope(env) currentTile.setPriorityEnv(env.replace(".shp", "_PRIO.shp")) fu.cpShapeFile(env.replace(".shp", ""), env.replace(".shp", "") + "_PRIO", [".prj", ".shp", ".dbf", ".shx"]) for i in range(len(ObjListTile)): currentTileEnv = ObjListTile[i].getEnvelope() for j in range(1 + i, len(ObjListTile)): NextTileEnv = ObjListTile[j].getEnvelope() if IsIntersect(currentTileEnv, NextTileEnv): InterName = ObjListTile[i].getName( ) + "_inter_" + ObjListTile[j].getName() intersection = fu.ClipVectorData(ObjListTile[i].getEnvelope(), ObjListTile[j].getEnvelope(), tmpFile, InterName) notDiag = erodeInter(ObjListTile[i], ObjListTile[j], intersection, buff, proj) if notDiag: tmpName = ObjListTile[i].getName() + "_TMP" subtractShape(ObjListTile[i].getPriorityEnv(), intersection, tmpFile, tmpName, proj) fu.removeShape( ObjListTile[i].getPriorityEnv().replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.cpShapeFile( tmpFile + "/" + tmpName.replace(".shp", ""), ObjListTile[i].getPriorityEnv().replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.removeShape(tmpFile + "/" + tmpName.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) ObjListTile.reverse() for i in range(len(ObjListTile)): currentTileEnv = ObjListTile[i].getEnvelope() for j in range(1 + i, len(ObjListTile)): NextTileEnv = ObjListTile[j].getEnvelope() if IsIntersect(currentTileEnv, NextTileEnv): if diag(ObjListTile[i], ObjListTile[j]): InterName = ObjListTile[i].getName( ) + "_inter_" + ObjListTile[j].getName() intersection = fu.ClipVectorData( ObjListTile[i].getEnvelope(), ObjListTile[j].getEnvelope(), tmpFile, InterName) erodeDiag(ObjListTile[i], ObjListTile[j], intersection, buff, tmpFile, proj) else: tmpName = ObjListTile[i].getName() + "_TMP" subtractShape(ObjListTile[i].getPriorityEnv(), ObjListTile[j].getPriorityEnv(), tmpFile, tmpName, proj) fu.removeShape( ObjListTile[i].getPriorityEnv().replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.cpShapeFile( tmpFile + "/" + tmpName.replace(".shp", ""), ObjListTile[i].getPriorityEnv().replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.removeShape(tmpFile + "/" + tmpName.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"])
def erodeDiag(currentTile, NextTile, intersection, buff, TMP, proj): xo, yo = currentTile.getOrigin() # tuile la plus prio xn, yn = NextTile.getOrigin() Extent = getShapeExtent(intersection) # [minX, maxX, minY, maxY] if yo > yn and xo > xn: minX = Extent[1] - buff maxX = Extent[1] minY = Extent[2] maxY = Extent[3] fu.removeShape(intersection.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) pathFolder = "/".join( intersection.split("/")[0:len(intersection.split("/")) - 1]) createShape(minX, minY, maxX, maxY, pathFolder, intersection.split("/")[-1].replace(".shp", ""), proj) tmpName = NextTile.getName() + "_TMP" subtractShape(NextTile.getPriorityEnv(), intersection, TMP, tmpName, proj) fu.removeShape(NextTile.getPriorityEnv().replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.cpShapeFile(TMP + "/" + tmpName.replace(".shp", ""), NextTile.getPriorityEnv().replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.removeShape(TMP + "/" + tmpName.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) tmpName = currentTile.getName() + "_TMP" subtractShape(currentTile.getPriorityEnv(), NextTile.getPriorityEnv(), TMP, tmpName, proj) fu.removeShape(currentTile.getPriorityEnv().replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.cpShapeFile(TMP + "/" + tmpName.replace(".shp", ""), currentTile.getPriorityEnv().replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.removeShape(TMP + "/" + tmpName.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) if yo > yn and xo < xn: tmpName = NextTile.getName() + "_TMP" subtractShape(NextTile.getPriorityEnv(), currentTile.getPriorityEnv(), TMP, tmpName, proj) fu.removeShape(NextTile.getPriorityEnv().replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.cpShapeFile(TMP + "/" + tmpName.replace(".shp", ""), NextTile.getPriorityEnv().replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.removeShape(TMP + "/" + tmpName.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"])
def ExtractData(pathToClip, shapeData, pathOut, pathFeat, pathConf, pathWd): """ Clip the shapeFile pathToClip with the shapeFile shapeData and store it in pathOut """ f = file(pathConf) cfg = Config(f) cloud_threshold = cfg.chain.cloud_threshold featuresPath = cfg.chain.featuresPath currentTile = pathToClip.split("_")[-1].split(".")[0] driver = ogr.GetDriverByName('ESRI Shapefile') dataSource = driver.Open(pathToClip, 0) # 0 means read-only. 1 means writeable. # Check to see if shapefile is found. if dataSource is None: print 'Could not open %s' % (pathToClip) else: layer = dataSource.GetLayer() featureCount = layer.GetFeatureCount() if featureCount != 0: tmpdir = "" pathName = pathWd command = "cp " suffix = " " + pathOut if pathWd == None: tmpdir = "/tmp" pathName = pathOut command = "rm " suffix = "" CloudMask = featuresPath + "/" + currentTile + "/CloudThreshold_" + cloud_threshold + ".shp" NbView.genNbView(featuresPath + "/" + currentTile, CloudMask, cloud_threshold, pathWd) path_tmp = fu.ClipVectorData( shapeData, pathFeat + "/" + currentTile + tmpdir + "/MaskCommunSL.shp", pathName) path_tmp2 = fu.ClipVectorData(path_tmp, pathToClip, pathName) path = fu.ClipVectorData(path_tmp2, CloudMask, pathName) if fu.multiSearch(path): NoMulti = path.replace(".shp", "_NoMulti.shp") fu.multiPolyToPoly(path, NoMulti) fu.removeShape(path.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.cpShapeFile(NoMulti.replace(".shp", ""), path.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.removeShape(NoMulti.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) if pathWd != None: fu.cpShapeFile( path.replace(".shp", ""), pathOut + "/" + path.split("/")[-1].replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) else: fu.removeShape(path_tmp.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"]) fu.removeShape(path_tmp2.replace(".shp", ""), [".prj", ".shp", ".dbf", ".shx"])