def getAllDatasByDate(currentTile, rasterPatterns, masksPatterns, arboRaster, arboMask, TileFolder, rasterInitValue, masksInitValues): #get all raster into the current tile and store it with init value buf = [] r = [] for currentPattern in rasterPatterns: r += fu.fileSearchRegEx(TileFolder + currentTile + arboRaster + currentPattern) for currentR in r: buf.append((currentR, rasterInitValue)) for currentPattern, currentMaskInit in zip(masksPatterns, masksInitValues): m = [] m += fu.fileSearchRegEx(TileFolder + currentTile + arboMask + currentPattern) for currentM in m: buf.append((currentM, currentMaskInit)) #sort it by date buff = [(getDateFromRaster(currentRaster[0]), currentRaster) for currentRaster in buf] buff = fu.sortByFirstElem(buff) allDates = [] allRasters = [] for date, rasters in buff: allDates.append(date) allRasters.append(rasters) return allRasters, allDates
def extraction(vectorFill, vectorSource, field, field_val, driversFill, driversSource): ogrDriversSource = ogr.GetDriverByName(driversSource) dataSourceSource = ogrDriversSource.Open(vectorSource, 0) layerSource = dataSourceSource.GetLayer() print("RECHERCHE DES FIDs") All_FID = [(currentFeat.GetField(field), str(currentFeat.GetFID())) for currentFeat in layerSource if currentFeat.GetField(field) in field_val] print("FIDs trouvée") layerSource.ResetReading() All_FID = fu.sortByFirstElem(All_FID) for currentClass, FID in All_FID: splits = fu.splitList(FID, len(vectorFill)) for currentSplit, currentVectorFill in zip(splits, vectorFill): cmd = "ogr2ogr -append " + currentVectorFill + " " + vectorSource + " -where \" fid in (" + ",".join( currentSplit) + ")\"" print(cmd) print("Ajout de " + str(currentClass) + " dans " + currentVectorFill.split("/")[-1]) os.system(cmd)
def getNbSample(shape, tile, dataField, valToFind, resol, region, coeff, current_seed, region_field, region_val="-1"): driver = ogr.GetDriverByName("ESRI Shapefile") buff = [] dataSource = driver.Open(shape, 0) layer = dataSource.GetLayer() for feature in layer: if str(feature.GetField(dataField)) in valToFind and str( feature.GetField(region_field)) == str(region_val) and str( feature.GetField("seed_" + str(current_seed))) == "learn": geom = feature.GetGeometryRef() buff.append((feature.GetField(dataField), geom.GetArea())) rep = fu.sortByFirstElem(buff) repDict = {} for currentClass, currentAreas in rep: array = np.asarray(currentAreas) totalArea = np.sum(array) repDict[currentClass] = int( (float(coeff) * totalArea) / (int(resol) * int(resol))) return repDict
def getIntersections(outGridPath, inGridPath, tileField_first, tileField_second): """ OUT AllIntersections [list of tuple] : [(S2Tile, [L8Tile, L8Tile, L8Tile]), (...), ...] """ driver = ogr.GetDriverByName("ESRI Shapefile") dataOut = driver.Open(outGridPath, 0) dataIn = driver.Open(inGridPath, 0) layerOut = dataOut.GetLayer() layerIn = dataIn.GetLayer() AllIntersections = [] #Ex : [[S2, [L8, L8, ..., L8]], [], ...] outTiles = [(outTile.GetGeometryRef().Clone(), outTile.GetField(tileField_first)) for outTile in layerOut] inTiles = [(inTile.GetGeometryRef().Clone(), inTile.GetField(tileField_second)) for inTile in layerIn] for outTileGeom, outTile in outTiles: for inTileGeom, inTile in inTiles: intersection = outTileGeom.Intersection(inTileGeom) if intersection.GetArea() != 0.0 and ( outTile, inTile) not in AllIntersections: AllIntersections.append((outTile, inTile)) return fu.sortByFirstElem(AllIntersections)
def getTileSameDate(folder): buf = [] content = os.listdir(folder) for currentContent in content: date = currentContent.split("_")[2].split("-")[0] buf.append((date, folder+"/"+currentContent)) buf = fu.sortByFirstElem(buf) out = [currentList for date, currentList in buf if len(currentList) > 1] return out
def mergeSubVector(inpath, classes="", inbase="dept_", outbase="departement_"): listout = fut.FileSearch_AND(inpath, True, inbase, ".shp", "chk") listofchkofzones = fut.sortByFirstElem([ ("_".join(x.split('_')[0:len(x.split('_')) - 1]), x) for x in listout ]) for zone in listofchkofzones: zoneval = zone[0].split('_')[len(zone[0].split('_')) - 1:len(zone[0].split('_'))] outfile = os.path.join(inpath, outbase + zoneval[0] + '.shp') mf.mergeVectors(zone[1], outfile) iota2Formatting(outfile, classes, outfile)
def concatenateDates(rasterList): """from a list of raster, find raster to concatenates (same acquisition date) """ from Common import FileUtils as fut date_position = 4 date_SAR = [] for raster in rasterList: vv, vh = raster.imageFilenamesList SAR_directory, SAR_name = os.path.split(vv) acquisition_date = SAR_name.split("-")[date_position].split("t")[0] date_SAR.append((acquisition_date, raster)) date_SAR = fut.sortByFirstElem(date_SAR) return [ tuple(listOfConcatenation) for date, listOfConcatenation in date_SAR ]
def GetDataAugmentationByCopyParameters(iota2_dir_samples): """ read the */learningSample* directory parse the IOTA2's directory */learningSamples* and return a list by seed in order to feed DataAugmentationByCopy function Parameters ---------- iota2_dir_samples : string absolute path to the /learningSamples IOTA2's directory Example ------- >>> os.listdir("/learningSamples") ["Samples_region_2_seed0_learn.sqlite", "Samples_region_2_seed1_learn.sqlite", "Samples_region_1_seed0_learn.sqlite", "Samples_region_1_seed1_learn.sqlite"] >>> GetSamplesSet("/learningSamples") [[Samples_region_1_seed0_learn.sqlite, Samples_region_2_seed0_learn.sqlite], [Samples_region_1_seed1_learn.sqlite, Samples_region_2_seed1_learn.sqlite]] Return ------ list a list of list where each inner list contains all samples for a given run """ seed_pos = 3 samples_set = [(os.path.basename(samples).split("_")[seed_pos], samples) for samples in fut.FileSearch_AND( iota2_dir_samples, True, "Samples_region", "sqlite")] samples_set = [ samplesSeed for seed, samplesSeed in fut.sortByFirstElem(samples_set) ] return samples_set
def extraction(shapeE, DriverE, field, field_val, nb_extrac, shapeS, fieldo, DriverS): driver = ogr.GetDriverByName(DriverE) dataSource = driver.Open(shapeE, 0) layer = dataSource.GetLayer() driver = ogr.GetDriverByName(DriverS) dataSourceS = driver.Open(shapeS, 1) layerS = dataSourceS.GetLayer() print "checking FID" All_FID = [(currentFeat.GetField(field), currentFeat.GetFID()) for currentFeat in layer if currentFeat.GetField(field) in field_val] All_FID = fu.sortByFirstElem(All_FID) print "FIDs found" # get Fieldo index featureDefnS = layerS.GetLayerDefn() indfieldo = featureDefnS.GetFieldIndex(fieldo) # Fields Lists listFieldIn = fu.getAllFieldsInShape(shapeE, DriverE) listFieldOut = fu.getAllFieldsInShape(shapeS, DriverS) numberOfFeatures = layerS.GetFIDColumn() # in case of not closed layers layerS.ResetReading() layer.ResetReading() i = 0 fid_ind = layerS for val in field_val: print "fill up " + str(val) + " values" # list of Fid of the current landcover type (val) listFid = [x[1] for x in All_FID if x[0] == val][0] # Random selection print len(listFid) nbExtraction = nb_extrac[i] if nbExtraction > len(listFid): nbExtraction = len(listFid) print "Warning : class " + str(val) + " extraction set to " + str( nbExtraction) sublistFid = random.sample(listFid, nbExtraction) chunkSublistFID = fu.splitList(sublistFid, 1 + int(len(sublistFid) / 1000)) filterFID = [] for chunk in chunkSublistFID: # Filter input shapefile filterFID.append("(" + " OR ".join([ layer.GetFIDColumn() + "=" + str(currentFID) for currentFID in chunk ]) + ")") ffilter = " OR ".join(filterFID) layer.SetAttributeFilter(ffilter) newfid = max([feat.GetFID() for feat in layerS]) # filtered input features into output shapefile for feature in layer: geom = feature.GetGeometryRef() dstfeature = ogr.Feature(layerS.GetLayerDefn()) dstfeature.SetGeometry(geom) dstfeature.SetFID(newfid + 1) newfid += 1 indIn = 0 while indIn < len(listFieldIn): dstfeature.SetField(listFieldOut[indIn], feature.GetField(listFieldIn[indIn])) indIn += 1 layerS.CreateFeature(dstfeature) dstfeature.Destroy() i += 1 layerS = layer = None print "DONE"
def searchCrownTile(inpath, raster, clump, ram, grid, outpath, nbcore = 4, ngrid = -1, logger=logger): """ in : inpath : working directory with datas raster : name of raster ram : ram for otb application grid : grid name for serialisation out : output path ngrid : tile number out : raster with normelized name (tile_ngrid.tif) """ begintime = time.time() if os.path.exists(os.path.join(outpath, "tile_%s.tif"%(ngrid))): logger.error("Output file '%s' already exists"%(os.path.join(outpath, \ "tile_%s.tif"%(ngrid)))) sys.exit() rasterfile = gdal.Open(clump, 0) clumpBand = rasterfile.GetRasterBand(1) xsize = rasterfile.RasterXSize ysize = rasterfile.RasterYSize clumpArray = clumpBand.ReadAsArray() clumpProps = regionprops(clumpArray) rasterfile = clumpBand = clumpArray = None # Get extent of all image clumps params = {x.label:x.bbox for x in clumpProps} timeextents = time.time() logger.info(" ".join([" : ".join(["Get extents of all entities", str(round(timeextents - begintime, 2))]), "seconds"])) # Open Grid file driver = ogr.GetDriverByName("ESRI Shapefile") shape = driver.Open(grid, 0) grid_layer = shape.GetLayer() allTile = False # for each tile for feature in grid_layer : if ngrid is None: ngrid = int(feature.GetField("FID")) allTile = True # get feature FID idtile = int(feature.GetField("FID")) # feature ID vs. requested tile (ngrid) if idtile == int(ngrid): logger.info("Tile : %s"%(idtile)) # manage environment if not os.path.exists(os.path.join(inpath, str(ngrid))): os.mkdir(os.path.join(inpath, str(ngrid))) # entities ID list of tile listTileId = listTileEntities(raster, outpath, feature) # if no entities in tile if len(listTileId) != 0 : timentities = time.time() logger.info(" ".join([" : ".join(["Entities ID list of tile", str(round(timentities - timeextents, 2))]), "seconds"])) logger.info(" : ".join(["Entities number", str(len(listTileId))])) # tile entities bounding box listExtent = ExtentEntitiesTile(listTileId, params, xsize, ysize, False) timeextent = time.time() logger.info(" ".join([" : ".join(["Compute geographical extent of entities", str(round(timeextent - timentities, 2))]), "seconds"])) # Extract classification raster on tile entities extent tifRasterExtract = os.path.join(inpath, str(ngrid), "tile_%s.tif"%(ngrid)) if os.path.exists(tifRasterExtract):os.remove(tifRasterExtract) xmin, ymax = pixToGeo(raster, listExtent[1], listExtent[0]) xmax, ymin = pixToGeo(raster, listExtent[3], listExtent[2]) command = "gdalwarp -q -multi -wo NUM_THREADS={} -te {} {} {} {} -ot UInt32 {} {}".format(nbcore,\ xmin, \ ymin, \ xmax, \ ymax, \ raster, \ tifRasterExtract) Utils.run(command) timeextract = time.time() logger.info(" ".join([" : ".join(["Extract classification raster on tile entities extent", str(round(timeextract - timeextent, 2))]), "seconds"])) # Crown entities research ds = gdal.Open(tifRasterExtract) idx = ds.ReadAsArray()[1] g = graph.RAG(idx.astype(int), connectivity = 2) # Create connection duplicates listelt = [] for elt in g.edges(): if elt[0] > 301 and elt[1] > 301: listelt.append(elt) listelt.append((elt[1], elt[0])) # group by tile entities id topo = dict(fu.sortByFirstElem(listelt)) # Flat list and remove tile entities flatneighbors = set(chain(*list(dict((key,value) for key, value in list(topo.items()) if key in listTileId).values()))) timecrownentities = time.time() logger.info(" ".join([" : ".join(["List crown entities", str(round(timecrownentities - timeextract, 2))]), "seconds"])) # Crown raster extraction listExtentneighbors = ExtentEntitiesTile(flatneighbors, params, xsize, ysize, False) xmin, ymax = pixToGeo(raster, listExtentneighbors[1], listExtentneighbors[0]) xmax, ymin = pixToGeo(raster, listExtentneighbors[3], listExtentneighbors[2]) rastEntitiesNeighbors = os.path.join(inpath, str(ngrid), "crown_%s.tif"%(ngrid)) if os.path.exists(rastEntitiesNeighbors):os.remove(rastEntitiesNeighbors) command = "gdalwarp -q -multi -wo NUM_THREADS={} -te {} {} {} {} -ot UInt32 {} {}".format(nbcore,\ xmin, \ ymin, \ xmax, \ ymax, \ raster, \ rastEntitiesNeighbors) Utils.run(command) timeextractcrown = time.time() logger.info(" ".join([" : ".join(["Extract classification raster on crown entities extent", str(round(timeextractcrown - timecrownentities, 2))]), "seconds"])) shutil.copy(rastEntitiesNeighbors, os.path.join(outpath, "crown_%s.tif"%(ngrid))) with open(os.path.join(inpath, str(ngrid), "listid_%s"%(ngrid)), 'wb') as fp: pickle.dump([listTileId + list(flatneighbors)], fp) shutil.copy(os.path.join(inpath, str(ngrid), "listid_%s"%(ngrid)), os.path.join(outpath, "listid_%s"%(ngrid))) shutil.rmtree(os.path.join(inpath, str(ngrid)), ignore_errors=True) if allTile: ngrid += 1
def extraction(vectorFill, vectorSource, field, field_val, driversFill, driversSource): ogrDriversFill = [ ogr.GetDriverByName(currentDriver) for currentDriver in driversFill ] ogrDriversSource = ogr.GetDriverByName(driversSource) dataSourceFill = [ currentDriver.Open(currentShape, 1) for currentDriver, currentShape in zip(ogrDriversFill, vectorFill) ] dataSourceSource = ogrDriversSource.Open(vectorSource, 0) layerFill = [ currentDataSource.GetLayer() for currentDataSource in dataSourceFill ] layerSource = dataSourceSource.GetLayer() FIDColumn = layerSource.GetFIDColumn() if FIDColumn == "": FIDColumn = "FID" FIDMAX = [ max([feat.GetFID() for feat in currentLayerToFill]) for currentLayerToFill in layerFill ] listFieldSource = fu.getAllFieldsInShape(vectorSource, driversSource) All_FID = [(currentFeat.GetField(field), currentFeat.GetFID()) for currentFeat in layerSource if currentFeat.GetField(field) in field_val] layerSource.ResetReading() for layerToFill in layerFill: layerToFill.ResetReading() All_FID = fu.sortByFirstElem(All_FID) for currentClass, FID in All_FID: splits = fu.splitList(FID, len(vectorFill)) i = 0 for currentSplit, layerToFill, fidMax in zip(splits, layerFill, FIDMAX): chunkSublistFID = fu.splitList(currentSplit, 1 + int(len(currentSplit) / 1000)) filterFID = "(" + " OR ".join([ "(" + " OR ".join([ FIDColumn + "=" + str(currentFID) for currentFID in chunk ]) + ")" for chunk in chunkSublistFID ]) + ")" layerSource.SetAttributeFilter(filterFID) newfid = fidMax print "Ajout de " + str(currentClass) + " dans " + vectorFill[ i] + " filter : " + filterFID for feature in layerSource: geom = feature.GetGeometryRef() print geom dstfeature = ogr.Feature(layerSource.GetLayerDefn()) dstfeature.SetGeometry(geom) dstfeature.SetFID(newfid + 1) newfid += 1 indIn = 0 while indIn < len(listFieldSource): dstfeature.SetField( listFieldSource[indIn], feature.GetField(listFieldSource[indIn])) indIn += 1 layerToFill.CreateFeature(dstfeature) dstfeature.Destroy() i += 1 for layerToFill in layerFill: layerToFill = None layerSource = None