def getAllDatasByDate(currentTile, rasterPatterns, masksPatterns, arboRaster,
arboMask, TileFolder, rasterInitValue, masksInitValues):
#get all raster into the current tile and store it with init value
buf = []
r = []
for currentPattern in rasterPatterns:
r += fu.fileSearchRegEx(TileFolder + currentTile + arboRaster +
currentPattern)
for currentR in r:
buf.append((currentR, rasterInitValue))
for currentPattern, currentMaskInit in zip(masksPatterns, masksInitValues):
m = []
m += fu.fileSearchRegEx(TileFolder + currentTile + arboMask +
currentPattern)
for currentM in m:
buf.append((currentM, currentMaskInit))
#sort it by date
buff = [(getDateFromRaster(currentRaster[0]), currentRaster)
for currentRaster in buf]
buff = fu.sortByFirstElem(buff)
allDates = []
allRasters = []
for date, rasters in buff:
allDates.append(date)
allRasters.append(rasters)
return allRasters, allDates
def extraction(vectorFill, vectorSource, field, field_val, driversFill,
driversSource):
ogrDriversSource = ogr.GetDriverByName(driversSource)
dataSourceSource = ogrDriversSource.Open(vectorSource, 0)
layerSource = dataSourceSource.GetLayer()
print("RECHERCHE DES FIDs")
All_FID = [(currentFeat.GetField(field), str(currentFeat.GetFID()))
for currentFeat in layerSource
if currentFeat.GetField(field) in field_val]
print("FIDs trouvée")
layerSource.ResetReading()
All_FID = fu.sortByFirstElem(All_FID)
for currentClass, FID in All_FID:
splits = fu.splitList(FID, len(vectorFill))
for currentSplit, currentVectorFill in zip(splits, vectorFill):
cmd = "ogr2ogr -append " + currentVectorFill + " " + vectorSource + " -where \" fid in (" + ",".join(
currentSplit) + ")\""
print(cmd)
print("Ajout de " + str(currentClass) + " dans " +
currentVectorFill.split("/")[-1])
os.system(cmd)
def getNbSample(shape,
tile,
dataField,
valToFind,
resol,
region,
coeff,
current_seed,
region_field,
region_val="-1"):
driver = ogr.GetDriverByName("ESRI Shapefile")
buff = []
dataSource = driver.Open(shape, 0)
layer = dataSource.GetLayer()
for feature in layer:
if str(feature.GetField(dataField)) in valToFind and str(
feature.GetField(region_field)) == str(region_val) and str(
feature.GetField("seed_" + str(current_seed))) == "learn":
geom = feature.GetGeometryRef()
buff.append((feature.GetField(dataField), geom.GetArea()))
rep = fu.sortByFirstElem(buff)
repDict = {}
for currentClass, currentAreas in rep:
array = np.asarray(currentAreas)
totalArea = np.sum(array)
repDict[currentClass] = int(
(float(coeff) * totalArea) / (int(resol) * int(resol)))
return repDict
def getIntersections(outGridPath, inGridPath, tileField_first,
tileField_second):
"""
OUT
AllIntersections [list of tuple] : [(S2Tile, [L8Tile, L8Tile, L8Tile]), (...), ...]
"""
driver = ogr.GetDriverByName("ESRI Shapefile")
dataOut = driver.Open(outGridPath, 0)
dataIn = driver.Open(inGridPath, 0)
layerOut = dataOut.GetLayer()
layerIn = dataIn.GetLayer()
AllIntersections = [] #Ex : [[S2, [L8, L8, ..., L8]], [], ...]
outTiles = [(outTile.GetGeometryRef().Clone(),
outTile.GetField(tileField_first)) for outTile in layerOut]
inTiles = [(inTile.GetGeometryRef().Clone(),
inTile.GetField(tileField_second)) for inTile in layerIn]
for outTileGeom, outTile in outTiles:
for inTileGeom, inTile in inTiles:
intersection = outTileGeom.Intersection(inTileGeom)
if intersection.GetArea() != 0.0 and (
outTile, inTile) not in AllIntersections:
AllIntersections.append((outTile, inTile))
return fu.sortByFirstElem(AllIntersections)
def getTileSameDate(folder):
buf = []
content = os.listdir(folder)
for currentContent in content:
date = currentContent.split("_")[2].split("-")[0]
buf.append((date, folder+"/"+currentContent))
buf = fu.sortByFirstElem(buf)
out = [currentList for date, currentList in buf if len(currentList) > 1]
return out
def mergeSubVector(inpath, classes="", inbase="dept_", outbase="departement_"):
listout = fut.FileSearch_AND(inpath, True, inbase, ".shp", "chk")
listofchkofzones = fut.sortByFirstElem([
("_".join(x.split('_')[0:len(x.split('_')) - 1]), x) for x in listout
])
for zone in listofchkofzones:
zoneval = zone[0].split('_')[len(zone[0].split('_')) -
1:len(zone[0].split('_'))]
outfile = os.path.join(inpath, outbase + zoneval[0] + '.shp')
mf.mergeVectors(zone[1], outfile)
iota2Formatting(outfile, classes, outfile)
def concatenateDates(rasterList):
"""from a list of raster, find raster to concatenates (same acquisition date)
"""
from Common import FileUtils as fut
date_position = 4
date_SAR = []
for raster in rasterList:
vv, vh = raster.imageFilenamesList
SAR_directory, SAR_name = os.path.split(vv)
acquisition_date = SAR_name.split("-")[date_position].split("t")[0]
date_SAR.append((acquisition_date, raster))
date_SAR = fut.sortByFirstElem(date_SAR)
return [
tuple(listOfConcatenation) for date, listOfConcatenation in date_SAR
]
def GetDataAugmentationByCopyParameters(iota2_dir_samples):
""" read the */learningSample* directory
parse the IOTA2's directory */learningSamples* and return a list by seed
in order to feed DataAugmentationByCopy function
Parameters
----------
iota2_dir_samples : string
absolute path to the /learningSamples IOTA2's directory
Example
-------
>>> os.listdir("/learningSamples")
["Samples_region_2_seed0_learn.sqlite",
"Samples_region_2_seed1_learn.sqlite",
"Samples_region_1_seed0_learn.sqlite",
"Samples_region_1_seed1_learn.sqlite"]
>>> GetSamplesSet("/learningSamples")
[[Samples_region_1_seed0_learn.sqlite, Samples_region_2_seed0_learn.sqlite],
[Samples_region_1_seed1_learn.sqlite, Samples_region_2_seed1_learn.sqlite]]
Return
------
list
a list of list where each inner list contains all samples for a given run
"""
seed_pos = 3
samples_set = [(os.path.basename(samples).split("_")[seed_pos], samples)
for samples in fut.FileSearch_AND(
iota2_dir_samples, True, "Samples_region", "sqlite")]
samples_set = [
samplesSeed for seed, samplesSeed in fut.sortByFirstElem(samples_set)
]
return samples_set
def extraction(shapeE, DriverE, field, field_val, nb_extrac, shapeS, fieldo,
DriverS):
driver = ogr.GetDriverByName(DriverE)
dataSource = driver.Open(shapeE, 0)
layer = dataSource.GetLayer()
driver = ogr.GetDriverByName(DriverS)
dataSourceS = driver.Open(shapeS, 1)
layerS = dataSourceS.GetLayer()
print "checking FID"
All_FID = [(currentFeat.GetField(field), currentFeat.GetFID())
for currentFeat in layer
if currentFeat.GetField(field) in field_val]
All_FID = fu.sortByFirstElem(All_FID)
print "FIDs found"
# get Fieldo index
featureDefnS = layerS.GetLayerDefn()
indfieldo = featureDefnS.GetFieldIndex(fieldo)
# Fields Lists
listFieldIn = fu.getAllFieldsInShape(shapeE, DriverE)
listFieldOut = fu.getAllFieldsInShape(shapeS, DriverS)
numberOfFeatures = layerS.GetFIDColumn()
# in case of not closed layers
layerS.ResetReading()
layer.ResetReading()
i = 0
fid_ind = layerS
for val in field_val:
print "fill up " + str(val) + " values"
# list of Fid of the current landcover type (val)
listFid = [x[1] for x in All_FID if x[0] == val][0]
# Random selection
print len(listFid)
nbExtraction = nb_extrac[i]
if nbExtraction > len(listFid):
nbExtraction = len(listFid)
print "Warning : class " + str(val) + " extraction set to " + str(
nbExtraction)
sublistFid = random.sample(listFid, nbExtraction)
chunkSublistFID = fu.splitList(sublistFid,
1 + int(len(sublistFid) / 1000))
filterFID = []
for chunk in chunkSublistFID:
# Filter input shapefile
filterFID.append("(" + " OR ".join([
layer.GetFIDColumn() + "=" + str(currentFID)
for currentFID in chunk
]) + ")")
ffilter = " OR ".join(filterFID)
layer.SetAttributeFilter(ffilter)
newfid = max([feat.GetFID() for feat in layerS])
# filtered input features into output shapefile
for feature in layer:
geom = feature.GetGeometryRef()
dstfeature = ogr.Feature(layerS.GetLayerDefn())
dstfeature.SetGeometry(geom)
dstfeature.SetFID(newfid + 1)
newfid += 1
indIn = 0
while indIn < len(listFieldIn):
dstfeature.SetField(listFieldOut[indIn],
feature.GetField(listFieldIn[indIn]))
indIn += 1
layerS.CreateFeature(dstfeature)
dstfeature.Destroy()
i += 1
layerS = layer = None
print "DONE"
def searchCrownTile(inpath, raster, clump, ram, grid, outpath, nbcore = 4, ngrid = -1, logger=logger):
"""
in :
inpath : working directory with datas
raster : name of raster
ram : ram for otb application
grid : grid name for serialisation
out : output path
ngrid : tile number
out :
raster with normelized name (tile_ngrid.tif)
"""
begintime = time.time()
if os.path.exists(os.path.join(outpath, "tile_%s.tif"%(ngrid))):
logger.error("Output file '%s' already exists"%(os.path.join(outpath, \
"tile_%s.tif"%(ngrid))))
sys.exit()
rasterfile = gdal.Open(clump, 0)
clumpBand = rasterfile.GetRasterBand(1)
xsize = rasterfile.RasterXSize
ysize = rasterfile.RasterYSize
clumpArray = clumpBand.ReadAsArray()
clumpProps = regionprops(clumpArray)
rasterfile = clumpBand = clumpArray = None
# Get extent of all image clumps
params = {x.label:x.bbox for x in clumpProps}
timeextents = time.time()
logger.info(" ".join([" : ".join(["Get extents of all entities", str(round(timeextents - begintime, 2))]), "seconds"]))
# Open Grid file
driver = ogr.GetDriverByName("ESRI Shapefile")
shape = driver.Open(grid, 0)
grid_layer = shape.GetLayer()
allTile = False
# for each tile
for feature in grid_layer :
if ngrid is None:
ngrid = int(feature.GetField("FID"))
allTile = True
# get feature FID
idtile = int(feature.GetField("FID"))
# feature ID vs. requested tile (ngrid)
if idtile == int(ngrid):
logger.info("Tile : %s"%(idtile))
# manage environment
if not os.path.exists(os.path.join(inpath, str(ngrid))):
os.mkdir(os.path.join(inpath, str(ngrid)))
# entities ID list of tile
listTileId = listTileEntities(raster, outpath, feature)
# if no entities in tile
if len(listTileId) != 0 :
timentities = time.time()
logger.info(" ".join([" : ".join(["Entities ID list of tile", str(round(timentities - timeextents, 2))]), "seconds"]))
logger.info(" : ".join(["Entities number", str(len(listTileId))]))
# tile entities bounding box
listExtent = ExtentEntitiesTile(listTileId, params, xsize, ysize, False)
timeextent = time.time()
logger.info(" ".join([" : ".join(["Compute geographical extent of entities", str(round(timeextent - timentities, 2))]), "seconds"]))
# Extract classification raster on tile entities extent
tifRasterExtract = os.path.join(inpath, str(ngrid), "tile_%s.tif"%(ngrid))
if os.path.exists(tifRasterExtract):os.remove(tifRasterExtract)
xmin, ymax = pixToGeo(raster, listExtent[1], listExtent[0])
xmax, ymin = pixToGeo(raster, listExtent[3], listExtent[2])
command = "gdalwarp -q -multi -wo NUM_THREADS={} -te {} {} {} {} -ot UInt32 {} {}".format(nbcore,\
xmin, \
ymin, \
xmax, \
ymax, \
raster, \
tifRasterExtract)
Utils.run(command)
timeextract = time.time()
logger.info(" ".join([" : ".join(["Extract classification raster on tile entities extent", str(round(timeextract - timeextent, 2))]), "seconds"]))
# Crown entities research
ds = gdal.Open(tifRasterExtract)
idx = ds.ReadAsArray()[1]
g = graph.RAG(idx.astype(int), connectivity = 2)
# Create connection duplicates
listelt = []
for elt in g.edges():
if elt[0] > 301 and elt[1] > 301:
listelt.append(elt)
listelt.append((elt[1], elt[0]))
# group by tile entities id
topo = dict(fu.sortByFirstElem(listelt))
# Flat list and remove tile entities
flatneighbors = set(chain(*list(dict((key,value) for key, value in list(topo.items()) if key in listTileId).values())))
timecrownentities = time.time()
logger.info(" ".join([" : ".join(["List crown entities", str(round(timecrownentities - timeextract, 2))]), "seconds"]))
# Crown raster extraction
listExtentneighbors = ExtentEntitiesTile(flatneighbors, params, xsize, ysize, False)
xmin, ymax = pixToGeo(raster, listExtentneighbors[1], listExtentneighbors[0])
xmax, ymin = pixToGeo(raster, listExtentneighbors[3], listExtentneighbors[2])
rastEntitiesNeighbors = os.path.join(inpath, str(ngrid), "crown_%s.tif"%(ngrid))
if os.path.exists(rastEntitiesNeighbors):os.remove(rastEntitiesNeighbors)
command = "gdalwarp -q -multi -wo NUM_THREADS={} -te {} {} {} {} -ot UInt32 {} {}".format(nbcore,\
xmin, \
ymin, \
xmax, \
ymax, \
raster, \
rastEntitiesNeighbors)
Utils.run(command)
timeextractcrown = time.time()
logger.info(" ".join([" : ".join(["Extract classification raster on crown entities extent", str(round(timeextractcrown - timecrownentities, 2))]), "seconds"]))
shutil.copy(rastEntitiesNeighbors, os.path.join(outpath, "crown_%s.tif"%(ngrid)))
with open(os.path.join(inpath, str(ngrid), "listid_%s"%(ngrid)), 'wb') as fp:
pickle.dump([listTileId + list(flatneighbors)], fp)
shutil.copy(os.path.join(inpath, str(ngrid), "listid_%s"%(ngrid)), os.path.join(outpath, "listid_%s"%(ngrid)))
shutil.rmtree(os.path.join(inpath, str(ngrid)), ignore_errors=True)
if allTile:
ngrid += 1
def extraction(vectorFill, vectorSource, field, field_val, driversFill,
driversSource):
ogrDriversFill = [
ogr.GetDriverByName(currentDriver) for currentDriver in driversFill
]
ogrDriversSource = ogr.GetDriverByName(driversSource)
dataSourceFill = [
currentDriver.Open(currentShape, 1)
for currentDriver, currentShape in zip(ogrDriversFill, vectorFill)
]
dataSourceSource = ogrDriversSource.Open(vectorSource, 0)
layerFill = [
currentDataSource.GetLayer() for currentDataSource in dataSourceFill
]
layerSource = dataSourceSource.GetLayer()
FIDColumn = layerSource.GetFIDColumn()
if FIDColumn == "":
FIDColumn = "FID"
FIDMAX = [
max([feat.GetFID() for feat in currentLayerToFill])
for currentLayerToFill in layerFill
]
listFieldSource = fu.getAllFieldsInShape(vectorSource, driversSource)
All_FID = [(currentFeat.GetField(field), currentFeat.GetFID())
for currentFeat in layerSource
if currentFeat.GetField(field) in field_val]
layerSource.ResetReading()
for layerToFill in layerFill:
layerToFill.ResetReading()
All_FID = fu.sortByFirstElem(All_FID)
for currentClass, FID in All_FID:
splits = fu.splitList(FID, len(vectorFill))
i = 0
for currentSplit, layerToFill, fidMax in zip(splits, layerFill,
FIDMAX):
chunkSublistFID = fu.splitList(currentSplit,
1 + int(len(currentSplit) / 1000))
filterFID = "(" + " OR ".join([
"(" + " OR ".join([
FIDColumn + "=" + str(currentFID) for currentFID in chunk
]) + ")" for chunk in chunkSublistFID
]) + ")"
layerSource.SetAttributeFilter(filterFID)
newfid = fidMax
print "Ajout de " + str(currentClass) + " dans " + vectorFill[
i] + " filter : " + filterFID
for feature in layerSource:
geom = feature.GetGeometryRef()
print geom
dstfeature = ogr.Feature(layerSource.GetLayerDefn())
dstfeature.SetGeometry(geom)
dstfeature.SetFID(newfid + 1)
newfid += 1
indIn = 0
while indIn < len(listFieldSource):
dstfeature.SetField(
listFieldSource[indIn],
feature.GetField(listFieldSource[indIn]))
indIn += 1
layerToFill.CreateFeature(dstfeature)
dstfeature.Destroy()
i += 1
for layerToFill in layerFill:
layerToFill = None
layerSource = None
