def launchFit(sensorName, cmdPath, workingDirectory, outDirectory, outGridPath,
outputGrid_tileNameField, outPixRes, outputProjection,
interpolator, inputGrid_tileNameField, inputGRID, arboMask,
masksInitValues, masksPatterns, arboRaster, rasterInitValue,
rasterPatterns, TileFolder):
createOutGrid(TileFolder, rasterPatterns[0], outputProjection,
outputGrid_tileNameField, outGridPath)
intersections = getIntersections(inputGRID, outGridPath,
inputGrid_tileNameField,
outputGrid_tileNameField)
allCmd = []
for refTile, tiles in intersections:
outFolder_tile = outDirectory + "/" + refTile
if not os.path.exists(outFolder_tile):
os.mkdir(outFolder_tile)
minX_ref, maxX_ref, minY_ref, maxY_ref = getTileEnvelope(
inputGRID, inputGrid_tileNameField, refTile)
for currentTile in tiles:
datas, dates = getAllDatasByDate(currentTile, rasterPatterns,
masksPatterns, arboRaster,
arboMask, TileFolder,
rasterInitValue, masksInitValues)
for currentDatas, currentDate in zip(datas, dates):
outFolderDate = outFolder_tile + "/" + sensorName + "_" + refTile + "_" + currentDate + "_" + currentTile
outFolderDateMask = outFolderDate + "/MASKS"
if not os.path.exists(outFolderDate):
os.mkdir(outFolderDate)
os.mkdir(outFolderDateMask)
for currentRaster in currentDatas:
folder = outFolderDate
print currentRaster
if checkMaskFromRaster(currentRaster[0], masksPatterns):
folder = outFolderDateMask
outFolder = folder
if workingDirectory:
folder = workingDirectory + "/"
outName = currentRaster[0].split("/")[-1].replace(
currentTile,
refTile).replace(".tif", "_" + currentTile + ".tif")
out = folder + "/" + outName
cmd = "gdalwarp -t_srs EPSG:" + outputProjection + " -wo INIT_DEST=" + currentRaster[
1] + " -te " + str(minX_ref) + " " + str(
minY_ref
) + " " + str(maxX_ref) + " " + str(
maxY_ref
) + " -tr " + outPixRes + " -" + outPixRes + " -r " + interpolator + " " + currentRaster[
0] + " " + out
if not os.path.exists(outFolder + "/" + outName):
allCmd.append(cmd)
run(cmd)
if workingDirectory:
shutil.copy(out, outFolder + "/" + outName)
os.remove(out)
fu.writeCmds(cmdPath, allCmd)
def createRegionsByTiles(shapeRegion,
field_Region,
pathToEnv,
pathOut,
pathWd,
logger_=logger):
"""
create a shapeFile into tile's envelope for each regions in shapeRegion and for each tiles
IN :
- shapeRegion : the shape which contains all regions
- field_Region : the field into the region's shape which describes each tile belong to which model
- pathToEnv : path to the tile's envelope with priority
- pathOut : path to store all resulting shapeFile
- pathWd : path to working directory (not mandatory, due to cluster's architecture default = None)
"""
pathName = pathWd
if pathWd == None:
#sequential case
pathName = pathOut
#getAllTiles
AllTiles = fu.FileSearch_AND(pathToEnv, True, ".shp")
regionList = fu.getFieldElement(shapeRegion, "ESRI Shapefile",
field_Region, "unique")
shpRegionList = splitVectorLayer(shapeRegion, field_Region, "int",
regionList, pathName)
AllClip = []
for shp in shpRegionList:
for tile in AllTiles:
logger_.info("Extract %s in %s", shp, tile)
pathToClip = fu.ClipVectorData(shp, tile, pathName)
AllClip.append(pathToClip)
if pathWd:
for clip in AllClip:
cmd = "cp " + clip.replace(".shp", "*") + " " + pathOut
run(cmd)
else:
for shp in shpRegionList:
path = shp.replace(".shp", "")
os.remove(path + ".shp")
os.remove(path + ".shx")
os.remove(path + ".dbf")
os.remove(path + ".prj")
return AllClip
def splitVectorLayer(shp_in, attribute, attribute_type, field_vals, pathOut):
"""
Split a vector layer in function of its attribute
ARGs:
INPUT:
- shp_in: input shapefile
- attribute: attribute to look for
- attribute_type: attribute type which could be "string" or "int"
OUTPUT
- shp_out_list: list of shapefile names
"""
short_shp_in = shp_in.split('.')
shp_out_list = []
name = shp_in.split("/")[-1].split(".")[0]
if attribute_type == "string":
for val in field_vals:
if val != "None":
shp_out = pathOut + "/" + name + "_region_" + str(val) + ".shp"
if (not os.path.isfile(shp_out)):
cmd = "ogr2ogr "
cmd += "-where '" + attribute + ' = "' + val + '"' + "' "
cmd += shp_out + " "
cmd += shp_in + " "
run(cmd)
shp_out_list.append(shp_out)
elif attribute_type == "int":
for val in field_vals:
shp_out = pathOut + "/" + name + "_region_" + str(val) + ".shp"
if (not os.path.isfile(shp_out)):
cmd = "ogr2ogr "
cmd += "-where '" + attribute + " = " + str(val) + "' "
cmd += shp_out + " "
cmd += shp_in + " "
run(cmd)
shp_out_list.append(shp_out)
else:
raise Exception("Error for attribute_type ", attribute_type,
'! Should be "string" or "int"')
return shp_out_list
def generateRaster(raster_path, features_path, driver, field, valField, output,
epsg):
"""
"""
driver = ogr.GetDriverByName(driver)
features = driver.Open(features_path, 0)
if features is None:
raise Exception("Could not open " + features_path)
lyr = features.GetLayer()
allGeom = getAllGeom(lyr, field, valField)
minX, maxX, minY, maxY = getBBox(raster_path, allGeom)
layerName = features_path.split("/")[-1].split(".")[0]
fieldType = getFieldType(lyr, field)
sep = ""
if not isinstance(fieldType, int): sep = "'"
valField = [
"(" + field + "=" + sep + currentValToKeep + sep + ")"
for currentValToKeep in valField
]
csql = " OR ".join(valField)
sizeX, sizeY = getRasterResolution(raster_path)
cmd = "gdalwarp -overwrite -t_srs EPSG:" + str(epsg) + " -tr " + str(
sizeX
) + " " + str(
sizeX
) + " -of GTiff -cl " + layerName + " -csql \"SELECT * FROM " + layerName + " WHERE (" + csql + ")\" -ot Byte -te " + str(
minX
) + " " + str(minY) + " " + str(maxX) + " " + str(
maxY
) + " -cutline " + features_path + " -crop_to_cutline " + raster_path + " " + output
run(cmd)
ds = gdal.Open(output, GA_Update)
proj = ds.GetProjection()
gt = ds.GetGeoTransform()
gt2 = (minX, sizeX, gt[2], maxY, gt[4], sizeY)
ds.SetGeoTransform(gt2)
def launchFit(noDataM, noDataR, tileFolder, currentTile, sensorName, S2Folder, S2Bands, masks):
S2Bands = S2Bands+masks
noDataM = dict(list(zip(masks, noDataM)))
try:
workingDirectory = os.environ["TMPDIR"]
except:
workingDirectory = None
content = os.listdir(tileFolder)
AllCmd = []
if os.path.isdir(tileFolder+"/"+currentTile):
folder = tileFolder+"/"+currentTile
sameDateFolder = getTileSameDate(folder)
for currentSameDate in sameDateFolder:
buf = []
cpt = 0
for currentBand in S2Bands:
print("Current Band : "+currentBand)
buf.append([])
AllTiles = []
for currentFolder in currentSameDate:
print(currentFolder)
path = FileSearch_fast(currentFolder, True, currentBand, ".tif")
tile = path.split("/")[-1].split("_")[-1].replace(".tif", "")
X, Y, inEPSG = getTileOrigin(tile, S2Folder)
X_conv, Y_conv = converCoord((X, Y), inEPSG, 2154)
buf[cpt].append((path, X_conv, Y_conv))
t = path.split("/")[-1].split("_")[-1].replace(".tif", "")# -> origin tile name in raster to fit
currentDate = path.split("/")[-1].split("_")[1].split("-")[0]#-> current date in raster to fit
currentDestTile = path.split("/")[-1].split("_")[3]#destination tile Name in raster name
if t not in AllTiles:
AllTiles.append(t)
outFolder = tileFolder+"/"+currentDestTile+"/"+sensorName+"_"+currentDestTile+"_"+currentDate+"_"+"_".join(AllTiles)
initVal = noDataR
if currentBand in masks:
outFolder = outFolder+"/MASKS"
initVal = noDataM[currentBand]
outName = "_".join(buf[cpt][0][0].split("/")[-1].split("_")[0:-1])+"_"+"_".join(AllTiles)+".tif"
#case 2 acquisitions same day -> remove hours information
tmp = []
tmpList = outName.split("_")
for i in range(len(tmpList)):
if i != rasterPosDate:
tmp.append(tmpList[i])
else:
tmp.append(currentDate)
outName = "_".join(tmp)
#############################
if not os.path.exists(outFolder):
os.mkdir(outFolder)
priorityPaths = " ".join([currentPath for currentPath, X, Y in sortTiles(buf[cpt])])
if not os.path.exists(outFolder+"/"+outName):
workingFolder = outFolder
if workingDirectory:
workingFolder = workingDirectory
cmd = "gdal_merge.py -init "+str(initVal)+" -n "+str(initVal)+" -o "+workingFolder+"/"+outName+" "+priorityPaths
cmd2 = "gdal_merge.py -init "+str(initVal)+" -n "+str(initVal)+" -o "+workingFolder+"/"+outName+" "+priorityPaths+" | "+outFolder+"/"+outName
AllCmd.append(cmd)
addLineToFile(cmdPath, cmd2)
run(cmd)
if workingDirectory:
shutil.copy(workingFolder+"/"+outName, outFolder+"/"+outName)
os.remove(workingFolder+"/"+outName)
cpt += 1
def mosaicFromShape(rasters, shape, rasterOut, workingDir=None):
"""
usage : from a shape clip rasters and mosaic them (upper left priority).
IN
rasters [list of string] : path to rasters
shape [string] : vector pathg
rasterOut [string] : output path
workingDir [string]
OUT
"""
#sort input raster by origin (upperLeft)
rasters_p = sorted([(raster, fut.getRasterExtent(raster))
for raster in rasters],
key=upperLeft)[::-1]
outputDirectory = os.path.split(rasterOut)[0]
wDir = outputDirectory
tmp_files_vec = []
tmp_files_raster = []
if workingDir:
wDir = workingDir
for raster, coordinates in rasters_p:
rasterName = os.path.splitext(os.path.split(raster)[-1])[0]
raster_footPrint = os.path.join(wDir, rasterName + ".shp")
if os.path.exists(raster_footPrint):
while os.path.exists(raster_footPrint):
raster_footPrint = raster_footPrint.replace(
".shp", "_" + str(random.randint(1, 1000)) + ".shp")
#raster footPrint
genRasterEnvelope(raster, raster_footPrint)
tmp_files_vec.append(raster_footPrint)
clip = raster_footPrint.replace(".shp", "_Clip.shp")
clip_raster = raster_footPrint.replace(".shp", "_Clip.tif")
inter = genIntersectionShape(raster_footPrint, shape, clip)
cmd = "gdalwarp -crop_to_cutline -cutline " + clip + " " + raster + " " + clip_raster
run(cmd)
tmp_files_raster.append(clip_raster)
tmp_files_vec.append(clip)
#mosaic using gdal_merge.py, the last image will be copied over earlier ones
rasters_clip = " ".join(tmp_files_raster[::-1])
cmd = "gdal_merge.py -o " + rasterOut + " -n 0 " + rasters_clip
run(cmd)
#clean tmp files
for vec in tmp_files_vec:
fut.removeShape(vec.replace(".shp", ""),
[".prj", ".shp", ".dbf", ".shx"])
for img in tmp_files_raster:
os.remove(img)
if workingDir:
shutil.copy(rasterOut, outputDirectory)