def combineClassif(orderClassif,
classifFolder,
legendFolder,
tempFolder,
outClassifName,
outLegendName,
legendPrefix='legend_',
legendExt='.txt',
legendDel='\t',
toNa=[]):
'''
Combine the classifications listed in the orderClassif file,
with the priorities listed in the file, and using the legends
in the legend folder provided.
The legends should have the same name as the classifications,
save for a potential prefix to be provided.
The output is a unified classification, with a unified legend,
where classes with the same (exact) name have been combined.
All values that are not in the legends will be passed to missing.
orderClassif (str): Address of the file with the classifications
names and the priorities for combining them.
The file should be .txt and tab delimited.
It should have two columns, the first is the priority number and
the second is the base name of the classif, without extension.
The first row is considered as the column names.
classifFolder (str): Address of the folder containing the
classifications to combine.
legendFolder (str): Address of the folder containing the
legends of the classifications. The legend files should
have the same name as the classifications, save for
a potential prefix.
CAPS do not matter, all category names will be passed
to lower case.
tempFolder (str): Address of the temporary folder where
intermediate classifications can be saved during processing.
outClassifName (str): Full address of the output combined
classification.
The function uses the value 255 as the no data value for the
output.
outLegendName (str): Full address of the output legend.
Should be a .txt.
legendPrefix (str): prefix used to differentiate the legends
names from the classifications names.
legendExt (str): Extension of the legend files.
legendDel (str): Delimiter of the legend files.
toNa (list of str): Categories in the classifications
to pass to no data in the output classification.
The names provided should match exactly the information
in the legend files.
These names should not include any trailing numbers
if several categories have been made (e.g. Cloud for Cloud1, Cloud2...)
'''
#Check the computer platform
ordi = checkPlatform()
if ordi == "Windows":
import gdal_merge as gm
#Import the classifications names along with the order
classifNames = []
priorities = []
with open(orderClassif, 'r') as f:
next(f) # skip headings
reader = csv.reader(f, delimiter='\t')
for priority, classif in reader:
priorities.append(int(priority))
classifNames.append(classif)
#Remove the raster extensions if there are any
classifNames = [re.sub('.(tif|TIF)$', '', c) for c in classifNames]
#Order the classifs in the correct order
classifNames = [
c for (p, c) in sorted(zip(priorities, classifNames),
key=lambda pair: pair[0])
]
#Import all the legends into a list of dictionaries
legends = []
for nm in classifNames:
dic = {} #Will hold the legend values in format Value:Classname
with open(os.path.join(legendFolder, legendPrefix + nm + legendExt),
'r') as f:
next(f) # skip headings
reader = csv.reader(f, delimiter=legendDel)
for v, c in reader:
dic[int(v)] = re.sub('[0-9]*$', '', c.lower())
#Add to the list
legends.append(dic)
#Establish a common legend for all the classifications
#Get the unique class names and remove any of the categories being mapped to NA
commonLegend = {v for l in legends for v in l.values() if v not in toNa}
commonLegend = list(commonLegend)
#Add a raster value by alphabetical order
commonLegend = {nm: i for i, nm in enumerate(sorted(commonLegend), 1)}
#Loop through the classifications, remove the clouds and shadows, and change the values to the common legend
for nm, legend in zip(classifNames, legends):
if (os.path.isfile(os.path.join(classifFolder, nm + '.tif'))):
tif = '.tif'
else:
tif = '.TIF'
try:
classifRaster = gdal.Open(os.path.join(classifFolder, nm + tif))
except RuntimeError, e:
do_message(
"ERROR", "PROCESS ABORTED: Error (2) opening raster file " +
nm + tif + "\n" + str(e))
return False
if classifRaster is None:
do_message(
"ERROR",
"PROCESS ABORTED: Error (3) opening raster file " + nm + tif)
return False
#Get the raster band
classifBand = classifRaster.GetRasterBand(1)
#Transform the band into a numpy array
band = classifBand.ReadAsArray()
band = band.astype(int)
#Map the old values to the new common values
for v in np.unique(band):
if v in legend:
#Get the class name for that value
c = legend[v]
if c in toNa:
replaceVal = 255
else:
replaceVal = commonLegend[c]
else:
replaceVal = 255
band[band == v] = replaceVal
#Create an empty raster to export the classification that was just mapped
outR = rt.newRasterFromBase(classifRaster,
tempFolder + '/reclass_' + nm + '.tif',
'GTiff',
255,
gdal.GDT_Byte,
bands=1)
#Write the re-classified values to the empty raster
outR.GetRasterBand(1).WriteArray(band)
#Close the rasters
outR.FlushCache()
outR = None
def main():
##########################################################################################################################
##########################################################################################################################
##PARAMETERS
#Address of the folder with the classifications
classifFolder = 'E:/gedata_current/jde_coffee/data/SDM/classifs'
#TAB delimited file with the priority order in which to combine the classifications
#The file should have two columns, the first is a priority number, the second the file name of the classification
orderClassif = 'E:/gedata_current/jde_coffee/data/SDM/classifs/classif_priorities_SDM.txt'
#Classes to pass to NA values in combining the classifications
#Leave an empty list if None
#These names should not include any trailing numbers if several categories have been made (e.g. Cloud1, Cloud2...)
toNa = ['Cloud','Shadow']
#Additional values in the classifications that are not in the legends
valuesNotInLegend = [0]
#Address of the folder with the legends
legendFolder = 'E:/gedata_current/jde_coffee/data/SDM/classifs/Legend'
#Legend file names prefix to add to the classification names
legendPrefix = 'legend_'
#Legend file extensions: could be '.csv' or '.txt'
legendExt = '.csv'
#Legend file delimiter
legendDelim = ','
#Address of the folder with the confusion matrices
matrixFolder = 'E:/gedata_current/jde_coffee/data/SDM/classifs/Matconf'
#Matrix file names prefix to add to the classification names
matrixPrefix = 'Matconf_'
#Matrix file extensions: could be '.csv' or '.txt'
matrixExt = '.csv'
#Legend file delimiter
matrixDelim = ','
#Address of the folder for the temporary files
tempFolder = 'E:/gedata_current/temp'
#Address and name of the grid shapefile
gridShape = 'E:/gedata_current/jde_coffee/data/SDM/grids/AOI_grid_SDM_2km.shp'
#Field to use for the ID of the cells
idField = 'serp_id'
#Projection epsg for the output.
#Leave None to use the one from the grid. Should be the same as the grid if the grid doesn't have a projection info!
epsgProject = 32723
#SRTM Mask option
srtmMask = False
srtmRaster = 'E:/gedata_current/jde_coffee/data/SDM/srtm/srtm_v3_SDMnet.tif'
breakValues = [550] #list of values used as break to separate the classifications
labelsElev = ['robusta','arabica']
#Share of clouds/missing per cell under which correct by simple proportional scaling
propCloud = 0.5
#Output name and address for the combined files and legend
exportClassifName = 'megot_classif.tif'
##########################################################################################################################
##########################################################################################################################
##CODE
#Combine the classifications without and with the clouds
combine_classifications(classifFolder, orderClassif, legendFolder, legendPrefix, legendExt,
legendDelim, tempFolder, 'nocloud_'+exportClassifName, toNa=toNa, valuesNotInLegend=valuesNotInLegend)
combine_classifications(classifFolder, orderClassif, legendFolder, legendPrefix, legendExt,
legendDelim, tempFolder, 'cloud_'+exportClassifName, toNa=[], valuesNotInLegend=valuesNotInLegend)
#Merge the two combined rasters
#Check the computer platform
ordi = checkPlatform()
if ordi == "Windows":
import gdal_merge as gm
#Prepare the arguments for gdal_merge.py -- They all need to be strings
args = ['-n', '255', '-a_nodata', '255', '-of', 'GTiff', '-ot', 'UInt32', '-co', 'COMPRESS:LZW', '-o', classifFolder+'/'+exportClassifName]
args.extend(['cloud_'+exportClassifName,'nocloud_'+exportClassifName])
#Do the merge
if ordi == "Linux":
args = ['gdal_merge.py','-of', "GTiff"]+args
p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
p.communicate()[0] #Waits until the subprocess is finished before running the rest of the code
elif ordi == "Windows":
#Windows makes an error when calling gdal_merge directly,
#So need to specify python for the executable and to make it run in the shell
sys.argv = args
gm.main()
#Combine the legends to have a single legend?
#Remove the intermediary files
for f in [classifFolder+'/cloud_'+exportClassifName,classifFolder+'/nocloud_'+exportClassifName,legendFolder+'/legend_nocloud_'+exportClassifName]:
os.remove(f)
#Import the raster with the combined classifications
baseClassif = gdal.Open(os.path.join(classifFolder,exportClassifName))
#Mask the combined raster with the srtm data if needed and demultiply the classification based on the elevation breaks
if not srtmMask:
processed = rt.newRasterFromBase(baseClassif, tempFolder+'/'+'masked0.tif', 'GTiff', -32768, gdal.GDT_Int16, bands=1)
processed.GetRasterBand(1).WriteArray(baseClassif.GetRasterBand(1).readAsArray())
#Number of pieces of the classification raster to tabulate
pieces = 1
baseClassif = None
processed = None
else:
#Import the elevation raster
elev = gdal.Open(srtmRaster)
nodata = elev.GetRasterBand(1).GetNoDataValue()
#Adapt the resolution of the smooth images to the baseline if needed
elev = rt.warpRaster(elev, baseClassif, resampleOption='average', outputURI=None, outFormat='MEM')
#Number of pieces of the classification raster to tabulate
pieces = len(breakValues) + 1
#Complete the breakValues
breakValues.insert(0, 0)
breakValues.append(10000)
#Get the size of the rasters to identify the limits of the blocks to loop through
band = baseClassif.GetRasterBand(1)
#Get the size of the raster
xsize = band.XSize
ysize = band.YSize
#Set the block size
BlockXSize = 256
BlockYSize = 256
#Get the number of blocks in x and y directions
xBlocks = int(round(xsize/BlockXSize)) + 1
yBlocks = int(round(ysize/BlockYSize)) + 1
processed = []
for i in range(pieces):
#Prepare empty rasters to hold the masked rasters
processed.append(rt.newRasterFromBase(baseClassif, tempFolder+'/'+'masked'+str(i)+'.tif', 'GTiff', -32768, gdal.GDT_Int16, bands=1))
for xStep in range(xBlocks):
for yStep in range(yBlocks):
#Import the block of the classification
baseBlock = rt.readRasterBlock(baseClassif, xStep*BlockXSize, yStep*BlockYSize, BlockXSize, BlockYSize)
#Import the block of the elevation raster
elevBlock = rt.readRasterBlock(elev, xStep*BlockXSize, yStep*BlockYSize, BlockXSize, BlockYSize)
for i in range(pieces):
#Mask the values based on the elevation raster
if not nodata:
outArray[np.logical_or(np.isnan(elevBlock),np.logical_or(elevBlock<breakValues[i],elevBlock>=breakValues[i+1]))] = -32768
else:
outArray[np.logical_or(np.logical_or(np.isnan(elevBlock),elevBlock==nodata),np.logical_or(elevBlock<breakValues[i],elevBlock>=breakValues[i+1]))] = -32768
#Write to output raster
processed[i].GetRasterBand(1).WriteArray(outArray, xStep*BlockXSize, yStep*BlockYSize)
#Close the rasters
for p in processed:
processed[p] = None
baseClassif = None
elev = None
'''
#Import the grid
driver = ogr.GetDriverByName('ESRI Shapefile')
dataSource = driver.Open(gridShape, 1) # 0 means read-only. 1 means writeable.
# Check if shapefile is correctly loaded.
if dataSource is None:
do_message("ERROR","PROCESS ABORTED: Error (1) opening shapefile " + gridShape)
return False
# Create layer
gridLayer = dataSource.GetLayer(0)
'''
tabulated = []
for i in len(pieces):
#Tabulate area by grid cell for each of the combined classifications
stat = img.tabulate_area(regionsFileName=gridShape, rasterFileName=os.path.join(tempFolder,'masked'+str(i)+'.tif'),
bands=1, polIdFieldName=idField, numPix=False, outTxt=None, prefix='cat', numOutDecs=0, alltouch=False)
#Add the tabulated values to the result list
#The result is a dictionary of dictionaries. First dictionary are the polygon ids, the second [value in raster: count]
tabulated.append(stat[0])
#Get all the polygon ids for each elevation zone
polygons = []
#Get all the categories
categories = set()
for piece in tabulated:
polygons.append(sorted(piece.keys()))
for k in piece.keys():
categories.update(k.keys())
categories = list(categories)
categories.sort()
#Create empty arrays and fill them with the values
tabArray = []
for l in range(len(polygons)):
tabArray.append(np.zeros((len(polygons),len(categories))))
#For each polygon cell and category, get the associated count
for p in range(len(polygons[l])):
for c in range(len(categories)):
try:
tabArray[l][p,c] = tabulated[l][polygons[l][p]][categories[c]]
except:
continue
def combine_classifications(classifFolder, orderClassif, legendFolder, legendPrefix, legendExt,
legendDelim, tempFolder, exportClassifName, toNa=[], valuesNotInLegend=[]):
#Check the computer platform
ordi = checkPlatform()
if ordi == "Windows":
import gdal_merge as gm
#Import the classifications names along with the order
classifNames=[]
priorities=[]
with open(orderClassif,'r') as f:
next(f) # skip headings
reader=csv.reader(f,delimiter='\t')
for priority,classif in reader:
priorities.append(int(priority))
classifNames.append(classif)
#Remove the raster extensions if there are any
classifNames = [re.sub('.(tif|TIF)$','',c) for c in classifNames]
#Order the classifs in the correct order
classifNames = [c for (p,c) in sorted(zip(priorities,classifNames), key=lambda pair: pair[0])]
#Import all the legends into a list of dictionaries
legends = []
newLegends = []
for nm in classifNames:
dict = {} #Will hold the legend values in format Value:Classname
with open(os.path.join(legendFolder,legendPrefix+nm+legendExt),'r') as f:
next(f) # skip headings
reader=csv.reader(f,delimiter=legendDelim)
for v,c in reader:
dict[int(v)] = re.sub('[0-9]*$','',c)
#Add to the list
legends.append(dict)
#Establish a legend combining the info for all the classifications
#Get the unique class names
commonLegend = {v for l in legends for v in l.values()}
commonLegend = {v:[] for v in commonLegend}
#Add the values corresponding to each class
i = 1
for l in legends:
for k,v in l.iteritems():
commonLegend[v].append(k+100*i)
i += 1 #The values of the rasters for each classification will be in a different 100s.
#Loop through the classifications, and change the values to the combined legend
i = 1
for nm,legend in zip(classifNames,legends):
if (os.path.isfile(os.path.join(classifFolder, nm+'.tif'))):
tif = '.tif'
else:
tif = '.TIF'
try:
classifRaster = gdal.Open(os.path.join(classifFolder, nm+tif))
except RuntimeError, e:
do_message("ERROR","PROCESS ABORTED: Error (2) opening raster file " + nm+tif + "\n" + str(e))
return False
if classifRaster is None:
do_message("ERROR","PROCESS ABORTED: Error (3) opening raster file " + nm+tif)
return False
#Get the raster band
classifBand = classifRaster.GetRasterBand(1)
#Get the no data value if any
nodata = int(classifBand.GetNoDataValue())
#Transform the band into a numpy array
band = classifBand.ReadAsArray()
band = band.astype(int)
#Get the values in the raster in sorted order
#(Get them all otherwise throws an error)
legendValues = legend.keys()
palette = list(legendValues)
if nodata:
palette.append(nodata)
if valuesNotInLegend:
for v in valuesNotInLegend:
palette.append(v)
palette = sorted(palette)
#Create the key that gives the new values the palette should be mapped to
mapkey = []
for v in palette:
if v in legendValues:
#Get the class name for that value
c = legend[v]
if c in toNa:
mapkey.append(255)
else:
mapkey.append(v+100*i)
else:
mapkey.append(255)
i += 1 #The values of the rasters for each classification will be in a different 100s.
mapkey = np.array(mapkey)
#Change the values in the raster
index = np.digitize(band, palette, right=True)
band = mapkey[index]
#Create an empty raster to export the classification that was just mapped
outR = rt.newRasterFromBase(classifRaster, tempFolder+'/reclass_'+nm+'.tif', 'GTiff', 255, gdal.GDT_UInt32, bands=1)
#Write the re-classified values to the empty raster
outR.GetRasterBand(1).WriteArray(band)
#Close the rasters
outR.FlushCache()
outR = None
mpsFill = p.map(pp, mps)
# print('time gapfillone parallel %s' % (time.time() - t0))
# Replace the fitted value in the raster
for z in mpsFill:
r[z[0], z[1]] = z[4]
# Prepare name for output
outName = os.path.join(
outFolder,
re.sub('.tif', '_' + suffix + '.tif',
os.path.basename(rasters[rIndex])))
if os.path.isfile(outName):
os.remove(outName)
# t0 = time.time()
outR = rt.newRasterFromBase(dst, outName, 'GTiff', nodataOut,
gdal.GDT_Float32)
outR.GetRasterBand(1).WriteArray(r)
outR.FlushCache()
outR = None
dst = None
# print('time export %s' % (time.time() - t0))
# Prepare the expansion values for export. These values
# provide some information on the difficulty to
# interpolate the pixels
outI.append(sum([z[5] for z in mpsFill]) / float(len(mpsFill)))
if parallel:
# Close the threads
p.close()
def extractModis(root,
regions,
varieties,
regionsIn,
outFolder,
masks,
startExtract,
endExtract,
shpName,
attr,
tempDir,
exportFormat,
epsgShp=None):
#Transform into date format
if not endExtract:
endExtract = datetime.now()
endExtract = endExtract.date()
else:
endExtract = datetime.strptime(endExtract, '%Y-%m-%d').date()
if not startExtract:
startExtract = endExtract - timedelta(days=30)
else:
startExtract = datetime.strptime(startExtract, '%Y-%m-%d').date()
#Open the shapefile
driver = ogr.GetDriverByName('ESRI Shapefile')
dataSource = driver.Open(shpName,
0) # 0 means read-only. 1 means writeable.
# Check if shapefile is correctly loaded.
if dataSource is None:
print("ERROR",
"PROCESS ABORTED: Error (1) opening shapefile " + shpName)
return False
# Get layer
inVecLayer = dataSource.GetLayer(0)
#Get projection if possible
inEpsg = proj.getProjection(dataSource)
if not inEpsg and not epsgShp:
print(
'Could not identify projection for the shapefile. Specify a value for epsgShp parameter'
)
return False
elif not inEpsg:
inEpsg = epsgShp
if not isinstance(inEpsg, int):
inEpsg = int(inEpsg)
if not inEpsg == 4326:
#Create the transform
prj1 = osr.SpatialReference()
prj1.ImportFromEPSG(inEpsg)
prj2 = osr.SpatialReference()
prj2.ImportFromEPSG(4326)
coordTrans = osr.CoordinateTransformation(prj1, prj2)
#create output datasource
outdriver = ogr.GetDriverByName('ESRI Shapefile')
srcOut = outdriver.CreateDataSource(os.path.join(tempDir, 'temp.shp'))
#Create a new layer
lyrOut = srcOut.CreateLayer("layer 1",
srs=prj2,
geom_type=inVecLayer.GetGeomType())
#Add fields
lyr1Defn = inVecLayer.GetLayerDefn()
for i in range(0, lyr1Defn.GetFieldCount()):
fieldDefn = lyr1Defn.GetFieldDefn(i)
lyrOut.CreateField(fieldDefn)
# get the output layer's feature definition
lyrOutDefn = lyrOut.GetLayerDefn()
# loop through the input features
inFeature = inVecLayer.GetNextFeature()
while inFeature:
# get the input geometry
geom = inFeature.GetGeometryRef()
# reproject the geometry
geom.Transform(coordTrans)
# create a new feature
outFeature = ogr.Feature(lyrOutDefn)
# set the geometry and attribute
outFeature.SetGeometry(geom)
for i in range(0, lyrOutDefn.GetFieldCount()):
outFeature.SetField(
lyrOutDefn.GetFieldDefn(i).GetNameRef(),
inFeature.GetField(i))
# add the feature to the shapefile
lyrOut.CreateFeature(outFeature)
# dereference the features and get the next input feature
outFeature = None
inFeature = inVecLayer.GetNextFeature()
#Close the two shapefiles
dataSource.Destroy()
inVecLayer = None
srcOut.Destroy()
lyrOut = None
#Empty dictionary to hold the results.
#Every municipality covered by one of the regions will have a coffee weight
coffeeWeights = {'arabica': {}, 'robusta': {}}
#Every municipality will have a total number of modis pixels
totPixels = {}
#Every municipality covered by one of the regions will have a share of pixels with coffee in it
coffeeShare = {'arabica': {}, 'robusta': {}}
#Every municipality covered by one of the regions will have a coffee count
#Every date will have a dictionary containing the municipalities and values
imgStats = {'arabica': {}, 'robusta': {}}
#Loop through the regions to extract the modis data for each
for r in range(len(regions)):
print('Extracting region ' + str(regions[r]) + '...')
#Import all the smooth modis images on disk
onDisk = [
f for f in os.listdir(os.path.join(root, regions[r], regionsIn))
if f.endswith('.tif')
]
#Dates of these files
datesAll = [
re.search('_([0-9]{4}-[0-9]{2}-[0-9]{2})', f).group(1)
for f in onDisk
]
#Transform into date format
datesAll = [datetime.strptime(d, '%Y-%m-%d').date() for d in datesAll]
#Keep only the files and dates within the dates to process
onDisk = [
f for f, d in zip(onDisk, datesAll)
if d >= startExtract and d <= endExtract
]
datesAll = [
d for d in datesAll if d >= startExtract and d <= endExtract
]
#Sort the two list by date
datesAll, onDisk = (list(x)
for x in zip(*sorted(zip(datesAll, onDisk))))
if not onDisk:
print('no modis images to process in ' + regions[r])
continue
#Get a base image as a reference for format and extent
baseImg = gdal.Open(
os.path.join(root, regions[r], regionsIn, onDisk[0]))
#Get the corner latitude and longitude for the raster
cols = baseImg.RasterXSize # width
rows = baseImg.RasterYSize # height
gt = baseImg.GetGeoTransform()
minLonR = gt[0]
minLatR = gt[3] + cols * gt[4] + rows * gt[5]
maxLonR = gt[0] + cols * gt[1] + rows * gt[2]
maxLatR = gt[3]
#Close the raster
baseImg = None
#Crop the shapefile by the extent of the raster
inShp = tempDir + '/temp.shp'
outShp = tempDir + '/temp_crop.shp'
#Call ogr2ogr using the os system command
cmd = 'ogr2ogr -f "ESRI Shapefile" %s %s -clipsrc %f %f %f %f -overwrite' % (
outShp, inShp, minLonR, minLatR, maxLonR, maxLatR)
os.system(cmd)
###Extract the number of modis pixels per cropped municipalities
#Import the mask
weightsRaster = gdal.Open(root + '/' + regions[r] + '/' + masks[r][0])
#Create an empty copy for storing the temporary weighted image
weighted = rt.newRasterFromBase(weightsRaster,
tempDir + '/temp.tif',
'GTiff',
0.,
gdal.GDT_Float32,
bands=1)
#Import coffee weights band as array
weightsBand = weightsRaster.GetRasterBand(1)
weightsArray = weightsBand.ReadAsArray()
#Recast the type to be sure
weightsArray = weightsArray.astype(np.float32)
#Replace all data by 1
weightsArray[:] = 1.
#Export to temporary raster
weighted.GetRasterBand(1).WriteArray(weightsArray)
#Close the temporary raster
weighted.FlushCache()
weighted = None
#Compute the count of pixels for each polygon of the cropped municipalities
stat = img.raster_stats(regionsFileName=os.path.join(
tempDir, 'temp_crop.shp'),
rasterFileName=tempDir + '/temp.tif',
polIdFieldName=attr,
statistics=['sum'],
outTxt=None,
addShp=False,
addSuffix='',
numOutDecs=2,
alltouch=True)
if not stat:
print('error extracting coffee sums per polygon')
continue
stat = stat[0]
#Add the counts to the total pixel counts
for polyid in stat.keys():
if not str(polyid) in totPixels:
totPixels[str(polyid)] = stat[polyid]['sum']
else:
totPixels[str(
polyid)] = totPixels[str(polyid)] + stat[polyid]['sum']
#Remove existing temporary raster if any
try:
os.remove(os.path.join(tempDir, 'temp.tif'))
except OSError:
pass
for v in range(len(varieties[r])):
for date in datesAll:
if not date.strftime('%Y-%m-%d') in imgStats[varieties[r][v]]:
imgStats[varieties[r][v]][date.strftime('%Y-%m-%d')] = {}
imgStats[varieties[r][v]][date.strftime(
'%Y-%m-%d')]['date'] = date.strftime('%Y-%m-%d')
###Extract the number of pixels with some coffee per cropped municipalities
#Import the mask
weightsRaster = gdal.Open(root + '/' + regions[r] + '/' +
masks[r][v])
#Create an empty copy for storing the temporary weighted image
weighted = rt.newRasterFromBase(weightsRaster,
tempDir + '/temp.tif',
'GTiff',
0.,
gdal.GDT_Float32,
bands=1)
#Import coffee weights band as array
weightsBand = weightsRaster.GetRasterBand(1)
weightsArray = weightsBand.ReadAsArray()
#Recast the type to be sure
weightsArray = weightsArray.astype(np.float32)
#Replace the no data value by 0 and all non zero by 1
nodataW = weightsBand.GetNoDataValue()
if not nodataW:
weightsArray[np.logical_or(
np.isnan(weightsArray),
np.logical_or(weightsArray > 1., weightsArray < 0.))] = 0
else:
weightsArray[np.logical_or(
np.logical_or(np.isnan(weightsArray),
weightsArray == nodataW),
np.logical_or(weightsArray > 1., weightsArray < 0.))] = 0
nodataW = None
#Replace all positive coffee values by 1
weightsArray[weightsArray > 0.] = 1
#Export to temporary raster
weighted.GetRasterBand(1).WriteArray(weightsArray)
#Close the temporary raster
weighted.FlushCache()
weighted = None
'''
if regions[r] == 'ES' and varieties[r][v] == 'robusta':
try:
os.remove(os.path.join(tempDir,'tempES.tif'))
except OSError:
pass
weighted = rt.newRasterFromBase(weightsRaster, tempDir+'/tempES.tif', 'GTiff', 0., gdal.GDT_Float32, bands=1)
#Export to temporary raster
weighted.GetRasterBand(1).WriteArray(weightsArray)
#Close the temporary raster
weighted.FlushCache()
weighted = None
'''
#Compute the count of pixels for each polygon of the cropped municipalities
stat = img.raster_stats(regionsFileName=os.path.join(
tempDir, 'temp_crop.shp'),
rasterFileName=tempDir + '/temp.tif',
polIdFieldName=attr,
statistics=['sum'],
outTxt=None,
addShp=False,
addSuffix='',
numOutDecs=2,
alltouch=True)
if not stat:
print('error extracting coffee sums per polygon')
continue
stat = stat[0]
#Add the sums to the share of coffee
for polyid in stat.keys():
#Do not consider the polygon if all values are missing or 0
if np.isnan(stat[polyid]['sum']) or stat[polyid]['sum'] == 0:
continue
if not str(polyid) in coffeeWeights[varieties[r][v]]:
coffeeShare[varieties[r][v]][str(
polyid)] = stat[polyid]['sum']
else:
coffeeShare[varieties[r][v]][str(polyid)] = coffeeShare[
varieties[r][v]][str(polyid)] + stat[polyid]['sum']
#Remove existing temporary raster if any
try:
os.remove(os.path.join(tempDir, 'temp.tif'))
except OSError:
pass
weightsRaster = None
###Extract the sum of coffee weights per cropped municipality
#Import the mask
weightsRaster = gdal.Open(root + '/' + regions[r] + '/' +
masks[r][v])
#Create an empty copy for storing the temporary weighted image
weighted = rt.newRasterFromBase(weightsRaster,
tempDir + '/temp.tif',
'GTiff',
0,
gdal.GDT_Float32,
bands=1)
#Import coffee weights band as array
weightsBand = weightsRaster.GetRasterBand(1)
weightsArray = weightsBand.ReadAsArray()
#Recast the type to be sure
weightsArray = weightsArray.astype(np.float32)
#Replace the no data values by 0
nodataW = weightsBand.GetNoDataValue()
if not nodataW:
weightsArray[np.logical_or(
np.isnan(weightsArray),
np.logical_or(weightsArray > 1., weightsArray < 0.))] = 0.
else:
weightsArray[np.logical_or(
np.logical_or(np.isnan(weightsArray),
weightsArray == nodataW),
np.logical_or(weightsArray > 1., weightsArray < 0.))] = 0.
nodataW = None
#Export to temporary raster
weighted.GetRasterBand(1).WriteArray(weightsArray)
#Close the temporary raster
weighted.FlushCache()
weighted = None
#Compute the sum for each polygon of the coffee
stat = img.raster_stats(regionsFileName=os.path.join(
tempDir, 'temp_crop.shp'),
rasterFileName=tempDir + '/temp.tif',
polIdFieldName=attr,
statistics=['sum'],
outTxt=None,
addShp=False,
addSuffix='',
numOutDecs=2,
alltouch=True)
if not stat:
print('error extracting coffee sums per polygon')
continue
stat = stat[0]
#Add the weights to the coffee weights dictionary
for polyid in stat.keys():
#Do not consider the polygon if all values are missing or 0
if np.isnan(stat[polyid]['sum']) or stat[polyid]['sum'] == 0:
continue
if not str(polyid) in coffeeWeights[varieties[r][v]]:
coffeeWeights[varieties[r][v]][str(
polyid)] = stat[polyid]['sum']
else:
coffeeWeights[varieties[r][v]][str(
polyid)] = coffeeWeights[varieties[r][v]][str(
polyid)] + stat[polyid]['sum']
#Loop through the images
for i, date in zip(onDisk, datesAll):
print(date)
#Import the image
baseImg = gdal.Open(
os.path.join(root, regions[r], regionsIn, i))
#Remove existing temporary raster if any
try:
os.remove(os.path.join(tempDir, 'temp.tif'))
except OSError:
pass
#Create an empty copy for storing the temporary weighted image
weighted = rt.newRasterFromBase(baseImg,
tempDir + '/temp.tif',
'GTiff',
np.nan,
gdal.GDT_Float32,
bands=1)
#Import image band as array
imgBand = baseImg.GetRasterBand(1)
imgArray = imgBand.ReadAsArray()
#Recast the type to be sure
imgArray = imgArray.astype(np.float32)
#Replace the no data values by 0
nodataI = imgBand.GetNoDataValue()
if not nodataI:
imgArray[np.logical_or(
np.isnan(imgArray),
np.logical_or(imgArray > 1., imgArray < -1.))] = 0.
else:
imgArray[np.logical_or(
np.logical_or(np.isnan(imgArray), imgArray == nodataI),
np.logical_or(imgArray > 1., imgArray < -1.))] = 0.
#Change the resolution of the raster to match the images if needed
geoSmooth = weightsRaster.GetGeoTransform()
geoBase = baseImg.GetGeoTransform()
reproject = [
1 for a, b in zip(geoSmooth, geoBase) if not a == b
]
if reproject:
weightsReproj = rt.warpRaster(weightsRaster,
baseImg,
resampleOption='nearest',
outputURI=None,
outFormat='MEM')
else:
weightsReproj = weightsRaster
#Import coffee weights band as array
weightsBand = weightsReproj.GetRasterBand(1)
weightsArray = weightsBand.ReadAsArray()
#Recast the type to be sure
weightsArray = weightsArray.astype(np.float32)
#Replace the no data values by 0
nodataW = weightsBand.GetNoDataValue()
if not nodataW:
weightsArray[np.logical_or(
np.isnan(weightsArray),
np.logical_or(weightsArray > 1.,
weightsArray < 0.))] = 0.
else:
weightsArray[np.logical_or(
np.logical_or(np.isnan(weightsArray),
weightsArray == nodataW),
np.logical_or(weightsArray > 1.,
weightsArray < 0.))] = 0.
#Estimate the weighted sum for each pixel
weightedArray = np.multiply(weightsArray, imgArray)
#Export to temporary raster
weighted.GetRasterBand(1).WriteArray(weightedArray)
#Close the temporary raster
weighted.FlushCache()
weighted = None
weightsReproj = None
stat = img.raster_stats(regionsFileName=os.path.join(
tempDir, 'temp_crop.shp'),
rasterFileName=tempDir + '/temp.tif',
polIdFieldName=attr,
statistics=['sum'],
outTxt=None,
addShp=False,
addSuffix='',
numOutDecs=2,
alltouch=True)
if not stat:
print('error extracting ndvi sums per polygon for date ' +
date.strftime('%Y-%m-%d'))
continue
stat = stat[0]
#Output is a list (here one element) where each element is a dictionary. Keys are the shp ids. Each then is
#also a dictionary, with each key being a summary statistics
for polyid in stat.keys():
#Do not consider the polygon if not contained in the raster bounding box or if there is no coffee
#if not polyid in insideFeat or np.isnan(stat[polyid]['sum']) or not str(polyid) in coffeeWeights[varieties[r][v]].keys():
if np.isnan(stat[polyid]['sum']) or not str(
polyid) in coffeeWeights[varieties[r][v]].keys():
continue
if not str(polyid) in imgStats[varieties[r][v]][
date.strftime('%Y-%m-%d')]:
imgStats[varieties[r][v]][date.strftime('%Y-%m-%d')][
str(polyid)] = stat[polyid]['sum']
else:
imgStats[varieties[r][v]][date.strftime('%Y-%m-%d')][
str(polyid)] = (imgStats[varieties[r][v]][
date.strftime('%Y-%m-%d')][str(polyid)] +
stat[polyid]['sum'])
#Remove the temporary file
try:
os.remove(os.path.join(tempDir, 'temp.tif'))
except OSError:
pass
#Close the mask
weightsRaster = None
#Remove the temporary shapefile
try:
driver.DeleteDataSource(os.path.join(tempDir, 'temp_crop.shp'))
except OSError:
pass
#Combine the data from the coffee weights and the sum of the ndvi
for v in ['arabica', 'robusta']:
if imgStats[v]:
for date in imgStats[v].keys():
for polyid in imgStats[v][date].keys():
if not polyid == 'date':
# Remove the region/municipality if the share of pixels with some coffee is
# less than 5% of the total pixels covered by MODIS
if not polyid in coffeeShare[v].keys(
) or coffeeShare[v][polyid] / totPixels[polyid] < 0.05:
del imgStats[v][date][polyid]
continue
imgStats[v][date][polyid] = imgStats[v][date][
polyid] / coffeeWeights[v][polyid]
#Export all the dates and polygon ids to a text file
outNm = 'Weighted_avgndvi_permicroregion_' + v + '_' + startExtract.strftime(
'%Y-%m-%d') + '_' + endExtract.strftime('%Y-%m-%d') + '.txt'
if exportFormat == 'wide':
#order the output by polygon id in a list
#Each element is a dictionary with the fields to be exported (polygon id, date and value)
transposed = {}
polyids = set()
for date in imgStats[v].keys():
polyids.update(imgStats[v][date].keys())
polyids.remove('date')
for polyid in polyids:
transposed[polyid] = {}
transposed[polyid]['id'] = polyid
for date in imgStats[v].keys():
keys = imgStats[v][date].keys()
keys.remove('date')
for polyid in keys:
transposed[polyid][date] = imgStats[v][date][polyid]
imgStats[v] = transposed
#order the output by date in a list. Each element is an element of the original dictionary and will be exported
out = []
colnames = set()
for k in imgStats[v].keys():
out.append(imgStats[v][k])
colnames.update(imgStats[v][k].keys())
colnames = list(colnames)
if 'date' in colnames:
colnames.remove('date')
colnames.sort(key=float)
colnames.insert(0, 'date')
else:
colnames.remove('id')
colnames.sort(
key=lambda d: datetime.strptime(d, '%Y-%m-%d'))
colnames.insert(0, 'id')
elif exportFormat == 'long':
out = []
for date in imgStats[v].keys():
polyids = imgStats[v][date].keys()
polyids.remove('date')
for polyid in polyids:
out.append({
'id': polyid,
'date': date,
'value': imgStats[v][date][polyid]
})
colnames = ['id', 'date', 'value']
with open(os.path.join(outFolder, outNm), "w") as f:
dict_writer = DictWriter(f,
colnames,
extrasaction='ignore',
delimiter="\t",
restval="0")
dict_writer.writeheader()
for p in out:
dict_writer.writerow(p)
#Remove the temporary shapefile
try:
driver.DeleteDataSource(os.path.join(tempDir, 'temp.shp'))
except OSError:
pass
img = gdal.Open(f)
# Get the no data value
noImg = img.GetRasterBand(1).GetNoDataValue()
if not noImg:
noImg = -99
img.GetRasterBand(1).SetNoDataValue(noImg)
# Transform into array
imgArray = img.GetRasterBand(1).ReadAsArray()
# Mask using the land data
imgArray[landArray == noLand] = noImg
out = rt.newRasterFromBase(img,
os.path.join(tempDir, os.path.basename(f)),
'GTiff', -99, gdal.GDT_Float32)
out.GetRasterBand(1).WriteArray(imgArray)
out.FlushCache()
out = None
# Write values back into file
# img.GetRasterBand(1).WriteArray(imgArray)
# img.FlushCache()
img = None
landMask = None
# Loop through the images and tabulate them