def combineClassif(orderClassif, classifFolder, legendFolder, tempFolder, outClassifName, outLegendName, legendPrefix='legend_', legendExt='.txt', legendDel='\t', toNa=[]): ''' Combine the classifications listed in the orderClassif file, with the priorities listed in the file, and using the legends in the legend folder provided. The legends should have the same name as the classifications, save for a potential prefix to be provided. The output is a unified classification, with a unified legend, where classes with the same (exact) name have been combined. All values that are not in the legends will be passed to missing. orderClassif (str): Address of the file with the classifications names and the priorities for combining them. The file should be .txt and tab delimited. It should have two columns, the first is the priority number and the second is the base name of the classif, without extension. The first row is considered as the column names. classifFolder (str): Address of the folder containing the classifications to combine. legendFolder (str): Address of the folder containing the legends of the classifications. The legend files should have the same name as the classifications, save for a potential prefix. CAPS do not matter, all category names will be passed to lower case. tempFolder (str): Address of the temporary folder where intermediate classifications can be saved during processing. outClassifName (str): Full address of the output combined classification. The function uses the value 255 as the no data value for the output. outLegendName (str): Full address of the output legend. Should be a .txt. legendPrefix (str): prefix used to differentiate the legends names from the classifications names. legendExt (str): Extension of the legend files. legendDel (str): Delimiter of the legend files. toNa (list of str): Categories in the classifications to pass to no data in the output classification. The names provided should match exactly the information in the legend files. These names should not include any trailing numbers if several categories have been made (e.g. Cloud for Cloud1, Cloud2...) ''' #Check the computer platform ordi = checkPlatform() if ordi == "Windows": import gdal_merge as gm #Import the classifications names along with the order classifNames = [] priorities = [] with open(orderClassif, 'r') as f: next(f) # skip headings reader = csv.reader(f, delimiter='\t') for priority, classif in reader: priorities.append(int(priority)) classifNames.append(classif) #Remove the raster extensions if there are any classifNames = [re.sub('.(tif|TIF)$', '', c) for c in classifNames] #Order the classifs in the correct order classifNames = [ c for (p, c) in sorted(zip(priorities, classifNames), key=lambda pair: pair[0]) ] #Import all the legends into a list of dictionaries legends = [] for nm in classifNames: dic = {} #Will hold the legend values in format Value:Classname with open(os.path.join(legendFolder, legendPrefix + nm + legendExt), 'r') as f: next(f) # skip headings reader = csv.reader(f, delimiter=legendDel) for v, c in reader: dic[int(v)] = re.sub('[0-9]*$', '', c.lower()) #Add to the list legends.append(dic) #Establish a common legend for all the classifications #Get the unique class names and remove any of the categories being mapped to NA commonLegend = {v for l in legends for v in l.values() if v not in toNa} commonLegend = list(commonLegend) #Add a raster value by alphabetical order commonLegend = {nm: i for i, nm in enumerate(sorted(commonLegend), 1)} #Loop through the classifications, remove the clouds and shadows, and change the values to the common legend for nm, legend in zip(classifNames, legends): if (os.path.isfile(os.path.join(classifFolder, nm + '.tif'))): tif = '.tif' else: tif = '.TIF' try: classifRaster = gdal.Open(os.path.join(classifFolder, nm + tif)) except RuntimeError, e: do_message( "ERROR", "PROCESS ABORTED: Error (2) opening raster file " + nm + tif + "\n" + str(e)) return False if classifRaster is None: do_message( "ERROR", "PROCESS ABORTED: Error (3) opening raster file " + nm + tif) return False #Get the raster band classifBand = classifRaster.GetRasterBand(1) #Transform the band into a numpy array band = classifBand.ReadAsArray() band = band.astype(int) #Map the old values to the new common values for v in np.unique(band): if v in legend: #Get the class name for that value c = legend[v] if c in toNa: replaceVal = 255 else: replaceVal = commonLegend[c] else: replaceVal = 255 band[band == v] = replaceVal #Create an empty raster to export the classification that was just mapped outR = rt.newRasterFromBase(classifRaster, tempFolder + '/reclass_' + nm + '.tif', 'GTiff', 255, gdal.GDT_Byte, bands=1) #Write the re-classified values to the empty raster outR.GetRasterBand(1).WriteArray(band) #Close the rasters outR.FlushCache() outR = None
def main(): ########################################################################################################################## ########################################################################################################################## ##PARAMETERS #Address of the folder with the classifications classifFolder = 'E:/gedata_current/jde_coffee/data/SDM/classifs' #TAB delimited file with the priority order in which to combine the classifications #The file should have two columns, the first is a priority number, the second the file name of the classification orderClassif = 'E:/gedata_current/jde_coffee/data/SDM/classifs/classif_priorities_SDM.txt' #Classes to pass to NA values in combining the classifications #Leave an empty list if None #These names should not include any trailing numbers if several categories have been made (e.g. Cloud1, Cloud2...) toNa = ['Cloud','Shadow'] #Additional values in the classifications that are not in the legends valuesNotInLegend = [0] #Address of the folder with the legends legendFolder = 'E:/gedata_current/jde_coffee/data/SDM/classifs/Legend' #Legend file names prefix to add to the classification names legendPrefix = 'legend_' #Legend file extensions: could be '.csv' or '.txt' legendExt = '.csv' #Legend file delimiter legendDelim = ',' #Address of the folder with the confusion matrices matrixFolder = 'E:/gedata_current/jde_coffee/data/SDM/classifs/Matconf' #Matrix file names prefix to add to the classification names matrixPrefix = 'Matconf_' #Matrix file extensions: could be '.csv' or '.txt' matrixExt = '.csv' #Legend file delimiter matrixDelim = ',' #Address of the folder for the temporary files tempFolder = 'E:/gedata_current/temp' #Address and name of the grid shapefile gridShape = 'E:/gedata_current/jde_coffee/data/SDM/grids/AOI_grid_SDM_2km.shp' #Field to use for the ID of the cells idField = 'serp_id' #Projection epsg for the output. #Leave None to use the one from the grid. Should be the same as the grid if the grid doesn't have a projection info! epsgProject = 32723 #SRTM Mask option srtmMask = False srtmRaster = 'E:/gedata_current/jde_coffee/data/SDM/srtm/srtm_v3_SDMnet.tif' breakValues = [550] #list of values used as break to separate the classifications labelsElev = ['robusta','arabica'] #Share of clouds/missing per cell under which correct by simple proportional scaling propCloud = 0.5 #Output name and address for the combined files and legend exportClassifName = 'megot_classif.tif' ########################################################################################################################## ########################################################################################################################## ##CODE #Combine the classifications without and with the clouds combine_classifications(classifFolder, orderClassif, legendFolder, legendPrefix, legendExt, legendDelim, tempFolder, 'nocloud_'+exportClassifName, toNa=toNa, valuesNotInLegend=valuesNotInLegend) combine_classifications(classifFolder, orderClassif, legendFolder, legendPrefix, legendExt, legendDelim, tempFolder, 'cloud_'+exportClassifName, toNa=[], valuesNotInLegend=valuesNotInLegend) #Merge the two combined rasters #Check the computer platform ordi = checkPlatform() if ordi == "Windows": import gdal_merge as gm #Prepare the arguments for gdal_merge.py -- They all need to be strings args = ['-n', '255', '-a_nodata', '255', '-of', 'GTiff', '-ot', 'UInt32', '-co', 'COMPRESS:LZW', '-o', classifFolder+'/'+exportClassifName] args.extend(['cloud_'+exportClassifName,'nocloud_'+exportClassifName]) #Do the merge if ordi == "Linux": args = ['gdal_merge.py','-of', "GTiff"]+args p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) p.communicate()[0] #Waits until the subprocess is finished before running the rest of the code elif ordi == "Windows": #Windows makes an error when calling gdal_merge directly, #So need to specify python for the executable and to make it run in the shell sys.argv = args gm.main() #Combine the legends to have a single legend? #Remove the intermediary files for f in [classifFolder+'/cloud_'+exportClassifName,classifFolder+'/nocloud_'+exportClassifName,legendFolder+'/legend_nocloud_'+exportClassifName]: os.remove(f) #Import the raster with the combined classifications baseClassif = gdal.Open(os.path.join(classifFolder,exportClassifName)) #Mask the combined raster with the srtm data if needed and demultiply the classification based on the elevation breaks if not srtmMask: processed = rt.newRasterFromBase(baseClassif, tempFolder+'/'+'masked0.tif', 'GTiff', -32768, gdal.GDT_Int16, bands=1) processed.GetRasterBand(1).WriteArray(baseClassif.GetRasterBand(1).readAsArray()) #Number of pieces of the classification raster to tabulate pieces = 1 baseClassif = None processed = None else: #Import the elevation raster elev = gdal.Open(srtmRaster) nodata = elev.GetRasterBand(1).GetNoDataValue() #Adapt the resolution of the smooth images to the baseline if needed elev = rt.warpRaster(elev, baseClassif, resampleOption='average', outputURI=None, outFormat='MEM') #Number of pieces of the classification raster to tabulate pieces = len(breakValues) + 1 #Complete the breakValues breakValues.insert(0, 0) breakValues.append(10000) #Get the size of the rasters to identify the limits of the blocks to loop through band = baseClassif.GetRasterBand(1) #Get the size of the raster xsize = band.XSize ysize = band.YSize #Set the block size BlockXSize = 256 BlockYSize = 256 #Get the number of blocks in x and y directions xBlocks = int(round(xsize/BlockXSize)) + 1 yBlocks = int(round(ysize/BlockYSize)) + 1 processed = [] for i in range(pieces): #Prepare empty rasters to hold the masked rasters processed.append(rt.newRasterFromBase(baseClassif, tempFolder+'/'+'masked'+str(i)+'.tif', 'GTiff', -32768, gdal.GDT_Int16, bands=1)) for xStep in range(xBlocks): for yStep in range(yBlocks): #Import the block of the classification baseBlock = rt.readRasterBlock(baseClassif, xStep*BlockXSize, yStep*BlockYSize, BlockXSize, BlockYSize) #Import the block of the elevation raster elevBlock = rt.readRasterBlock(elev, xStep*BlockXSize, yStep*BlockYSize, BlockXSize, BlockYSize) for i in range(pieces): #Mask the values based on the elevation raster if not nodata: outArray[np.logical_or(np.isnan(elevBlock),np.logical_or(elevBlock<breakValues[i],elevBlock>=breakValues[i+1]))] = -32768 else: outArray[np.logical_or(np.logical_or(np.isnan(elevBlock),elevBlock==nodata),np.logical_or(elevBlock<breakValues[i],elevBlock>=breakValues[i+1]))] = -32768 #Write to output raster processed[i].GetRasterBand(1).WriteArray(outArray, xStep*BlockXSize, yStep*BlockYSize) #Close the rasters for p in processed: processed[p] = None baseClassif = None elev = None ''' #Import the grid driver = ogr.GetDriverByName('ESRI Shapefile') dataSource = driver.Open(gridShape, 1) # 0 means read-only. 1 means writeable. # Check if shapefile is correctly loaded. if dataSource is None: do_message("ERROR","PROCESS ABORTED: Error (1) opening shapefile " + gridShape) return False # Create layer gridLayer = dataSource.GetLayer(0) ''' tabulated = [] for i in len(pieces): #Tabulate area by grid cell for each of the combined classifications stat = img.tabulate_area(regionsFileName=gridShape, rasterFileName=os.path.join(tempFolder,'masked'+str(i)+'.tif'), bands=1, polIdFieldName=idField, numPix=False, outTxt=None, prefix='cat', numOutDecs=0, alltouch=False) #Add the tabulated values to the result list #The result is a dictionary of dictionaries. First dictionary are the polygon ids, the second [value in raster: count] tabulated.append(stat[0]) #Get all the polygon ids for each elevation zone polygons = [] #Get all the categories categories = set() for piece in tabulated: polygons.append(sorted(piece.keys())) for k in piece.keys(): categories.update(k.keys()) categories = list(categories) categories.sort() #Create empty arrays and fill them with the values tabArray = [] for l in range(len(polygons)): tabArray.append(np.zeros((len(polygons),len(categories)))) #For each polygon cell and category, get the associated count for p in range(len(polygons[l])): for c in range(len(categories)): try: tabArray[l][p,c] = tabulated[l][polygons[l][p]][categories[c]] except: continue
def combine_classifications(classifFolder, orderClassif, legendFolder, legendPrefix, legendExt, legendDelim, tempFolder, exportClassifName, toNa=[], valuesNotInLegend=[]): #Check the computer platform ordi = checkPlatform() if ordi == "Windows": import gdal_merge as gm #Import the classifications names along with the order classifNames=[] priorities=[] with open(orderClassif,'r') as f: next(f) # skip headings reader=csv.reader(f,delimiter='\t') for priority,classif in reader: priorities.append(int(priority)) classifNames.append(classif) #Remove the raster extensions if there are any classifNames = [re.sub('.(tif|TIF)$','',c) for c in classifNames] #Order the classifs in the correct order classifNames = [c for (p,c) in sorted(zip(priorities,classifNames), key=lambda pair: pair[0])] #Import all the legends into a list of dictionaries legends = [] newLegends = [] for nm in classifNames: dict = {} #Will hold the legend values in format Value:Classname with open(os.path.join(legendFolder,legendPrefix+nm+legendExt),'r') as f: next(f) # skip headings reader=csv.reader(f,delimiter=legendDelim) for v,c in reader: dict[int(v)] = re.sub('[0-9]*$','',c) #Add to the list legends.append(dict) #Establish a legend combining the info for all the classifications #Get the unique class names commonLegend = {v for l in legends for v in l.values()} commonLegend = {v:[] for v in commonLegend} #Add the values corresponding to each class i = 1 for l in legends: for k,v in l.iteritems(): commonLegend[v].append(k+100*i) i += 1 #The values of the rasters for each classification will be in a different 100s. #Loop through the classifications, and change the values to the combined legend i = 1 for nm,legend in zip(classifNames,legends): if (os.path.isfile(os.path.join(classifFolder, nm+'.tif'))): tif = '.tif' else: tif = '.TIF' try: classifRaster = gdal.Open(os.path.join(classifFolder, nm+tif)) except RuntimeError, e: do_message("ERROR","PROCESS ABORTED: Error (2) opening raster file " + nm+tif + "\n" + str(e)) return False if classifRaster is None: do_message("ERROR","PROCESS ABORTED: Error (3) opening raster file " + nm+tif) return False #Get the raster band classifBand = classifRaster.GetRasterBand(1) #Get the no data value if any nodata = int(classifBand.GetNoDataValue()) #Transform the band into a numpy array band = classifBand.ReadAsArray() band = band.astype(int) #Get the values in the raster in sorted order #(Get them all otherwise throws an error) legendValues = legend.keys() palette = list(legendValues) if nodata: palette.append(nodata) if valuesNotInLegend: for v in valuesNotInLegend: palette.append(v) palette = sorted(palette) #Create the key that gives the new values the palette should be mapped to mapkey = [] for v in palette: if v in legendValues: #Get the class name for that value c = legend[v] if c in toNa: mapkey.append(255) else: mapkey.append(v+100*i) else: mapkey.append(255) i += 1 #The values of the rasters for each classification will be in a different 100s. mapkey = np.array(mapkey) #Change the values in the raster index = np.digitize(band, palette, right=True) band = mapkey[index] #Create an empty raster to export the classification that was just mapped outR = rt.newRasterFromBase(classifRaster, tempFolder+'/reclass_'+nm+'.tif', 'GTiff', 255, gdal.GDT_UInt32, bands=1) #Write the re-classified values to the empty raster outR.GetRasterBand(1).WriteArray(band) #Close the rasters outR.FlushCache() outR = None
mpsFill = p.map(pp, mps) # print('time gapfillone parallel %s' % (time.time() - t0)) # Replace the fitted value in the raster for z in mpsFill: r[z[0], z[1]] = z[4] # Prepare name for output outName = os.path.join( outFolder, re.sub('.tif', '_' + suffix + '.tif', os.path.basename(rasters[rIndex]))) if os.path.isfile(outName): os.remove(outName) # t0 = time.time() outR = rt.newRasterFromBase(dst, outName, 'GTiff', nodataOut, gdal.GDT_Float32) outR.GetRasterBand(1).WriteArray(r) outR.FlushCache() outR = None dst = None # print('time export %s' % (time.time() - t0)) # Prepare the expansion values for export. These values # provide some information on the difficulty to # interpolate the pixels outI.append(sum([z[5] for z in mpsFill]) / float(len(mpsFill))) if parallel: # Close the threads p.close()
def extractModis(root, regions, varieties, regionsIn, outFolder, masks, startExtract, endExtract, shpName, attr, tempDir, exportFormat, epsgShp=None): #Transform into date format if not endExtract: endExtract = datetime.now() endExtract = endExtract.date() else: endExtract = datetime.strptime(endExtract, '%Y-%m-%d').date() if not startExtract: startExtract = endExtract - timedelta(days=30) else: startExtract = datetime.strptime(startExtract, '%Y-%m-%d').date() #Open the shapefile driver = ogr.GetDriverByName('ESRI Shapefile') dataSource = driver.Open(shpName, 0) # 0 means read-only. 1 means writeable. # Check if shapefile is correctly loaded. if dataSource is None: print("ERROR", "PROCESS ABORTED: Error (1) opening shapefile " + shpName) return False # Get layer inVecLayer = dataSource.GetLayer(0) #Get projection if possible inEpsg = proj.getProjection(dataSource) if not inEpsg and not epsgShp: print( 'Could not identify projection for the shapefile. Specify a value for epsgShp parameter' ) return False elif not inEpsg: inEpsg = epsgShp if not isinstance(inEpsg, int): inEpsg = int(inEpsg) if not inEpsg == 4326: #Create the transform prj1 = osr.SpatialReference() prj1.ImportFromEPSG(inEpsg) prj2 = osr.SpatialReference() prj2.ImportFromEPSG(4326) coordTrans = osr.CoordinateTransformation(prj1, prj2) #create output datasource outdriver = ogr.GetDriverByName('ESRI Shapefile') srcOut = outdriver.CreateDataSource(os.path.join(tempDir, 'temp.shp')) #Create a new layer lyrOut = srcOut.CreateLayer("layer 1", srs=prj2, geom_type=inVecLayer.GetGeomType()) #Add fields lyr1Defn = inVecLayer.GetLayerDefn() for i in range(0, lyr1Defn.GetFieldCount()): fieldDefn = lyr1Defn.GetFieldDefn(i) lyrOut.CreateField(fieldDefn) # get the output layer's feature definition lyrOutDefn = lyrOut.GetLayerDefn() # loop through the input features inFeature = inVecLayer.GetNextFeature() while inFeature: # get the input geometry geom = inFeature.GetGeometryRef() # reproject the geometry geom.Transform(coordTrans) # create a new feature outFeature = ogr.Feature(lyrOutDefn) # set the geometry and attribute outFeature.SetGeometry(geom) for i in range(0, lyrOutDefn.GetFieldCount()): outFeature.SetField( lyrOutDefn.GetFieldDefn(i).GetNameRef(), inFeature.GetField(i)) # add the feature to the shapefile lyrOut.CreateFeature(outFeature) # dereference the features and get the next input feature outFeature = None inFeature = inVecLayer.GetNextFeature() #Close the two shapefiles dataSource.Destroy() inVecLayer = None srcOut.Destroy() lyrOut = None #Empty dictionary to hold the results. #Every municipality covered by one of the regions will have a coffee weight coffeeWeights = {'arabica': {}, 'robusta': {}} #Every municipality will have a total number of modis pixels totPixels = {} #Every municipality covered by one of the regions will have a share of pixels with coffee in it coffeeShare = {'arabica': {}, 'robusta': {}} #Every municipality covered by one of the regions will have a coffee count #Every date will have a dictionary containing the municipalities and values imgStats = {'arabica': {}, 'robusta': {}} #Loop through the regions to extract the modis data for each for r in range(len(regions)): print('Extracting region ' + str(regions[r]) + '...') #Import all the smooth modis images on disk onDisk = [ f for f in os.listdir(os.path.join(root, regions[r], regionsIn)) if f.endswith('.tif') ] #Dates of these files datesAll = [ re.search('_([0-9]{4}-[0-9]{2}-[0-9]{2})', f).group(1) for f in onDisk ] #Transform into date format datesAll = [datetime.strptime(d, '%Y-%m-%d').date() for d in datesAll] #Keep only the files and dates within the dates to process onDisk = [ f for f, d in zip(onDisk, datesAll) if d >= startExtract and d <= endExtract ] datesAll = [ d for d in datesAll if d >= startExtract and d <= endExtract ] #Sort the two list by date datesAll, onDisk = (list(x) for x in zip(*sorted(zip(datesAll, onDisk)))) if not onDisk: print('no modis images to process in ' + regions[r]) continue #Get a base image as a reference for format and extent baseImg = gdal.Open( os.path.join(root, regions[r], regionsIn, onDisk[0])) #Get the corner latitude and longitude for the raster cols = baseImg.RasterXSize # width rows = baseImg.RasterYSize # height gt = baseImg.GetGeoTransform() minLonR = gt[0] minLatR = gt[3] + cols * gt[4] + rows * gt[5] maxLonR = gt[0] + cols * gt[1] + rows * gt[2] maxLatR = gt[3] #Close the raster baseImg = None #Crop the shapefile by the extent of the raster inShp = tempDir + '/temp.shp' outShp = tempDir + '/temp_crop.shp' #Call ogr2ogr using the os system command cmd = 'ogr2ogr -f "ESRI Shapefile" %s %s -clipsrc %f %f %f %f -overwrite' % ( outShp, inShp, minLonR, minLatR, maxLonR, maxLatR) os.system(cmd) ###Extract the number of modis pixels per cropped municipalities #Import the mask weightsRaster = gdal.Open(root + '/' + regions[r] + '/' + masks[r][0]) #Create an empty copy for storing the temporary weighted image weighted = rt.newRasterFromBase(weightsRaster, tempDir + '/temp.tif', 'GTiff', 0., gdal.GDT_Float32, bands=1) #Import coffee weights band as array weightsBand = weightsRaster.GetRasterBand(1) weightsArray = weightsBand.ReadAsArray() #Recast the type to be sure weightsArray = weightsArray.astype(np.float32) #Replace all data by 1 weightsArray[:] = 1. #Export to temporary raster weighted.GetRasterBand(1).WriteArray(weightsArray) #Close the temporary raster weighted.FlushCache() weighted = None #Compute the count of pixels for each polygon of the cropped municipalities stat = img.raster_stats(regionsFileName=os.path.join( tempDir, 'temp_crop.shp'), rasterFileName=tempDir + '/temp.tif', polIdFieldName=attr, statistics=['sum'], outTxt=None, addShp=False, addSuffix='', numOutDecs=2, alltouch=True) if not stat: print('error extracting coffee sums per polygon') continue stat = stat[0] #Add the counts to the total pixel counts for polyid in stat.keys(): if not str(polyid) in totPixels: totPixels[str(polyid)] = stat[polyid]['sum'] else: totPixels[str( polyid)] = totPixels[str(polyid)] + stat[polyid]['sum'] #Remove existing temporary raster if any try: os.remove(os.path.join(tempDir, 'temp.tif')) except OSError: pass for v in range(len(varieties[r])): for date in datesAll: if not date.strftime('%Y-%m-%d') in imgStats[varieties[r][v]]: imgStats[varieties[r][v]][date.strftime('%Y-%m-%d')] = {} imgStats[varieties[r][v]][date.strftime( '%Y-%m-%d')]['date'] = date.strftime('%Y-%m-%d') ###Extract the number of pixels with some coffee per cropped municipalities #Import the mask weightsRaster = gdal.Open(root + '/' + regions[r] + '/' + masks[r][v]) #Create an empty copy for storing the temporary weighted image weighted = rt.newRasterFromBase(weightsRaster, tempDir + '/temp.tif', 'GTiff', 0., gdal.GDT_Float32, bands=1) #Import coffee weights band as array weightsBand = weightsRaster.GetRasterBand(1) weightsArray = weightsBand.ReadAsArray() #Recast the type to be sure weightsArray = weightsArray.astype(np.float32) #Replace the no data value by 0 and all non zero by 1 nodataW = weightsBand.GetNoDataValue() if not nodataW: weightsArray[np.logical_or( np.isnan(weightsArray), np.logical_or(weightsArray > 1., weightsArray < 0.))] = 0 else: weightsArray[np.logical_or( np.logical_or(np.isnan(weightsArray), weightsArray == nodataW), np.logical_or(weightsArray > 1., weightsArray < 0.))] = 0 nodataW = None #Replace all positive coffee values by 1 weightsArray[weightsArray > 0.] = 1 #Export to temporary raster weighted.GetRasterBand(1).WriteArray(weightsArray) #Close the temporary raster weighted.FlushCache() weighted = None ''' if regions[r] == 'ES' and varieties[r][v] == 'robusta': try: os.remove(os.path.join(tempDir,'tempES.tif')) except OSError: pass weighted = rt.newRasterFromBase(weightsRaster, tempDir+'/tempES.tif', 'GTiff', 0., gdal.GDT_Float32, bands=1) #Export to temporary raster weighted.GetRasterBand(1).WriteArray(weightsArray) #Close the temporary raster weighted.FlushCache() weighted = None ''' #Compute the count of pixels for each polygon of the cropped municipalities stat = img.raster_stats(regionsFileName=os.path.join( tempDir, 'temp_crop.shp'), rasterFileName=tempDir + '/temp.tif', polIdFieldName=attr, statistics=['sum'], outTxt=None, addShp=False, addSuffix='', numOutDecs=2, alltouch=True) if not stat: print('error extracting coffee sums per polygon') continue stat = stat[0] #Add the sums to the share of coffee for polyid in stat.keys(): #Do not consider the polygon if all values are missing or 0 if np.isnan(stat[polyid]['sum']) or stat[polyid]['sum'] == 0: continue if not str(polyid) in coffeeWeights[varieties[r][v]]: coffeeShare[varieties[r][v]][str( polyid)] = stat[polyid]['sum'] else: coffeeShare[varieties[r][v]][str(polyid)] = coffeeShare[ varieties[r][v]][str(polyid)] + stat[polyid]['sum'] #Remove existing temporary raster if any try: os.remove(os.path.join(tempDir, 'temp.tif')) except OSError: pass weightsRaster = None ###Extract the sum of coffee weights per cropped municipality #Import the mask weightsRaster = gdal.Open(root + '/' + regions[r] + '/' + masks[r][v]) #Create an empty copy for storing the temporary weighted image weighted = rt.newRasterFromBase(weightsRaster, tempDir + '/temp.tif', 'GTiff', 0, gdal.GDT_Float32, bands=1) #Import coffee weights band as array weightsBand = weightsRaster.GetRasterBand(1) weightsArray = weightsBand.ReadAsArray() #Recast the type to be sure weightsArray = weightsArray.astype(np.float32) #Replace the no data values by 0 nodataW = weightsBand.GetNoDataValue() if not nodataW: weightsArray[np.logical_or( np.isnan(weightsArray), np.logical_or(weightsArray > 1., weightsArray < 0.))] = 0. else: weightsArray[np.logical_or( np.logical_or(np.isnan(weightsArray), weightsArray == nodataW), np.logical_or(weightsArray > 1., weightsArray < 0.))] = 0. nodataW = None #Export to temporary raster weighted.GetRasterBand(1).WriteArray(weightsArray) #Close the temporary raster weighted.FlushCache() weighted = None #Compute the sum for each polygon of the coffee stat = img.raster_stats(regionsFileName=os.path.join( tempDir, 'temp_crop.shp'), rasterFileName=tempDir + '/temp.tif', polIdFieldName=attr, statistics=['sum'], outTxt=None, addShp=False, addSuffix='', numOutDecs=2, alltouch=True) if not stat: print('error extracting coffee sums per polygon') continue stat = stat[0] #Add the weights to the coffee weights dictionary for polyid in stat.keys(): #Do not consider the polygon if all values are missing or 0 if np.isnan(stat[polyid]['sum']) or stat[polyid]['sum'] == 0: continue if not str(polyid) in coffeeWeights[varieties[r][v]]: coffeeWeights[varieties[r][v]][str( polyid)] = stat[polyid]['sum'] else: coffeeWeights[varieties[r][v]][str( polyid)] = coffeeWeights[varieties[r][v]][str( polyid)] + stat[polyid]['sum'] #Loop through the images for i, date in zip(onDisk, datesAll): print(date) #Import the image baseImg = gdal.Open( os.path.join(root, regions[r], regionsIn, i)) #Remove existing temporary raster if any try: os.remove(os.path.join(tempDir, 'temp.tif')) except OSError: pass #Create an empty copy for storing the temporary weighted image weighted = rt.newRasterFromBase(baseImg, tempDir + '/temp.tif', 'GTiff', np.nan, gdal.GDT_Float32, bands=1) #Import image band as array imgBand = baseImg.GetRasterBand(1) imgArray = imgBand.ReadAsArray() #Recast the type to be sure imgArray = imgArray.astype(np.float32) #Replace the no data values by 0 nodataI = imgBand.GetNoDataValue() if not nodataI: imgArray[np.logical_or( np.isnan(imgArray), np.logical_or(imgArray > 1., imgArray < -1.))] = 0. else: imgArray[np.logical_or( np.logical_or(np.isnan(imgArray), imgArray == nodataI), np.logical_or(imgArray > 1., imgArray < -1.))] = 0. #Change the resolution of the raster to match the images if needed geoSmooth = weightsRaster.GetGeoTransform() geoBase = baseImg.GetGeoTransform() reproject = [ 1 for a, b in zip(geoSmooth, geoBase) if not a == b ] if reproject: weightsReproj = rt.warpRaster(weightsRaster, baseImg, resampleOption='nearest', outputURI=None, outFormat='MEM') else: weightsReproj = weightsRaster #Import coffee weights band as array weightsBand = weightsReproj.GetRasterBand(1) weightsArray = weightsBand.ReadAsArray() #Recast the type to be sure weightsArray = weightsArray.astype(np.float32) #Replace the no data values by 0 nodataW = weightsBand.GetNoDataValue() if not nodataW: weightsArray[np.logical_or( np.isnan(weightsArray), np.logical_or(weightsArray > 1., weightsArray < 0.))] = 0. else: weightsArray[np.logical_or( np.logical_or(np.isnan(weightsArray), weightsArray == nodataW), np.logical_or(weightsArray > 1., weightsArray < 0.))] = 0. #Estimate the weighted sum for each pixel weightedArray = np.multiply(weightsArray, imgArray) #Export to temporary raster weighted.GetRasterBand(1).WriteArray(weightedArray) #Close the temporary raster weighted.FlushCache() weighted = None weightsReproj = None stat = img.raster_stats(regionsFileName=os.path.join( tempDir, 'temp_crop.shp'), rasterFileName=tempDir + '/temp.tif', polIdFieldName=attr, statistics=['sum'], outTxt=None, addShp=False, addSuffix='', numOutDecs=2, alltouch=True) if not stat: print('error extracting ndvi sums per polygon for date ' + date.strftime('%Y-%m-%d')) continue stat = stat[0] #Output is a list (here one element) where each element is a dictionary. Keys are the shp ids. Each then is #also a dictionary, with each key being a summary statistics for polyid in stat.keys(): #Do not consider the polygon if not contained in the raster bounding box or if there is no coffee #if not polyid in insideFeat or np.isnan(stat[polyid]['sum']) or not str(polyid) in coffeeWeights[varieties[r][v]].keys(): if np.isnan(stat[polyid]['sum']) or not str( polyid) in coffeeWeights[varieties[r][v]].keys(): continue if not str(polyid) in imgStats[varieties[r][v]][ date.strftime('%Y-%m-%d')]: imgStats[varieties[r][v]][date.strftime('%Y-%m-%d')][ str(polyid)] = stat[polyid]['sum'] else: imgStats[varieties[r][v]][date.strftime('%Y-%m-%d')][ str(polyid)] = (imgStats[varieties[r][v]][ date.strftime('%Y-%m-%d')][str(polyid)] + stat[polyid]['sum']) #Remove the temporary file try: os.remove(os.path.join(tempDir, 'temp.tif')) except OSError: pass #Close the mask weightsRaster = None #Remove the temporary shapefile try: driver.DeleteDataSource(os.path.join(tempDir, 'temp_crop.shp')) except OSError: pass #Combine the data from the coffee weights and the sum of the ndvi for v in ['arabica', 'robusta']: if imgStats[v]: for date in imgStats[v].keys(): for polyid in imgStats[v][date].keys(): if not polyid == 'date': # Remove the region/municipality if the share of pixels with some coffee is # less than 5% of the total pixels covered by MODIS if not polyid in coffeeShare[v].keys( ) or coffeeShare[v][polyid] / totPixels[polyid] < 0.05: del imgStats[v][date][polyid] continue imgStats[v][date][polyid] = imgStats[v][date][ polyid] / coffeeWeights[v][polyid] #Export all the dates and polygon ids to a text file outNm = 'Weighted_avgndvi_permicroregion_' + v + '_' + startExtract.strftime( '%Y-%m-%d') + '_' + endExtract.strftime('%Y-%m-%d') + '.txt' if exportFormat == 'wide': #order the output by polygon id in a list #Each element is a dictionary with the fields to be exported (polygon id, date and value) transposed = {} polyids = set() for date in imgStats[v].keys(): polyids.update(imgStats[v][date].keys()) polyids.remove('date') for polyid in polyids: transposed[polyid] = {} transposed[polyid]['id'] = polyid for date in imgStats[v].keys(): keys = imgStats[v][date].keys() keys.remove('date') for polyid in keys: transposed[polyid][date] = imgStats[v][date][polyid] imgStats[v] = transposed #order the output by date in a list. Each element is an element of the original dictionary and will be exported out = [] colnames = set() for k in imgStats[v].keys(): out.append(imgStats[v][k]) colnames.update(imgStats[v][k].keys()) colnames = list(colnames) if 'date' in colnames: colnames.remove('date') colnames.sort(key=float) colnames.insert(0, 'date') else: colnames.remove('id') colnames.sort( key=lambda d: datetime.strptime(d, '%Y-%m-%d')) colnames.insert(0, 'id') elif exportFormat == 'long': out = [] for date in imgStats[v].keys(): polyids = imgStats[v][date].keys() polyids.remove('date') for polyid in polyids: out.append({ 'id': polyid, 'date': date, 'value': imgStats[v][date][polyid] }) colnames = ['id', 'date', 'value'] with open(os.path.join(outFolder, outNm), "w") as f: dict_writer = DictWriter(f, colnames, extrasaction='ignore', delimiter="\t", restval="0") dict_writer.writeheader() for p in out: dict_writer.writerow(p) #Remove the temporary shapefile try: driver.DeleteDataSource(os.path.join(tempDir, 'temp.shp')) except OSError: pass
img = gdal.Open(f) # Get the no data value noImg = img.GetRasterBand(1).GetNoDataValue() if not noImg: noImg = -99 img.GetRasterBand(1).SetNoDataValue(noImg) # Transform into array imgArray = img.GetRasterBand(1).ReadAsArray() # Mask using the land data imgArray[landArray == noLand] = noImg out = rt.newRasterFromBase(img, os.path.join(tempDir, os.path.basename(f)), 'GTiff', -99, gdal.GDT_Float32) out.GetRasterBand(1).WriteArray(imgArray) out.FlushCache() out = None # Write values back into file # img.GetRasterBand(1).WriteArray(imgArray) # img.FlushCache() img = None landMask = None # Loop through the images and tabulate them