#os.system ("gdal_translate -of GTIFF -projwin 33.8 5.2 42.3 -5 -projwin_srs EPSG:4326 GHS_BUILT_LDSMT_GLOBE_R2015B_3857_38_v1_0.vrt Kenya.tif")
sFileGHSL4326 = sFileGHSL.split(".")[0]+"4326.tif"
os.system ("gdalwarp -t_srs EPSG:4326 "+sFileGHSL+" "+sFileGHSL4326)
match_geotrans, match_proj = rasterRegrid(sFileGHSL4326, sFileFinalGrid, sFileGHSL_regrid ,"nearest")
[xsize, ysize, geotransform, geoproj, data_GHSL] = readFile(sFileGHSL_regrid)
data_GHSL [data_GHSL <3 ] =0
data_GHSL [data_GHSL >6 ] =0
data_GHSL [data_GHSL ==3] =1
data_GHSL [data_GHSL ==4] =1
data_GHSL [data_GHSL ==5] =1
data_GHSL [data_GHSL ==6] =1
#Save GHSL [0,1]
data_GHSL = data_GHSL * data_grid
writeGeotiffSingleBand(sFileGHSL_regrid, geotransform, geoproj, data_GHSL)
data_GUF_GHSL = (data_GUF + data_GHSL)
data_GUF_GHSL [data_GUF_GHSL >= 1] = 1
data_GUF_GHSL [data_GUF_GHSL < 1] = 0
#Save merge GUF + GHSL
data_GUF_GHSL = data_GUF_GHSL * data_grid
writeGeotiffSingleBand(sFileGHSL_GUF_regrid, geotransform, geoproj, data_GUF_GHSL)
del data_GUF, data_GHSL
#regrid world pop to 90m
match_geotrans, match_proj = rasterRegrid(sFileWorldPop, sFileFinalGrid, sFileWorldPop_regrid ,"nearest")
#read world pop to 90m
[xsize, ysize, geotransform, geoproj, data_WorldPop] = readFile(sFileWorldPop_regrid)
__author__ = 'lauro'
import os
from Geotiff import writeGeotiffSingleBand, readFile
dir1 = '/share/africa/Global_datasets/Builtup/AFR_buiA_GUF_100m_cuts_20180312'
dir2 = '/share/africa/Global_datasets/Builtup/WRL_buiA_GHSL_100m_nearest_20180312'
#origineDir = '/Users/lauro/Downloads'
dirOut = '/share/africa/Global_datasets/Builtup/AFR_buiA_merged_100m_20180312'
for f in os.listdir(dir1):
if f.endswith('.tif'):
#print (f)
filename, ext = os.path.splitext(f)
#print filename
#if filename == "UR_Tanzania_tanga":
[xsize, ysize, geotransform, geoproj,
data1] = readFile(os.path.join(dir1, f))
[xsize, ysize, geotransform, geoproj,
data2] = readFile(os.path.join(dir2, filename + "_100m_nearest.tif"))
#print(filename+"_100m_nearest.tif")
sFileOut = os.path.join(dirOut, filename + '_GHSL_GUF_merged.tif')
print("writing... " + sFileOut)
writeGeotiffSingleBand(sFileOut, geotransform, geoproj, data1 + data2)
def downscale_crop(sFileMapSpamLowResTot, sFileMask, sFileClipper):
sCropType = get_crop(sFileMapSpamLowResTot)
sCountry = get_country(sFileMask)
#making dir out
dir_out = os.path.join(os.path.dirname(sFileMapSpamLowResTot),
sCountry + "_outputs")
if not os.path.exists(dir_out):
os.mkdir(dir_out)
sFileMapSpamLowRes = os.path.join(
dir_out,
os.path.basename(
sFileMapSpamLowResTot.split('.')[0] + "_" + sCountry + ".tif"))
warp = '''gdalwarp -co "COMPRESS=DEFLATE" -dstnodata {noDataValue} -crop_to_cutline -overwrite -cutline {clipper} "{infile}" "{outfile}"'''.format(
noDataValue=-9999,
clipper=sFileClipper,
infile=sFileMapSpamLowResTot,
outfile=sFileMapSpamLowRes)
print(warp)
os.system(warp)
[xsize_orig, ysize_orig, geotransform, geoproj,
data_low] = readFile_withNoData(sFileMapSpamLowRes)
log_print("Original MAPSPAM map size: (" + str(ysize_orig) + "," +
str(xsize_orig) + ") (rows,cols)")
#TODO verify
total_exposure = np.nansum(data_low)
#making unique values
data_low = np.reshape(non_unique(data_low.ravel()), data_low.shape)
writeGeotiffSingleBand(sFileMapSpamLowRes, geotransform, geoproj, data_low)
if bDisplayImages: plot_image(data_low)
log_print("Total original production value for crop type=" + sCropType +
": %.2f" % np.nansum(data_low))
del data_low
sFileMapSpamHiRes = os.path.join(
dir_out,
sFileMapSpamLowRes.split('.')[0] + "_regrid.tif")
match_geotrans, match_proj = rasterRegrid(sFileMapSpamLowRes, sFileMask,
sFileMapSpamHiRes, "nearest")
[xsize, ysize, geotransform_high, geoproj_high,
data_high] = readFile_withNoData(sFileMapSpamHiRes)
[xsize, ysize, geotransform_high, geoproj_high,
data_mask] = readFile_withNoData(sFileMask)
data_high[np.isnan(data_high)] = 0
data_mask[np.isnan(data_mask)] = 0
data_high_masked = data_high * data_mask
if bDisplayImages:
plot_image(data_mask)
plot_image(data_high)
plot_image(data_high_masked)
data_high_masked[np.isnan(data_high_masked)] = 0
#debug
#plot_image(data_high_masked)
log_print("Starting counting...")
dictCounter = Counter(data_high_masked.ravel())
log_print("Data counted")
data_mask_spam = np.copy(data_high)
ratio = abs(
(geotransform[1] * geotransform[5]) /
(geotransform_high[1] *
geotransform_high[5])) #number of high res pixels in low res pixel
i = 0
data_gar_excluded = []
#non posso leggere data low perche cambia nel salvarli
data_high_unique = np.unique(data_high.ravel())
for key in data_high_unique:
if key not in dictCounter.keys() and key != 0:
print(key)
dictCounter[key] = ratio
data_mask_spam[data_mask_spam == key] = -9999
data_gar_excluded.append(key)
excluded_exposure = sum(data_gar_excluded)
log_print("MAPSPAM production value not overlapping mask: %.2f (%.1f %%)" %
(excluded_exposure, excluded_exposure / total_exposure * 100))
#building mask (union of the original mask + non-zero values from MapSpam)
data_mask_spam[data_mask_spam != -9999] = 0
data_mask_spam[data_mask_spam == -9999] = 1
data_mask = data_mask_spam + data_mask
data_mask[data_mask > 0] = 1
data_high_masked = data_high * data_mask
del data_mask, data_high, data_mask_spam
if bVerbose: log_print("Counter length: " + str(dictCounter.__len__()))
if bVerbose:
log_print("Unique length: " + str(len(np.unique(data_high_masked))))
for key, value in dictCounter.items():
data_high_masked[data_high_masked == key] = key / value
if bVerbose:
print('Amount of pixel for crop' + str(key) + ': ' + str(value))
if bDisplayImages:
fig = plt.figure()
cax = plt.imshow(data_high_masked[:, :])
plt.colormaps()
cbar = fig.colorbar(
cax, orientation='vertical') # vertically oriented colorbar
#cbar.ax.set_yticklabels(['< -1', '0', 1, 2,'> 10'])
plt.show()
sFileDownscaled = os.path.join(dir_out,
sCountry + "_" + sCropType + "_HighRes.tif")
description = 'Production [tons] of ' + sCropType + ' and price [USD] per tons'
bandMetadata = {}
bandMetadata[('unit', '1')] = 'tons'
iNbands = 1
writeFile(sFileDownscaled, match_geotrans, match_proj, data_high_masked,
bandMetadata, description, {'Crop type': sCropType}, iNbands)
log_print("Total downscaled production value for crop type=" + sCropType +
": %.2f" % np.nansum(data_high_masked))
log_print("Exposure downscaled. Final result save in file: " +
sFileDownscaled)
os.mkdir(dir_out)
# regridded outputs
sFileOriginalRaster_regrid = join(dir_out, "hrsl_bw_pop_sum_90m_regridded.tif")
sFileOriginalRaster_regrid_m = join(dir_out,
"hrsl_bw_buiA_mask_90m_regridded.tif")
#########################################
#########################################
# START
# regrid and read GRID
[xsize, ysize, geotransform, geoproj,
data_grid] = readFile_withNoData(sFileFinalGrid)
# regrid and read OriginalRaster
match_geotrans, match_proj = rasterRegrid(sFileOriginalRaster, sFileFinalGrid,
sFileOriginalRaster_regrid,
"nearest")
[xsize, ysize, geotransform, geoproj,
data_OriginalRaster_regrid] = readFile_withNoData(sFileOriginalRaster_regrid)
writeGeotiffSingleBand(sFileOriginalRaster_regrid, geotransform, geoproj,
data_OriginalRaster_regrid)
data_OriginalRaster_regrid[data_OriginalRaster_regrid > 0] = 1
writeGeotiffSingleBand(sFileOriginalRaster_regrid_m, geotransform, geoproj,
data_OriginalRaster_regrid)
sFileMask = "E:/africa_downscaling/Rwanda/hrsl_rwa.tif"
sFileFinalGrid = "E:/africa_downscaling/Rwanda/rw_box.tif"
# output directory
dir_out="E:/africa_downscaling/rw_2/"
if not os.path.exists (dir_out):
os.mkdir(dir_out)
# regridded outputs
sFileWorldPop_regrid = join(dir_out, "rw_mask_facebook.tif")
#########################################
#########################################
#########################################
# START
[xsize, ysize, geotransform, geoproj, data_mask] = readFile_withNoData_band2(sFileMask)
writeGeotiffSingleBand(sFileMask+'_2.tif', geotransform, geoproj, data_mask)
[xsize, ysize, geotransform, geoproj, data_grid] = readFile_withNoData(sFileFinalGrid)
#regrid world pop to 90m
match_geotrans, match_proj = rasterRegrid(sFileMask+'_2.tif', sFileFinalGrid, sFileWorldPop_regrid ,"nearest")
def downscale_pop(sFileGARPoint, sFileMask, sExposureType):
#sBuildingType = "UFB"
#sBuildingType = "UCB"
bDisplayImages = False
#sFileGARPoint = '/Users/lauro/Desktop/EU_ACP_UNISDR_Africa/DATA/UR_Tanzania/AFR_buiA_GAR_5km_20180313/africa_tza.shp'
#sFileMask = '/Users/lauro/Desktop/EU_ACP_UNISDR_Africa/DATA/UR_Tanzania/AFR_buiA_GUF_100m_nations_20180315/UR_Tanzania.tif'
#sFileGARPoint = '/Users/lauro/Desktop/EU_ACP_UNISDR_Africa/DATA/Swaziland/AFR_buiA_GAR_5km_20180313/africa_swz.shp'
#sFileMask = '/Users/lauro/Desktop/EU_ACP_UNISDR_Africa/DATA/Swaziland/AFR_buiA_GUF_100m_nations_20180315/Swaziland.tif'
#os.system('ogr2ogr -overwrite -f "ESRI Shapefile" -where "curve=\''+sBuildingType+'\'" ' + sFileGARPointCurveType + ' ' + sFileGARPoint)
inDriver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = inDriver.Open(sFileGARPoint, 0)
inLayer = inDataSource.GetLayer()
inLayerDefn = inLayer.GetLayerDefn()
log_print("Reading GAR shape file...")
log_print("Total features: " + str(inLayer.GetFeatureCount()))
#inLayer.SetAttributeFilter("curve = \''+sBuildingType+'\'")
(fXmin, fXmax, fYmin, fYmax) = inLayer.GetExtent()
iXLowRes = 0.04167
iYLowRes = 0.04167
fXmin = fXmin - iXLowRes / 2
fYmin = fYmin - iYLowRes / 2
fXmax = fXmax + iXLowRes / 2
fYmax = fYmax + iYLowRes / 2
sFileGARPointBase = os.path.basename(sFileGARPoint)
dir_out = os.path.join(os.path.dirname(sFileGARPoint),
sFileGARPointBase.split('.')[0] + "_" + "outputs")
if not os.path.exists(dir_out):
os.mkdir(dir_out)
sFileGARPointPop = os.path.join(
dir_out,
sFileGARPointBase.split('.')[0] + "_" + sExposureType + ".shp")
outDriver = ogr.GetDriverByName("ESRI Shapefile")
outDataSource = outDriver.CreateDataSource(sFileGARPointPop)
proj = inLayer.GetSpatialRef()
##Creating the layer with its fields
outLayer = outDataSource.CreateLayer(sFileGARPointPop, proj, ogr.wkbPoint)
field_exposure = ogr.FieldDefn(sExposureType, ogr.OFTReal)
outLayer.CreateField(field_exposure)
pop = []
geoms = []
for feature in inLayer:
if feature.GetGeometryRef().GetPoint() not in geoms:
geoms.append(feature.GetGeometryRef().GetPoint())
pop.append(feature.GetField(sExposureType))
else:
id = geoms.index(feature.GetGeometryRef().GetPoint())
pop[id] = pop[id] + feature.GetField(sExposureType)
for i in range(len(geoms)):
outFeature = ogr.Feature(outLayer.GetLayerDefn())
outFeature.SetField(sExposureType, pop[i])
point = ogr.Geometry(ogr.wkbPoint)
point.AddPoint(*geoms[i])
outFeature.SetGeometry(point)
outLayer.CreateFeature(outFeature)
total_exposure = sum(pop)
log_print("Total unique features in outLayer: " +
str(outLayer.GetFeatureCount()))
log_print("Total original exposure value (for curve type=" + sCurveType +
", exposure type=" + sExposureType + "): %.2f" % total_exposure)
inLayer.ResetReading()
inDataSource = None
outDataSource = None
# for i in range (0, len(geoms)):
log_print("Converting points to raster...")
sFileGARRasterEXPLowRes = os.path.join(
dir_out,
sFileGARPointBase.split('.')[0] + "_" + sExposureType + ".tif")
if bVerbose:
print('gdal_rasterize -co compress=DEFLATE -a ' + sExposureType +
' -l ' + os.path.basename(sFileGARPointPop).split('.')[0] +
' -tr ' + str(iXLowRes) + ' ' + str(iYLowRes) + ' -te ' +
str(fXmin) + ' ' + str(fYmin) + ' ' + str(fXmax) + ' ' +
str(fYmax) + ' ' + sFileGARPointPop + ' ' +
sFileGARRasterEXPLowRes)
os.system('gdal_rasterize -co compress=DEFLATE -a ' + sExposureType +
' -l ' + os.path.basename(sFileGARPointPop).split('.')[0] +
' -tr ' + str(iXLowRes) + ' ' + str(iYLowRes) + ' -te ' +
str(fXmin) + ' ' + str(fYmin) + ' ' + str(fXmax) + ' ' +
str(fYmax) + ' ' + sFileGARPointPop + ' ' +
sFileGARRasterEXPLowRes)
# if bVerbose: print('gdal_rasterize -co compress=DEFLATE -a '+sExposureType+' -where "curve=\''+sBuildingType+'\'" -l ' + os.path.basename(sFileGARPoint).split('.')[0] + ' -tr ' + str(iXLowRes) + ' ' + str(iYLowRes) + ' -te ' + str(fXmin) + ' ' + str(fYmin) + ' ' + str(fXmax) + ' ' + str(fYmax) + ' ' + sFileGARPoint + ' ' +sFileGARRasterEXPLowRes)
# os.system('gdal_rasterize -co compress=DEFLATE -a '+sExposureType+' -where "curve=\''+sBuildingType+'\'" -l ' + os.path.basename(sFileGARPoint).split('.')[0] + ' -tr ' + str(iXLowRes) + ' ' + str(iYLowRes) + ' -te ' + str(fXmin) + ' ' + str(fYmin) + ' ' + str(fXmax) + ' ' + str(fYmax) + ' ' + sFileGARPoint + ' ' +sFileGARRasterEXPLowRes)
[xsize_orig, ysize_orig, geotransform, geoproj,
data_low] = readFile(sFileGARRasterEXPLowRes)
log_print("Original GAR map size: (" + str(ysize_orig) + "," +
str(xsize_orig) + ") (rows,cols)")
uniq = non_unique(data_low.ravel().tolist())
data_low = np.reshape(uniq, data_low.shape)
writeGeotiffSingleBand(sFileGARRasterEXPLowRes,
geotransform,
geoproj,
data_low,
nan_value=NAN_VALUE)
if bDisplayImages: plot_image(data_low)
# DEBUG
#b = np.unique(data_low.ravel()).tolist()
del data_low
sFileGARRasterEXPHiRes = os.path.join(
dir_out,
sFileGARPointBase.split('.')[0] + "_" + sCurveType + "_" +
sExposureType + "_regrid.tif")
match_geotrans, match_proj = rasterRegrid(sFileGARRasterEXPLowRes,
sFileMask,
sFileGARRasterEXPHiRes,
"nearest")
[xsize, ysize, geotransform_high, geoproj,
data_high] = readFile(sFileGARRasterEXPHiRes)
[xsize, ysize, geotransform_high, geoproj,
data_mask] = readFile(sFileMask, fix_nan_value=255)
## DEBUG
#c = np.unique(data_high.ravel()).tolist()
#print (len(b),len(c))
data_high_masked = data_high * data_mask
if bDisplayImages:
plot_image(data_mask)
plot_image(data_high)
plot_image(data_high_masked)
log_print("Starting counting...")
dictCounter = Counter(data_high_masked.ravel().tolist())
log_print("Data counted")
data_mask_gar = np.copy(data_high)
ratio = abs(
(geotransform[1] * geotransform[5]) /
(geotransform_high[1] *
geotransform_high[5])) #number of high res pixels in low res pixel
data_gar_excluded = []
for key in data_high.ravel().tolist(
): # for name, age in list.items(): (for Python 3.x)
if key not in dictCounter.keys() and key != 0:
dictCounter[key] = ratio
data_mask_gar[data_mask_gar == key] = -9999
data_gar_excluded.append(key)
excluded_exposure = sum(data_gar_excluded)
log_print("GAR exposure value not overlapping mask: %.2f (%.1f %%)" %
(excluded_exposure, excluded_exposure / total_exposure * 100))
del data_gar_excluded
#building mask (union of the original mask + non-zero values form GAR)
data_mask_gar[data_mask_gar != -9999] = 0
data_mask_gar[data_mask_gar == -9999] = 1
data_mask = data_mask_gar + data_mask
data_mask[data_mask > 0] = 1
data_high_masked = data_high * data_mask
del data_mask, data_high, data_mask_gar
if bVerbose: log_print("Counter length: " + str(dictCounter.__len__()))
if bVerbose:
log_print("Unique length: " + str(len(np.unique(data_high_masked))))
for key, value in dictCounter.items():
data_high_masked[data_high_masked == key] = key / value
if bVerbose:
log_print('Amount of pixel for population ' + str(key) + ': ' +
str(value))
if bDisplayImages:
fig = plt.figure()
cax = plt.imshow(data_high_masked[:, :])
plt.colormaps()
cbar = fig.colorbar(
cax, orientation='vertical') # vertically oriented colorbar
#cbar.ax.set_yticklabels(['< -1', '0', 1, 2,'> 10'])
plt.show()
sFileDownscaled = os.path.join(
dir_out,
sFileGARPointBase.split('.')[0] + "_" + sCurveType + "_" +
sExposureType + "_HighRes.tif")
bandMetadata = {}
bandMetadata[('unit', '1')] = 'people'
#bandMetadata[('unit', '2')] = 'USD/tons'
if sExposureType == "VALHUM":
description = 'VALHUM in ' + sCurveType + ' (downscaling of GAR2015 data)'
elif sExposureType == "VALFIS":
description = 'VALFIS in ' + sCurveType + ' (downscaling of GAR2015 data)'
iNbands = 1
writeFile(sFileDownscaled,
match_geotrans,
match_proj,
data_high_masked,
bandMetadata,
description, {'Building_type': sCurveType},
iNbands,
nan_value=NAN_VALUE)
log_print("Total downscaled exposure value (for curve type=" + sCurveType +
", exposure type=" + sExposureType +
"): %.2f" % np.sum(data_high_masked))
log_print("Exposure downscaled. Final result save in file: " +
sFileDownscaled)