def single_value_to_population(value, file_mask):
"""
read mask file, calculate population count and rescale population/pixel value according to 'value'
writes population_{value}_highres.tiff file
"""
dir_out = os.path.join(os.path.dirname(file_mask), "outputs")
os.makedirs(dir_out, exist_ok=True)
xsize, ysize, geotransform, geoproj, mask_data = readFile(file_mask)
pop_count = np.sum(mask_data)
scaling_coeff = value / pop_count
new_data = mask_data * scaling_coeff
file_downscaled = os.path.join(dir_out,
"population_" + str(value) + "_HighRes.tif")
description = 'Downscaled value'
bandMetadata = {}
bandMetadata[('unit', '1')] = 'people'
writeFile(file_downscaled,
geotransform,
geoproj,
new_data,
bandMetadata,
description, {},
1,
nan_value=NAN_VALUE)
def downscale_crop(sFileMapSpamLowResTot, sFileMask, sFileClipper):
sCropType = get_crop(sFileMapSpamLowResTot)
sCountry = get_country(sFileMask)
#making dir out
dir_out = os.path.join(os.path.dirname(sFileMapSpamLowResTot),
sCountry + "_outputs")
if not os.path.exists(dir_out):
os.mkdir(dir_out)
sFileMapSpamLowRes = os.path.join(
dir_out,
os.path.basename(
sFileMapSpamLowResTot.split('.')[0] + "_" + sCountry + ".tif"))
warp = '''gdalwarp -co "COMPRESS=DEFLATE" -dstnodata {noDataValue} -crop_to_cutline -overwrite -cutline {clipper} "{infile}" "{outfile}"'''.format(
noDataValue=-9999,
clipper=sFileClipper,
infile=sFileMapSpamLowResTot,
outfile=sFileMapSpamLowRes)
print(warp)
os.system(warp)
[xsize_orig, ysize_orig, geotransform, geoproj,
data_low] = readFile_withNoData(sFileMapSpamLowRes)
log_print("Original MAPSPAM map size: (" + str(ysize_orig) + "," +
str(xsize_orig) + ") (rows,cols)")
#TODO verify
total_exposure = np.nansum(data_low)
#making unique values
data_low = np.reshape(non_unique(data_low.ravel()), data_low.shape)
writeGeotiffSingleBand(sFileMapSpamLowRes, geotransform, geoproj, data_low)
if bDisplayImages: plot_image(data_low)
log_print("Total original production value for crop type=" + sCropType +
": %.2f" % np.nansum(data_low))
del data_low
sFileMapSpamHiRes = os.path.join(
dir_out,
sFileMapSpamLowRes.split('.')[0] + "_regrid.tif")
match_geotrans, match_proj = rasterRegrid(sFileMapSpamLowRes, sFileMask,
sFileMapSpamHiRes, "nearest")
[xsize, ysize, geotransform_high, geoproj_high,
data_high] = readFile_withNoData(sFileMapSpamHiRes)
[xsize, ysize, geotransform_high, geoproj_high,
data_mask] = readFile_withNoData(sFileMask)
data_high[np.isnan(data_high)] = 0
data_mask[np.isnan(data_mask)] = 0
data_high_masked = data_high * data_mask
if bDisplayImages:
plot_image(data_mask)
plot_image(data_high)
plot_image(data_high_masked)
data_high_masked[np.isnan(data_high_masked)] = 0
#debug
#plot_image(data_high_masked)
log_print("Starting counting...")
dictCounter = Counter(data_high_masked.ravel())
log_print("Data counted")
data_mask_spam = np.copy(data_high)
ratio = abs(
(geotransform[1] * geotransform[5]) /
(geotransform_high[1] *
geotransform_high[5])) #number of high res pixels in low res pixel
i = 0
data_gar_excluded = []
#non posso leggere data low perche cambia nel salvarli
data_high_unique = np.unique(data_high.ravel())
for key in data_high_unique:
if key not in dictCounter.keys() and key != 0:
print(key)
dictCounter[key] = ratio
data_mask_spam[data_mask_spam == key] = -9999
data_gar_excluded.append(key)
excluded_exposure = sum(data_gar_excluded)
log_print("MAPSPAM production value not overlapping mask: %.2f (%.1f %%)" %
(excluded_exposure, excluded_exposure / total_exposure * 100))
#building mask (union of the original mask + non-zero values from MapSpam)
data_mask_spam[data_mask_spam != -9999] = 0
data_mask_spam[data_mask_spam == -9999] = 1
data_mask = data_mask_spam + data_mask
data_mask[data_mask > 0] = 1
data_high_masked = data_high * data_mask
del data_mask, data_high, data_mask_spam
if bVerbose: log_print("Counter length: " + str(dictCounter.__len__()))
if bVerbose:
log_print("Unique length: " + str(len(np.unique(data_high_masked))))
for key, value in dictCounter.items():
data_high_masked[data_high_masked == key] = key / value
if bVerbose:
print('Amount of pixel for crop' + str(key) + ': ' + str(value))
if bDisplayImages:
fig = plt.figure()
cax = plt.imshow(data_high_masked[:, :])
plt.colormaps()
cbar = fig.colorbar(
cax, orientation='vertical') # vertically oriented colorbar
#cbar.ax.set_yticklabels(['< -1', '0', 1, 2,'> 10'])
plt.show()
sFileDownscaled = os.path.join(dir_out,
sCountry + "_" + sCropType + "_HighRes.tif")
description = 'Production [tons] of ' + sCropType + ' and price [USD] per tons'
bandMetadata = {}
bandMetadata[('unit', '1')] = 'tons'
iNbands = 1
writeFile(sFileDownscaled, match_geotrans, match_proj, data_high_masked,
bandMetadata, description, {'Crop type': sCropType}, iNbands)
log_print("Total downscaled production value for crop type=" + sCropType +
": %.2f" % np.nansum(data_high_masked))
log_print("Exposure downscaled. Final result save in file: " +
sFileDownscaled)
def downscale_pop(sFileGARPoint, sFileMask, sExposureType):
#sBuildingType = "UFB"
#sBuildingType = "UCB"
bDisplayImages = False
#sFileGARPoint = '/Users/lauro/Desktop/EU_ACP_UNISDR_Africa/DATA/UR_Tanzania/AFR_buiA_GAR_5km_20180313/africa_tza.shp'
#sFileMask = '/Users/lauro/Desktop/EU_ACP_UNISDR_Africa/DATA/UR_Tanzania/AFR_buiA_GUF_100m_nations_20180315/UR_Tanzania.tif'
#sFileGARPoint = '/Users/lauro/Desktop/EU_ACP_UNISDR_Africa/DATA/Swaziland/AFR_buiA_GAR_5km_20180313/africa_swz.shp'
#sFileMask = '/Users/lauro/Desktop/EU_ACP_UNISDR_Africa/DATA/Swaziland/AFR_buiA_GUF_100m_nations_20180315/Swaziland.tif'
#os.system('ogr2ogr -overwrite -f "ESRI Shapefile" -where "curve=\''+sBuildingType+'\'" ' + sFileGARPointCurveType + ' ' + sFileGARPoint)
inDriver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = inDriver.Open(sFileGARPoint, 0)
inLayer = inDataSource.GetLayer()
inLayerDefn = inLayer.GetLayerDefn()
log_print("Reading GAR shape file...")
log_print("Total features: " + str(inLayer.GetFeatureCount()))
#inLayer.SetAttributeFilter("curve = \''+sBuildingType+'\'")
(fXmin, fXmax, fYmin, fYmax) = inLayer.GetExtent()
iXLowRes = 0.04167
iYLowRes = 0.04167
fXmin = fXmin - iXLowRes / 2
fYmin = fYmin - iYLowRes / 2
fXmax = fXmax + iXLowRes / 2
fYmax = fYmax + iYLowRes / 2
sFileGARPointBase = os.path.basename(sFileGARPoint)
dir_out = os.path.join(os.path.dirname(sFileGARPoint),
sFileGARPointBase.split('.')[0] + "_" + "outputs")
if not os.path.exists(dir_out):
os.mkdir(dir_out)
sFileGARPointPop = os.path.join(
dir_out,
sFileGARPointBase.split('.')[0] + "_" + sExposureType + ".shp")
outDriver = ogr.GetDriverByName("ESRI Shapefile")
outDataSource = outDriver.CreateDataSource(sFileGARPointPop)
proj = inLayer.GetSpatialRef()
##Creating the layer with its fields
outLayer = outDataSource.CreateLayer(sFileGARPointPop, proj, ogr.wkbPoint)
field_exposure = ogr.FieldDefn(sExposureType, ogr.OFTReal)
outLayer.CreateField(field_exposure)
pop = []
geoms = []
for feature in inLayer:
if feature.GetGeometryRef().GetPoint() not in geoms:
geoms.append(feature.GetGeometryRef().GetPoint())
pop.append(feature.GetField(sExposureType))
else:
id = geoms.index(feature.GetGeometryRef().GetPoint())
pop[id] = pop[id] + feature.GetField(sExposureType)
for i in range(len(geoms)):
outFeature = ogr.Feature(outLayer.GetLayerDefn())
outFeature.SetField(sExposureType, pop[i])
point = ogr.Geometry(ogr.wkbPoint)
point.AddPoint(*geoms[i])
outFeature.SetGeometry(point)
outLayer.CreateFeature(outFeature)
total_exposure = sum(pop)
log_print("Total unique features in outLayer: " +
str(outLayer.GetFeatureCount()))
log_print("Total original exposure value (for curve type=" + sCurveType +
", exposure type=" + sExposureType + "): %.2f" % total_exposure)
inLayer.ResetReading()
inDataSource = None
outDataSource = None
# for i in range (0, len(geoms)):
log_print("Converting points to raster...")
sFileGARRasterEXPLowRes = os.path.join(
dir_out,
sFileGARPointBase.split('.')[0] + "_" + sExposureType + ".tif")
if bVerbose:
print('gdal_rasterize -co compress=DEFLATE -a ' + sExposureType +
' -l ' + os.path.basename(sFileGARPointPop).split('.')[0] +
' -tr ' + str(iXLowRes) + ' ' + str(iYLowRes) + ' -te ' +
str(fXmin) + ' ' + str(fYmin) + ' ' + str(fXmax) + ' ' +
str(fYmax) + ' ' + sFileGARPointPop + ' ' +
sFileGARRasterEXPLowRes)
os.system('gdal_rasterize -co compress=DEFLATE -a ' + sExposureType +
' -l ' + os.path.basename(sFileGARPointPop).split('.')[0] +
' -tr ' + str(iXLowRes) + ' ' + str(iYLowRes) + ' -te ' +
str(fXmin) + ' ' + str(fYmin) + ' ' + str(fXmax) + ' ' +
str(fYmax) + ' ' + sFileGARPointPop + ' ' +
sFileGARRasterEXPLowRes)
# if bVerbose: print('gdal_rasterize -co compress=DEFLATE -a '+sExposureType+' -where "curve=\''+sBuildingType+'\'" -l ' + os.path.basename(sFileGARPoint).split('.')[0] + ' -tr ' + str(iXLowRes) + ' ' + str(iYLowRes) + ' -te ' + str(fXmin) + ' ' + str(fYmin) + ' ' + str(fXmax) + ' ' + str(fYmax) + ' ' + sFileGARPoint + ' ' +sFileGARRasterEXPLowRes)
# os.system('gdal_rasterize -co compress=DEFLATE -a '+sExposureType+' -where "curve=\''+sBuildingType+'\'" -l ' + os.path.basename(sFileGARPoint).split('.')[0] + ' -tr ' + str(iXLowRes) + ' ' + str(iYLowRes) + ' -te ' + str(fXmin) + ' ' + str(fYmin) + ' ' + str(fXmax) + ' ' + str(fYmax) + ' ' + sFileGARPoint + ' ' +sFileGARRasterEXPLowRes)
[xsize_orig, ysize_orig, geotransform, geoproj,
data_low] = readFile(sFileGARRasterEXPLowRes)
log_print("Original GAR map size: (" + str(ysize_orig) + "," +
str(xsize_orig) + ") (rows,cols)")
uniq = non_unique(data_low.ravel().tolist())
data_low = np.reshape(uniq, data_low.shape)
writeGeotiffSingleBand(sFileGARRasterEXPLowRes,
geotransform,
geoproj,
data_low,
nan_value=NAN_VALUE)
if bDisplayImages: plot_image(data_low)
# DEBUG
#b = np.unique(data_low.ravel()).tolist()
del data_low
sFileGARRasterEXPHiRes = os.path.join(
dir_out,
sFileGARPointBase.split('.')[0] + "_" + sCurveType + "_" +
sExposureType + "_regrid.tif")
match_geotrans, match_proj = rasterRegrid(sFileGARRasterEXPLowRes,
sFileMask,
sFileGARRasterEXPHiRes,
"nearest")
[xsize, ysize, geotransform_high, geoproj,
data_high] = readFile(sFileGARRasterEXPHiRes)
[xsize, ysize, geotransform_high, geoproj,
data_mask] = readFile(sFileMask, fix_nan_value=255)
## DEBUG
#c = np.unique(data_high.ravel()).tolist()
#print (len(b),len(c))
data_high_masked = data_high * data_mask
if bDisplayImages:
plot_image(data_mask)
plot_image(data_high)
plot_image(data_high_masked)
log_print("Starting counting...")
dictCounter = Counter(data_high_masked.ravel().tolist())
log_print("Data counted")
data_mask_gar = np.copy(data_high)
ratio = abs(
(geotransform[1] * geotransform[5]) /
(geotransform_high[1] *
geotransform_high[5])) #number of high res pixels in low res pixel
data_gar_excluded = []
for key in data_high.ravel().tolist(
): # for name, age in list.items(): (for Python 3.x)
if key not in dictCounter.keys() and key != 0:
dictCounter[key] = ratio
data_mask_gar[data_mask_gar == key] = -9999
data_gar_excluded.append(key)
excluded_exposure = sum(data_gar_excluded)
log_print("GAR exposure value not overlapping mask: %.2f (%.1f %%)" %
(excluded_exposure, excluded_exposure / total_exposure * 100))
del data_gar_excluded
#building mask (union of the original mask + non-zero values form GAR)
data_mask_gar[data_mask_gar != -9999] = 0
data_mask_gar[data_mask_gar == -9999] = 1
data_mask = data_mask_gar + data_mask
data_mask[data_mask > 0] = 1
data_high_masked = data_high * data_mask
del data_mask, data_high, data_mask_gar
if bVerbose: log_print("Counter length: " + str(dictCounter.__len__()))
if bVerbose:
log_print("Unique length: " + str(len(np.unique(data_high_masked))))
for key, value in dictCounter.items():
data_high_masked[data_high_masked == key] = key / value
if bVerbose:
log_print('Amount of pixel for population ' + str(key) + ': ' +
str(value))
if bDisplayImages:
fig = plt.figure()
cax = plt.imshow(data_high_masked[:, :])
plt.colormaps()
cbar = fig.colorbar(
cax, orientation='vertical') # vertically oriented colorbar
#cbar.ax.set_yticklabels(['< -1', '0', 1, 2,'> 10'])
plt.show()
sFileDownscaled = os.path.join(
dir_out,
sFileGARPointBase.split('.')[0] + "_" + sCurveType + "_" +
sExposureType + "_HighRes.tif")
bandMetadata = {}
bandMetadata[('unit', '1')] = 'people'
#bandMetadata[('unit', '2')] = 'USD/tons'
if sExposureType == "VALHUM":
description = 'VALHUM in ' + sCurveType + ' (downscaling of GAR2015 data)'
elif sExposureType == "VALFIS":
description = 'VALFIS in ' + sCurveType + ' (downscaling of GAR2015 data)'
iNbands = 1
writeFile(sFileDownscaled,
match_geotrans,
match_proj,
data_high_masked,
bandMetadata,
description, {'Building_type': sCurveType},
iNbands,
nan_value=NAN_VALUE)
log_print("Total downscaled exposure value (for curve type=" + sCurveType +
", exposure type=" + sExposureType +
"): %.2f" % np.sum(data_high_masked))
log_print("Exposure downscaled. Final result save in file: " +
sFileDownscaled)
sFileClipper = arg
elif opt in ("-f", "--filter"):
sAttributeName = arg
elif opt in ("-g", "--group"):
sGroup = arg
elif opt in ("-l", "--level"):
sLevel = arg
else:
print('\nPlease provide an input files...')
displayHelp()
sys.exit(2)
listFilesOut = []
listFilesOut = clip_africa(sFileIn, sFileClipper, sAttributeName,
sFileOutCore, sGroup, sLevel)
#print (a2sFileOut)
for f in listFilesOut:
print("masking: " + f)
[xsize, ysize, geotransform, geoproj, data] = readFile(f)
data = np.array(data)
data[data < 255] = 0 #data[data < 3] = 0 ### in case of GHSL
data[data == 255] = 1 #data[data >= 3] = 1 ### in case of GHSL
bandMetadata = {}
bandMetadata[('built-up', '1')] = '1'
description = 'Built up layer form GHSL'
#writeFile(os.path.splitext(f)[0]+"_masked.tif", geotransform, geoproj, data, bandMetadata, description, "built-up", 1)
writeFile(f, geotransform, geoproj, data, bandMetadata, description,
"built-up", 1)
print(str(datetime.datetime.now()) + ' - Process ended.\n')