def plot_image(data):
fig = plt.figure()
cax = plt.imshow(data, cmap='Blues')
plt.colormaps()
#plt.clim(0,400)
cbar = fig.colorbar(cax, orientation='vertical')
#cbar.ax.set_yticklabels(['< -1', '0', 1, 2,'> 10'])# vertically oriented colorbar
plt.show()
def simple_plot(a):
fig, ax = plt.subplots()
im = plt.imshow(a, interpolation='none')
plt.colormaps()
#cbar = fig.colorbar(cax,ticks=[-1, 0, 1, 2, 10],orientation='vertical')
cbar = fig.colorbar(im, orientation='vertical')
#plt.clim(0,1)
#cbar.ax.set_yticklabels(['< -1', '0', 1, 2,'> 10'])# vertically oriented colorbar
#ax.format_coord = Formatter(im)
plt.show()
def get_cmap(cmap_name):
"""Return matplotlib colormap object.
From matplotlib.cm or cmocean.
Additional custom colormap for salinity is provided:
- "custom_salinity1"
"""
cm.register_cmap(name='cubehelix3',
data=mpl._cm.cubehelix(gamma=1.0, s=2.0, r=1.0, h=3))
if cmap_name in cmo.cmapnames:
colormap = cmo.cmap_d[cmap_name]
elif cmap_name in plt.colormaps():
colormap = plt.get_cmap(cmap_name)
elif cmap_name == "custom_salinity1":
colormap = shiftedcolormap(cm.get_cmap("cubehelix3"),
start=0,
midpoint=0.89,
stop=0.9,
name='shiftedcmap')
else:
raise ValueError('Get unrecognised name for the colormap `{}`.\
Colormaps should be from standard matplotlib \
set or from cmocean package.'.format(cmap_name))
return colormap
def get_cmap(cmap=None):
"""Get the color map.
Parameters
----------
cmap: str, mpl.colors.Colormap
The colormap can be provided as the name (should be in matplotlib or cmocean colormaps),
or as matplotlib colormap object.
Returns
-------
colormap:mpl.colors.Colormap
Matplotlib colormap object.
"""
if cmap:
if isinstance(cmap, (mpl.colors.Colormap)):
colormap = cmap
elif cmap in cmo.cmapnames:
colormap = cmo.cmap_d[cmap]
elif cmap in plt.colormaps():
colormap = plt.get_cmap(cmap)
else:
raise ValueError(
"Get unrecognised name for the colormap `{}`. Colormaps should be from standard matplotlib set of from cmocean package."
.format(cmap))
else:
colormap = plt.get_cmap("Spectral_r")
return colormap
def _colormaps(self, environ, start_response, path):
return response.respond_ok(
[c for c in colormaps() if not c.endswith('_r')], start_response)
def downscale_crop(sFileMapSpamLowResTot, sFileMask, sFileClipper):
sCropType = get_crop(sFileMapSpamLowResTot)
sCountry = get_country(sFileMask)
#making dir out
dir_out = os.path.join(os.path.dirname(sFileMapSpamLowResTot),
sCountry + "_outputs")
if not os.path.exists(dir_out):
os.mkdir(dir_out)
sFileMapSpamLowRes = os.path.join(
dir_out,
os.path.basename(
sFileMapSpamLowResTot.split('.')[0] + "_" + sCountry + ".tif"))
warp = '''gdalwarp -co "COMPRESS=DEFLATE" -dstnodata {noDataValue} -crop_to_cutline -overwrite -cutline {clipper} "{infile}" "{outfile}"'''.format(
noDataValue=-9999,
clipper=sFileClipper,
infile=sFileMapSpamLowResTot,
outfile=sFileMapSpamLowRes)
print(warp)
os.system(warp)
[xsize_orig, ysize_orig, geotransform, geoproj,
data_low] = readFile_withNoData(sFileMapSpamLowRes)
log_print("Original MAPSPAM map size: (" + str(ysize_orig) + "," +
str(xsize_orig) + ") (rows,cols)")
#TODO verify
total_exposure = np.nansum(data_low)
#making unique values
data_low = np.reshape(non_unique(data_low.ravel()), data_low.shape)
writeGeotiffSingleBand(sFileMapSpamLowRes, geotransform, geoproj, data_low)
if bDisplayImages: plot_image(data_low)
log_print("Total original production value for crop type=" + sCropType +
": %.2f" % np.nansum(data_low))
del data_low
sFileMapSpamHiRes = os.path.join(
dir_out,
sFileMapSpamLowRes.split('.')[0] + "_regrid.tif")
match_geotrans, match_proj = rasterRegrid(sFileMapSpamLowRes, sFileMask,
sFileMapSpamHiRes, "nearest")
[xsize, ysize, geotransform_high, geoproj_high,
data_high] = readFile_withNoData(sFileMapSpamHiRes)
[xsize, ysize, geotransform_high, geoproj_high,
data_mask] = readFile_withNoData(sFileMask)
data_high[np.isnan(data_high)] = 0
data_mask[np.isnan(data_mask)] = 0
data_high_masked = data_high * data_mask
if bDisplayImages:
plot_image(data_mask)
plot_image(data_high)
plot_image(data_high_masked)
data_high_masked[np.isnan(data_high_masked)] = 0
#debug
#plot_image(data_high_masked)
log_print("Starting counting...")
dictCounter = Counter(data_high_masked.ravel())
log_print("Data counted")
data_mask_spam = np.copy(data_high)
ratio = abs(
(geotransform[1] * geotransform[5]) /
(geotransform_high[1] *
geotransform_high[5])) #number of high res pixels in low res pixel
i = 0
data_gar_excluded = []
#non posso leggere data low perche cambia nel salvarli
data_high_unique = np.unique(data_high.ravel())
for key in data_high_unique:
if key not in dictCounter.keys() and key != 0:
print(key)
dictCounter[key] = ratio
data_mask_spam[data_mask_spam == key] = -9999
data_gar_excluded.append(key)
excluded_exposure = sum(data_gar_excluded)
log_print("MAPSPAM production value not overlapping mask: %.2f (%.1f %%)" %
(excluded_exposure, excluded_exposure / total_exposure * 100))
#building mask (union of the original mask + non-zero values from MapSpam)
data_mask_spam[data_mask_spam != -9999] = 0
data_mask_spam[data_mask_spam == -9999] = 1
data_mask = data_mask_spam + data_mask
data_mask[data_mask > 0] = 1
data_high_masked = data_high * data_mask
del data_mask, data_high, data_mask_spam
if bVerbose: log_print("Counter length: " + str(dictCounter.__len__()))
if bVerbose:
log_print("Unique length: " + str(len(np.unique(data_high_masked))))
for key, value in dictCounter.items():
data_high_masked[data_high_masked == key] = key / value
if bVerbose:
print('Amount of pixel for crop' + str(key) + ': ' + str(value))
if bDisplayImages:
fig = plt.figure()
cax = plt.imshow(data_high_masked[:, :])
plt.colormaps()
cbar = fig.colorbar(
cax, orientation='vertical') # vertically oriented colorbar
#cbar.ax.set_yticklabels(['< -1', '0', 1, 2,'> 10'])
plt.show()
sFileDownscaled = os.path.join(dir_out,
sCountry + "_" + sCropType + "_HighRes.tif")
description = 'Production [tons] of ' + sCropType + ' and price [USD] per tons'
bandMetadata = {}
bandMetadata[('unit', '1')] = 'tons'
iNbands = 1
writeFile(sFileDownscaled, match_geotrans, match_proj, data_high_masked,
bandMetadata, description, {'Crop type': sCropType}, iNbands)
log_print("Total downscaled production value for crop type=" + sCropType +
": %.2f" % np.nansum(data_high_masked))
log_print("Exposure downscaled. Final result save in file: " +
sFileDownscaled)
vect_embedded = tsne.fit_transform(dist_mat)
#%%
plt.figure(figsize=[10, 10])
plt.scatter(vect_embedded[:, 0], vect_embedded[:, 1], 25, "blue", alpha=0.6)
plt.xlabel("tSNE1")
plt.ylabel("tSNE2")
plt.axis("equal")
plt.savefig(join(homedir, "tSNE_DimenRed_ClassEmbed.png"))
plt.show()
#%%
plt.figure(figsize=[10, 10])
plt.pcolor(dist_mat)
plt.xlabel("ImageNet Class #")
plt.ylabel("ImageNet Class #")
plt.title("Distance Matrix between the 128d embed vectors")
plt.colormaps()
plt.colorbar()
plt.axis("image")
plt.savefig(join(homedir, "DistanceMatrix.png"))
#%%
plt.figure()
plt.hist(dist_mat[dist_mat != 0], bins=50)
plt.title("Distribution of Distance Matrix of BigGAN-Deep256")
plt.xlabel("L2 Distance")
plt.savefig(join(homedir, "EmbedVectDistanceDist.png"))
plt.show()
#%%
#%%
from nltk.corpus import wordnet as wn
import pickle, urllib
ImageNet_Classname = pickle.load(
def _colormaps(self,environ,start_response,path):
return response.respond_ok([c for c in colormaps() if not c.endswith('_r')],start_response)
def downscale_pop(sFileGARPoint, sFileMask, sExposureType):
#sBuildingType = "UFB"
#sBuildingType = "UCB"
bDisplayImages = False
#sFileGARPoint = '/Users/lauro/Desktop/EU_ACP_UNISDR_Africa/DATA/UR_Tanzania/AFR_buiA_GAR_5km_20180313/africa_tza.shp'
#sFileMask = '/Users/lauro/Desktop/EU_ACP_UNISDR_Africa/DATA/UR_Tanzania/AFR_buiA_GUF_100m_nations_20180315/UR_Tanzania.tif'
#sFileGARPoint = '/Users/lauro/Desktop/EU_ACP_UNISDR_Africa/DATA/Swaziland/AFR_buiA_GAR_5km_20180313/africa_swz.shp'
#sFileMask = '/Users/lauro/Desktop/EU_ACP_UNISDR_Africa/DATA/Swaziland/AFR_buiA_GUF_100m_nations_20180315/Swaziland.tif'
#os.system('ogr2ogr -overwrite -f "ESRI Shapefile" -where "curve=\''+sBuildingType+'\'" ' + sFileGARPointCurveType + ' ' + sFileGARPoint)
inDriver = ogr.GetDriverByName("ESRI Shapefile")
inDataSource = inDriver.Open(sFileGARPoint, 0)
inLayer = inDataSource.GetLayer()
inLayerDefn = inLayer.GetLayerDefn()
log_print("Reading GAR shape file...")
log_print("Total features: " + str(inLayer.GetFeatureCount()))
#inLayer.SetAttributeFilter("curve = \''+sBuildingType+'\'")
(fXmin, fXmax, fYmin, fYmax) = inLayer.GetExtent()
iXLowRes = 0.04167
iYLowRes = 0.04167
fXmin = fXmin - iXLowRes / 2
fYmin = fYmin - iYLowRes / 2
fXmax = fXmax + iXLowRes / 2
fYmax = fYmax + iYLowRes / 2
sFileGARPointBase = os.path.basename(sFileGARPoint)
dir_out = os.path.join(os.path.dirname(sFileGARPoint),
sFileGARPointBase.split('.')[0] + "_" + "outputs")
if not os.path.exists(dir_out):
os.mkdir(dir_out)
sFileGARPointPop = os.path.join(
dir_out,
sFileGARPointBase.split('.')[0] + "_" + sExposureType + ".shp")
outDriver = ogr.GetDriverByName("ESRI Shapefile")
outDataSource = outDriver.CreateDataSource(sFileGARPointPop)
proj = inLayer.GetSpatialRef()
##Creating the layer with its fields
outLayer = outDataSource.CreateLayer(sFileGARPointPop, proj, ogr.wkbPoint)
field_exposure = ogr.FieldDefn(sExposureType, ogr.OFTReal)
outLayer.CreateField(field_exposure)
pop = []
geoms = []
for feature in inLayer:
if feature.GetGeometryRef().GetPoint() not in geoms:
geoms.append(feature.GetGeometryRef().GetPoint())
pop.append(feature.GetField(sExposureType))
else:
id = geoms.index(feature.GetGeometryRef().GetPoint())
pop[id] = pop[id] + feature.GetField(sExposureType)
for i in range(len(geoms)):
outFeature = ogr.Feature(outLayer.GetLayerDefn())
outFeature.SetField(sExposureType, pop[i])
point = ogr.Geometry(ogr.wkbPoint)
point.AddPoint(*geoms[i])
outFeature.SetGeometry(point)
outLayer.CreateFeature(outFeature)
total_exposure = sum(pop)
log_print("Total unique features in outLayer: " +
str(outLayer.GetFeatureCount()))
log_print("Total original exposure value (for curve type=" + sCurveType +
", exposure type=" + sExposureType + "): %.2f" % total_exposure)
inLayer.ResetReading()
inDataSource = None
outDataSource = None
# for i in range (0, len(geoms)):
log_print("Converting points to raster...")
sFileGARRasterEXPLowRes = os.path.join(
dir_out,
sFileGARPointBase.split('.')[0] + "_" + sExposureType + ".tif")
if bVerbose:
print('gdal_rasterize -co compress=DEFLATE -a ' + sExposureType +
' -l ' + os.path.basename(sFileGARPointPop).split('.')[0] +
' -tr ' + str(iXLowRes) + ' ' + str(iYLowRes) + ' -te ' +
str(fXmin) + ' ' + str(fYmin) + ' ' + str(fXmax) + ' ' +
str(fYmax) + ' ' + sFileGARPointPop + ' ' +
sFileGARRasterEXPLowRes)
os.system('gdal_rasterize -co compress=DEFLATE -a ' + sExposureType +
' -l ' + os.path.basename(sFileGARPointPop).split('.')[0] +
' -tr ' + str(iXLowRes) + ' ' + str(iYLowRes) + ' -te ' +
str(fXmin) + ' ' + str(fYmin) + ' ' + str(fXmax) + ' ' +
str(fYmax) + ' ' + sFileGARPointPop + ' ' +
sFileGARRasterEXPLowRes)
# if bVerbose: print('gdal_rasterize -co compress=DEFLATE -a '+sExposureType+' -where "curve=\''+sBuildingType+'\'" -l ' + os.path.basename(sFileGARPoint).split('.')[0] + ' -tr ' + str(iXLowRes) + ' ' + str(iYLowRes) + ' -te ' + str(fXmin) + ' ' + str(fYmin) + ' ' + str(fXmax) + ' ' + str(fYmax) + ' ' + sFileGARPoint + ' ' +sFileGARRasterEXPLowRes)
# os.system('gdal_rasterize -co compress=DEFLATE -a '+sExposureType+' -where "curve=\''+sBuildingType+'\'" -l ' + os.path.basename(sFileGARPoint).split('.')[0] + ' -tr ' + str(iXLowRes) + ' ' + str(iYLowRes) + ' -te ' + str(fXmin) + ' ' + str(fYmin) + ' ' + str(fXmax) + ' ' + str(fYmax) + ' ' + sFileGARPoint + ' ' +sFileGARRasterEXPLowRes)
[xsize_orig, ysize_orig, geotransform, geoproj,
data_low] = readFile(sFileGARRasterEXPLowRes)
log_print("Original GAR map size: (" + str(ysize_orig) + "," +
str(xsize_orig) + ") (rows,cols)")
uniq = non_unique(data_low.ravel().tolist())
data_low = np.reshape(uniq, data_low.shape)
writeGeotiffSingleBand(sFileGARRasterEXPLowRes,
geotransform,
geoproj,
data_low,
nan_value=NAN_VALUE)
if bDisplayImages: plot_image(data_low)
# DEBUG
#b = np.unique(data_low.ravel()).tolist()
del data_low
sFileGARRasterEXPHiRes = os.path.join(
dir_out,
sFileGARPointBase.split('.')[0] + "_" + sCurveType + "_" +
sExposureType + "_regrid.tif")
match_geotrans, match_proj = rasterRegrid(sFileGARRasterEXPLowRes,
sFileMask,
sFileGARRasterEXPHiRes,
"nearest")
[xsize, ysize, geotransform_high, geoproj,
data_high] = readFile(sFileGARRasterEXPHiRes)
[xsize, ysize, geotransform_high, geoproj,
data_mask] = readFile(sFileMask, fix_nan_value=255)
## DEBUG
#c = np.unique(data_high.ravel()).tolist()
#print (len(b),len(c))
data_high_masked = data_high * data_mask
if bDisplayImages:
plot_image(data_mask)
plot_image(data_high)
plot_image(data_high_masked)
log_print("Starting counting...")
dictCounter = Counter(data_high_masked.ravel().tolist())
log_print("Data counted")
data_mask_gar = np.copy(data_high)
ratio = abs(
(geotransform[1] * geotransform[5]) /
(geotransform_high[1] *
geotransform_high[5])) #number of high res pixels in low res pixel
data_gar_excluded = []
for key in data_high.ravel().tolist(
): # for name, age in list.items(): (for Python 3.x)
if key not in dictCounter.keys() and key != 0:
dictCounter[key] = ratio
data_mask_gar[data_mask_gar == key] = -9999
data_gar_excluded.append(key)
excluded_exposure = sum(data_gar_excluded)
log_print("GAR exposure value not overlapping mask: %.2f (%.1f %%)" %
(excluded_exposure, excluded_exposure / total_exposure * 100))
del data_gar_excluded
#building mask (union of the original mask + non-zero values form GAR)
data_mask_gar[data_mask_gar != -9999] = 0
data_mask_gar[data_mask_gar == -9999] = 1
data_mask = data_mask_gar + data_mask
data_mask[data_mask > 0] = 1
data_high_masked = data_high * data_mask
del data_mask, data_high, data_mask_gar
if bVerbose: log_print("Counter length: " + str(dictCounter.__len__()))
if bVerbose:
log_print("Unique length: " + str(len(np.unique(data_high_masked))))
for key, value in dictCounter.items():
data_high_masked[data_high_masked == key] = key / value
if bVerbose:
log_print('Amount of pixel for population ' + str(key) + ': ' +
str(value))
if bDisplayImages:
fig = plt.figure()
cax = plt.imshow(data_high_masked[:, :])
plt.colormaps()
cbar = fig.colorbar(
cax, orientation='vertical') # vertically oriented colorbar
#cbar.ax.set_yticklabels(['< -1', '0', 1, 2,'> 10'])
plt.show()
sFileDownscaled = os.path.join(
dir_out,
sFileGARPointBase.split('.')[0] + "_" + sCurveType + "_" +
sExposureType + "_HighRes.tif")
bandMetadata = {}
bandMetadata[('unit', '1')] = 'people'
#bandMetadata[('unit', '2')] = 'USD/tons'
if sExposureType == "VALHUM":
description = 'VALHUM in ' + sCurveType + ' (downscaling of GAR2015 data)'
elif sExposureType == "VALFIS":
description = 'VALFIS in ' + sCurveType + ' (downscaling of GAR2015 data)'
iNbands = 1
writeFile(sFileDownscaled,
match_geotrans,
match_proj,
data_high_masked,
bandMetadata,
description, {'Building_type': sCurveType},
iNbands,
nan_value=NAN_VALUE)
log_print("Total downscaled exposure value (for curve type=" + sCurveType +
", exposure type=" + sExposureType +
"): %.2f" % np.sum(data_high_masked))
log_print("Exposure downscaled. Final result save in file: " +
sFileDownscaled)