def test_plotlibs():
import matplotlib
matplotlib.use('Agg') # use this if no xserver is available
from matplotlib import pyplot as plt
from matplotlib.colors import Normalize
from cartopy import config as cartopy_config
from cartopy.util import add_cyclic_point
import cartopy.crs as ccrs
from cartopy.io.shapereader import Reader
from cartopy.feature import ShapelyFeature
from PyPDF2 import PdfFileWriter, PdfFileReader
from reportlab.pdfgen import canvas
from flyingpigeon import config
DIR_SHP = config.shapefiles_dir()
def plot_polygons(regions):
"""
extract the polygon coordinate and plot it on a worldmap
:param regions: list of ISO abreviations for polygons
:return png: map_graphic.png
"""
from cartopy.io.shapereader import Reader
from cartopy.feature import ShapelyFeature
from os.path import curdir, abspath
from flyingpigeon import config
DIR_SHP = config.shapefiles_dir()
if type(regions) == str:
regions = list([regions])
fname = join(DIR_SHP, "countries.shp")
geos = Reader(fname).geometries()
records = Reader(fname).records()
LOGGER.debug('')
fig = plt.figure(figsize=(10, 10), facecolor='w',
edgecolor='k') # dpi=600,
projection = ccrs.Orthographic(central_longitude=0.0,
central_latitude=0.0,
globe=None) # Robinson()
ax = plt.axes(projection=projection)
for r in records:
geo = geos.next()
if r.attributes['ISO_A3'] in regions:
shape_feature = ShapelyFeature(geo,
ccrs.PlateCarree(),
edgecolor='black')
ax.add_feature(shape_feature)
ax.coastlines()
# ax.set_global()
o1, map_graphic = mkstemp(dir=abspath(curdir), suffix='.png')
fig.savefig(map_graphic)
plt.close()
return map_graphic
from cdo import Cdo
from tempfile import mkstemp
import os
from flyingpigeon.utils import drs_filename, get_variable, calc_grouping , sort_by_filename
from flyingpigeon import config
import logging
logger = logging.getLogger(__name__)
DIR_MASKS = config.masks_dir()
DIR_SHP = config.shapefiles_dir()
def countries():
"""
:return: a list of all countries codes.
"""
countries = _COUNTRIES_.keys()
countries.sort()
return countries
def countries_longname():
"""
:return: the longname of all countries.
"""
longname = ''
for country in countries():
longname = longname + "%s : %s \n" % (country, _COUNTRIES_[country]['longname'])
return longname
def masking(resource, mask, prefix=None, dir_output=None):
from os.path import join, abspath, dirname, getsize, curdir
from netCDF4 import Dataset
from flyingpigeon import config
import logging
LOGGER = logging.getLogger("PYWPS")
DIR_SHP = config.shapefiles_dir()
def has_Lambert_Conformal(resource):
"""
Check if grid is organised as Lambert_Conformal
:param resource: file to be checked
:return Boolean: True/False
"""
if type(resource) != list:
resource = [resource]
for nc in resource:
ds = Dataset(nc)
if 'Lambert_Conformal' not in ds.variables.keys():
return False
return True
def call(resource=[],
variable=None,
dimension_map=None,
calc=None,
calc_grouping=None,
def get_data(variable,
polygon=None,
dir_output=None,
start = 1950,
end = 2014
):
import ocgis
from os import rename, path, makedirs
from flyingpigeon import utils
from flyingpigeon import subset as sb
from flyingpigeon import clipping
try:
ocgis.env.OVERWRITE=True
ocgis.env.DIR_SHPCABINET = config.shapefiles_dir()
geoms = '50m_country'
# sci = ShpCabinetIterator(geoms)
if dir_output != None and path.exists(dir_output) == False:
makedirs(dir_output)
if polygon != None:
ugid = clipping.select_ugid(polygon)
else:
ugid = None
url = get_url(variable)
dimension_map = sb.get_dimension_map(url)
time_region = {'year':range(start,end+1)}
if variable == 'tg':
var = 'tas'
unit = 'K'
elif variable == 'tn':
var = 'tasmin'
unit = 'K'
elif variable == 'tx':
var = 'tasmax'
unit = 'K'
elif variable == 'rr':
var = 'pr'
unit = 'kg m-2 s-1'
except Exception as e:
logger.error('could not set processing environment')
if variable == 'rr':
try:
rd = ocgis.RequestDataset(url,
variable,
dimension_map = dimension_map,
time_region = time_region)
calc = 'rr=rr/84600'#
EOBS_file = ocgis.OcgOperations(dataset=rd,
calc=calc,
geom=geoms,
select_ugid=ugid,
output_format='nc',
dir_output=dir_output,
add_auxiliary_files=False,
# time_subset_func=func_june_july
).execute()
except Exception as e:
logger.error('ocgis failed for rr with url : %s' %(url))
else:
try:
unit = 'K'
rd = ocgis.RequestDataset(url,
variable,
conform_units_to=unit,
dimension_map = dimension_map,
time_region = time_region)
calc = [{'func': 'moving_window', 'name': 'ma', 'kwds': {'k': 3, 'operation': 'mean'}}]
EOBS_file = ocgis.OcgOperations(dataset=rd,
geom=geoms,
select_ugid=ugid,
output_format='nc',
calc=calc,
dir_output=dir_output,
add_auxiliary_files=False,
time_subset_func=func_june_july
).execute()
except Exception as e:
logger.error('ocgis failed for tg, tx or tn with url : %s' %(url))
try:
if polygon == None:
domain = att_dict['CORDEX_domain']
else:
domain = att_dict['CORDEX_domain'].replace('EUR', polygon)
EOBS_filename = '%s_%s_%s_%s_%s_%s_%s_%s_%s-%s.nc' % (var,
domain,
att_dict['driving_model_id'],
att_dict['experiment_id'],
att_dict['driving_model_ensemble_member'],
att_dict['model_id'],
att_dict['rcm_version_id'],
att_dict['frequency'],
start,
end)
fpath, basename = path.split(EOBS_file)
# set_attributes(EOBS_file, variable)
rename(EOBS_file, path.join(fpath, EOBS_filename))
except Exception as e:
logger.error('attributes not set for : %s' %(EOBS_file))
return path.join(fpath, EOBS_filename)
kwds = {'percentile': percentile, 'window_width': window_width}
calc = [{'func': 'daily_perc', 'name': 'dp', 'kwds': kwds}]
ops = OcgOperations(dataset=rd,
calc=calc,
output_format='nc',
prefix='wholedomain'
).execute()
print ops
from flyingpigeon.config import shapefiles_dir
from ocgis import env
from flyingpigeon.subset import get_geom, get_ugid
env.DIR_SHPCABINET = shapefiles_dir()
geom = get_geom('CMR')
ugid = get_ugid(polygons='CMR', geom=geom)
ops = OcgOperations(dataset=rd,
calc=calc,
geom=geom,
select_ugid=ugid,
output_format='nc',
prefix='polygonsubset'
).execute()
print ops
from flyingpigeon import subset
ops = subset.clipping(resource=resource,
variable=None,
def execute(self):
self.status.set('starting anopholes ...', 0)
nc_files = self.getInputValues(identifier='netcdf_file')
ocgis.env.DIR_SHPCABINET = config.shapefiles_dir()
ocgis.env.DIR_OUTPUT = os.curdir
ocgis.env.OVERWRITE = True
sc = ocgis.ShpCabinet()
geoms = 'continent'
select_ugid = [1] # UGID for Africa
self.status.set('got ShapeCabinet selected ugid : %s ...'% (select_ugid), 12)
# guess var names of files
for nc_file in nc_files:
ds = Dataset(nc_file)
if "tas" in ds.variables.keys():
file_tas = nc_file
elif "huss" in ds.variables.keys():
file_huss = nc_file
elif "ps" in ds.variables.keys():
file_ps = nc_file
elif "pr" in ds.variables.keys():
file_pr = nc_file
elif "evspsblpot" in ds.variables.keys():
file_evspsblpot = nc_file # Dataset(nc_file , 'r')
else:
raise Exception('input netcdf file has not variable tas|hurs|pr|evspsblpot')
self.status.set('sort files to appropriate variable names done' , 15)
#file_land_sea_mask = self.land_sea_mask.getValue()
#logging.debug('get landseamask ... done')
# build the n4 out variable based on pr
rd = ocgis.RequestDataset(file_pr, 'pr') # time_range=[dt1, dt2]
file_n4 = None
try :
(fp_tar, file_n4) = tempfile.mkstemp(dir=".", suffix='.nc')
prefix=os.path.splitext(os.path.basename(file_n4))[0]
ops = ocgis.OcgOperations(dataset=rd, geom=geoms, prefix=prefix, output_format='nc',select_ugid=select_ugid)
file_n4 = ops.execute()
self.status.set('created N4 outfile : %s ...'% (file_n4), 15)
except Exception as e:
self.status.set('"Something awful happened! Africa polygon subset failed for %s' % (file_n4, e), 15)
logging.exception("Something awful happened! Africa polygon subset failed for %s" % (file_n4), e )
nc_tas = Dataset(file_tas,'r')
nc_pr = Dataset(file_pr,'r')
nc_ps = Dataset(file_ps,'r')
nc_huss = Dataset(file_huss,'r')
nc_evspsblpot = Dataset(file_evspsblpot,'r')
nc_n4 = Dataset(file_n4,'a')
#nc_land_sea_mask = Dataset(file_land_sea_mask,'r')
logging.debug('open all files ... done')
#change attributes und variable name here
# att.put.nc(nc_n4, "n4", "units", "NC_FLOAT", -9e+33)
## read in values
tas = np.squeeze(nc_tas.variables["tas"])
pr = np.squeeze(nc_pr.variables["pr"])
ps = np.squeeze(nc_ps.variables["ps"])
huss = np.squeeze(nc_huss.variables["huss"])
evspsblpot = np.squeeze(nc_evspsblpot.variables["evspsblpot"])
logging.debug('read in all variables ... done')
var_n4 = nc_n4.variables["pr"]
n4 = np.zeros(pr.shape, dtype='f')
logging.debug('opended n4 file ... done')
# define some constatnts:
Increase_Ta = 0
#Evaporation (> -8)
Increase_Et = 0
#Rainfall (> -150)
Increase_Rt = 0
#Relative Humidity in (-97,+39)
Increase_RH = 0
## Text
deltaT = 6.08
h0 = 97
AT = 1.79*10**6
lamb = 1.5
m = 1000
De = 37.1
Te = 7.7
Nep = 120
alpha1 = 280.486
alpha2 = 0.025616
#if (abs(deltaT)<4):
#b = 0.89
#else:
b = 0.88
logging.debug('configuration ... done; start main loop, now!')
for x in range(0,tas.shape[1],1): #tas.shape[1]
for y in range(0,tas.shape[2],1): #tas.shape[2]
#if (var_n4[0,x,y] >= 0):
#try:
## get the appropriate values
#RH = hurs[:,x,y] * 100
Ta = tas[:,x,y] -273.15
Rt = pr[:,x,y] * 86400.
Et = np.fabs(evspsblpot[:,x,y] * 86400.) # in case evspsblpot ist stored as negaitve value
# calculation of rel. humidity
e_ = ((ps[:,x,y] * huss[:,x,y])/62.2)
es = 6.1078*10.**(7.5*(tas[:,x,y]-273.16)/(237.3+(tas[:,x,y]-273.16)))
RH = (e_ / es) * 100.
#calulation of water temperature
Tw = Ta + deltaT
## Check for Values out of range
Rt[Rt + Increase_Rt < 0] = 0
Et[Rt + Increase_Rt < 0] = 0
RH[RH + Increase_RH < 0] = 0
RH[RH + Increase_RH > 100] = 100
# create appropriate variabels
D = np.zeros(Ta.size)
Vt = np.zeros(Ta.size)
p4 = np.zeros(Ta.size)
ft = np.zeros(Ta.size)
Gc_Ta = np.zeros(Ta.size)
F4 = np.zeros(Ta.size)
N23 = np.zeros(Ta.size)
p_DD = np.zeros(Ta.size)
p_Tw = np.zeros([Ta.size,3])
p_Rt = np.zeros([Ta.size,3])
p_D = np.zeros([Ta.size,3])
G = np.zeros([Ta.size,3])
P = np.zeros([Ta.size,4])
p = np.zeros([Ta.size,4])
d = np.zeros([Ta.size,4])
N = np.zeros([Ta.size,4])
## initialize the model
Vt[0] = 1000.
N[0,0] = N[0,1] = N[0,2] = N[0,3] = 100.
# pdb.set_trace()
for t in range(0, (Ta.size -1) ,1):
#print x, y, t
if (Vt[t] == 0) & (Rt[t] == 0):
Vt[t+1] = 0
else:
Vt[t+1] = (Vt[t] + AT*Rt[t]/1000.)*(1 - 3.*Et[t]/h0* (Vt[0]/(Vt[t]+AT*Rt[t]/1000))**(1./3.))
if((Vt[t] == 0) & (Rt[t] == 0)):
Vt[t+1] = 0
else:
Vt[t+1] = (Vt[t] + AT*Rt[t]/1000.)*(1 - 3.*Et[t]/h0*(Vt[0]/(Vt[t]+AT*Rt[t]/1000))**(1./3.))
if(Vt[t+1] <= 0):
Vt[t+1] = 0
if (Vt[t+1] == 0):
D[t+1] = D[t] + 1
else:
D[t+1] = 0
beta2 = 4*10**(-6)*RH**2 - 1.09*10**(-3)*RH - 0.0255
beta1 = -2.32 * 10.**(-4.)* RH**2. + 0.0515*RH + 1.06
beta0 = 1.13*10**(-3)*RH**2 - 0.158*RH - 6.61
p4 = np.exp(-1/(beta2*Ta**2. + beta1*Ta + beta0))
d[:,0] = np.where(Vt != 0, 1.011 + 20.212*(1 + (Tw/12.096)**4.839)**(-1), 1.011 + 20.212*(1 + (Ta/12.096)**4.839)**(-1))
d[:,1] = np.where(Vt != 0, 8.130 + 13.794*(1 + (Tw/20.742)**8.946)**(-1) - d[:,0], 8.130 + 13.794*(1 + (Ta/20.742)**8.946)**(-1) - d[:,0])
d[:,2] = np.where(Vt != 0, 8.560 + 20.654*(1 + (Tw/19.759)**6.827)**(-1) - d[:,1] - d[:,0] , 8.560 + 20.654*(1 + (Ta/19.759)**6.827)**(-1) - d[:,1] - d[:,0])
d[:,3] = -1/np.log(p4)
p_Tw[:,0] = np.where(Vt != 0,np.where((Ta >= 14) & (Ta <= 40),np.exp(-1/d[:,0]),0),np.where((Ta >= 25) & (Ta <= 35),np.exp(-1./d[:,0]),0))
p_Tw[:,1] = np.where(Vt != 0,np.where((Tw >= 18) & (Tw <= 32),np.exp(-1/d[:,1]),0),np.where((Tw >= 18) & (Tw <= 32),np.exp(-1/d[:,1]),0))
p_Tw[:,2] = np.where(Vt != 0,np.where((Tw >= 18) & (Tw <= 32),np.exp(-1/d[:,2]),0),np.where((Tw >= 18) & (Tw <= 32),np.exp(-1/d[:,2]),0))
p_Rt[:,0] = np.exp(-0.0242*Rt)
p_Rt[:,1] = np.exp(-0.0127*Rt)
p_Rt[:,2] = np.exp(-0.00618*Rt)
p_D[:,0] = 2*np.exp(-0.405*D)/(1 + np.exp(-0.405*D))
p_D[:,1] = 2*np.exp(-0.855*D)/(1 + np.exp(-0.855*D))
p_D[:,2] = 2*np.exp(-0.602*D)/(1 + np.exp(-0.602*D))
for t in range(0,Rt.size -1,1): #tas.shape[0]
if(Vt[t] != 0):
p_DD[t] = (b*m/(1000*(N[t,1]+N[t,2])/Vt[t])) * (1 - (lamb**lamb/(lamb +(1000*(N[t,1]+N[t,2])/Vt[t])/m)**lamb))
else:
p_DD[t] = 1
p[t,0]= p_Tw[t,0]*p_Rt[t,0]*p_D[t,0]
p[t,1]= p_Tw[t,1]*p_Rt[t,1]*p_D[t,1]*p_DD[t]
p[t,2]= p_Tw[t,2]*p_Rt[t,2]*p_D[t,2]*p_DD[t]
p[t,3]= p4[t]
for j in range(0,4,1):
P[t,j] = (p[t,j] - p[t,j]**(d[t,j]))/(1. - p[t,j]**d[t,j])
for j in range(0,3,1):
G[t,j] = (1. - p[t,j])/(1. - p[t,j]**d[t,j])*p[t,j]**d[t,j]
ft[t] = 0.518*np.exp(-6.*(N[t,1]/Vt[t] - 0.317)**2.) + 0.192
Gc_Ta[t] = 1. + De/(Ta[t] - Te)
F4[t] = ft[t]*Nep/Gc_Ta[t]
N[t+1,0] = (P[t,0] * N[t,0] + (alpha1 * F4[t]) * N[t,3])
N[t+1,1] = (P[t,1] * N[t,1] + G[t,0] * N[t,0])
N[t+1,2] = (P[t,2] * N[t,2] + G[t,1] * N[t,1])
N[t+1,3] = (P[t,3] * N[t,3] + G[t,2] * N[t,2])
N[np.isnan(N)] = 0
n4[:,x,y] = N[:,3] #p4[t] # p_D[t,2] #N[t,3]
process = (((x+1) * tas.shape[2] + y ) / ((tas.shape[1] * tas.shape[2]) / 100 ))
logging.debug('Calculation process: %d % \r ' % (process))
#except Exception as e:
#logging.warn('Gridbox not calculated. Error= %s ' % (e))
#else:
# n4[:,x,y] = float('NaN')
# var_n4.assignValue(np.zeros(var_n4.shape))
self.status.set("anopheles done", 100)
self.output.setValue( file_n4 )