def test_plotlibs(): import matplotlib matplotlib.use('Agg') # use this if no xserver is available from matplotlib import pyplot as plt from matplotlib.colors import Normalize from cartopy import config as cartopy_config from cartopy.util import add_cyclic_point import cartopy.crs as ccrs from cartopy.io.shapereader import Reader from cartopy.feature import ShapelyFeature from PyPDF2 import PdfFileWriter, PdfFileReader from reportlab.pdfgen import canvas from flyingpigeon import config DIR_SHP = config.shapefiles_dir()
def plot_polygons(regions): """ extract the polygon coordinate and plot it on a worldmap :param regions: list of ISO abreviations for polygons :return png: map_graphic.png """ from cartopy.io.shapereader import Reader from cartopy.feature import ShapelyFeature from os.path import curdir, abspath from flyingpigeon import config DIR_SHP = config.shapefiles_dir() if type(regions) == str: regions = list([regions]) fname = join(DIR_SHP, "countries.shp") geos = Reader(fname).geometries() records = Reader(fname).records() LOGGER.debug('') fig = plt.figure(figsize=(10, 10), facecolor='w', edgecolor='k') # dpi=600, projection = ccrs.Orthographic(central_longitude=0.0, central_latitude=0.0, globe=None) # Robinson() ax = plt.axes(projection=projection) for r in records: geo = geos.next() if r.attributes['ISO_A3'] in regions: shape_feature = ShapelyFeature(geo, ccrs.PlateCarree(), edgecolor='black') ax.add_feature(shape_feature) ax.coastlines() # ax.set_global() o1, map_graphic = mkstemp(dir=abspath(curdir), suffix='.png') fig.savefig(map_graphic) plt.close() return map_graphic
from cdo import Cdo from tempfile import mkstemp import os from flyingpigeon.utils import drs_filename, get_variable, calc_grouping , sort_by_filename from flyingpigeon import config import logging logger = logging.getLogger(__name__) DIR_MASKS = config.masks_dir() DIR_SHP = config.shapefiles_dir() def countries(): """ :return: a list of all countries codes. """ countries = _COUNTRIES_.keys() countries.sort() return countries def countries_longname(): """ :return: the longname of all countries. """ longname = '' for country in countries(): longname = longname + "%s : %s \n" % (country, _COUNTRIES_[country]['longname']) return longname def masking(resource, mask, prefix=None, dir_output=None):
from os.path import join, abspath, dirname, getsize, curdir from netCDF4 import Dataset from flyingpigeon import config import logging LOGGER = logging.getLogger("PYWPS") DIR_SHP = config.shapefiles_dir() def has_Lambert_Conformal(resource): """ Check if grid is organised as Lambert_Conformal :param resource: file to be checked :return Boolean: True/False """ if type(resource) != list: resource = [resource] for nc in resource: ds = Dataset(nc) if 'Lambert_Conformal' not in ds.variables.keys(): return False return True def call(resource=[], variable=None, dimension_map=None, calc=None, calc_grouping=None,
def get_data(variable, polygon=None, dir_output=None, start = 1950, end = 2014 ): import ocgis from os import rename, path, makedirs from flyingpigeon import utils from flyingpigeon import subset as sb from flyingpigeon import clipping try: ocgis.env.OVERWRITE=True ocgis.env.DIR_SHPCABINET = config.shapefiles_dir() geoms = '50m_country' # sci = ShpCabinetIterator(geoms) if dir_output != None and path.exists(dir_output) == False: makedirs(dir_output) if polygon != None: ugid = clipping.select_ugid(polygon) else: ugid = None url = get_url(variable) dimension_map = sb.get_dimension_map(url) time_region = {'year':range(start,end+1)} if variable == 'tg': var = 'tas' unit = 'K' elif variable == 'tn': var = 'tasmin' unit = 'K' elif variable == 'tx': var = 'tasmax' unit = 'K' elif variable == 'rr': var = 'pr' unit = 'kg m-2 s-1' except Exception as e: logger.error('could not set processing environment') if variable == 'rr': try: rd = ocgis.RequestDataset(url, variable, dimension_map = dimension_map, time_region = time_region) calc = 'rr=rr/84600'# EOBS_file = ocgis.OcgOperations(dataset=rd, calc=calc, geom=geoms, select_ugid=ugid, output_format='nc', dir_output=dir_output, add_auxiliary_files=False, # time_subset_func=func_june_july ).execute() except Exception as e: logger.error('ocgis failed for rr with url : %s' %(url)) else: try: unit = 'K' rd = ocgis.RequestDataset(url, variable, conform_units_to=unit, dimension_map = dimension_map, time_region = time_region) calc = [{'func': 'moving_window', 'name': 'ma', 'kwds': {'k': 3, 'operation': 'mean'}}] EOBS_file = ocgis.OcgOperations(dataset=rd, geom=geoms, select_ugid=ugid, output_format='nc', calc=calc, dir_output=dir_output, add_auxiliary_files=False, time_subset_func=func_june_july ).execute() except Exception as e: logger.error('ocgis failed for tg, tx or tn with url : %s' %(url)) try: if polygon == None: domain = att_dict['CORDEX_domain'] else: domain = att_dict['CORDEX_domain'].replace('EUR', polygon) EOBS_filename = '%s_%s_%s_%s_%s_%s_%s_%s_%s-%s.nc' % (var, domain, att_dict['driving_model_id'], att_dict['experiment_id'], att_dict['driving_model_ensemble_member'], att_dict['model_id'], att_dict['rcm_version_id'], att_dict['frequency'], start, end) fpath, basename = path.split(EOBS_file) # set_attributes(EOBS_file, variable) rename(EOBS_file, path.join(fpath, EOBS_filename)) except Exception as e: logger.error('attributes not set for : %s' %(EOBS_file)) return path.join(fpath, EOBS_filename)
kwds = {'percentile': percentile, 'window_width': window_width} calc = [{'func': 'daily_perc', 'name': 'dp', 'kwds': kwds}] ops = OcgOperations(dataset=rd, calc=calc, output_format='nc', prefix='wholedomain' ).execute() print ops from flyingpigeon.config import shapefiles_dir from ocgis import env from flyingpigeon.subset import get_geom, get_ugid env.DIR_SHPCABINET = shapefiles_dir() geom = get_geom('CMR') ugid = get_ugid(polygons='CMR', geom=geom) ops = OcgOperations(dataset=rd, calc=calc, geom=geom, select_ugid=ugid, output_format='nc', prefix='polygonsubset' ).execute() print ops from flyingpigeon import subset ops = subset.clipping(resource=resource, variable=None,
def execute(self): self.status.set('starting anopholes ...', 0) nc_files = self.getInputValues(identifier='netcdf_file') ocgis.env.DIR_SHPCABINET = config.shapefiles_dir() ocgis.env.DIR_OUTPUT = os.curdir ocgis.env.OVERWRITE = True sc = ocgis.ShpCabinet() geoms = 'continent' select_ugid = [1] # UGID for Africa self.status.set('got ShapeCabinet selected ugid : %s ...'% (select_ugid), 12) # guess var names of files for nc_file in nc_files: ds = Dataset(nc_file) if "tas" in ds.variables.keys(): file_tas = nc_file elif "huss" in ds.variables.keys(): file_huss = nc_file elif "ps" in ds.variables.keys(): file_ps = nc_file elif "pr" in ds.variables.keys(): file_pr = nc_file elif "evspsblpot" in ds.variables.keys(): file_evspsblpot = nc_file # Dataset(nc_file , 'r') else: raise Exception('input netcdf file has not variable tas|hurs|pr|evspsblpot') self.status.set('sort files to appropriate variable names done' , 15) #file_land_sea_mask = self.land_sea_mask.getValue() #logging.debug('get landseamask ... done') # build the n4 out variable based on pr rd = ocgis.RequestDataset(file_pr, 'pr') # time_range=[dt1, dt2] file_n4 = None try : (fp_tar, file_n4) = tempfile.mkstemp(dir=".", suffix='.nc') prefix=os.path.splitext(os.path.basename(file_n4))[0] ops = ocgis.OcgOperations(dataset=rd, geom=geoms, prefix=prefix, output_format='nc',select_ugid=select_ugid) file_n4 = ops.execute() self.status.set('created N4 outfile : %s ...'% (file_n4), 15) except Exception as e: self.status.set('"Something awful happened! Africa polygon subset failed for %s' % (file_n4, e), 15) logging.exception("Something awful happened! Africa polygon subset failed for %s" % (file_n4), e ) nc_tas = Dataset(file_tas,'r') nc_pr = Dataset(file_pr,'r') nc_ps = Dataset(file_ps,'r') nc_huss = Dataset(file_huss,'r') nc_evspsblpot = Dataset(file_evspsblpot,'r') nc_n4 = Dataset(file_n4,'a') #nc_land_sea_mask = Dataset(file_land_sea_mask,'r') logging.debug('open all files ... done') #change attributes und variable name here # att.put.nc(nc_n4, "n4", "units", "NC_FLOAT", -9e+33) ## read in values tas = np.squeeze(nc_tas.variables["tas"]) pr = np.squeeze(nc_pr.variables["pr"]) ps = np.squeeze(nc_ps.variables["ps"]) huss = np.squeeze(nc_huss.variables["huss"]) evspsblpot = np.squeeze(nc_evspsblpot.variables["evspsblpot"]) logging.debug('read in all variables ... done') var_n4 = nc_n4.variables["pr"] n4 = np.zeros(pr.shape, dtype='f') logging.debug('opended n4 file ... done') # define some constatnts: Increase_Ta = 0 #Evaporation (> -8) Increase_Et = 0 #Rainfall (> -150) Increase_Rt = 0 #Relative Humidity in (-97,+39) Increase_RH = 0 ## Text deltaT = 6.08 h0 = 97 AT = 1.79*10**6 lamb = 1.5 m = 1000 De = 37.1 Te = 7.7 Nep = 120 alpha1 = 280.486 alpha2 = 0.025616 #if (abs(deltaT)<4): #b = 0.89 #else: b = 0.88 logging.debug('configuration ... done; start main loop, now!') for x in range(0,tas.shape[1],1): #tas.shape[1] for y in range(0,tas.shape[2],1): #tas.shape[2] #if (var_n4[0,x,y] >= 0): #try: ## get the appropriate values #RH = hurs[:,x,y] * 100 Ta = tas[:,x,y] -273.15 Rt = pr[:,x,y] * 86400. Et = np.fabs(evspsblpot[:,x,y] * 86400.) # in case evspsblpot ist stored as negaitve value # calculation of rel. humidity e_ = ((ps[:,x,y] * huss[:,x,y])/62.2) es = 6.1078*10.**(7.5*(tas[:,x,y]-273.16)/(237.3+(tas[:,x,y]-273.16))) RH = (e_ / es) * 100. #calulation of water temperature Tw = Ta + deltaT ## Check for Values out of range Rt[Rt + Increase_Rt < 0] = 0 Et[Rt + Increase_Rt < 0] = 0 RH[RH + Increase_RH < 0] = 0 RH[RH + Increase_RH > 100] = 100 # create appropriate variabels D = np.zeros(Ta.size) Vt = np.zeros(Ta.size) p4 = np.zeros(Ta.size) ft = np.zeros(Ta.size) Gc_Ta = np.zeros(Ta.size) F4 = np.zeros(Ta.size) N23 = np.zeros(Ta.size) p_DD = np.zeros(Ta.size) p_Tw = np.zeros([Ta.size,3]) p_Rt = np.zeros([Ta.size,3]) p_D = np.zeros([Ta.size,3]) G = np.zeros([Ta.size,3]) P = np.zeros([Ta.size,4]) p = np.zeros([Ta.size,4]) d = np.zeros([Ta.size,4]) N = np.zeros([Ta.size,4]) ## initialize the model Vt[0] = 1000. N[0,0] = N[0,1] = N[0,2] = N[0,3] = 100. # pdb.set_trace() for t in range(0, (Ta.size -1) ,1): #print x, y, t if (Vt[t] == 0) & (Rt[t] == 0): Vt[t+1] = 0 else: Vt[t+1] = (Vt[t] + AT*Rt[t]/1000.)*(1 - 3.*Et[t]/h0* (Vt[0]/(Vt[t]+AT*Rt[t]/1000))**(1./3.)) if((Vt[t] == 0) & (Rt[t] == 0)): Vt[t+1] = 0 else: Vt[t+1] = (Vt[t] + AT*Rt[t]/1000.)*(1 - 3.*Et[t]/h0*(Vt[0]/(Vt[t]+AT*Rt[t]/1000))**(1./3.)) if(Vt[t+1] <= 0): Vt[t+1] = 0 if (Vt[t+1] == 0): D[t+1] = D[t] + 1 else: D[t+1] = 0 beta2 = 4*10**(-6)*RH**2 - 1.09*10**(-3)*RH - 0.0255 beta1 = -2.32 * 10.**(-4.)* RH**2. + 0.0515*RH + 1.06 beta0 = 1.13*10**(-3)*RH**2 - 0.158*RH - 6.61 p4 = np.exp(-1/(beta2*Ta**2. + beta1*Ta + beta0)) d[:,0] = np.where(Vt != 0, 1.011 + 20.212*(1 + (Tw/12.096)**4.839)**(-1), 1.011 + 20.212*(1 + (Ta/12.096)**4.839)**(-1)) d[:,1] = np.where(Vt != 0, 8.130 + 13.794*(1 + (Tw/20.742)**8.946)**(-1) - d[:,0], 8.130 + 13.794*(1 + (Ta/20.742)**8.946)**(-1) - d[:,0]) d[:,2] = np.where(Vt != 0, 8.560 + 20.654*(1 + (Tw/19.759)**6.827)**(-1) - d[:,1] - d[:,0] , 8.560 + 20.654*(1 + (Ta/19.759)**6.827)**(-1) - d[:,1] - d[:,0]) d[:,3] = -1/np.log(p4) p_Tw[:,0] = np.where(Vt != 0,np.where((Ta >= 14) & (Ta <= 40),np.exp(-1/d[:,0]),0),np.where((Ta >= 25) & (Ta <= 35),np.exp(-1./d[:,0]),0)) p_Tw[:,1] = np.where(Vt != 0,np.where((Tw >= 18) & (Tw <= 32),np.exp(-1/d[:,1]),0),np.where((Tw >= 18) & (Tw <= 32),np.exp(-1/d[:,1]),0)) p_Tw[:,2] = np.where(Vt != 0,np.where((Tw >= 18) & (Tw <= 32),np.exp(-1/d[:,2]),0),np.where((Tw >= 18) & (Tw <= 32),np.exp(-1/d[:,2]),0)) p_Rt[:,0] = np.exp(-0.0242*Rt) p_Rt[:,1] = np.exp(-0.0127*Rt) p_Rt[:,2] = np.exp(-0.00618*Rt) p_D[:,0] = 2*np.exp(-0.405*D)/(1 + np.exp(-0.405*D)) p_D[:,1] = 2*np.exp(-0.855*D)/(1 + np.exp(-0.855*D)) p_D[:,2] = 2*np.exp(-0.602*D)/(1 + np.exp(-0.602*D)) for t in range(0,Rt.size -1,1): #tas.shape[0] if(Vt[t] != 0): p_DD[t] = (b*m/(1000*(N[t,1]+N[t,2])/Vt[t])) * (1 - (lamb**lamb/(lamb +(1000*(N[t,1]+N[t,2])/Vt[t])/m)**lamb)) else: p_DD[t] = 1 p[t,0]= p_Tw[t,0]*p_Rt[t,0]*p_D[t,0] p[t,1]= p_Tw[t,1]*p_Rt[t,1]*p_D[t,1]*p_DD[t] p[t,2]= p_Tw[t,2]*p_Rt[t,2]*p_D[t,2]*p_DD[t] p[t,3]= p4[t] for j in range(0,4,1): P[t,j] = (p[t,j] - p[t,j]**(d[t,j]))/(1. - p[t,j]**d[t,j]) for j in range(0,3,1): G[t,j] = (1. - p[t,j])/(1. - p[t,j]**d[t,j])*p[t,j]**d[t,j] ft[t] = 0.518*np.exp(-6.*(N[t,1]/Vt[t] - 0.317)**2.) + 0.192 Gc_Ta[t] = 1. + De/(Ta[t] - Te) F4[t] = ft[t]*Nep/Gc_Ta[t] N[t+1,0] = (P[t,0] * N[t,0] + (alpha1 * F4[t]) * N[t,3]) N[t+1,1] = (P[t,1] * N[t,1] + G[t,0] * N[t,0]) N[t+1,2] = (P[t,2] * N[t,2] + G[t,1] * N[t,1]) N[t+1,3] = (P[t,3] * N[t,3] + G[t,2] * N[t,2]) N[np.isnan(N)] = 0 n4[:,x,y] = N[:,3] #p4[t] # p_D[t,2] #N[t,3] process = (((x+1) * tas.shape[2] + y ) / ((tas.shape[1] * tas.shape[2]) / 100 )) logging.debug('Calculation process: %d % \r ' % (process)) #except Exception as e: #logging.warn('Gridbox not calculated. Error= %s ' % (e)) #else: # n4[:,x,y] = float('NaN') # var_n4.assignValue(np.zeros(var_n4.shape)) self.status.set("anopheles done", 100) self.output.setValue( file_n4 )