def test_unrotate_pole():
ncs = [
local_path(TESTDATA['cordex_tasmax_2006_nc']),
local_path(TESTDATA['cordex_tasmax_2007_nc'])
]
lats, lons = utils.unrotate_pole(ncs)
assert lats.shape == (103, 106)
def get_indices(resources, indices):
"""
calculating indices (netCDF files) defined in _SDMINDICES_
:param resources:
:param indices: indices defined in _SDMINDICES_
:return list: list of filepathes to netCDF files
"""
from flyingpigeon.utils import sort_by_filename, calc_grouping, drs_filename, unrotate_pole
# from flyingpigeon.ocgis_module import call
from flyingpigeon.indices import indice_variable, calc_indice_simple
# names = [drs_filename(nc, skip_timestamp=False, skip_format=False,
# variable=None, rename_file=True, add_file_path=True) for nc in resources]
ncs = sort_by_filename(resources, historical_concatination=True)
ncs_indices = []
logger.info('resources sorted found %s datasets' % len(ncs.keys()))
for key in ncs.keys():
for indice in indices:
try:
name, month = indice.split('_')
variable = key.split('_')[0]
# print name, month , variable
if variable == indice_variable(name):
logger.info('calculating indice %s ' % indice)
prefix = key.replace(variable, name).replace('_day_', '_%s_' % month)
nc = calc_indice_simple(resource=ncs[key],
variable=variable,
polygons=['Europe', 'Africa', 'Asia', 'North America', 'Oceania',
'South America', 'Antarctica'],
mosaic=True,
prefix=prefix, indices=name, groupings=month)
if nc is not None:
coords = unrotate_pole(nc[0], write_to_file=True)
ncs_indices.append(nc[0])
logger.info('Successful calculated indice %s %s' % (key, indice))
else:
msg = 'failed to calculate indice %s %s' % (key, indice)
logger.exception(msg)
except:
msg = 'failed to calculate indice %s %s' % (key, indice)
logger.exception(msg)
raise
return ncs_indices
def get_extent(resource):
"""
returns the spatial extention of the values
:param resource: NetCDF file
:return: min_lat, max_lat, min_lon, max_lon
"""
ds = Dataset(resource)
if 'lat' in ds.variables:
lats = ds.variables['lat']
lons = ds.variables['lon']
ds.close()
else:
ds.close()
from flyingpigeon import utils
lats, lons = utils.unrotate_pole(resource)
min_lat = lats[:].min()
max_lat = lats[:].max()
min_lon = lons[:].min()
max_lon = lons[:].max()
return min_lat, max_lat, min_lon, max_lon
def get_extent(resource):
"""
returns the spatial extention of the values
:param resource: NetCDF file
:return: min_lat, max_lat, min_lon, max_lon
"""
ds = Dataset(resource)
if 'lat' in ds.variables:
lats = ds.variables['lat']
lons = ds.variables['lon']
ds.close()
else:
ds.close()
from flyingpigeon import utils
lats, lons = utils.unrotate_pole(resource)
min_lat = lats[:].min()
max_lat = lats[:].max()
min_lon = lons[:].min()
max_lon = lons[:].max()
return min_lat, max_lat, min_lon, max_lon
def get_gam(ncs_indices, coordinate):
from netCDF4 import Dataset
from os.path import basename
from shapely.geometry import Point
from numpy import squeeze, ravel, isnan, nan, array, reshape
from flyingpigeon.utils import get_variable, get_values, unrotate_pole
from flyingpigeon.ocgis_module import call
try:
from rpy2.robjects.packages import importr
import rpy2.robjects as ro
import rpy2.robjects.numpy2ri
rpy2.robjects.numpy2ri.activate()
base = importr("base")
stats = importr("stats")
mgcv = importr("mgcv")
logger.info('rpy2 modules imported')
except Exception as e:
msg = 'failed to import rpy2 modules %s' % e
logger.debug(msg)
raise Exception(msg)
for i, ncs in enumerate(ncs_indices):
# ocgis need unrotated coordinates to extract points
# unrotate_pole writes lats lons into the file.
# ACHTUNG: will fail if the data is stored on a file system with no write permissions
try:
lats, lons = unrotate_pole(ncs, write_to_file=True)
point = Point(float(coordinate[0]), float(coordinate[1]))
# get the values
variable = get_variable(ncs)
agg = basename(ncs).split('_')[-2]
indice = '%s_%s' % (variable, agg)
timeseries = call(resource=ncs, geom=point, select_nearest=True)
ts = Dataset(timeseries)
vals = squeeze(ts.variables[variable][:])
from numpy import min, max, mean, append, zeros, ones
dif = max(vals) - min(vals)
a = append(vals - dif ,vals)
vals = append(a, vals+dif)
if i == 0 :
from numpy import zeros, ones
a = append (zeros(len(vals)) , ones(len(vals)) )
PA = append(a , zeros(len(vals)))
data = {'PA': ro.FloatVector(PA)}
data[str(indice)] = ro.FloatVector(vals)
form = 'PA ~ '
form = form + 's(%s, k=3)' % indice
else:
form = form + ' + s(%s, k=3)' % indice
data[str(indice)] = ro.FloatVector(vals)
except Exception as e:
msg = 'Failed to prepare data %s' % e
logger.debug(msg)
try:
logger.info(data)
dataf = ro.DataFrame(data)
eq = ro.Formula(str(form))
gam_model = mgcv.gam(base.eval(eq), data=dataf, family=stats.binomial(), scale=-1, na_action=stats.na_exclude) #
logger.info('GAM model trained')
except Exception as e:
msg = 'Failed to generate GAM model %s' % e
logger.debug(msg)
# ### ###########################
# # plot response curves
# ### ###########################
try:
from flyingpigeon.visualisation import concat_images
from tempfile import mkstemp
grdevices = importr('grDevices')
graphicDev = importr('Cairo')
infos = []
for i in range(1,len(ncs_indices)+1):
ip, info = mkstemp(dir='.',suffix='.png')
#grdevices.png(filename=info)
#graphicDev.CairoPDF(info, width = 7, height = 7, pointsize = 12)
graphicDev.CairoPNG(info, width = 640 , height = 480, pointsize = 12) # 640, 480) #, pointsize = 12 width = 30, height = 30,
print 'file opened!'
infos.append(info)
#grdevices.png(filename=info)
ylim = ro.IntVector([-6,6])
trans = ro.r('function(x){exp(x)/(1+exp(x))}')
mgcv.plot_gam(gam_model, trans=trans, shade='T',
col='black',select=i,ylab='Predicted Probability',rug=False ,
cex_lab = 1.4, cex_axis = 1.4, ) #
print 'gam plotted ;-)'
#ylim=ylim, ,
grdevices.dev_off()
#graphicDev.dev_off()
#graphicDev.Cairo_onSave( dev_cur(), onSave=True )
print(' %s plots generated ' % len(infos))
infos_concat = concat_images(infos, orientation='h')
except Exception as e:
msg = 'Failed to plot statistical graphic %s' % e
logger.debug(msg)
raise Exception(msg)
return gam_model, infos_concat
def get_gam(ncs_indices, coordinate):
from netCDF4 import Dataset
from os.path import basename
from shapely.geometry import Point
from numpy import squeeze, ravel, isnan, nan, array, reshape
from flyingpigeon.utils import get_variable, get_values, unrotate_pole
from flyingpigeon.ocgis_module import call
try:
from rpy2.robjects.packages import importr
import rpy2.robjects as ro
import rpy2.robjects.numpy2ri
rpy2.robjects.numpy2ri.activate()
base = importr("base")
stats = importr("stats")
mgcv = importr("mgcv")
logger.info('rpy2 modules imported')
except Exception as e:
msg = 'failed to import rpy2 modules %s' % e
logger.debug(msg)
raise Exception(msg)
for i, ncs in enumerate(ncs_indices):
# ocgis need unrotated coordinates to extract points
# unrotate_pole writes lats lons into the file.
# ACHTUNG: will fail if the data is stored on a file system with no write permissions
try:
lats, lons = unrotate_pole(ncs, write_to_file=True)
point = Point(float(coordinate[0]), float(coordinate[1]))
# get the values
variable = get_variable(ncs)
agg = basename(ncs).split('_')[-2]
indice = '%s_%s' % (variable, agg)
timeseries = call(resource=ncs, geom=point, select_nearest=True)
ts = Dataset(timeseries)
vals = squeeze(ts.variables[variable][:])
from numpy import min, max, mean, append, zeros, ones
dif = max(vals) - min(vals)
a = append(vals - dif, vals)
vals = append(a, vals + dif)
if i == 0:
from numpy import zeros, ones
a = append(zeros(len(vals)), ones(len(vals)))
PA = append(a, zeros(len(vals)))
data = {'PA': ro.FloatVector(PA)}
data[str(indice)] = ro.FloatVector(vals)
form = 'PA ~ '
form = form + 's(%s, k=3)' % indice
else:
form = form + ' + s(%s, k=3)' % indice
data[str(indice)] = ro.FloatVector(vals)
except Exception as e:
msg = 'Failed to prepare data %s' % e
logger.debug(msg)
try:
logger.info(data)
dataf = ro.DataFrame(data)
eq = ro.Formula(str(form))
gam_model = mgcv.gam(base.eval(eq),
data=dataf,
family=stats.binomial(),
scale=-1,
na_action=stats.na_exclude) #
logger.info('GAM model trained')
except Exception as e:
msg = 'Failed to generate GAM model %s' % e
logger.debug(msg)
# ### ###########################
# # plot response curves
# ### ###########################
try:
from flyingpigeon.visualisation import concat_images
from tempfile import mkstemp
grdevices = importr('grDevices')
graphicDev = importr('Cairo')
infos = []
for i in range(1, len(ncs_indices) + 1):
ip, info = mkstemp(dir='.', suffix='.png')
#grdevices.png(filename=info)
#graphicDev.CairoPDF(info, width = 7, height = 7, pointsize = 12)
graphicDev.CairoPNG(
info, width=640, height=480, pointsize=12
) # 640, 480) #, pointsize = 12 width = 30, height = 30,
print 'file opened!'
infos.append(info)
#grdevices.png(filename=info)
ylim = ro.IntVector([-6, 6])
trans = ro.r('function(x){exp(x)/(1+exp(x))}')
mgcv.plot_gam(
gam_model,
trans=trans,
shade='T',
col='black',
select=i,
ylab='Predicted Probability',
rug=False,
cex_lab=1.4,
cex_axis=1.4,
) #
print 'gam plotted ;-)'
#ylim=ylim, ,
grdevices.dev_off()
#graphicDev.dev_off()
#graphicDev.Cairo_onSave( dev_cur(), onSave=True )
print(' %s plots generated ' % len(infos))
infos_concat = concat_images(infos, orientation='h')
except Exception as e:
msg = 'Failed to plot statistical graphic %s' % e
logger.debug(msg)
raise Exception(msg)
return gam_model, infos_concat
