def test_get_values():
values = utils.get_values(local_path(TESTDATA['cmip5_tasmax_2007_nc']))
assert 12 == len(values)
values = utils.get_values(local_path(TESTDATA['cordex_tasmax_2007_nc']))
assert 12 == len(values)
values = utils.get_values([local_path(TESTDATA['cordex_tasmax_2006_nc']),
local_path(TESTDATA['cordex_tasmax_2007_nc'])])
assert 23 == len(values)
def test_get_values():
values = utils.get_values(local_path(TESTDATA["cmip5_tasmax_2007_nc"]))
assert 12 == len(values)
values = utils.get_values(local_path(TESTDATA["cordex_tasmax_2007_nc"]))
assert 12 == len(values)
values = utils.get_values(
[local_path(TESTDATA["cordex_tasmax_2006_nc"]), local_path(TESTDATA["cordex_tasmax_2007_nc"])]
)
assert 23 == len(values)
def execute(self):
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import sort_by_filename, archive, get_values, get_time
ncs = self.getInputValues(identifier='resource')
logger.info("ncs: %s " % ncs)
coords = self.getInputValues(identifier='coords')
logger.info("coords %s", coords)
filenames = []
nc_exp = sort_by_filename(ncs, historical_concatination=True)
from numpy import savetxt, column_stack
from shapely.geometry import Point
for key in nc_exp.keys():
try:
logger.info('start calculation for %s ' % key)
ncs = nc_exp[key]
times = get_time(ncs, format='%Y-%m-%d_%H:%M:%S')
concat_vals = times # ['%s-%02d-%02d_%02d:%02d:%02d' %
# (t.year, t.month, t.day, t.hour, t.minute, t.second) for t in times]
header = 'date_time'
filename = '%s.csv' % key
filenames.append(filename)
for p in coords:
try:
self.status.set('processing point : {0}'.format(p), 20)
# define the point:
p = p.split(',')
point = Point(float(p[0]), float(p[1]))
# get the values
timeseries = call(resource=ncs,
geom=point,
select_nearest=True)
vals = get_values(timeseries)
# concatenation of values
header = header + ',%s-%s' % (p[0], p[1])
concat_vals = column_stack([concat_vals, vals])
except Exception as e:
logger.debug('failed for point %s %s' % (p, e))
self.status.set(
'*** all points processed for {0} ****'.format(key), 50)
savetxt(filename,
concat_vals,
fmt='%s',
delimiter=',',
header=header)
except Exception as e:
logger.debug('failed for %s %s' % (key, e))
# set the outputs
self.status.set('*** creating output tar archive ****', 90)
tarout_file = archive(filenames)
self.tarout.setValue(tarout_file)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
ncs = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
LOGGER.info('ncs: {}'.format(ncs))
coords = []
for coord in request.inputs['coords']:
coords.append(coord.data)
LOGGER.info('coords {}'.format(coords))
filenames = []
nc_exp = sort_by_filename(ncs, historical_concatination=True)
for key in nc_exp.keys():
try:
LOGGER.info('start calculation for {}'.format(key))
ncs = nc_exp[key]
times = get_time(ncs) # , format='%Y-%m-%d_%H:%M:%S')
concat_vals = times # ['%s-%02d-%02d_%02d:%02d:%02d' %
# (t.year, t.month, t.day, t.hour, t.minute, t.second) for t in times]
header = 'date_time'
filename = '{}.csv'.format(key)
filenames.append(filename)
for p in coords:
try:
response.update_status('processing point: {}'.format(p), 20)
# define the point:
p = p.split(',')
point = Point(float(p[0]), float(p[1]))
# get the values
timeseries = call(resource=ncs, geom=point, select_nearest=True)
vals = get_values(timeseries)
# concatenation of values
header = header + ',{}-{}'.format(p[0], p[1])
concat_vals = column_stack([concat_vals, vals])
except Exception as e:
LOGGER.debug('failed for point {} {}'.format(p, e))
response.update_status('*** all points processed for {0} ****'.format(key), 50)
# TODO: Ascertain whether this 'savetxt' is a valid command without string formatting argument: '%s'
savetxt(filename, concat_vals, fmt='%s', delimiter=',', header=header)
except Exception as ex:
LOGGER.debug('failed for {}: {}'.format(key, str(ex)))
# set the outputs
response.update_status('*** creating output tar archive ****', 90)
tarout_file = archive(filenames)
response.outputs['tarout'].file = tarout_file
return response
def execute(self):
from flyingpigeon.ocgis_module import call
from flyingpigeon.utils import sort_by_filename, archive, get_values, get_time
ncs = self.getInputValues(identifier='netcdf_file')
logger.info("ncs: %s " % ncs)
coords = self.getInputValues(identifier='coords')
logger.info("coords %s", coords)
filenames = []
nc_exp = sort_by_filename(ncs, historical_concatination=True)
#(fp_tar, tarout_file) = tempfile.mkstemp(dir=".", suffix='.tar')
#tar = tarfile.open(tarout_file, "w")
from numpy import savetxt, column_stack
from shapely.geometry import Point
for key in nc_exp.keys():
try:
logger.info('start calculation for %s ' % key )
ncs = nc_exp[key]
times = get_time(ncs)
concat_vals = ['%s-%02d-%02d_%02d:%02d:%02d' %
(t.year, t.month, t.day, t.hour, t.minute, t.second) for t in times]
header = 'date_time'
filename = '%s.csv' % key
filenames.append(filename)
for p in coords:
try:
self.status.set('processing point : {0}'.format(p), 20)
# define the point:
p = p.split(',')
point = Point(float(p[0]), float(p[1]))
# get the values
timeseries = call(resource=ncs, geom=point, select_nearest=True)
vals = get_values(timeseries)
# concatination of values
header = header + ',%s-%s' % (p[0], p[1])
concat_vals = column_stack([concat_vals, vals])
except Exception as e:
logger.debug('failed for point %s %s' % (p , e))
self.status.set('*** all points processed for {0} ****'.format(key), 50)
savetxt(filename, concat_vals, fmt='%s', delimiter=',', header=header)
except Exception as e:
logger.debug('failed for %s %s' % (key, e))
### set the outputs
self.status.set('*** creating output tar archive ****',90)
tarout_file = archive(filenames)
self.tarout.setValue( tarout_file )
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
ncs = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
LOGGER.info("ncs: %s " % ncs)
coords = request.inputs['coords'] # self.getInputValues(identifier='coords')
LOGGER.info("coords %s", coords)
filenames = []
nc_exp = sort_by_filename(ncs, historical_concatination=True)
for key in nc_exp.keys():
try:
LOGGER.info('start calculation for %s ' % key)
ncs = nc_exp[key]
times = get_time(ncs, format='%Y-%m-%d_%H:%M:%S')
concat_vals = times # ['%s-%02d-%02d_%02d:%02d:%02d' %
# (t.year, t.month, t.day, t.hour, t.minute, t.second) for t in times]
header = 'date_time'
filename = '%s.csv' % key
filenames.append(filename)
for p in coords:
try:
response.update_status('processing point : {0}'.format(p), 20)
# define the point:
p = p.split(',')
point = Point(float(p[0]), float(p[1]))
# get the values
timeseries = call(resource=ncs, geom=point, select_nearest=True)
vals = get_values(timeseries)
# concatenation of values
header = header + ',%s-%s' % (p[0], p[1])
concat_vals = column_stack([concat_vals, vals])
except Exception as e:
LOGGER.debug('failed for point %s %s' % (p, e))
response.update_status('*** all points processed for {0} ****'.format(key), 50)
savetxt(filename, concat_vals, fmt='%s', delimiter=',', header=header)
except Exception as e:
LOGGER.debug('failed for %s %s' % (key, e))
# set the outputs
response.update_status('*** creating output tar archive ****', 90)
tarout_file = archive(filenames)
response.outputs['tarout'].file = tarout_file
return response
def get_PAmask(coordinates=[], nc=None):
"""
generates a matrix with 1/0 values over land areas. (NaN for water regions)
:param coordinates: 2D array with lat lon coordinates representing tree observation
:param domain: region (default='EUR-11')
:return : PAmask
"""
from scipy import spatial
import numpy as np
import numpy.ma as ma
# from netCDF4 import Dataset
# from flyingpigeon import config
# DIR_MASKS = config.masks_dir()
from flyingpigeon.utils import get_variable
from flyingpigeon.utils import get_coordinates, get_values # unrotate_pole,
lats, lons = np.array(get_coordinates(nc)) # unrotate_pole(nc, write_to_file=False))
sftlf = get_values(nc)[0, :, :]
#
# sftlf[sftlf.mask is True] = 0
# sftlf[sftlf.mask is False] = np.nan
domain = sftlf.shape
lats1D = np.array(lats).ravel()
lons1D = np.array(lons).ravel()
tree = spatial.KDTree(zip(lats1D, lons1D))
l, i = tree.query(coordinates)
PA = np.zeros(len(lats1D))
PA[i] = 1
#
PAmask = PA.reshape(domain)
PAmask[sftlf.mask] = np.nan
return PAmask
def fieldmean(resource):
"""
calculating of a weighted field mean
:param resource: str or list of str containing the netCDF files pathes
:return list: averaged values
"""
from flyingpigeon.utils import get_values, get_coordinates
from numpy import radians, average, cos, sqrt
data = get_values(resource) # np.squeeze(ds.variables[variable][:])
dim = data.shape
if len(data.shape) == 3:
# TODO if data.shape == 2 , 4 ...
lats, lons = get_coordinates(resource, unrotate=True)
if len(lats.shape) == 2:
# TODO: calcult weighed average with 2D lats (rotated pole coordinates)
lats, lons = get_coordinates(resource)
if dim[0] == len(lats):
lat_index = 0
elif dim[1] == len(lats):
lat_index = 1
elif dim[2] == len(lats):
lat_index = 2
else:
LOGGER.exception(
'length of latitude is not matching values dimensions')
lat_w = sqrt(cos(lats * radians(1)))
meanLon = average(data, axis=lat_index, weights=lat_w)
meanTimeserie = average(meanLon, axis=1)
print('fieldmean calculated')
else:
print('not 3D shaped data. Average can not be calculated')
return meanTimeserie
def fieldmean(resource):
"""
calculating of a weighted field mean
:param resource: str or list of str containing the netCDF files pathes
:return list: timeseries of the averaged values per timepstep
"""
from flyingpigeon.utils import get_values, get_coordinates, get_index_lat
from numpy import radians, average, cos, sqrt, array
data = get_values(resource) # np.squeeze(ds.variables[variable][:])
dim = data.shape
LOGGER.debug(data.shape)
if len(data.shape) == 3:
# TODO if data.shape == 2 , 4 ...
lats, lons = get_coordinates(resource, unrotate=False)
lats = array(lats)
if len(lats.shape) == 2:
lats = lats[:, 0]
else:
LOGGER.debug('Latitudes not reduced to 1D')
# TODO: calculat weighed average with 2D lats (rotated pole coordinates)
# lats, lons = get_coordinates(resource, unrotate=False)
# if len(lats.shape) == 2:
# lats, lons = get_coordinates(resource)
lat_index = get_index_lat(resource)
LOGGER.debug('lats dimension %s ' % len(lats.shape))
LOGGER.debug('lats index %s' % lat_index)
lat_w = sqrt(cos(lats * radians(1)))
meanLon = average(data, axis=lat_index, weights=lat_w)
meanTimeserie = average(meanLon, axis=1)
LOGGER.debug('fieldmean calculated')
else:
LOGGER.error('not 3D shaped data. Average can not be calculated')
return meanTimeserie
def calc_indice_percentile(resources=[],
variable=None,
prefix=None,
indices='TG90p',
refperiod=None,
groupings='yr',
polygons=None,
percentile=90,
mosaic=False,
dir_output=None,
dimension_map=None):
"""
Calculates given indices for suitable files in the appropriate time grouping and polygon.
:param resource: list of filenames in data reference syntax (DRS) convention (netcdf)
:param variable: variable name to be selected in the in netcdf file (default=None)
:param indices: list of indices (default ='TG90p')
:param prefix: filename prefix
:param refperiod: reference period tuple = (start,end)
:param grouping: indices time aggregation (default='yr')
:param dir_output: output directory for result file (netcdf)
:param dimension_map: optional dimension map if different to standard (default=None)
:return: list of netcdf files with calculated indices. Files are saved into out_dir.
"""
from os.path import join, dirname, exists
from os import remove
import uuid
from numpy import ma
from datetime import datetime as dt
from flyingpigeon.ocgis_module import call
from flyingpigeon.subset import clipping
from flyingpigeon.utils import get_values, get_time
if type(resources) != list:
resources = list([resources])
if type(indices) != list:
indices = list([indices])
if type(groupings) != list:
groupings = list([groupings])
if type(refperiod) == list:
refperiod = refperiod[0]
if refperiod is None:
start = dt.strptime(refperiod.split('-')[0], '%Y%m%d')
end = dt.strptime(refperiod.split('-')[1], '%Y%m%d')
time_range = [start, end]
else:
time_range = None
if dir_output is None:
if not exists(dir_output):
makedirs(dir_output)
################################################
# Compute a custom percentile basis using ICCLIM
################################################
from ocgis.contrib import library_icclim as lic
nc_indices = []
nc_dic = sort_by_filename(resources)
for grouping in groupings:
calc_group = calc_grouping(grouping)
for key in nc_dic.keys():
resource = nc_dic[key]
if variable is None:
variable = get_variable(resource)
if polygons is None:
nc_reference = call(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
dir_output=dir_output)
else:
nc_reference = clipping(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
polygons=polygons,
dir_output=dir_output,
mosaic=mosaic)
arr = get_values(resource=nc_reference)
dt_arr = get_time(resource=nc_reference)
arr = ma.masked_array(arr)
dt_arr = ma.masked_array(dt_arr)
percentile = percentile
window_width = 5
for indice in indices:
name = indice.replace('_', str(percentile))
var = indice.split('_')[0]
operation = None
if 'T' in var:
if percentile >= 50:
operation = 'Icclim%s90p' % var
func = 'icclim_%s90p' % var # icclim_TG90p
else:
operation = 'Icclim%s10p' % var
func = 'icclim_%s10p' % var
################################
# load the appropriate operation
################################
ops = [op for op in dir(lic) if operation in op]
if len(ops) == 0:
raise Exception("operator does not exist %s", operation)
exec "percentile_dict = lic.%s.get_percentile_dict(arr, dt_arr, percentile, window_width)" % ops[
0]
calc = [{
'func': func,
'name': name,
'kwds': {
'percentile_dict': percentile_dict
}
}]
if polygons is None:
nc_indices.extend(
call(resource=resource,
prefix=key.replace(variable, name).replace(
'_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output))
else:
nc_indices.extend(
clipping(
resource=resource,
prefix=key.replace(variable, name).replace(
'_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output,
polygons=polygons,
mosaic=mosaic,
))
if len(nc_indices) is 0:
logger.debug('No indices are calculated')
return None
return nc_indices
def map_spatial_analog(ncfile, variable='dissimilarity', cmap='viridis', title='Spatial analog'):
"""Return a matplotlib Figure instance showing a map of the dissimilarity measure.
"""
import netCDF4 as nc
from flyingpigeon import utils
from mpl_toolkits.axes_grid import make_axes_locatable
import matplotlib.axes as maxes
try:
var = utils.get_values(ncfile, variable)
LOGGER.info('Data loaded')
lats, lons = utils.get_coordinates(ncfile, variable=variable, unrotate=False)
if len(lats.shape) == 1:
cyclic_var, cyclic_lons = add_cyclic_point(var, coord=lons)
lons = cyclic_lons.data
var = cyclic_var
with nc.Dataset(ncfile) as D:
V = D.variables[variable]
lon, lat = map(float, V.target_location.split(','))
LOGGER.info('Lat and lon loaded')
except Exception as e:
msg = 'Failed to get data for plotting: {0}\n{1}'.format(ncfile, e)
LOGGER.exception(msg)
raise Exception(msg)
try:
fig = plt.figure(facecolor='w', edgecolor='k')
fig.subplots_adjust(top=.95, bottom=.05, left=.03, right=.95)
ax = plt.axes(
projection=ccrs.Robinson(central_longitude=int(np.mean(lons))))
divider = make_axes_locatable(ax)
cax = divider.new_horizontal("4%", pad=0.15, axes_class=maxes.Axes)
fig.add_axes(cax)
ax.plot(lon, lat, marker='o', mfc='#292421', ms=13, transform=ccrs.PlateCarree())
ax.plot(lon, lat, marker='o', mfc='#ffffff', ms=7, transform=ccrs.PlateCarree())
cs = ax.contourf(lons, lats, var, 60,
transform=ccrs.PlateCarree(),
cmap=cmap, interpolation='nearest')
ax.coastlines(color='k', linewidth=.8)
ax.set_title(title)
cb = plt.colorbar(cs, cax=cax, orientation='vertical')
cb.set_label(u"– Dissimilarity +") # ha='left', va='center')
cb.set_ticks([])
except:
msg = 'failed to plot graphic'
LOGGER.exception(msg)
LOGGER.info('Plot created and figure saved')
return fig
def map_robustness(signal, high_agreement_mask, low_agreement_mask,
variable=None, cmap='seismic', title=None,
file_extension='png'):
"""
generates a graphic for the output of the ensembleRobustness process for a lat/long file.
:param signal: netCDF file containing the signal difference over time
:param highagreement:
:param lowagreement:
:param variable:
:param cmap: default='seismic',
:param title: default='Model agreement of signal'
:returns str: path/to/file.png
"""
from flyingpigeon import utils
from numpy import mean, ma
if variable is None:
variable = utils.get_variable(signal)
try:
var_signal = utils.get_values(signal)
mask_l = utils.get_values(low_agreement_mask)
mask_h = utils.get_values(high_agreement_mask)
# mask_l = ma.masked_where(low < 0.5, low)
# mask_h = ma.masked_where(high < 0.5, high)
# mask_l[mask_l == 0] = np.nan
# mask_h[mask_h == 0] = np.nan
LOGGER.info('data loaded')
lats, lons = utils.get_coordinates(signal, unrotate=True)
if len(lats.shape) == 1:
cyclic_var, cyclic_lons = add_cyclic_point(var_signal, coord=lons)
mask_l, cyclic_lons = add_cyclic_point(mask_l, coord=lons)
mask_h, cyclic_lons = add_cyclic_point(mask_h, coord=lons)
lons = cyclic_lons.data
var_signal = cyclic_var
LOGGER.info('lat lon loaded')
minval = round(np.nanmin(var_signal))
maxval = round(np.nanmax(var_signal)+.5)
LOGGER.info('prepared data for plotting')
except:
msg = 'failed to get data for plotting'
LOGGER.exception(msg)
raise Exception(msg)
try:
fig = plt.figure(facecolor='w', edgecolor='k')
ax = plt.axes(projection=ccrs.Robinson(central_longitude=int(mean(lons))))
norm = MidpointNormalize(midpoint=0)
cs = plt.contourf(lons, lats, var_signal, 60, norm=norm, transform=ccrs.PlateCarree(),
cmap=cmap, interpolation='nearest')
cl = plt.contourf(lons, lats, mask_l, 1, transform=ccrs.PlateCarree(), colors='none', hatches=[None, '/'])
ch = plt.contourf(lons, lats, mask_h, 1, transform=ccrs.PlateCarree(), colors='none', hatches=[None, '.'])
# artists, labels = ch.legend_elements()
# plt.legend(artists, labels, handleheight=2)
# plt.clim(minval,maxval)
ax.coastlines()
ax.gridlines()
# ax.set_global()
if title is None:
plt.title('%s with Agreement' % variable)
else:
plt.title(title)
plt.colorbar(cs)
plt.annotate('// = low model ensemble agreement', (0, 0), (0, -10),
xycoords='axes fraction', textcoords='offset points', va='top')
plt.annotate('.. = high model ensemble agreement', (0, 0), (0, -20),
xycoords='axes fraction', textcoords='offset points', va='top')
graphic = fig2plot(fig=fig, file_extension=file_extension)
plt.close()
LOGGER.info('Plot created and figure saved')
except:
msg = 'failed to plot graphic'
LOGGER.exception(msg)
return graphic
def calc_indice_percentile(resources=[], variable=None,
prefix=None, indices='TG90p', refperiod=None,
groupings='yr', polygons=None, percentile=90, mosaik = False,
dir_output=None, dimension_map = None):
"""
Calculates given indices for suitable files in the appopriate time grouping and polygon.
:param resource: list of filenames in drs convention (netcdf)
:param variable: variable name to be selected in the in netcdf file (default=None)
:param indices: list of indices (default ='TG90p')
:param prefix: filename prefix
:param refperiod: reference refperiod touple = (start,end)
:param grouping: indices time aggregation (default='yr')
:param dir_output: output directory for result file (netcdf)
:param dimension_map: optional dimension map if different to standard (default=None)
:return: list of netcdf files with calculated indices. Files are saved into out_dir
"""
from os.path import join, dirname, exists
from os import remove
import uuid
from numpy import ma
from datetime import datetime as dt
from flyingpigeon.ocgis_module import call
from flyingpigeon.subset import clipping
from flyingpigeon.utils import get_values, get_time
if type(resources) != list:
resources = list([resources])
if type(indices) != list:
indices = list([indices])
if type(groupings) != list:
groupings = list([groupings])
if type(refperiod) == list:
refperiod = refperiod[0]
if refperiod != None:
start = dt.strptime(refperiod.split('-')[0] , '%Y%m%d')
end = dt.strptime(refperiod.split('-')[1] , '%Y%m%d')
time_range = [start, end]
else:
time_range = None
if dir_output != None:
if not exists(dir_output):
makedirs(dir_output)
########################################################################################################################
# Compute a custom percentile basis using ICCLIM. ######################################################################
########################################################################################################################
from ocgis.contrib import library_icclim as lic
nc_indices = []
nc_dic = sort_by_filename(resources)
for grouping in groupings:
calc_group = calc_grouping(grouping)
for key in nc_dic.keys():
resource = nc_dic[key]
if variable == None:
variable = get_variable(resource)
if polygons == None:
nc_reference = call(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
dir_output=dir_output)
else:
nc_reference = clipping(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
polygons=polygons,
dir_output=dir_output,
mosaik = mosaik)
arr = get_values(nc_files=nc_reference)
dt_arr = get_time(nc_files=nc_reference)
arr = ma.masked_array(arr)
dt_arr = ma.masked_array(dt_arr)
percentile = percentile
window_width = 5
for indice in indices:
name = indice.replace('_', str(percentile))
var = indice.split('_')[0]
operation = None
if 'T' in var:
if percentile >= 50:
operation = 'Icclim%s90p' % var
func = 'icclim_%s90p' % var # icclim_TG90p
else:
operation = 'Icclim%s10p' % var
func = 'icclim_%s10p' % var
################################
# load the appropriate operation
################################
ops = [op for op in dir(lic) if operation in op]
if len(ops) == 0:
raise Exception("operator does not exist %s", operation)
exec "percentile_dict = lic.%s.get_percentile_dict(arr, dt_arr, percentile, window_width)" % ops[0]
calc = [{'func': func, 'name': name, 'kwds': {'percentile_dict': percentile_dict}}]
if polygons == None:
nc_indices.append(call(resource=resource,
prefix=key.replace(variable,name).replace('_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output))
else:
nc_indices.extend(clipping(resource=resource,
prefix=key.replace(variable,name).replace('_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output,
polygons=polygons,
mosaik = mosaik,
))
return nc_indices
def map_robustness(signal,
high_agreement_mask,
low_agreement_mask,
cmap='seismic',
title=None):
"""
generates a graphic for the output of the ensembleRobustness process for a lat/long file.
:param signal: netCDF file containing the signal difference over time
:param highagreement:
:param lowagreement:
:param variable:
:param cmap: default='seismic',
:param title: default='Model agreement of signal'
:returns str: path/to/file.png
"""
try:
# from flyingpigeon.utils import get_values
from cartopy.util import add_cyclic_point
# from flyingpigeon.utils import get_coordinates
var_signal = utils.get_values(signal)
mask_l = utils.get_values(low_agreement_mask)
mask_h = utils.get_values(high_agreement_mask)
mask_l.mask = var_signal.mask
mask_h.mask = var_signal.mask
mask_l.mask[mask_l.data is 0] = False
mask_h.mask[mask_h.data is 0] = False
LOGGER.debug('values loaded')
lats, lons = utils.get_coordinates(signal)
#
# cyclic_var, cyclic_lons = add_cyclic_point(var_signal, coord=lons)
# mask_l, cyclic_lons = add_cyclic_point(mask_l, coord=lons)
# mask_h, cyclic_lons = add_cyclic_point(mask_h, coord=lons)
#
# lons = cyclic_lons.data
# var_signal = cyclic_var
#
LOGGER.debug('coordinates loaded')
#
minval = round(np.nanmin(var_signal))
maxval = round(np.nanmax(var_signal) + .5)
LOGGER.info('prepared data for plotting')
except:
msg = 'failed to get data for plotting'
LOGGER.exception(msg)
try:
fig = plt.figure(facecolor='w',
edgecolor='k') # figsize=(20,10), dpi=600,
ax = plt.axes(projection=ccrs.Robinson(central_longitude=0))
norm = MidpointNormalize(midpoint=0)
cs = plt.contourf(lons,
lats,
var_signal,
60,
transform=ccrs.PlateCarree(),
norm=norm,
cmap=cmap,
interpolation='nearest')
cl = plt.contourf(lons,
lats,
mask_l,
60,
transform=ccrs.PlateCarree(),
colors='none',
hatches=['//'])
ch = plt.contourf(lons,
lats,
mask_h,
60,
transform=ccrs.PlateCarree(),
colors='none',
hatches=['.'])
# plt.clim(minval, maxval)
ax.coastlines()
# ax.set_global()
if title is None:
plt.title('Robustness')
else:
plt.title(title)
plt.colorbar(cs)
plt.annotate('// = low model ensemble agreement', (0, 0), (0, -10),
xycoords='axes fraction',
textcoords='offset points',
va='top')
plt.annotate('.. = high model ensemble agreement', (0, 0), (0, -20),
xycoords='axes fraction',
textcoords='offset points',
va='top')
_, graphic = mkstemp(dir='.', suffix='.png')
fig.savefig(graphic)
plt.close()
LOGGER.info('Plot created and figure saved')
except:
msg = 'failed to plot graphic'
LOGGER.exception(msg)
_, graphic = mkstemp(dir='.', suffix='.png')
return graphic
def calc_indice_percentile(resources=[], variable=None,
prefix=None, indices='TG90p', refperiod=None,
groupings='yr', polygons=None, percentile=90, mosaic = False,
dir_output=None, dimension_map = None):
"""
Calculates given indices for suitable files in the appropriate time grouping and polygon.
:param resource: list of filenames in data reference syntax (DRS) convention (netcdf)
:param variable: variable name to be selected in the in netcdf file (default=None)
:param indices: list of indices (default ='TG90p')
:param prefix: filename prefix
:param refperiod: reference period tuple = (start,end)
:param grouping: indices time aggregation (default='yr')
:param dir_output: output directory for result file (netcdf)
:param dimension_map: optional dimension map if different to standard (default=None)
:return: list of netcdf files with calculated indices. Files are saved into out_dir.
"""
from os.path import join, dirname, exists
from os import remove
import uuid
from numpy import ma
from datetime import datetime as dt
from flyingpigeon.ocgis_module import call
from flyingpigeon.subset import clipping
from flyingpigeon.utils import get_values, get_time
if type(resources) != list:
resources = list([resources])
if type(indices) != list:
indices = list([indices])
if type(groupings) != list:
groupings = list([groupings])
if type(refperiod) == list:
refperiod = refperiod[0]
if refperiod != None:
start = dt.strptime(refperiod.split('-')[0] , '%Y%m%d')
end = dt.strptime(refperiod.split('-')[1] , '%Y%m%d')
time_range = [start, end]
else:
time_range = None
if dir_output != None:
if not exists(dir_output):
makedirs(dir_output)
########################################################################################################################
# Compute a custom percentile basis using ICCLIM. ######################################################################
########################################################################################################################
from ocgis.contrib import library_icclim as lic
nc_indices = []
nc_dic = sort_by_filename(resources)
for grouping in groupings:
calc_group = calc_grouping(grouping)
for key in nc_dic.keys():
resource = nc_dic[key]
if variable == None:
variable = get_variable(resource)
if polygons == None:
nc_reference = call(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
dir_output=dir_output)
else:
nc_reference = clipping(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
polygons=polygons,
dir_output=dir_output,
mosaic = mosaic)
arr = get_values(resource=nc_reference)
dt_arr = get_time(resource=nc_reference)
arr = ma.masked_array(arr)
dt_arr = ma.masked_array(dt_arr)
percentile = percentile
window_width = 5
for indice in indices:
name = indice.replace('_', str(percentile))
var = indice.split('_')[0]
operation = None
if 'T' in var:
if percentile >= 50:
operation = 'Icclim%s90p' % var
func = 'icclim_%s90p' % var # icclim_TG90p
else:
operation = 'Icclim%s10p' % var
func = 'icclim_%s10p' % var
################################
# load the appropriate operation
################################
ops = [op for op in dir(lic) if operation in op]
if len(ops) == 0:
raise Exception("operator does not exist %s", operation)
exec "percentile_dict = lic.%s.get_percentile_dict(arr, dt_arr, percentile, window_width)" % ops[0]
calc = [{'func': func, 'name': name, 'kwds': {'percentile_dict': percentile_dict}}]
if polygons == None:
nc_indices.append(call(resource=resource,
prefix=key.replace(variable,name).replace('_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output))
else:
nc_indices.extend(clipping(resource=resource,
prefix=key.replace(variable,name).replace('_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output,
polygons=polygons,
mosaic = mosaic,
))
return nc_indices
#def calc_indice_unconventional(resource=[], variable=None, prefix=None,
#indices=None, polygons=None, groupings=None,
#dir_output=None, dimension_map = None):
#"""
#Calculates given indices for suitable files in the appropriate time grouping and polygon.
#:param resource: list of filenames in data reference syntax (DRS) convention (netcdf)
#:param variable: variable name to be selected in the in netcdf file (default=None)
#:param indices: list of indices (default ='TGx')
#:param polygons: list of polygons (default =None)
#:param grouping: indices time aggregation (default='yr')
#:param out_dir: output directory for result file (netcdf)
#:param dimension_map: optional dimension map if different to standard (default=None)
#:return: list of netcdf files with calculated indices. Files are saved into dir_output
#"""
#from os.path import join, dirname, exists
#from os import remove
#import uuid
#from flyingpigeon import ocgis_module
#from flyingpigeon.subset import get_ugid, get_geom
#if type(resource) != list:
#resource = list([resource])
#if type(indices) != list:
#indices = list([indices])
#if type(polygons) != list and polygons != None:
#polygons = list([polygons])
#elif polygons == None:
#polygons = [None]
#else:
#logger.error('Polygons not found')
#if type(groupings) != list:
#groupings = list([groupings])
#if dir_output != None:
#if not exists(dir_output):
#makedirs(dir_output)
#experiments = sort_by_filename(resource)
#outputs = []
#print('environment for calc_indice_unconventional set')
#logger.info('environment for calc_indice_unconventional set')
#for key in experiments:
#if variable == None:
#variable = get_variable(experiments[key][0])
#try:
#ncs = experiments[key]
#for indice in indices:
#logger.info('indice: %s' % indice)
#try:
#for grouping in groupings:
#logger.info('grouping: %s' % grouping)
#try:
#calc_group = calc_grouping(grouping)
#logger.info('calc_group: %s' % calc_group)
#for polygon in polygons:
#try:
#domain = key.split('_')[1].split('-')[0]
#if polygon == None:
#if prefix == None:
#prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping )
#geom = None
#ugid = None
#else:
#if prefix == None:
#prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping ).replace(domain,polygon)
#geom = get_geom(polygon=polygon)
#ugid = get_ugid(polygons=polygon, geom=geom)
#if indice == 'TGx':
#calc=[{'func': 'max', 'name': 'TGx'}]
#tmp = ocgis_module.call(resource=ncs,# conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, calc_grouping=calc_group, prefix=prefix,
#dir_output=dir_output, geom=geom, select_ugid=ugid)
#elif indice == 'TGn':
#calc=[{'func': 'min', 'name': 'TGn'}]
#tmp = ocgis_module.call(resource=ncs, #conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, calc_grouping= calc_group, prefix=prefix,
#dir_output=dir_output, geom=geom, select_ugid = ugid)
#elif indice == 'TGx5day':
#calc = [{'func': 'moving_window', 'name': 'TGx5day', 'kwds': {'k': 5, 'operation': 'mean', 'mode': 'same' }}]
#tmp2 = ocgis_module.call(resource=ncs, #conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, prefix=str(uuid.uuid4()),
#geom=geom, select_ugid = ugid)
#calc=[{'func': 'max', 'name': 'TGx5day'}]
#logger.info('moving window calculated : %s' % tmp2)
#tmp = ocgis_module.call(resource=tmp2,
#variable=indice, dimension_map=dimension_map,
#calc=calc, calc_grouping=calc_group, prefix=prefix,
#dir_output=dir_output)
#remove(tmp2)
#elif indice == 'TGn5day':
#calc = [{'func': 'moving_window', 'name': 'TGn5day', 'kwds': {'k': 5, 'operation': 'mean', 'mode': 'same' }}]
#tmp2 = ocgis_module.call(resource=ncs, #conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, prefix=str(uuid.uuid4()),
#geom=geom, select_ugid = ugid)
#calc=[{'func': 'min', 'name': 'TGn5day'}]
#logger.info('moving window calculated : %s' % tmp2)
#tmp = ocgis_module.call(resource=tmp2,
#variable=indice, dimension_map=dimension_map,
#calc=calc, calc_grouping=calc_group, prefix=prefix,
#dir_output=dir_output)
#remove(tmp2)
#else:
#logger.error('Indice %s is not a known inidce' % (indice))
#outputs.append(tmp)
#logger.info('indice file calcualted %s ' % (tmp))
#except Exception as e:
#logger.debug('could not calc indice %s for key %s, polygon %s and calc_grouping %s : %s' % (indice, key, polygon, grouping, e ))
#except Exception as e:
#logger.debug('could not calc indice %s for key %s and calc_grouping %s : %s' % ( indice, key, polygon, e ))
#except Exception as e:
#logger.debug('could not calc indice %s for key %s: %s'% (indice, key, e ))
#except Exception as e:
#logger.debug('could not calc key %s: %s' % (key, e))
#return outputs
def calc_indice_percentile(resources=[],
variable=None,
prefix=None,
indices='TG90p',
refperiod=None,
groupings='yr',
polygons=None,
percentile=90,
mosaic=False,
dir_output=None,
dimension_map=None):
"""
Calculates given indices for suitable files in the appropriate time grouping and polygon.
:param resource: list of filenames in data reference syntax (DRS) convention (netcdf)
:param variable: variable name to be selected in the in netcdf file (default=None)
:param indices: list of indices (default ='TG90p')
:param prefix: filename prefix
:param refperiod: reference period tuple = (start,end)
:param grouping: indices time aggregation (default='yr')
:param dir_output: output directory for result file (netcdf)
:param dimension_map: optional dimension map if different to standard (default=None)
:return: list of netcdf files with calculated indices. Files are saved into out_dir.
"""
from os.path import join, dirname, exists
from os import remove
import uuid
from numpy import ma
from datetime import datetime as dt
from flyingpigeon.ocgis_module import call
from flyingpigeon.subset import clipping
from flyingpigeon.utils import get_values, get_time
if type(resources) != list:
resources = list([resources])
if type(indices) != list:
indices = list([indices])
if type(groupings) != list:
groupings = list([groupings])
if type(refperiod) == list:
refperiod = refperiod[0]
if refperiod != None:
start = dt.strptime(refperiod.split('-')[0], '%Y%m%d')
end = dt.strptime(refperiod.split('-')[1], '%Y%m%d')
time_range = [start, end]
else:
time_range = None
if dir_output != None:
if not exists(dir_output):
makedirs(dir_output)
########################################################################################################################
# Compute a custom percentile basis using ICCLIM. ######################################################################
########################################################################################################################
from ocgis.contrib import library_icclim as lic
nc_indices = []
nc_dic = sort_by_filename(resources)
for grouping in groupings:
calc_group = calc_grouping(grouping)
for key in nc_dic.keys():
resource = nc_dic[key]
if variable == None:
variable = get_variable(resource)
if polygons == None:
nc_reference = call(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
dir_output=dir_output)
else:
nc_reference = clipping(resource=resource,
prefix=str(uuid.uuid4()),
time_range=time_range,
output_format='nc',
polygons=polygons,
dir_output=dir_output,
mosaic=mosaic)
arr = get_values(resource=nc_reference)
dt_arr = get_time(resource=nc_reference)
arr = ma.masked_array(arr)
dt_arr = ma.masked_array(dt_arr)
percentile = percentile
window_width = 5
for indice in indices:
name = indice.replace('_', str(percentile))
var = indice.split('_')[0]
operation = None
if 'T' in var:
if percentile >= 50:
operation = 'Icclim%s90p' % var
func = 'icclim_%s90p' % var # icclim_TG90p
else:
operation = 'Icclim%s10p' % var
func = 'icclim_%s10p' % var
################################
# load the appropriate operation
################################
ops = [op for op in dir(lic) if operation in op]
if len(ops) == 0:
raise Exception("operator does not exist %s", operation)
exec "percentile_dict = lic.%s.get_percentile_dict(arr, dt_arr, percentile, window_width)" % ops[
0]
calc = [{
'func': func,
'name': name,
'kwds': {
'percentile_dict': percentile_dict
}
}]
if polygons == None:
nc_indices.append(
call(resource=resource,
prefix=key.replace(variable, name).replace(
'_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output))
else:
nc_indices.extend(
clipping(
resource=resource,
prefix=key.replace(variable, name).replace(
'_day_', '_%s_' % grouping),
calc=calc,
calc_grouping=calc_group,
output_format='nc',
dir_output=dir_output,
polygons=polygons,
mosaic=mosaic,
))
return nc_indices
#def calc_indice_unconventional(resource=[], variable=None, prefix=None,
#indices=None, polygons=None, groupings=None,
#dir_output=None, dimension_map = None):
#"""
#Calculates given indices for suitable files in the appropriate time grouping and polygon.
#:param resource: list of filenames in data reference syntax (DRS) convention (netcdf)
#:param variable: variable name to be selected in the in netcdf file (default=None)
#:param indices: list of indices (default ='TGx')
#:param polygons: list of polygons (default =None)
#:param grouping: indices time aggregation (default='yr')
#:param out_dir: output directory for result file (netcdf)
#:param dimension_map: optional dimension map if different to standard (default=None)
#:return: list of netcdf files with calculated indices. Files are saved into dir_output
#"""
#from os.path import join, dirname, exists
#from os import remove
#import uuid
#from flyingpigeon import ocgis_module
#from flyingpigeon.subset import get_ugid, get_geom
#if type(resource) != list:
#resource = list([resource])
#if type(indices) != list:
#indices = list([indices])
#if type(polygons) != list and polygons != None:
#polygons = list([polygons])
#elif polygons == None:
#polygons = [None]
#else:
#logger.error('Polygons not found')
#if type(groupings) != list:
#groupings = list([groupings])
#if dir_output != None:
#if not exists(dir_output):
#makedirs(dir_output)
#experiments = sort_by_filename(resource)
#outputs = []
#print('environment for calc_indice_unconventional set')
#logger.info('environment for calc_indice_unconventional set')
#for key in experiments:
#if variable == None:
#variable = get_variable(experiments[key][0])
#try:
#ncs = experiments[key]
#for indice in indices:
#logger.info('indice: %s' % indice)
#try:
#for grouping in groupings:
#logger.info('grouping: %s' % grouping)
#try:
#calc_group = calc_grouping(grouping)
#logger.info('calc_group: %s' % calc_group)
#for polygon in polygons:
#try:
#domain = key.split('_')[1].split('-')[0]
#if polygon == None:
#if prefix == None:
#prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping )
#geom = None
#ugid = None
#else:
#if prefix == None:
#prefix = key.replace(variable, indice).replace('_day_','_%s_' % grouping ).replace(domain,polygon)
#geom = get_geom(polygon=polygon)
#ugid = get_ugid(polygons=polygon, geom=geom)
#if indice == 'TGx':
#calc=[{'func': 'max', 'name': 'TGx'}]
#tmp = ocgis_module.call(resource=ncs,# conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, calc_grouping=calc_group, prefix=prefix,
#dir_output=dir_output, geom=geom, select_ugid=ugid)
#elif indice == 'TGn':
#calc=[{'func': 'min', 'name': 'TGn'}]
#tmp = ocgis_module.call(resource=ncs, #conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, calc_grouping= calc_group, prefix=prefix,
#dir_output=dir_output, geom=geom, select_ugid = ugid)
#elif indice == 'TGx5day':
#calc = [{'func': 'moving_window', 'name': 'TGx5day', 'kwds': {'k': 5, 'operation': 'mean', 'mode': 'same' }}]
#tmp2 = ocgis_module.call(resource=ncs, #conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, prefix=str(uuid.uuid4()),
#geom=geom, select_ugid = ugid)
#calc=[{'func': 'max', 'name': 'TGx5day'}]
#logger.info('moving window calculated : %s' % tmp2)
#tmp = ocgis_module.call(resource=tmp2,
#variable=indice, dimension_map=dimension_map,
#calc=calc, calc_grouping=calc_group, prefix=prefix,
#dir_output=dir_output)
#remove(tmp2)
#elif indice == 'TGn5day':
#calc = [{'func': 'moving_window', 'name': 'TGn5day', 'kwds': {'k': 5, 'operation': 'mean', 'mode': 'same' }}]
#tmp2 = ocgis_module.call(resource=ncs, #conform_units_to='celcius',
#variable=variable, dimension_map=dimension_map,
#calc=calc, prefix=str(uuid.uuid4()),
#geom=geom, select_ugid = ugid)
#calc=[{'func': 'min', 'name': 'TGn5day'}]
#logger.info('moving window calculated : %s' % tmp2)
#tmp = ocgis_module.call(resource=tmp2,
#variable=indice, dimension_map=dimension_map,
#calc=calc, calc_grouping=calc_group, prefix=prefix,
#dir_output=dir_output)
#remove(tmp2)
#else:
#logger.error('Indice %s is not a known inidce' % (indice))
#outputs.append(tmp)
#logger.info('indice file calcualted %s ' % (tmp))
#except Exception as e:
#logger.debug('could not calc indice %s for key %s, polygon %s and calc_grouping %s : %s' % (indice, key, polygon, grouping, e ))
#except Exception as e:
#logger.debug('could not calc indice %s for key %s and calc_grouping %s : %s' % ( indice, key, polygon, e ))
#except Exception as e:
#logger.debug('could not calc indice %s for key %s: %s'% (indice, key, e ))
#except Exception as e:
#logger.debug('could not calc key %s: %s' % (key, e))
#return outputs
