def robustness_cc_signal(variable_mean,
standard_deviation=None,
variable=None):
"""
Claculating the Climate Change signal based on the output of robustness_stats.
:param variable_mean: list of two 2D spatial netCDF files
in the order of [refenence, projection]
:param standard_deviation: according to variable_mean files 2D netCDF files of the standard deviation
:return netCDF files: cc_signal.nc, mean_std.nc
"""
basename_ref = basename(variable_mean[0]).split('_')
basename_proj = basename(variable_mean[1]).split('_')
# ensstd_tg_mean_1981-01-01-2010-12-31.nc'
if variable is None:
variable = get_variable(variable_mean[0])
ds = Dataset(variable_mean[0])
vals_ref = np.squeeze(ds[variable][:])
ds.close()
ds = Dataset(variable_mean[1])
vals_proj = np.squeeze(ds[variable][:])
ds.close()
if standard_deviation is not None:
ds = Dataset(standard_deviation[0])
std_ref = np.squeeze(ds[variable][:])
ds.close()
ds = Dataset(standard_deviation[1])
std_proj = np.squeeze(ds[variable][:])
ds.close()
bn_mean_std = 'mean-std_{}_{}_{}'.format(basename_ref[1],
basename_ref[-2],
basename_proj[-1])
out_mean_std = copyfile(standard_deviation[0], bn_mean_std)
ds_median_std = Dataset(out_mean_std, mode='a')
ds_median_std[variable][:] = (std_ref + std_proj) / 2
ds_median_std.close()
else:
out_mean_std = None
bn_cc_signal = 'cc-signal_{}_{}_{}'.format(basename_ref[1],
basename_ref[-2],
basename_proj[-1])
out_cc_signal = copyfile(variable_mean[0], bn_cc_signal)
ds_cc = Dataset(out_cc_signal, mode='a')
ds_cc[variable][:] = np.squeeze(vals_proj - vals_ref)
ds_cc.close()
return out_cc_signal, out_mean_std
def _handler(self, request, response):
# init_process_logger('log.txt')
# response.outputs['output_log'].file = 'log.txt'
ncfiles = extract_archive(
resources=[inpt.file for inpt in request.inputs['resource']],
dir_output=self.workdir)
if 'variable' in request.inputs:
var = request.inputs['variable'][0].data
else:
var = get_variable(ncfiles[0])
# var = ncfiles[0].split("_")[0]
response.update_status('plotting variable {}'.format(var), 10)
try:
plotout_spagetti_file = plt_ncdata.spaghetti(ncfiles,
variable=var,
title='Field mean of {}'.format(var),
dir_output=self.workdir,
)
LOGGER.info("spagetti plot done")
response.update_status('Spagetti plot for %s %s files done' % (len(ncfiles), var), 50)
response.outputs['plotout_spagetti'].file = plotout_spagetti_file
except Exception as e:
raise Exception("spagetti plot failed : {}".format(e))
try:
plotout_uncertainty_file = plt_ncdata.uncertainty(ncfiles,
variable=var,
title='Ensemble uncertainty for {}'.format(var),
dir_output=self.workdir,
)
response.update_status('Uncertainty plot for {} {} files done'.format(len(ncfiles), var), 90)
response.outputs['plotout_uncertainty'].file = plotout_uncertainty_file
LOGGER.info("uncertainty plot done")
except Exception as err:
raise Exception("uncertainty plot failed {}".format(err.message))
response.update_status('visualisation done', 100)
return response
def parse_variable(self, request, path):
"""Parse variables specified in request and confirm they are present in file.
If no variable is specified in the request, guess the variables from the file content.
:return: List of variable names.
"""
ds = nc.Dataset(path)
if 'variable' in request.inputs:
var_names = [v.data for v in request.inputs['variable']]
for var in var_names:
if var not in ds.variables:
raise ValueError("{} not in {}".format(var, path))
else:
var_names = get_variable(ds)
ds.close()
return var_names
def robustness_stats(resources,
time_range=[None, None],
dir_output=None,
variable=None):
"""
calculating the spatial mean and corresponding standard deviation for an ensemble of consistent datasets containing one variableself.
If a time range is given the statistical values are calculated only in the disired timeperiod.
:param resources: str or list of str containing the netCDF files paths
:param time_range: sequence of two datetime.datetime objects to mark start and end point
:param dir_output: path to folder to store ouput files (default= curdir)
:param variable: variable name containing in netCDF file. If not set, variable name gets detected
:return netCDF files: out_ensmean.nc, out_ensstd.nc
"""
from ocgis import OcgOperations, RequestDataset, env
env.OVERWRITE = True
if variable is None:
variable = get_variable(resources[0])
out_means = []
for resource in resources:
rd = RequestDataset(resource, variable)
prefix = basename(resource).replace('.nc', '')
LOGGER.debug('processing mean of {}'.format(prefix))
calc = [{
'func': 'median',
'name': variable
}] # {'func': 'median', 'name': 'monthly_median'}
ops = OcgOperations(dataset=rd,
calc=calc,
calc_grouping=['all'],
output_format='nc',
prefix='median_' + prefix,
time_range=time_range,
dir_output=dir_output)
out_means.append(ops.execute())
# nc_out = call(resource=resources, calc=[{'func': 'mean', 'name': 'ens_mean'}],
# calc_grouping='all', # time_region=time_region,
# dir_output=dir_output, output_format='nc')
####
# read in numpy array
for i, out_mean in enumerate(out_means):
if i == 0:
ds = Dataset(out_mean)
var = ds[variable][:]
dims = [len(out_means), var[:].shape[-2], var[:].shape[-1]]
vals = np.empty(dims)
vals[i, :, :] = np.squeeze(var[:])
ds.close()
else:
ds = Dataset(out_mean)
vals[i, :, :] = np.squeeze(ds[variable][:])
ds.close()
####
# calc median, std
val_median = np.nanmedian(vals, axis=0)
val_std = np.nanstd(vals, axis=0)
#####
# prepare files by copying ...
ensmean_file = 'ensmean_{}_{}_{}.nc'.format(
variable, dt.strftime(time_range[0], '%Y-%m-%d'),
dt.strftime(time_range[1], '%Y-%m-%d'))
out_ensmean = copyfile(out_means[0], join(dir_output, ensmean_file))
ensstd_file = 'ensstd_{}_{}_{}.nc'.format(
variable, dt.strftime(time_range[0], '%Y-%m-%d'),
dt.strftime(time_range[1], '%Y-%m-%d'))
out_ensstd = copyfile(out_means[0], join(dir_output, ensstd_file))
####
# write values to files
ds_median = Dataset(out_ensmean, mode='a')
ds_median[variable][:] = val_median
ds_median.close()
ds_std = Dataset(out_ensstd, mode='a')
ds_std[variable][:] = val_std
ds_std.close()
LOGGER.info('processing the overall ensemble statistical mean ')
# prefix = 'ensmean_tg-mean_{}-{}'.format(dt.strftime(time_range[0], '%Y-%m-%d'),
# dt.strftime(time_range[1], '%Y-%m-%d'))
# rd = RequestDataset(out_means, var)
# calc = [{'func': 'mean', 'name': 'mean'}] # {'func': 'median', 'name': 'monthly_median'}
# ops = OcgOperations(dataset=rd, calc=calc, calc_grouping=['all'],
# output_format=output_format, prefix='mean_'+prefix, time_range=time_range)
# ensmean = ops.execute()
return out_ensmean, out_ensstd
def plot_map_ccsignal(signal, robustness=None,
variable=None, cmap=None, title=None,
file_extension='png', vmin=None, vmax=None): # 'seismic'
"""
generates a graphic for the output of the ensembleRobustness process for a lat/long file.
:param signal: netCDF file containing the signal difference over time
:param robustness: netCDF file containing 1 and 0 corresponding to signal robustness
:param variable: variable containing the netCDF files
:param cmap: default='seismic',
:param title: default='Model agreement of signal'
:returns str: path/to/file.png
"""
# from flyingpigeon import utils
from numpy import mean, ma
if variable is None:
variable = get_variable(signal)
print('found variable in file {}'.format(variable))
try:
ds = Dataset(signal)
var_signal = ds.variables[variable]
val_signal = np.squeeze(ds.variables[variable])
lon_att = var_signal.dimensions[-1]
lat_att = var_signal.dimensions[-2]
lon = ds.variables[lon_att][:]
lat = ds.variables[lat_att][:]
lons, lats = meshgrid(lon, lat)
ds.close()
if robustness is not None:
ds = Dataset(robustness)
var_rob = get_variable(robustness)
val_rob = np.squeeze(ds.variables[var_rob][:])
ds.close()
# mask = val_signal[:] # [val_signal[:]<val_std[:]]
# mask_h = np.empty(list(val_signal[:].shape)) # [[val_signal[:] > val_std[:]]] = 1
# mask_h[(val_signal >= (val_std / 4.))] = 1 #[:]
#
# mask_l = np.empty(list(val_signal[:].shape)) # [[val_signal[:] > val_std[:]]] = 1
# mask_l[mask_h != 1] = 1
# cyclic_var, cyclic_lons = add_cyclic_point(var_signal, coord=lons)
# mask, cyclic_lons = add_cyclic_point(mask, coord=lons)
#
# lons = cyclic_lons
# var_signal = cyclic_var
LOGGER.info('prepared data for plotting')
except Exception as e:
msg = 'failed to get data for plotting: {}'.format(e)
LOGGER.exception(msg)
raise Exception(msg)
try:
fig = plt.figure(figsize=(20, 10), facecolor='w', edgecolor='k')
ax = plt.axes(projection=ccrs.PlateCarree())
# ax = plt.axes(projection=ccrs.Robinson(central_longitude=int(mean(lons))))
# minval = round(np.nanmin(var_signal))
if cmap is None:
if variable in ['pr', 'prAdjust',
'prcptot', 'rx1day',
'wetdays', # 'cdd',
'cwd', 'sdii',
'max_5_day_precipitation_amount']:
cmap = 'BrBG'
if variable in ['tas', 'tasAdjust', 'tg', 'tg_mean']:
cmap = 'seismic'
else:
cmap = 'viridis'
LOGGER.debug('variable {} not found to set the colormap'.format(variable))
maxval = round(np.nanmax(val_signal)+.5)
minval = round(np.nanmin(val_signal))
norm = MidpointNormalize( vmin=minval, vcenter=0, vmax=maxval) # ) vcenter=0,,
cs = plt.pcolormesh(lons, lats, val_signal, transform=ccrs.PlateCarree(), cmap=cmap, norm=norm, vmin=vmin, vmax=vmax)
# 60,
plt.colorbar(cs)
if robustness != None:
ch = plt.contourf(lons, lats, val_rob, transform=ccrs.PlateCarree(), hatches=[None, '/', '.'], alpha=0, colors='none', cmap=None) # colors='white'
# cl = plt.contourf(lons, lats, mask_l, 1, transform=ccrs.PlateCarree(), hatches=[None, '/'], alpha=0, colors='none', cmap=None) # ,
plt.annotate('// = low model ensemble agreement', (0, 0), (0, -10),
xycoords='axes fraction', textcoords='offset points', va='top')
plt.annotate('.. = high model ensemble agreement', (0, 0), (0, -20),
xycoords='axes fraction', textcoords='offset points', va='top')
# ch = plt.contourf(lons, lats, mask, 1, transform=ccrs.PlateCarree(), colors='none', hatches=[None,'.' ])
ax.add_feature(cfeature.BORDERS, linewidth=2, linestyle='--')
ax.add_feature(cfeature.COASTLINE, linewidth=2,) # coastlines()
gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True,
linewidth=2, color='gray', alpha=0.5, linestyle='--')
gl.xlabels_top = False
gl.ylables_right = False
gl.xlabel_style = {'size': 15, 'color': 'black'}
gl.ylabel_style = {'size': 15, 'color': 'black'}
if title != None:
plt.title(title, fontsize=20 )
plt.xticks(fontsize=16, rotation=45)
plt.yticks(fontsize=16, ) # rotation=90
# # artists, labels = ch.legend_elements()
# # plt.legend(artists, labels, handleheight=2)
#
# ax.gridlines()
# # ax.set_global()
graphic = fig2plot(fig=fig, file_extension=file_extension)
plt.close()
LOGGER.info('Plot created and figure saved')
except:
msg = 'failed to plot graphic'
LOGGER.exception(msg)
return graphic
def plot_map_timemean(resource, variable=None, time_range=None,
title=None, delta=0, cmap=None, vmin=None, vmax=None, figsize=(15, 15),
file_extension='png', dir_output='.'):
"""
creates a spatial map with the mean over the timestepps.
If multiple files are provided, a mean over all files are condidered.
:param resource: netCDF file(s) containng spatial values to be plotted.
:param variable: variable to be visualised. If None (default), variable will be detected
:param title: string to be used as title
:param delta: set a delta for the values e.g. -273.15 to convert Kelvin to Celsius
:param figsize: figure size defult=(15,15)
:param vmin: colorbar minimum
:param vmax: colorbar maximum
:param file_extension: file extinction for the graphic
:param dir_output: output directory to store the output graphic
:returns str: path/to/file.png
"""
# from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER
try:
LOGGER.debug('plot_map function read in values for {}'.format(resource))
# get values of netcdf file
if type(resource) == str:
ds = Dataset(resource)
if variable is None:
variable = get_variable(resource)
lat = ds.variables['rlat']
lon = ds.variables['rlon']
lons, lats = meshgrid(lon, lat)
var = ds.variables[variable]
var = get_values(f, time_range=time_range, variable=variable).data
var_mean = np.nanmean(var, axis=0) + delta # mean over whole periode 30 Years 1981-2010 and transform to Celsius
else:
for i, f in enumerate(resource):
if i == 0:
ds = Dataset(f)
lat = ds.variables['rlat']
lon = ds.variables['rlon']
lons, lats = meshgrid(lon, lat)
vals = get_values(f, time_range=time_range, variable=variable).data
else:
vals = np.append(vals, get_values(f, time_range=time_range, variable=variable).data, axis=0)
var_mean = np.nanmean(vals, axis=0) + delta
# prepare plot
LOGGER.info('preparing matplotlib figure')
fig = plt.figure(figsize=figsize, facecolor='w', edgecolor='k')
ax = plt.axes(projection=ccrs.PlateCarree())
cs = plt.pcolormesh(lons, lats, var_mean,
transform=ccrs.PlateCarree(), cmap=cmap,
vmin=vmin, vmax=vmax,
)
# extent=(-0,17,10.5,24)
# ax.set_extent(extent)
ax.add_feature(cfeature.BORDERS, linewidth=2, linestyle='--')
# ax.add_feature(cfeature.RIVERS)
# ax.stock_img()
# ax.gridlines(draw_labels=False)
gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True,
linewidth=2, color='gray', alpha=0.5, linestyle='--')
gl.xlabels_top = False
gl.ylables_right = False
gl.ylables_left = False
# gl.xlines = False
# gl.xlocator = mticker.FixedLocator([0, 2,4,6,8,10,12,14,16] )
# gl.xformatter = LONGITUDE_FORMATTER
# gl.yformatter = LATITUDE_FORMATTER
gl.xlabel_style = {'size': 15, 'color': 'black', 'weight': 'bold'}
gl.ylabel_style = {'size': 15, 'color': 'black', 'weight': 'bold'}
if cmap is None:
if variable in ['pr', 'prAdjust',
'prcptot', 'rx1day', 'wetdays',
'cdd', 'cwd', 'sdii',
'max_5_day_precipitation_amount']:
cmap = 'Blues'
if variable in ['tas', 'tasAdjust', 'tg', 'tg_mean']:
cmap = 'seismic'
plt.title(title, fontsize=25)
cax = fig.add_axes([ax.get_position().x1 + 0.1, ax.get_position().y0,
0.02, ax.get_position().height])
cbar = plt.colorbar(cs, cax=cax) # Similar to fig.colorbar(im, cax = cax)
cbar.ax.tick_params(labelsize=20)
# ticklabs = cbar.ax.get_yticklabels()
# cbar.ax.set_yticklabels(ticklabs, fontsize=15)
# cb = add_colorbar(cs)
LOGGER.info('Matplotlib pcolormesh plot done')
output_png = fig2plot(fig=fig, file_extension='png',
dir_output=dir_output)
plt.close()
LOGGER.debug('Plot done for %s' % variable)
except Exception as err:
raise Exception('failed to calculate quantiles. %s' % err)
return output_png
def plot_ts_uncertainty(resource, variable=None, ylim=None, title=None,
file_extension='png', delta=0, window=None, dir_output='.',
figsize=(10,10)):
"""
creates a png file containing the appropriate uncertainty plot.
:param resource: list of files containing the same variable
:param variable: variable to be visualised. If None (default), variable will be detected
:param ylim: Y-axis limitations: tuple(min,max)
:param title: string to be used as title
:param figsize: figure size defult=(10,10)
:param window: windowsize of the rolling mean
:returns str: path/to/file.png
"""
LOGGER.debug('Start visualisation uncertainty plot')
import pandas as pd
import numpy as np
from datetime import datetime as dt
from os.path import basename
#
# from flyingpigeon.utils import get_time, sort_by_filename
# from flyingpigeon.calculation import fieldmean
# from flyingpigeon.metadata import get_frequency
# === prepare invironment
if type(resource) == str:
resource = list([resource])
if variable is None:
variable = get_variable(resource[0])
if title is None:
title = "Field mean of %s " % variable
LOGGER.info('variable %s found in resource.' % variable)
try:
fig = plt.figure(figsize=figsize, facecolor='w', edgecolor='k')
dic = sort_by_filename(resource, historical_concatination=True)
# Create index out of existing timestemps
for i, key in enumerate(dic.keys()):
for nc in dic[key]:
ds = Dataset(nc)
ts = get_time(nc)
if i == 0:
dates = pd.DatetimeIndex(ts)
else:
dates = dates.union(ts)
# create empty DataFrame according existing timestemps
df = pd.DataFrame(columns=list(dic.keys()), index=dates)
for key in dic.keys():
try:
for nc in dic[key]:
ds = Dataset(nc)
var = get_variable(nc)
ts = get_time(nc)
tg_val = np.squeeze(ds.variables[var][:])
d2 = np.nanmean(tg_val, axis=1)
data = np.nanmean(d2, axis=1) + delta
df[key].loc[ts] = data
# data = fieldmean(dic[key]) # get_values(f)
# ts = get_time(dic[key])
# ds = pd.Series(data=data, index=ts, name=key)
# # ds_yr = ds.resample('12M', ).mean() # yearly mean loffset='6M'
# df[key] = ds
LOGGER.info('read in pandas series timeseries for: {}'.format(key))
except Exception:
LOGGER.exception('failed to calculate timeseries for %s ' % (key))
frq = get_frequency(resource[0])
if window is None:
# if frq == 'day':
# window = 1095 # 1
# elif frq == 'man':
# window = 35 # 9
# elif frq == 'sem':
# window = 11 # 9
# elif frq == 'yr':
# window = 3 # 0
# else:
# LOGGER.debug('frequency %s is not included' % frq)
window = 10
print('frequency: {}, window: {}'.format(frq, window))
if len(df.index.values) >= window * 2:
# TODO: calculate windowsize according to timestapms (day,mon,yr ... with get_frequency)
df_smooth = df.rolling(window=window, center=True).mean()
LOGGER.info('rolling mean calculated for all input data')
else:
df_smooth = df.copy()
LOGGER.debug('timeseries too short for moving mean')
fig.text(0.95, 0.05, '!!! timeseries too short for moving mean over 30years !!!',
fontsize=20, color='red',
ha='right', va='bottom', alpha=0.5)
try:
rmean = np.squeeze(df_smooth.quantile([0.5], axis=1,).values)
# skipna=False quantile([0.5], axis=1, numeric_only=False )
q05 = np.squeeze(df_smooth.quantile([0.10], axis=1,).values) # numeric_only=False)
q33 = np.squeeze(df_smooth.quantile([0.33], axis=1,).values) # numeric_only=False)
q66 = np.squeeze(df_smooth.quantile([0.66], axis=1,).values) # numeric_only=False)
q95 = np.squeeze(df_smooth.quantile([0.90], axis=1,).values) # numeric_only=False)
LOGGER.info('quantile calculated for all input data')
except Exception as e:
LOGGER.exception('failed to calculate quantiles: {}'.format(e))
try:
x = pd.to_datetime(df.index.values)
x1 = x[x<=dt.strptime('2005-12-31', "%Y-%m-%d")]
x2 = x[len(x1)-1:] # -1 to catch up with the last historical value
plt.fill_between(x, q05, q95, alpha=0.5, color='grey')
plt.fill_between(x, q33, q66, alpha=0.5, color='grey')
plt.plot(x1, rmean[:len(x1)], c='blue', lw=3)
plt.plot(x2, rmean[len(x1)-1:], c='r', lw=3)
# plt.xlim(min(df.index.values), max(df.index.values))
plt.ylim(ylim)
plt.xticks(fontsize=16, rotation=45)
plt.yticks(fontsize=16, ) # rotation=90
plt.title(title, fontsize=20)
plt.grid() # .grid_line_alpha=0.3
output_png = fig2plot(fig=fig, file_extension=file_extension, dir_output=dir_output)
plt.close()
LOGGER.debug('timeseries uncertainty plot done for %s' % variable)
except Exception as e:
raise Exception('failed to calculate quantiles. {}'.format(e))
except Exception as e:
LOGGER.exception('uncertainty plot failed for {}: {}'.format(variable, e))
_, output_png = mkstemp(dir='.', suffix='.png')
return output_png
def plot_ts_spaghetti(resource, variable=None, ylim=None, title=None,
file_extension='png', delta=0, dir_output='.',
figsize=(10, 10)):
"""
creates a png file containing the appropriate spaghetti plot as a
field mean of the values.
:param resource: list of files containing the same variable
:param variable: variable to be visualised. If None (default), variable will be detected
:param title: string to be used as title
:param ylim: Y-axis limitations: tuple(min,max)
:param figsize: figure size defult=(10,10)
:retruns str: path to png file
"""
# from eggshell.nc.calculation import fieldmean
try:
fig = plt.figure(figsize=figsize, dpi=600, facecolor='w', edgecolor='k')
LOGGER.debug('Start visualisation spaghetti plot')
# === prepare invironment
if type(resource) != list:
resource = [resource]
if variable is None:
variable = get_variable(resource[0])
LOGGER.info('plot values preparation done')
except Exception as ex:
msg = "plot values preparation failed {}".format(ex)
LOGGER.exception(msg)
raise Exception(msg)
try:
for c, nc in enumerate(resource):
try:
# dt = get_time(nc)
# ts = fieldmean(nc)
if 'historical' in nc:
col = 'grey'
elif 'evaluation' in nc:
col = 'black'
elif 'rcp26' in nc:
col = 'blue'
elif 'rcp85' in nc:
col = 'red'
else:
col = 'green'
dt = get_time(nc) # [datetime.strptime(elem, '%Y-%m-%d') for elem in strDate[0]]
# ts = fieldmean(nc)
ds = Dataset(nc)
var = get_variable(nc)
tg_val = np.squeeze(ds.variables[var][:])
d2 = np.nanmean(tg_val, axis=1)
ts = np.nanmean(d2, axis=1)
plt.plot(dt, ts, col )
plt.grid()
plt.title(title)
#
# plt.plot(dt, ts)
# fig.line( dt,ts )
except Exception as e:
msg = "spaghetti plot failed for {} : {}".format(nc, e)
LOGGER.exception(msg)
plt.title(title, fontsize=20)
plt.ylim(ylim)
plt.xticks(fontsize=16, rotation=45)
plt.yticks(fontsize=16, ) # rotation=90
plt.grid()
output_png = fig2plot(fig=fig, file_extension=file_extension, dir_output=dir_output)
plt.close()
LOGGER.info('timeseries spaghetti plot done for %s with %s lines.' % (variable, c))
except Exception as ex:
msg = 'matplotlib spaghetti plot failed: {}'.format(ex)
LOGGER.exception(msg)
return output_png