def robustness_cc_signal(variable_mean, standard_deviation=None, variable=None): """ Claculating the Climate Change signal based on the output of robustness_stats. :param variable_mean: list of two 2D spatial netCDF files in the order of [refenence, projection] :param standard_deviation: according to variable_mean files 2D netCDF files of the standard deviation :return netCDF files: cc_signal.nc, mean_std.nc """ basename_ref = basename(variable_mean[0]).split('_') basename_proj = basename(variable_mean[1]).split('_') # ensstd_tg_mean_1981-01-01-2010-12-31.nc' if variable is None: variable = get_variable(variable_mean[0]) ds = Dataset(variable_mean[0]) vals_ref = np.squeeze(ds[variable][:]) ds.close() ds = Dataset(variable_mean[1]) vals_proj = np.squeeze(ds[variable][:]) ds.close() if standard_deviation is not None: ds = Dataset(standard_deviation[0]) std_ref = np.squeeze(ds[variable][:]) ds.close() ds = Dataset(standard_deviation[1]) std_proj = np.squeeze(ds[variable][:]) ds.close() bn_mean_std = 'mean-std_{}_{}_{}'.format(basename_ref[1], basename_ref[-2], basename_proj[-1]) out_mean_std = copyfile(standard_deviation[0], bn_mean_std) ds_median_std = Dataset(out_mean_std, mode='a') ds_median_std[variable][:] = (std_ref + std_proj) / 2 ds_median_std.close() else: out_mean_std = None bn_cc_signal = 'cc-signal_{}_{}_{}'.format(basename_ref[1], basename_ref[-2], basename_proj[-1]) out_cc_signal = copyfile(variable_mean[0], bn_cc_signal) ds_cc = Dataset(out_cc_signal, mode='a') ds_cc[variable][:] = np.squeeze(vals_proj - vals_ref) ds_cc.close() return out_cc_signal, out_mean_std
def _handler(self, request, response): # init_process_logger('log.txt') # response.outputs['output_log'].file = 'log.txt' ncfiles = extract_archive( resources=[inpt.file for inpt in request.inputs['resource']], dir_output=self.workdir) if 'variable' in request.inputs: var = request.inputs['variable'][0].data else: var = get_variable(ncfiles[0]) # var = ncfiles[0].split("_")[0] response.update_status('plotting variable {}'.format(var), 10) try: plotout_spagetti_file = plt_ncdata.spaghetti(ncfiles, variable=var, title='Field mean of {}'.format(var), dir_output=self.workdir, ) LOGGER.info("spagetti plot done") response.update_status('Spagetti plot for %s %s files done' % (len(ncfiles), var), 50) response.outputs['plotout_spagetti'].file = plotout_spagetti_file except Exception as e: raise Exception("spagetti plot failed : {}".format(e)) try: plotout_uncertainty_file = plt_ncdata.uncertainty(ncfiles, variable=var, title='Ensemble uncertainty for {}'.format(var), dir_output=self.workdir, ) response.update_status('Uncertainty plot for {} {} files done'.format(len(ncfiles), var), 90) response.outputs['plotout_uncertainty'].file = plotout_uncertainty_file LOGGER.info("uncertainty plot done") except Exception as err: raise Exception("uncertainty plot failed {}".format(err.message)) response.update_status('visualisation done', 100) return response
def parse_variable(self, request, path): """Parse variables specified in request and confirm they are present in file. If no variable is specified in the request, guess the variables from the file content. :return: List of variable names. """ ds = nc.Dataset(path) if 'variable' in request.inputs: var_names = [v.data for v in request.inputs['variable']] for var in var_names: if var not in ds.variables: raise ValueError("{} not in {}".format(var, path)) else: var_names = get_variable(ds) ds.close() return var_names
def robustness_stats(resources, time_range=[None, None], dir_output=None, variable=None): """ calculating the spatial mean and corresponding standard deviation for an ensemble of consistent datasets containing one variableself. If a time range is given the statistical values are calculated only in the disired timeperiod. :param resources: str or list of str containing the netCDF files paths :param time_range: sequence of two datetime.datetime objects to mark start and end point :param dir_output: path to folder to store ouput files (default= curdir) :param variable: variable name containing in netCDF file. If not set, variable name gets detected :return netCDF files: out_ensmean.nc, out_ensstd.nc """ from ocgis import OcgOperations, RequestDataset, env env.OVERWRITE = True if variable is None: variable = get_variable(resources[0]) out_means = [] for resource in resources: rd = RequestDataset(resource, variable) prefix = basename(resource).replace('.nc', '') LOGGER.debug('processing mean of {}'.format(prefix)) calc = [{ 'func': 'median', 'name': variable }] # {'func': 'median', 'name': 'monthly_median'} ops = OcgOperations(dataset=rd, calc=calc, calc_grouping=['all'], output_format='nc', prefix='median_' + prefix, time_range=time_range, dir_output=dir_output) out_means.append(ops.execute()) # nc_out = call(resource=resources, calc=[{'func': 'mean', 'name': 'ens_mean'}], # calc_grouping='all', # time_region=time_region, # dir_output=dir_output, output_format='nc') #### # read in numpy array for i, out_mean in enumerate(out_means): if i == 0: ds = Dataset(out_mean) var = ds[variable][:] dims = [len(out_means), var[:].shape[-2], var[:].shape[-1]] vals = np.empty(dims) vals[i, :, :] = np.squeeze(var[:]) ds.close() else: ds = Dataset(out_mean) vals[i, :, :] = np.squeeze(ds[variable][:]) ds.close() #### # calc median, std val_median = np.nanmedian(vals, axis=0) val_std = np.nanstd(vals, axis=0) ##### # prepare files by copying ... ensmean_file = 'ensmean_{}_{}_{}.nc'.format( variable, dt.strftime(time_range[0], '%Y-%m-%d'), dt.strftime(time_range[1], '%Y-%m-%d')) out_ensmean = copyfile(out_means[0], join(dir_output, ensmean_file)) ensstd_file = 'ensstd_{}_{}_{}.nc'.format( variable, dt.strftime(time_range[0], '%Y-%m-%d'), dt.strftime(time_range[1], '%Y-%m-%d')) out_ensstd = copyfile(out_means[0], join(dir_output, ensstd_file)) #### # write values to files ds_median = Dataset(out_ensmean, mode='a') ds_median[variable][:] = val_median ds_median.close() ds_std = Dataset(out_ensstd, mode='a') ds_std[variable][:] = val_std ds_std.close() LOGGER.info('processing the overall ensemble statistical mean ') # prefix = 'ensmean_tg-mean_{}-{}'.format(dt.strftime(time_range[0], '%Y-%m-%d'), # dt.strftime(time_range[1], '%Y-%m-%d')) # rd = RequestDataset(out_means, var) # calc = [{'func': 'mean', 'name': 'mean'}] # {'func': 'median', 'name': 'monthly_median'} # ops = OcgOperations(dataset=rd, calc=calc, calc_grouping=['all'], # output_format=output_format, prefix='mean_'+prefix, time_range=time_range) # ensmean = ops.execute() return out_ensmean, out_ensstd
def plot_map_ccsignal(signal, robustness=None, variable=None, cmap=None, title=None, file_extension='png', vmin=None, vmax=None): # 'seismic' """ generates a graphic for the output of the ensembleRobustness process for a lat/long file. :param signal: netCDF file containing the signal difference over time :param robustness: netCDF file containing 1 and 0 corresponding to signal robustness :param variable: variable containing the netCDF files :param cmap: default='seismic', :param title: default='Model agreement of signal' :returns str: path/to/file.png """ # from flyingpigeon import utils from numpy import mean, ma if variable is None: variable = get_variable(signal) print('found variable in file {}'.format(variable)) try: ds = Dataset(signal) var_signal = ds.variables[variable] val_signal = np.squeeze(ds.variables[variable]) lon_att = var_signal.dimensions[-1] lat_att = var_signal.dimensions[-2] lon = ds.variables[lon_att][:] lat = ds.variables[lat_att][:] lons, lats = meshgrid(lon, lat) ds.close() if robustness is not None: ds = Dataset(robustness) var_rob = get_variable(robustness) val_rob = np.squeeze(ds.variables[var_rob][:]) ds.close() # mask = val_signal[:] # [val_signal[:]<val_std[:]] # mask_h = np.empty(list(val_signal[:].shape)) # [[val_signal[:] > val_std[:]]] = 1 # mask_h[(val_signal >= (val_std / 4.))] = 1 #[:] # # mask_l = np.empty(list(val_signal[:].shape)) # [[val_signal[:] > val_std[:]]] = 1 # mask_l[mask_h != 1] = 1 # cyclic_var, cyclic_lons = add_cyclic_point(var_signal, coord=lons) # mask, cyclic_lons = add_cyclic_point(mask, coord=lons) # # lons = cyclic_lons # var_signal = cyclic_var LOGGER.info('prepared data for plotting') except Exception as e: msg = 'failed to get data for plotting: {}'.format(e) LOGGER.exception(msg) raise Exception(msg) try: fig = plt.figure(figsize=(20, 10), facecolor='w', edgecolor='k') ax = plt.axes(projection=ccrs.PlateCarree()) # ax = plt.axes(projection=ccrs.Robinson(central_longitude=int(mean(lons)))) # minval = round(np.nanmin(var_signal)) if cmap is None: if variable in ['pr', 'prAdjust', 'prcptot', 'rx1day', 'wetdays', # 'cdd', 'cwd', 'sdii', 'max_5_day_precipitation_amount']: cmap = 'BrBG' if variable in ['tas', 'tasAdjust', 'tg', 'tg_mean']: cmap = 'seismic' else: cmap = 'viridis' LOGGER.debug('variable {} not found to set the colormap'.format(variable)) maxval = round(np.nanmax(val_signal)+.5) minval = round(np.nanmin(val_signal)) norm = MidpointNormalize( vmin=minval, vcenter=0, vmax=maxval) # ) vcenter=0,, cs = plt.pcolormesh(lons, lats, val_signal, transform=ccrs.PlateCarree(), cmap=cmap, norm=norm, vmin=vmin, vmax=vmax) # 60, plt.colorbar(cs) if robustness != None: ch = plt.contourf(lons, lats, val_rob, transform=ccrs.PlateCarree(), hatches=[None, '/', '.'], alpha=0, colors='none', cmap=None) # colors='white' # cl = plt.contourf(lons, lats, mask_l, 1, transform=ccrs.PlateCarree(), hatches=[None, '/'], alpha=0, colors='none', cmap=None) # , plt.annotate('// = low model ensemble agreement', (0, 0), (0, -10), xycoords='axes fraction', textcoords='offset points', va='top') plt.annotate('.. = high model ensemble agreement', (0, 0), (0, -20), xycoords='axes fraction', textcoords='offset points', va='top') # ch = plt.contourf(lons, lats, mask, 1, transform=ccrs.PlateCarree(), colors='none', hatches=[None,'.' ]) ax.add_feature(cfeature.BORDERS, linewidth=2, linestyle='--') ax.add_feature(cfeature.COASTLINE, linewidth=2,) # coastlines() gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True, linewidth=2, color='gray', alpha=0.5, linestyle='--') gl.xlabels_top = False gl.ylables_right = False gl.xlabel_style = {'size': 15, 'color': 'black'} gl.ylabel_style = {'size': 15, 'color': 'black'} if title != None: plt.title(title, fontsize=20 ) plt.xticks(fontsize=16, rotation=45) plt.yticks(fontsize=16, ) # rotation=90 # # artists, labels = ch.legend_elements() # # plt.legend(artists, labels, handleheight=2) # # ax.gridlines() # # ax.set_global() graphic = fig2plot(fig=fig, file_extension=file_extension) plt.close() LOGGER.info('Plot created and figure saved') except: msg = 'failed to plot graphic' LOGGER.exception(msg) return graphic
def plot_map_timemean(resource, variable=None, time_range=None, title=None, delta=0, cmap=None, vmin=None, vmax=None, figsize=(15, 15), file_extension='png', dir_output='.'): """ creates a spatial map with the mean over the timestepps. If multiple files are provided, a mean over all files are condidered. :param resource: netCDF file(s) containng spatial values to be plotted. :param variable: variable to be visualised. If None (default), variable will be detected :param title: string to be used as title :param delta: set a delta for the values e.g. -273.15 to convert Kelvin to Celsius :param figsize: figure size defult=(15,15) :param vmin: colorbar minimum :param vmax: colorbar maximum :param file_extension: file extinction for the graphic :param dir_output: output directory to store the output graphic :returns str: path/to/file.png """ # from cartopy.mpl.gridliner import LONGITUDE_FORMATTER, LATITUDE_FORMATTER try: LOGGER.debug('plot_map function read in values for {}'.format(resource)) # get values of netcdf file if type(resource) == str: ds = Dataset(resource) if variable is None: variable = get_variable(resource) lat = ds.variables['rlat'] lon = ds.variables['rlon'] lons, lats = meshgrid(lon, lat) var = ds.variables[variable] var = get_values(f, time_range=time_range, variable=variable).data var_mean = np.nanmean(var, axis=0) + delta # mean over whole periode 30 Years 1981-2010 and transform to Celsius else: for i, f in enumerate(resource): if i == 0: ds = Dataset(f) lat = ds.variables['rlat'] lon = ds.variables['rlon'] lons, lats = meshgrid(lon, lat) vals = get_values(f, time_range=time_range, variable=variable).data else: vals = np.append(vals, get_values(f, time_range=time_range, variable=variable).data, axis=0) var_mean = np.nanmean(vals, axis=0) + delta # prepare plot LOGGER.info('preparing matplotlib figure') fig = plt.figure(figsize=figsize, facecolor='w', edgecolor='k') ax = plt.axes(projection=ccrs.PlateCarree()) cs = plt.pcolormesh(lons, lats, var_mean, transform=ccrs.PlateCarree(), cmap=cmap, vmin=vmin, vmax=vmax, ) # extent=(-0,17,10.5,24) # ax.set_extent(extent) ax.add_feature(cfeature.BORDERS, linewidth=2, linestyle='--') # ax.add_feature(cfeature.RIVERS) # ax.stock_img() # ax.gridlines(draw_labels=False) gl = ax.gridlines(crs=ccrs.PlateCarree(), draw_labels=True, linewidth=2, color='gray', alpha=0.5, linestyle='--') gl.xlabels_top = False gl.ylables_right = False gl.ylables_left = False # gl.xlines = False # gl.xlocator = mticker.FixedLocator([0, 2,4,6,8,10,12,14,16] ) # gl.xformatter = LONGITUDE_FORMATTER # gl.yformatter = LATITUDE_FORMATTER gl.xlabel_style = {'size': 15, 'color': 'black', 'weight': 'bold'} gl.ylabel_style = {'size': 15, 'color': 'black', 'weight': 'bold'} if cmap is None: if variable in ['pr', 'prAdjust', 'prcptot', 'rx1day', 'wetdays', 'cdd', 'cwd', 'sdii', 'max_5_day_precipitation_amount']: cmap = 'Blues' if variable in ['tas', 'tasAdjust', 'tg', 'tg_mean']: cmap = 'seismic' plt.title(title, fontsize=25) cax = fig.add_axes([ax.get_position().x1 + 0.1, ax.get_position().y0, 0.02, ax.get_position().height]) cbar = plt.colorbar(cs, cax=cax) # Similar to fig.colorbar(im, cax = cax) cbar.ax.tick_params(labelsize=20) # ticklabs = cbar.ax.get_yticklabels() # cbar.ax.set_yticklabels(ticklabs, fontsize=15) # cb = add_colorbar(cs) LOGGER.info('Matplotlib pcolormesh plot done') output_png = fig2plot(fig=fig, file_extension='png', dir_output=dir_output) plt.close() LOGGER.debug('Plot done for %s' % variable) except Exception as err: raise Exception('failed to calculate quantiles. %s' % err) return output_png
def plot_ts_uncertainty(resource, variable=None, ylim=None, title=None, file_extension='png', delta=0, window=None, dir_output='.', figsize=(10,10)): """ creates a png file containing the appropriate uncertainty plot. :param resource: list of files containing the same variable :param variable: variable to be visualised. If None (default), variable will be detected :param ylim: Y-axis limitations: tuple(min,max) :param title: string to be used as title :param figsize: figure size defult=(10,10) :param window: windowsize of the rolling mean :returns str: path/to/file.png """ LOGGER.debug('Start visualisation uncertainty plot') import pandas as pd import numpy as np from datetime import datetime as dt from os.path import basename # # from flyingpigeon.utils import get_time, sort_by_filename # from flyingpigeon.calculation import fieldmean # from flyingpigeon.metadata import get_frequency # === prepare invironment if type(resource) == str: resource = list([resource]) if variable is None: variable = get_variable(resource[0]) if title is None: title = "Field mean of %s " % variable LOGGER.info('variable %s found in resource.' % variable) try: fig = plt.figure(figsize=figsize, facecolor='w', edgecolor='k') dic = sort_by_filename(resource, historical_concatination=True) # Create index out of existing timestemps for i, key in enumerate(dic.keys()): for nc in dic[key]: ds = Dataset(nc) ts = get_time(nc) if i == 0: dates = pd.DatetimeIndex(ts) else: dates = dates.union(ts) # create empty DataFrame according existing timestemps df = pd.DataFrame(columns=list(dic.keys()), index=dates) for key in dic.keys(): try: for nc in dic[key]: ds = Dataset(nc) var = get_variable(nc) ts = get_time(nc) tg_val = np.squeeze(ds.variables[var][:]) d2 = np.nanmean(tg_val, axis=1) data = np.nanmean(d2, axis=1) + delta df[key].loc[ts] = data # data = fieldmean(dic[key]) # get_values(f) # ts = get_time(dic[key]) # ds = pd.Series(data=data, index=ts, name=key) # # ds_yr = ds.resample('12M', ).mean() # yearly mean loffset='6M' # df[key] = ds LOGGER.info('read in pandas series timeseries for: {}'.format(key)) except Exception: LOGGER.exception('failed to calculate timeseries for %s ' % (key)) frq = get_frequency(resource[0]) if window is None: # if frq == 'day': # window = 1095 # 1 # elif frq == 'man': # window = 35 # 9 # elif frq == 'sem': # window = 11 # 9 # elif frq == 'yr': # window = 3 # 0 # else: # LOGGER.debug('frequency %s is not included' % frq) window = 10 print('frequency: {}, window: {}'.format(frq, window)) if len(df.index.values) >= window * 2: # TODO: calculate windowsize according to timestapms (day,mon,yr ... with get_frequency) df_smooth = df.rolling(window=window, center=True).mean() LOGGER.info('rolling mean calculated for all input data') else: df_smooth = df.copy() LOGGER.debug('timeseries too short for moving mean') fig.text(0.95, 0.05, '!!! timeseries too short for moving mean over 30years !!!', fontsize=20, color='red', ha='right', va='bottom', alpha=0.5) try: rmean = np.squeeze(df_smooth.quantile([0.5], axis=1,).values) # skipna=False quantile([0.5], axis=1, numeric_only=False ) q05 = np.squeeze(df_smooth.quantile([0.10], axis=1,).values) # numeric_only=False) q33 = np.squeeze(df_smooth.quantile([0.33], axis=1,).values) # numeric_only=False) q66 = np.squeeze(df_smooth.quantile([0.66], axis=1,).values) # numeric_only=False) q95 = np.squeeze(df_smooth.quantile([0.90], axis=1,).values) # numeric_only=False) LOGGER.info('quantile calculated for all input data') except Exception as e: LOGGER.exception('failed to calculate quantiles: {}'.format(e)) try: x = pd.to_datetime(df.index.values) x1 = x[x<=dt.strptime('2005-12-31', "%Y-%m-%d")] x2 = x[len(x1)-1:] # -1 to catch up with the last historical value plt.fill_between(x, q05, q95, alpha=0.5, color='grey') plt.fill_between(x, q33, q66, alpha=0.5, color='grey') plt.plot(x1, rmean[:len(x1)], c='blue', lw=3) plt.plot(x2, rmean[len(x1)-1:], c='r', lw=3) # plt.xlim(min(df.index.values), max(df.index.values)) plt.ylim(ylim) plt.xticks(fontsize=16, rotation=45) plt.yticks(fontsize=16, ) # rotation=90 plt.title(title, fontsize=20) plt.grid() # .grid_line_alpha=0.3 output_png = fig2plot(fig=fig, file_extension=file_extension, dir_output=dir_output) plt.close() LOGGER.debug('timeseries uncertainty plot done for %s' % variable) except Exception as e: raise Exception('failed to calculate quantiles. {}'.format(e)) except Exception as e: LOGGER.exception('uncertainty plot failed for {}: {}'.format(variable, e)) _, output_png = mkstemp(dir='.', suffix='.png') return output_png
def plot_ts_spaghetti(resource, variable=None, ylim=None, title=None, file_extension='png', delta=0, dir_output='.', figsize=(10, 10)): """ creates a png file containing the appropriate spaghetti plot as a field mean of the values. :param resource: list of files containing the same variable :param variable: variable to be visualised. If None (default), variable will be detected :param title: string to be used as title :param ylim: Y-axis limitations: tuple(min,max) :param figsize: figure size defult=(10,10) :retruns str: path to png file """ # from eggshell.nc.calculation import fieldmean try: fig = plt.figure(figsize=figsize, dpi=600, facecolor='w', edgecolor='k') LOGGER.debug('Start visualisation spaghetti plot') # === prepare invironment if type(resource) != list: resource = [resource] if variable is None: variable = get_variable(resource[0]) LOGGER.info('plot values preparation done') except Exception as ex: msg = "plot values preparation failed {}".format(ex) LOGGER.exception(msg) raise Exception(msg) try: for c, nc in enumerate(resource): try: # dt = get_time(nc) # ts = fieldmean(nc) if 'historical' in nc: col = 'grey' elif 'evaluation' in nc: col = 'black' elif 'rcp26' in nc: col = 'blue' elif 'rcp85' in nc: col = 'red' else: col = 'green' dt = get_time(nc) # [datetime.strptime(elem, '%Y-%m-%d') for elem in strDate[0]] # ts = fieldmean(nc) ds = Dataset(nc) var = get_variable(nc) tg_val = np.squeeze(ds.variables[var][:]) d2 = np.nanmean(tg_val, axis=1) ts = np.nanmean(d2, axis=1) plt.plot(dt, ts, col ) plt.grid() plt.title(title) # # plt.plot(dt, ts) # fig.line( dt,ts ) except Exception as e: msg = "spaghetti plot failed for {} : {}".format(nc, e) LOGGER.exception(msg) plt.title(title, fontsize=20) plt.ylim(ylim) plt.xticks(fontsize=16, rotation=45) plt.yticks(fontsize=16, ) # rotation=90 plt.grid() output_png = fig2plot(fig=fig, file_extension=file_extension, dir_output=dir_output) plt.close() LOGGER.info('timeseries spaghetti plot done for %s with %s lines.' % (variable, c)) except Exception as ex: msg = 'matplotlib spaghetti plot failed: {}'.format(ex) LOGGER.exception(msg) return output_png