class ProjectContent(QToolBox):
"""Display the content in the current project
This toolbox contains several :class:`PlotterList` that show the content
of the current main and subproject"""
#: :class:`OrderedDict` containing the :class:`PlotterList` instances
#: of the different selection attributes
lists = OrderedDict()
@property
def current_names(self):
return [self.itemText(i) for i in range(self.count())]
def __init__(self, *args, **kwargs):
super(ProjectContent, self).__init__(*args, **kwargs)
self.lists = OrderedDict()
for attr in chain(['All'], sorted(Project._registered_plotters)):
item = self.add_plotterlist(attr, force=(attr == 'All'))
self.lists[attr] = item
self.currentChanged.connect(self.update_current_list)
Project.oncpchange.connect(self.update_lists)
def enable_list(self, list_widget):
"""Enable a given list widget based upon whether it is empty or not"""
i = self.indexOf(list_widget)
if i != -1:
self.setItemEnabled(i, not list_widget.is_empty)
def add_plotterlist(self, identifier, force=False):
"""Create a :class:`PlotterList` from an identifier from the
:class:`psyplot.project.Project` class"""
attr = identifier if identifier != 'All' else None
item = PlotterList(attr)
if not item.can_import_plotter:
return item
if force or not item.is_empty:
item.setParent(self)
item.updated_from_project.connect(self.enable_list)
self.addItem(item, identifier)
i = self.indexOf(item)
self.setItemEnabled(i, not item.is_empty)
return item
def update_current_list(self):
"""Update the current list from the current main and sub project"""
self.currentWidget().update_from_project(gcp(True))
self.currentWidget().update_from_project(gcp())
def update_lists(self, p):
# check new lists
current_items = self.current_names
for name, l in self.lists.items():
if not p.is_main:
l.update_from_project(p.main)
l.update_from_project(p)
if l.is_empty:
l.disconnect_items()
if name != 'All' and l.is_empty:
i = self.indexOf(l)
self.removeItem(i)
elif not l.is_empty and name not in current_items:
self.addItem(l, name)
('regular', QtGui.QFont.Normal),
('book', QtGui.QFont.Normal),
('medium', QtGui.QFont.Normal),
('roman', QtGui.QFont.Normal),
('semibold', QtGui.QFont.DemiBold),
('demibold', QtGui.QFont.DemiBold),
('demi', QtGui.QFont.DemiBold),
('bold', QtGui.QFont.Bold),
('heavy', QtGui.QFont.Bold),
('extra bold', QtGui.QFont.Black),
('black', QtGui.QFont.Black),
])
weights_qt2mpl = OrderedDict(
map(reversed,
utils.unique_everseen(weights_mpl2qt.items(), key=lambda t: t[1])))
def mpl_weight2qt(weight):
"""Convert a weight from matplotlib definition to a Qt weight
Parameters
----------
weight: int or string
Either an integer between 1 and 1000 or a string out of
:attr:`weights_mpl2qt`
Returns
-------
int
One type of the PyQt5.QtGui.QFont.Weight"""
class FuncArgParser(ArgumentParser):
"""Subclass of an argument parser that get's parts of the information
from a given function"""
def __init__(self, *args, **kwargs):
super(FuncArgParser, self).__init__(*args, **kwargs)
self.__arguments = OrderedDict()
self.__funcs = []
self.__main = None
def setup_args(self, func):
"""Add the parameters from the given `func` to the parameter settings
"""
self.__funcs.append(func)
args_dict = self.__arguments
args, varargs, varkw, defaults = inspect.getargspec(func)
full_doc = inspect.getdoc(func)
doc = docstrings._get_section(full_doc, 'Parameters') + '\n'
doc += docstrings._get_section(full_doc, 'Other Parameters')
doc = doc.rstrip()
default_min = len(args) - len(defaults)
for i, arg in enumerate(args):
if arg == 'self' or arg in args_dict:
continue
arg_doc = docstrings._keep_params(doc, [arg]) or \
docstrings._keep_types(doc, [arg])
args_dict[arg] = d = {'dest': arg, 'short': arg, 'long': arg}
if arg_doc:
lines = arg_doc.splitlines()
d['help'] = '\n'.join(lines[1:])
metavar = lines[0].split(':', 1)
if i >= default_min:
d['default'] = defaults[i - default_min]
if len(metavar) > 1:
dtype = metavar[1].strip()
if dtype == 'bool' and 'default' in d:
d['action'] = 'store_false' if d['default'] else \
'store_true'
else:
d['metavar'] = metavar[1].strip()
else:
d['positional'] = True
def update_arg(self, arg, if_existent=True, **kwargs):
"""Update the `add_argument` data for the given parameter
"""
if not if_existent:
self.__arguments.setdefault(arg, kwargs)
self.__arguments[arg].update(kwargs)
def pop_key(self, arg, key, *args, **kwargs):
"""Delete a previously defined key for the `add_argument`
"""
return self.__arguments[arg].pop(key, *args, **kwargs)
def create_arguments(self):
"""Create and add the arguments"""
ret = []
for arg, d in self.__arguments.items():
try:
is_positional = d.pop('positional', False)
short = d.pop('short')
long_name = d.pop('long', None)
if short == long_name:
long_name = None
args = [short, long_name] if long_name else [short]
if not is_positional:
for i, arg in enumerate(args):
args[i] = '-' * (i + 1) + arg
else:
d.pop('dest', None)
group = d.pop('group', self)
ret.append(group.add_argument(*args, **d))
except Exception:
print('Error while creating argument %s' % arg)
raise
return ret
@docstrings.get_sectionsf('FuncArgParser.parse_to_func')
@docstrings.dedent
def parse_to_func(self, args=None):
"""
Parse the given arguments to the main function
Parameters
----------
args: list
The list of arguments given to the
:meth:`ArgumentParser.parse_args` function. If None, the sys.argv
is used."""
if args is not None:
(self.__main or self.__funcs[0])(**vars(self.parse_args(args)))
else:
(self.__main or self.__funcs[0])(**vars(self.parse_args()))
@docstrings.dedent
def parse_known_to_func(self, args=None):
"""
Parse the known arguments from the given to the main function
Parameters
----------
%(FuncArgParser.parse_to_func.parameters)s"""
if args is not None:
(self.__main or self.__funcs[0])(
**vars(self.parse_known_args(args)[0]))
else:
(self.__main or self.__funcs[0])(
**vars(self.parse_known_args()[0]))
def set_main(self, func):
"""Set the function that is called by the :meth:`parse_to_func` and
:meth:`parse_known_to_func` function"""
self.__main = func
def recursive_processing(self, base_dir, target_dir, it):
"""Method to recursivly process the notebooks in the `base_dir`
Parameters
----------
base_dir: str
Path to the base example directory (see the `examples_dir`
parameter for the :class:`Gallery` class)
target_dir: str
Path to the output directory for the rst files (see the
`gallery_dirs` parameter for the :class:`Gallery` class)
it: iterable
The iterator over the subdirectories and files in `base_dir`
generated by the :func:`os.walk` function"""
try:
file_dir, dirs, files = next(it)
except StopIteration:
return '', []
readme_files = {'README.md', 'README.rst', 'README.txt'}
if readme_files.intersection(files):
foutdir = file_dir.replace(base_dir, target_dir)
create_dirs(foutdir)
this_nbps = [
NotebookProcessor(
infile=f,
outfile=os.path.join(foutdir, os.path.basename(f)),
disable_warnings=self.disable_warnings,
preprocess=(
(self.preprocess is True or f in self.preprocess) and
not (self.dont_preprocess is True or
f in self.dont_preprocess)),
clear=((self.clear is True or f in self.clear) and not
(self.dont_clear is True or f in self.dont_clear)))
for f in map(lambda f: os.path.join(file_dir, f),
filter(self.pattern.match, files))]
readme_file = next(iter(readme_files.intersection(files)))
else:
return '', []
labels = OrderedDict()
this_label = 'gallery_' + foutdir.replace(os.path.sep, '_')
if this_label.endswith('_'):
this_label = this_label[:-1]
for d in dirs:
label, nbps = self.recursive_processing(
base_dir, target_dir, it)
if label:
labels[label] = nbps
s = ".. _%s:\n\n" % this_label
with open(os.path.join(file_dir, readme_file)) as f:
s += f.read().rstrip() + '\n\n'
s += "\n\n.. toctree::\n\n"
s += ''.join(' %s\n' % os.path.splitext(os.path.basename(
nbp.get_out_file()))[0] for nbp in this_nbps)
for d in dirs:
findex = os.path.join(d, 'index.rst')
if os.path.exists(os.path.join(foutdir, findex)):
s += ' %s\n' % os.path.splitext(findex)[0]
s += '\n'
for nbp in this_nbps:
s += nbp.thumbnail_div + '\n'
s += "\n.. raw:: html\n\n <div style='clear:both'></div>\n"
for label, nbps in labels.items():
s += '\n.. only:: html\n\n .. rubric:: :ref:`%s`\n\n' % (
label)
for nbp in nbps:
s += nbp.thumbnail_div + '\n'
s += "\n.. raw:: html\n\n <div style='clear:both'></div>\n"
s += '\n'
with open(os.path.join(foutdir, 'index.rst'), 'w') as f:
f.write(s)
return this_label, list(chain(this_nbps, *labels.values()))
class HelpExplorer(QWidget, DockMixin):
"""A widget for showing the documentation. It behaves somewhat similar
to spyders object inspector plugin and can show restructured text either
as html (if sphinx is installed) or as plain text. It furthermore has a
browser to show html content
Warnings
--------
The :class:`HelpBrowser` class is known to crash under PyQt4 when new web
page domains are loaded. Hence you should disable the browsing to different
remote websites and even disable intersphinx"""
#: The viewer classes used by the help explorer. :class:`HelpExplorer`
#: instances replace this attribute with the corresponding HelpMixin
#: instance
viewers = OrderedDict([('HTML help', UrlHelp), ('Plain text', TextHelp)])
def __init__(self, *args, **kwargs):
super(HelpExplorer, self).__init__(*args, **kwargs)
self.vbox = vbox = QVBoxLayout()
self.combo = QComboBox(parent=self)
vbox.addWidget(self.combo)
if _viewers:
self.viewers = _viewers.copy()
for w in self.viewers.values():
w.setParent(self)
else:
self.viewers = OrderedDict(
[(key, cls(parent=self)) for key, cls in six.iteritems(
self.viewers)])
# save the UrlHelp because QWebEngineView creates child processes
# that are not properly closed by PyQt and as such use too much
# memory
if is_running_tests():
for key, val in self.viewers.items():
_viewers[key] = val
for key, ini in six.iteritems(self.viewers):
self.combo.addItem(key)
ini.hide()
vbox.addWidget(ini)
self.viewer = next(six.itervalues(self.viewers))
self.viewer.show()
self.combo.currentIndexChanged[str].connect(self.set_viewer)
self.setLayout(vbox)
def set_viewer(self, name):
"""Sets the current documentation viewer
Parameters
----------
name: str or object
A string must be one of the :attr:`viewers` attribute. An object
can be one of the values in the :attr:`viewers` attribute"""
if isstring(name) and asstring(name) not in self.viewers:
raise ValueError("Don't have a viewer named %s" % (name, ))
elif not isstring(name):
viewer = name
else:
name = asstring(name)
viewer = self.viewers[name]
self.viewer.hide()
self.viewer = viewer
self.viewer.show()
if (isstring(name) and
not self.combo.currentText() == name):
self.combo.setCurrentIndex(list(self.viewers).index(name))
@docstrings.dedent
def show_help(self, obj, oname='', files=None):
"""
Show the documentaion of the given object
We first try to use the current viewer based upon it's
:attr:`HelpMixin.can_document_object` attribute. If this does not work,
we check the other viewers
Parameters
----------
%(HelpMixin.show_help.parameters)s"""
oname = asstring(oname)
ret = None
if self.viewer.can_document_object:
try:
ret = self.viewer.show_help(obj, oname=oname, files=files)
except Exception:
logger.debug("Could not document %s with %s viewer!",
oname, self.combo.currentText(), exc_info=True)
else:
curr_i = self.combo.currentIndex()
for i, (viewername, viewer) in enumerate(
six.iteritems(self.viewers)):
if i != curr_i and viewer.can_document_object:
self.set_viewer(viewername)
self.combo.blockSignals(True)
self.combo.setCurrentIndex(i)
self.combo.blockSignals(False)
try:
ret = viewer.show_help(obj, oname=oname, files=files)
except Exception:
logger.debug("Could not document %s with %s viewer!",
oname, viewername, exc_info=True)
if ret:
self.parent().raise_()
return ret
@docstrings.dedent
def show_rst(self, text, oname='', files=None):
"""
Show restructured text
We first try to use the current viewer based upon it's
:attr:`HelpMixin.can_show_rst` attribute. If this does not work,
we check the other viewers
Parameters
----------
%(HelpMixin.show_rst.parameters)s"""
ret = None
if self.viewer.can_show_rst:
ret = self.viewer.show_rst(text, oname=oname, files=files)
else:
for viewer in six.itervalues(self.viewers):
if viewer.can_show_rst:
self.set_viewer(viewer)
ret = viewer.show_rst(text, oname=oname, files=files)
break
if ret:
self.parent().raise_()
return ret
@docstrings.dedent
def show_intro(self, text=''):
"""
Show an intro text
We first try to use the current viewer based upon it's
:attr:`HelpMixin.can_show_rst` attribute. If this does not work,
we check the other viewers
Parameters
----------
%(HelpMixin.show_intro.parameters)s"""
found = False
for i, viewer in enumerate(six.itervalues(self.viewers)):
viewer.show_intro(text)
if not found and viewer.can_show_rst:
if i:
self.set_viewer(viewer)
found = True
def close(self, *args, **kwargs):
try:
self.viewers['HTML help'].close(*args, **kwargs)
except AttributeError:
pass
return super(HelpExplorer, self).close(*args, **kwargs)
class QuantileEvaluation(Evaluator):
"""Evaluator to evaluate specific quantiles"""
name = 'quants'
summary = 'Compare the quantiles of simulation and observation'
names = OrderedDict([('prcp', {
'long_name': 'Precipitation',
'units': 'mm'
}), ('tmin', {
'long_name': 'Min. Temperature',
'units': 'degC'
}), ('tmax', {
'long_name': 'Max. Temperature',
'units': 'degC'
}), ('mean_cloud', {
'long_name': 'Cloud fraction',
'units': '-'
}), ('wind', {
'long_name': 'Wind Speed',
'units': 'm/s'
})])
@property
def all_variables(self):
return [[v + '_ref', v + '_sim'] for v in self.names]
setup_requires = ['prepare', 'output']
has_run = True
_datafile = 'quantile_evaluation.csv'
dbname = 'quantile_evaluation'
#: default formatoptions for the
#: :class:`psyplot.plotter.linreg.DensityRegPlotter` plotter
fmt = kwargs = dict(
legend={'loc': 'upper left'},
cmap='w_Reds',
title='%(pctl)sth percentile',
xlabel='%(type)s {desc}',
ylabel='%(type)s {desc}',
xrange=(['minmax', 1], ['minmax', 99]),
yrange=(['minmax', 1], ['minmax', 99]),
legendlabels=['$R^2$ = %(rsquared)s'],
bounds=['minmax', 11, 0, 99],
cbar='',
bins=10,
ideal=[0, 1],
id_color='r',
sym_lims='max',
)
@property
def default_config(self):
return default_quantile_config()._replace(
**super(QuantileEvaluation, self).default_config._asdict())
@property
def ds(self):
"""The dataset of the quantiles"""
import xarray as xr
data = self.data.reset_index()
ds = xr.Dataset.from_dataframe(
data.set_index('pctl', append=True).swaplevel())
full = xr.Dataset.from_dataframe(data).drop(
list(chain(self.data.index.names, ['pctl'])))
idx_name = full[next(v for v in self.names) + '_sim'].dims[-1]
full.rename({idx_name: 'full_index'}, inplace=True)
for vref, vsim in self.all_variables:
full.rename({
vref: 'all_' + vref,
vsim: 'all_' + vsim
},
inplace=True)
ds.merge(full, inplace=True)
for orig, attrs, (vref, vsim) in zip(self.names, self.names.values(),
self.all_variables):
for prefix in ['', 'all_']:
ds[prefix + vsim].attrs.update(attrs)
ds[prefix + vref].attrs.update(attrs)
ds[prefix + vsim].attrs['standard_name'] = orig
ds[prefix + vref].attrs['standard_name'] = orig
ds[prefix + vref].attrs['type'] = 'observed'
ds[prefix + vsim].attrs['type'] = 'simulated'
ds['all_' + vsim].attrs['pctl'] = 'All'
ds['all_' + vsim].attrs['pctl'] = 'All'
ds.pctl.attrs['long_name'] = 'Percentile'
return ds
def __init__(self, *args, **kwargs):
super(QuantileEvaluation, self).__init__(*args, **kwargs)
names = self.task_config.names
if names is not None:
self.names = OrderedDict(t for t in self.names.items()
if t[0] in names)
def setup_from_file(self, *args, **kwargs):
kwargs['index_col'] = ['id', 'year']
return super(QuantileEvaluation, self).setup_from_file(*args, **kwargs)
def setup_from_db(self, *args, **kwargs):
kwargs['index_col'] = ['id', 'year']
return super(QuantileEvaluation, self).setup_from_db(*args, **kwargs)
def setup_from_scratch(self):
df_ref = self.prepare.reference_data
dfi = self.prepare.input_data.reset_index(['lon', 'lat'])
# create simulation dataframe
df_sim = self.output.data
if len(df_ref) == 0 or len(df_sim) == 0:
self.logger.debug(
'Skipping %s because reference data contains no information!',
self.name)
return
names = self.names
# load observed precision
if self.task_config.no_rounding:
for name in names:
df_sim[name].values[:] = self.round_to_ref_prec(
df_ref[name].values, df_sim[name].values)
# merge reference and simulation into one single dataframe
df = df_ref.merge(df_sim,
left_index=True,
right_index=True,
suffixes=['_ref', '_sim'])
if {'mean_cloud', 'wind'}.intersection(names):
df.reset_index('day', inplace=True)
df = df.merge(dfi[['mean_cloud', 'wind']],
left_index=True,
right_index=True)
# mask out non-complete months for cloud validation and months with
# 0 or 1 cloud fraction
if 'mean_cloud' in names:
df.ix[df['mean_cloud_ref'].isnull().values |
(df['mean_cloud'] == 0.0) | (df['mean_cloud'] == 1.0),
['mean_cloud_sim', 'mean_cloud_ref']] = np.nan
# mask out non-complete wind for wind validation and months with
# a mean wind speed of 0
if 'wind' in names:
df.ix[df['wind_ref'].isnull().values | (df['wind'] == 0.0),
['wind_sim', 'wind_ref']] = np.nan
df.drop(['mean_cloud', 'wind'], 1, inplace=True)
df.set_index('day', append=True, inplace=True)
# transform wind
if self.task_config.transform_wind and 'wind' in names:
df['wind_ref'] **= 0.5
df['wind_sim'] **= 0.5
# calculate the percentiles for each station and month
g = df.sort_index().groupby(level=['id', 'year'])
self.logger.debug('Done with basic setup')
data = g.apply(self.calc)
if len(data):
data.index = data.index.droplevel(2)
self.data = data
@classmethod
def _modify_parser(cls, parser):
parser.setup_args(default_ks_config)
parser.setup_args(default_quantile_config)
parser, setup_grp, run_grp = super(QuantileEvaluation,
cls)._modify_parser(parser)
parser.update_arg('quantiles',
short='q',
group=run_grp,
type=utils.str_ranges,
metavar='f1[,f21[-f22[-f23]]]',
help=docstrings.dedents("""
The quantiles to use for calculating the percentiles.
%(str_ranges.s_help)s."""))
parser.pop_key('quantiles', 'nargs', None)
parser.update_arg('no_rounding', short='nr', group=run_grp)
parser.update_arg('names',
short='n',
group=setup_grp,
nargs='+',
metavar='variable',
choices=list(cls.names))
parser.update_arg('transform_wind', short='tw', group=setup_grp)
return parser, setup_grp, run_grp
def create_project(self, ds):
import psyplot.project as psy
import seaborn as sns
sns.set_style('white')
for name, (vref, vsim) in zip(self.names, self.all_variables):
self.logger.debug('Creating plots of %s', vsim)
kwargs = dict(precision=0.1) if vref.startswith('prcp') else {}
psy.plot.densityreg(ds,
name='all_' + vsim,
coord='all_' + vref,
fmt=self.fmt,
title='All percentiles',
arr_names=['%s_all' % name],
**kwargs)
arr_names = ['%s_%1.2f' % (name, p) for p in ds.pctl.values]
psy.plot.densityreg(ds,
name=vsim,
coord=vref,
fmt=self.fmt,
arr_names=arr_names,
pctl=range(ds.pctl.size),
**kwargs)
return psy.gcp(True)[:]
def make_run_config(self, sp, info):
for orig in self.names:
info[orig] = d = OrderedDict()
for plotter in sp(standard_name=orig).plotters:
d[plotter.data.pctl if plotter.data.name.
startswith('all') else int(plotter.data.pctl.values
)] = pctl_d = OrderedDict()
for key in ['rsquared', 'slope', 'intercept']:
val = plotter.plot_data[1].attrs.get(key)
if val is not None:
pctl_d[key] = float(val)
return info
def calc(self, group):
def calc_percentiles(vname):
arr = group[vname].values
arr = arr[~np.isnan(arr)]
if vname.startswith('prcp'):
arr = arr[arr > 0]
if len(arr) == 0:
return np.array([np.nan] * len(quantiles))
else:
return np.percentile(arr, quantiles)
quantiles = self.task_config.quantiles
df = pd.DataFrame.from_dict(
dict(
zip(chain(*self.all_variables),
map(calc_percentiles, chain(*self.all_variables)))))
df['pctl'] = quantiles
df.set_index('pctl')
return df
@staticmethod
def round_to_ref_prec(ref, sim, func=np.ceil):
"""Round one array to the precision of another
Parameters
----------
ref: np.ndarray
The reference array to get the precision from
sim: np.ndarray
The simulated array to round
func: function
The rounding function to use
Returns
-------
np.ndarray
Rounded `sim`"""
ref_sorted = np.unique(ref)
if len(ref_sorted) < 2:
return sim
precision = (ref_sorted[1:] - ref_sorted[:-1]).min()
return func((sim / precision) * precision)