def __init__(self, *args, **kwargs):
super(HelpExplorer, self).__init__(*args, **kwargs)
self.vbox = vbox = QVBoxLayout()
self.combo = QComboBox(parent=self)
vbox.addWidget(self.combo)
if _viewers:
self.viewers = _viewers.copy()
for w in self.viewers.values():
w.setParent(self)
else:
self.viewers = OrderedDict(
[(key, cls(parent=self)) for key, cls in six.iteritems(
self.viewers)])
# save the UrlHelp because QWebEngineView creates child processes
# that are not properly closed by PyQt and as such use too much
# memory
if is_running_tests():
for key, val in self.viewers.items():
_viewers[key] = val
for key, ini in six.iteritems(self.viewers):
self.combo.addItem(key)
ini.hide()
vbox.addWidget(ini)
self.viewer = next(six.itervalues(self.viewers))
self.viewer.show()
self.combo.currentIndexChanged[str].connect(self.set_viewer)
self.setLayout(vbox)
def __init__(self, *args, **kwargs):
super(ProjectContent, self).__init__(*args, **kwargs)
self.lists = OrderedDict()
for attr in chain(['All'], sorted(Project._registered_plotters)):
item = self.add_plotterlist(attr, force=(attr == 'All'))
self.lists[attr] = item
self.currentChanged.connect(self.update_current_list)
Project.oncpchange.connect(self.update_lists)
def make_run_config(self, sp, info):
for orig in self.names:
info[orig] = d = OrderedDict()
for plotter in sp(standard_name=orig).plotters:
d[plotter.data.pctl if plotter.data.name.
startswith('all') else int(plotter.data.pctl.values
)] = pctl_d = OrderedDict()
for key in ['rsquared', 'slope', 'intercept']:
val = plotter.plot_data[1].attrs.get(key)
if val is not None:
pctl_d[key] = float(val)
return info
def fill_combos_from_project(self, project):
"""Fill :attr:`group_combo` and :attr:`fmt_combo` from a project
Parameters
----------
project: psyplot.project.Project
The project to use"""
current_text = self.group_combo.currentText()
with self.no_fmtos_update:
self.group_combo.clear()
if project is None or project.is_main or not len(project.plotters):
self.fmt_combo.clear()
self.groups = []
self.fmtos = []
self.line_edit.setEnabled(False)
return
self.line_edit.setEnabled(True)
# get dimensions
coords = sorted(project.coords_intersect)
coords_name = [COORDSGROUP] if coords else []
coords_verbose = ['Dimensions'] if coords else []
coords = [coords] if coords else []
# get formatoptions and group them alphabetically
grouped_fmts = defaultdict(list)
for fmto in project._fmtos:
grouped_fmts[fmto.group].append(fmto)
for val in six.itervalues(grouped_fmts):
val.sort(key=self.get_name)
grouped_fmts = OrderedDict(
sorted(six.iteritems(grouped_fmts),
key=lambda t: psyp.groups.get(t[0], t[0])))
fmt_groups = list(grouped_fmts.keys())
# save original names
self.groups = coords_name + [ALLGROUP] + fmt_groups
# save verbose group names (which are used in the combo box)
self.groupnames = coords_verbose + ['All formatoptions'] + list(
map(lambda s: psyp.groups.get(s, s), fmt_groups))
# save formatoptions
fmtos = list(grouped_fmts.values())
self.fmtos = coords + [sorted(chain(*fmtos), key=self.get_name)
] + fmtos
self.group_combo.addItems(self.groupnames)
ind = self.group_combo.findText(current_text)
self.group_combo.setCurrentIndex(ind if ind >= 0 else 0)
self.fill_fmt_combo(self.group_combo.currentIndex())
def __init__(self, *args, **kwargs):
super(HelpExplorer, self).__init__(*args, **kwargs)
self.vbox = vbox = QVBoxLayout()
self.combo = QComboBox(parent=self)
vbox.addWidget(self.combo)
self.viewers = OrderedDict([
(key, cls(parent=self)) for key, cls in six.iteritems(self.viewers)
])
for key, ini in six.iteritems(self.viewers):
self.combo.addItem(key)
ini.hide()
vbox.addWidget(ini)
self.viewer = next(six.itervalues(self.viewers))
self.viewer.show()
self.combo.currentIndexChanged[str].connect(self.set_viewer)
self.setLayout(vbox)
def __init__(self, parent, fmto, project):
"""
Parameters
----------
parent: psyplot_gui.fmt_widget.FormatoptionWidget
The formatoption widget where this widget is inserted
fmto: psyplot.plotter.Formatoption
The formatoption that is represented by this widget
project: psyplot.project.Project
The current psyplot subproject"""
QWidget.__init__(self, parent)
hbox = QHBoxLayout()
# Create a combo box for the rcParams 'labels' key
label_combo = DictCombo(fmto.rc['labels'], parent, modulo_style=False)
hbox.addWidget(label_combo)
# Create a combo for the :attr:`enhanced_attrs`
attrs = OrderedDict(
sorted(
utils.join_dicts([
getattr(plotter, fmto.key).enhanced_attrs
for plotter in project.plotters
],
delimiter=', ').items()))
attr_combo = DictCombo(attrs, parent)
hbox.addWidget(attr_combo)
fmtos = [
# add a button to change to the properties formatoption
getattr(fmto.plotter, fmto.key + 'props', None),
getattr(fmto.plotter, fmto.key + 'size', None),
getattr(fmto.plotter, fmto.key + 'weight', None)
]
fmtos = list(filter(None, fmtos))
if fmtos:
hbox.addWidget(Switch2FmtButton(parent, *fmtos))
self.setLayout(hbox)
def run(self, info):
logger = self.logger
logger.info('Calculating %s evaluation', self.name)
quants_info = self.config['evaluation'].get('quants')
ks_info = self.config['evaluation'].get('ks')
missing = []
if quants_info is None:
missing.append('quants task')
if ks_info is None:
missing.append('ks task')
if missing:
raise ValueError("%s requires that %s has been run before!" %
(self.name, ' and '.join(missing)))
quantiles = self.task_config.quantiles
if quantiles is None:
quantiles = slice(None)
possible_names = {
'wind', 'prcp', 'tmin', 'tmax', 'mean_cloud', 'cloud'
}
for v, v_ks in ks_info.items():
if v not in quants_info or v not in possible_names:
continue
#: Dataframe with intercept, rsquared and slope on index and
#: quantiles as columns
df = pd.DataFrame(quants_info[v]).loc[:, quantiles]
try:
del df['All']
except KeyError:
pass
slope = float((1 - np.abs(1 - df.loc['slope'].values)).mean())
rsquared = float(df.loc['rsquared'].values.mean())
info[v] = OrderedDict([
('rsquared', rsquared), ('slope', slope), ('ks', v_ks / 100.),
('quality', float(np.mean([rsquared, slope, v_ks / 100.])))
])
if logger.isEnabledFor(logging.DEBUG):
logger.debug('Simulation quality:')
for name, val in info.items():
logger.debug(' %s: %6.3f' % (name, val['quality']))
def run(self, info):
self.__setup = False
if self.setup_from == 'scratch':
df = self.data
# we may use a parallel setup which requires a weighted average
g = df.groupby(level='station_id')
total_counts = g.counts.transform("sum")
df['lat'] = df.counts / total_counts * df.lat
df['lon'] = df.counts / total_counts * df.lon
df['lat_std'] = (df.counts / total_counts) * df.lat_std**2
df['lon_std'] = (df.counts / total_counts) * df.lon_std**2
eecra = g.agg(
OrderedDict([('lat', 'sum'), ('lon', 'sum'),
('lat_std', 'sum'), ('lon_std', 'sum'),
('year', ('min', 'max')), ('counts', 'sum')]))
eecra.columns = [
'lat', 'lon', 'lat_std', 'lon_std', 'firstyear', 'lastyear',
'counts'
]
eecra[['lat_std', 'lon_std']] **= 0.5
use_xstall = self.task_config.xstall
if use_xstall:
to_replace = self.xstall_df
# keep only matching stations
to_replace = to_replace.join(eecra[[]], how='inner')
eecra.loc[to_replace.index, ['lat', 'lon']] = to_replace
self.data = eecra
if self.task_config.to_csv:
self.write2file()
if self.task_config.to_db:
self.write2db()
from psyplot import rcParams
import psyplot.config as psyc
try:
from textwrap import indent
except ImportError:
def indent(text, prefix, predicate=None): # python2
return '\n'.join(prefix + s if predicate is None or predicate(s) else s
for s in text.splitlines())
docstrings = psyp.docstrings
psyc.setup_logging(osp.join(osp.dirname(__file__), 'logging.yml'))
results = OrderedDict()
class TestFormatoption(psyp.Formatoption):
@property
def default(self):
try:
return super(TestFormatoption, self).default
except KeyError:
return ''
_validate = str
def update(self, value):
key = '%s.%s' % (self.plotter.data.psy.arr_name, self.key)
if not value:
from functools import partial
if with_qt5:
from PyQt5.QtWidgets import QSpinBox, QFontDialog, QColorDialog
else:
from PyQt4.QtGui import QSpinBox, QFontDialog, QColorDialog
if with_qt5:
weights_mpl2qt = OrderedDict([
('ultralight', QtGui.QFont.ExtraLight),
('light', QtGui.QFont.Light),
('normal', QtGui.QFont.Normal),
('regular', QtGui.QFont.Normal),
('book', QtGui.QFont.Normal),
('medium', QtGui.QFont.Medium),
('roman', QtGui.QFont.Medium),
('semibold', QtGui.QFont.DemiBold),
('demibold', QtGui.QFont.DemiBold),
('demi', QtGui.QFont.DemiBold),
('bold', QtGui.QFont.Bold),
('heavy', QtGui.QFont.Bold),
('extra bold', QtGui.QFont.ExtraBold),
('black', QtGui.QFont.Black),
])
else:
weights_mpl2qt = OrderedDict([
('ultralight', QtGui.QFont.Light),
('light', QtGui.QFont.Light),
('normal', QtGui.QFont.Normal),
('regular', QtGui.QFont.Normal),
('book', QtGui.QFont.Normal),
('medium', QtGui.QFont.Normal),
def __getitem__(self, key):
try:
return OrderedDict.__getitem__(self, key)
except KeyError:
return self.__missing__(key)
def __repr__(self):
return 'DefaultOrderedDict(%s, %s)' % (self.default_factory,
OrderedDict.__repr__(self))
def __init__(self, default_factory=None, *a, **kw):
if (default_factory is not None and not callable(default_factory)):
raise TypeError('first argument must be callable')
OrderedDict.__init__(self, *a, **kw)
self.default_factory = default_factory
def __init__(self, *args, **kwargs):
super(QuantileEvaluation, self).__init__(*args, **kwargs)
names = self.task_config.names
if names is not None:
self.names = OrderedDict(t for t in self.names.items()
if t[0] in names)
class HelpExplorer(QWidget, DockMixin):
"""A widget for showing the documentation. It behaves somewhat similar
to spyders object inspector plugin and can show restructured text either
as html (if sphinx is installed) or as plain text. It furthermore has a
browser to show html content
Warnings
--------
The :class:`HelpBrowser` class is known to crash under PyQt4 when new web
page domains are loaded. Hence you should disable the browsing to different
remote websites and even disable intersphinx"""
#: The viewer classes used by the help explorer. :class:`HelpExplorer`
#: instances replace this attribute with the corresponding HelpMixin
#: instance
viewers = OrderedDict([('HTML help', UrlHelp), ('Plain text', TextHelp)])
def __init__(self, *args, **kwargs):
super(HelpExplorer, self).__init__(*args, **kwargs)
self.vbox = vbox = QVBoxLayout()
self.combo = QComboBox(parent=self)
vbox.addWidget(self.combo)
self.viewers = OrderedDict([
(key, cls(parent=self)) for key, cls in six.iteritems(self.viewers)
])
for key, ini in six.iteritems(self.viewers):
self.combo.addItem(key)
ini.hide()
vbox.addWidget(ini)
self.viewer = next(six.itervalues(self.viewers))
self.viewer.show()
self.combo.currentIndexChanged[str].connect(self.set_viewer)
self.setLayout(vbox)
def set_viewer(self, name):
"""Sets the current documentation viewer
Parameters
----------
name: str or object
A string must be one of the :attr:`viewers` attribute. An object
can be one of the values in the :attr:`viewers` attribute"""
if isstring(name) and asstring(name) not in self.viewers:
raise ValueError("Don't have a viewer named %s" % (name, ))
elif not isstring(name):
viewer = name
else:
name = asstring(name)
viewer = self.viewers[name]
self.viewer.hide()
self.viewer = viewer
self.viewer.show()
if (isstring(name) and not self.combo.currentText() == name):
self.combo.setCurrentIndex(list(self.viewers).index(name))
@docstrings.dedent
def show_help(self, obj, oname='', files=None):
"""
Show the documentaion of the given object
We first try to use the current viewer based upon it's
:attr:`HelpMixin.can_document_object` attribute. If this does not work,
we check the other viewers
Parameters
----------
%(HelpMixin.show_help.parameters)s"""
oname = asstring(oname)
ret = None
if self.viewer.can_document_object:
try:
ret = self.viewer.show_help(obj, oname=oname, files=files)
except Exception:
logger.debug("Could not document %s with %s viewer!",
oname,
self.combo.currentText(),
exc_info=True)
else:
curr_i = self.combo.currentIndex()
for i, (viewername,
viewer) in enumerate(six.iteritems(self.viewers)):
if i != curr_i and viewer.can_document_object:
self.set_viewer(viewername)
self.combo.blockSignals(True)
self.combo.setCurrentIndex(i)
self.combo.blockSignals(False)
try:
ret = viewer.show_help(obj, oname=oname, files=files)
except Exception:
logger.debug("Could not document %s with %s viewer!",
oname,
viewername,
exc_info=True)
if ret:
self.parent().raise_()
return ret
@docstrings.dedent
def show_rst(self, text, oname='', files=None):
"""
Show restructured text
We first try to use the current viewer based upon it's
:attr:`HelpMixin.can_show_rst` attribute. If this does not work,
we check the other viewers
Parameters
----------
%(HelpMixin.show_rst.parameters)s"""
ret = None
if self.viewer.can_show_rst:
ret = self.viewer.show_rst(text, oname=oname, files=files)
else:
for viewer in six.itervalues(self.viewers):
if viewer.can_show_rst:
self.set_viewer(viewer)
ret = viewer.show_rst(text, oname=oname, files=files)
break
if ret:
self.parent().raise_()
return ret
@docstrings.dedent
def show_intro(self, text=''):
"""
Show an intro text
We first try to use the current viewer based upon it's
:attr:`HelpMixin.can_show_rst` attribute. If this does not work,
we check the other viewers
Parameters
----------
%(HelpMixin.show_intro.parameters)s"""
found = False
for i, viewer in enumerate(six.itervalues(self.viewers)):
viewer.show_intro(text)
if not found and viewer.can_show_rst:
if i:
self.set_viewer(viewer)
found = True
def close(self, *args, **kwargs):
self.viewers['HTML help'].close(*args, **kwargs)
return super(HelpExplorer, self).close(*args, **kwargs)
def fill_combos_from_project(self, project):
"""Fill :attr:`group_combo` and :attr:`fmt_combo` from a project
Parameters
----------
project: psyplot.project.Project
The project to use"""
if rcParams['fmt.sort_by_key']:
def sorter(fmto):
return fmto.key
else:
sorter = self.get_name
current_text = self.group_combo.currentText()
with self.no_fmtos_update:
self.group_combo.clear()
if project is None or project.is_main or not len(project):
self.fmt_combo.clear()
self.groups = []
self.fmtos = []
self.line_edit.setEnabled(False)
return
self.line_edit.setEnabled(True)
# get dimensions
it_vars = chain.from_iterable(
arr.psy.iter_base_variables for arr in project.arrays)
dims = next(it_vars).dims
sdims = set(dims)
for var in it_vars:
sdims.intersection_update(var.dims)
coords = [d for d in dims if d in sdims]
coords_name = [COORDSGROUP] if coords else []
coords_verbose = ['Dimensions'] if coords else []
coords = [coords] if coords else []
if len(project.plotters):
# get formatoptions and group them alphabetically
grouped_fmts = defaultdict(list)
for fmto in project._fmtos:
grouped_fmts[fmto.group].append(fmto)
for val in six.itervalues(grouped_fmts):
val.sort(key=sorter)
grouped_fmts = OrderedDict(
sorted(six.iteritems(grouped_fmts),
key=lambda t: psyp.groups.get(t[0], t[0])))
fmt_groups = list(grouped_fmts.keys())
# save original names
self.groups = coords_name + [ALLGROUP] + fmt_groups
# save verbose group names (which are used in the combo box)
self.groupnames = (
coords_verbose + ['All formatoptions'] + list(
map(lambda s: psyp.groups.get(s, s), fmt_groups)))
# save formatoptions
fmtos = list(grouped_fmts.values())
self.fmtos = coords + [sorted(
chain(*fmtos), key=sorter)] + fmtos
else:
self.groups = coords_name
self.groupnames = coords_verbose
self.fmtos = coords
self.group_combo.addItems(self.groupnames)
ind = self.group_combo.findText(current_text)
self.group_combo.setCurrentIndex(ind if ind >= 0 else 0)
self.fill_fmt_combo(self.group_combo.currentIndex())
return pathlib.Path(fname).as_uri()
except ImportError:
def file2html(fname):
return 'file://' + fname
def html2file(url):
p = urlparse(asstring(url))
# skip the first '/' on windows platform
return osp.abspath(osp.join(p.netloc,
p.path[int(sys.platform == 'win32'):]))
_viewers = OrderedDict()
logger = logging.getLogger(__name__)
class UrlCombo(QComboBox):
"""A editable ComboBox with autocompletion"""
def __init__(self, *args, **kwargs):
super(UrlCombo, self).__init__(*args, **kwargs)
self.setInsertPolicy(self.InsertAtTop)
self.setFocusPolicy(Qt.StrongFocus)
self.setEditable(True)
self.completer = QCompleter(self)
class ProjectContent(QToolBox):
"""Display the content in the current project
This toolbox contains several :class:`PlotterList` that show the content
of the current main and subproject"""
#: :class:`OrderedDict` containing the :class:`PlotterList` instances
#: of the different selection attributes
lists = OrderedDict()
@property
def current_names(self):
return [self.itemText(i) for i in range(self.count())]
def __init__(self, *args, **kwargs):
super(ProjectContent, self).__init__(*args, **kwargs)
self.lists = OrderedDict()
for attr in chain(['All'], sorted(Project._registered_plotters)):
item = self.add_plotterlist(attr, force=(attr == 'All'))
self.lists[attr] = item
self.currentChanged.connect(self.update_current_list)
Project.oncpchange.connect(self.update_lists)
def enable_list(self, list_widget):
"""Enable a given list widget based upon whether it is empty or not"""
i = self.indexOf(list_widget)
if i != -1:
self.setItemEnabled(i, not list_widget.is_empty)
def add_plotterlist(self, identifier, force=False):
"""Create a :class:`PlotterList` from an identifier from the
:class:`psyplot.project.Project` class"""
attr = identifier if identifier != 'All' else None
item = PlotterList(attr)
if not item.can_import_plotter:
return item
if force or not item.is_empty:
item.setParent(self)
item.updated_from_project.connect(self.enable_list)
self.addItem(item, identifier)
i = self.indexOf(item)
self.setItemEnabled(i, not item.is_empty)
return item
def update_current_list(self):
"""Update the current list from the current main and sub project"""
self.currentWidget().update_from_project(gcp(True))
self.currentWidget().update_from_project(gcp())
def update_lists(self, p):
# check new lists
current_items = self.current_names
for name, l in self.lists.items():
if not p.is_main:
l.update_from_project(p.main)
l.update_from_project(p)
if l.is_empty:
l.disconnect_items()
if name != 'All' and l.is_empty:
i = self.indexOf(l)
self.removeItem(i)
elif not l.is_empty and name not in current_items:
self.addItem(l, name)
def recursive_processing(self, base_dir, target_dir, it):
"""Method to recursivly process the notebooks in the `base_dir`
Parameters
----------
base_dir: str
Path to the base example directory (see the `examples_dir`
parameter for the :class:`Gallery` class)
target_dir: str
Path to the output directory for the rst files (see the
`gallery_dirs` parameter for the :class:`Gallery` class)
it: iterable
The iterator over the subdirectories and files in `base_dir`
generated by the :func:`os.walk` function"""
try:
file_dir, dirs, files = next(it)
except StopIteration:
return '', []
readme_files = {'README.md', 'README.rst', 'README.txt'}
if readme_files.intersection(files):
foutdir = file_dir.replace(base_dir, target_dir)
create_dirs(foutdir)
this_nbps = [
NotebookProcessor(
infile=f,
outfile=os.path.join(foutdir, os.path.basename(f)),
disable_warnings=self.disable_warnings,
preprocess=(
(self.preprocess is True or f in self.preprocess) and
not (self.dont_preprocess is True or
f in self.dont_preprocess)),
clear=((self.clear is True or f in self.clear) and not
(self.dont_clear is True or f in self.dont_clear)))
for f in map(lambda f: os.path.join(file_dir, f),
filter(self.pattern.match, files))]
readme_file = next(iter(readme_files.intersection(files)))
else:
return '', []
labels = OrderedDict()
this_label = 'gallery_' + foutdir.replace(os.path.sep, '_')
if this_label.endswith('_'):
this_label = this_label[:-1]
for d in dirs:
label, nbps = self.recursive_processing(
base_dir, target_dir, it)
if label:
labels[label] = nbps
s = ".. _%s:\n\n" % this_label
with open(os.path.join(file_dir, readme_file)) as f:
s += f.read().rstrip() + '\n\n'
s += "\n\n.. toctree::\n\n"
s += ''.join(' %s\n' % os.path.splitext(os.path.basename(
nbp.get_out_file()))[0] for nbp in this_nbps)
for d in dirs:
findex = os.path.join(d, 'index.rst')
if os.path.exists(os.path.join(foutdir, findex)):
s += ' %s\n' % os.path.splitext(findex)[0]
s += '\n'
for nbp in this_nbps:
s += nbp.thumbnail_div + '\n'
s += "\n.. raw:: html\n\n <div style='clear:both'></div>\n"
for label, nbps in labels.items():
s += '\n.. only:: html\n\n .. rubric:: :ref:`%s`\n\n' % (
label)
for nbp in nbps:
s += nbp.thumbnail_div + '\n'
s += "\n.. raw:: html\n\n <div style='clear:both'></div>\n"
s += '\n'
with open(os.path.join(foutdir, 'index.rst'), 'w') as f:
f.write(s)
return this_label, list(chain(this_nbps, *labels.values()))
class UrlBrowser(QFrame):
"""Very simple browser with session history and autocompletion based upon
the :class:`PyQt5.QtWebEngineWidgets.QWebEngineView` class
Warnings
--------
This class is known to crash under PyQt4 when new web page domains are
loaded. Hence it should be handled with care"""
completed = _temp_bool_prop(
'completed',
"Boolean whether the html page loading is completed.",
default=True)
url_like_re = re.compile('^\w+://')
doc_urls = OrderedDict([
('startpage', 'https://startpage.com/'),
('psyplot', 'http://psyplot.readthedocs.org/en/latest/'),
('pyplot', 'http://matplotlib.org/api/pyplot_api.html'),
('seaborn', 'http://stanford.edu/~mwaskom/software/seaborn/api.html'),
('cartopy', 'http://scitools.org.uk/cartopy/docs/latest/index.html'),
('xarray', 'http://xarray.pydata.org/en/stable/'),
('pandas', 'http://pandas.pydata.org/pandas-docs/stable/'),
('numpy', 'https://docs.scipy.org/doc/numpy/reference/routines.html'),
])
#: The initial url showed in the webview. If None, nothing will be
#: displayed
default_url = None
#: adress line
tb_url = None
#: button to go to previous url
bt_back = None
#: button to go to next url
bt_ahead = None
#: refresh the current url
bt_refresh = None
#: button to go lock to the current url
bt_lock = None
#: button to disable browsing in www
bt_url_lock = None
#: The upper part of the browser containing all the buttons
button_box = None
#: The upper most layout aranging the button box and the html widget
vbox = None
def __init__(self, *args, **kwargs):
super(UrlBrowser, self).__init__(*args, **kwargs)
# ---------------------------------------------------------------------
# ---------------------------- upper buttons --------------------------
# ---------------------------------------------------------------------
# adress line
self.tb_url = UrlCombo(self)
# button to go to previous url
self.bt_back = QToolButton(self)
# button to go to next url
self.bt_ahead = QToolButton(self)
# refresh the current url
self.bt_refresh = QToolButton(self)
# button to go lock to the current url
self.bt_lock = QToolButton(self)
# button to disable browsing in www
self.bt_url_lock = QToolButton(self)
# ---------------------------- buttons settings -----------------------
self.bt_back.setIcon(QIcon(get_icon('previous.png')))
self.bt_back.setToolTip('Go back one page')
self.bt_ahead.setIcon(QIcon(get_icon('next.png')))
self.bt_back.setToolTip('Go forward one page')
self.bt_refresh.setIcon(QIcon(get_icon('refresh.png')))
self.bt_refresh.setToolTip('Refresh the current page')
self.bt_lock.setCheckable(True)
self.bt_url_lock.setCheckable(True)
if not with_qt5 and rcParams['help_explorer.online'] is None:
# We now that the browser can crash with Qt4, therefore we disable
# the browing in the internet
self.bt_url_lock.click()
rcParams['help_explorer.online'] = False
elif rcParams['help_explorer.online'] is False:
self.bt_url_lock.click()
elif rcParams['help_explorer.online'] is None:
rcParams['help_explorer.online'] = True
rcParams.connect('help_explorer.online', self.update_url_lock_from_rc)
self.bt_url_lock.clicked.connect(self.toogle_url_lock)
self.bt_lock.clicked.connect(self.toogle_lock)
# tooltip and icons of lock and url_lock are set in toogle_lock and
# toogle_url_lock
self.toogle_lock()
self.toogle_url_lock()
# ---------------------------------------------------------------------
# --------- initialization and connection of the web view -------------
# ---------------------------------------------------------------------
#: The actual widget showing the html content
self.html = QWebEngineView(parent=self)
self.html.loadStarted.connect(self.completed)
self.html.loadFinished.connect(self.completed)
self.tb_url.currentIndexChanged[str].connect(self.browse)
self.bt_back.clicked.connect(self.html.back)
self.bt_ahead.clicked.connect(self.html.forward)
self.bt_refresh.clicked.connect(self.html.reload)
self.html.urlChanged.connect(self.url_changed)
# ---------------------------------------------------------------------
# ---------------------------- layouts --------------------------------
# ---------------------------------------------------------------------
# The upper part of the browser containing all the buttons
self.button_box = button_box = QHBoxLayout()
button_box.addWidget(self.bt_back)
button_box.addWidget(self.bt_ahead)
button_box.addWidget(self.tb_url)
button_box.addWidget(self.bt_refresh)
button_box.addWidget(self.bt_lock)
button_box.addWidget(self.bt_url_lock)
# The upper most layout aranging the button box and the html widget
self.vbox = vbox = QVBoxLayout()
self.vbox.setContentsMargins(0, 0, 0, 0)
vbox.addLayout(button_box)
vbox.addWidget(self.html)
self.setLayout(vbox)
if self.default_url is not None:
self.tb_url.addItem(self.default_url)
def browse(self, url):
"""Make a web browse on the given url and show the page on the Webview
widget. """
if self.bt_lock.isChecked():
return
if not self.url_like_re.match(url):
url = 'https://' + url
if self.bt_url_lock.isChecked() and url.startswith('http'):
return
if not self.completed:
logger.debug('Stopping current load...')
self.html.stop()
self.completed = True
logger.debug('Loading %s', url)
# we use :meth:`PyQt5.QtWebEngineWidgets.QWebEngineView.setUrl` instead
# of :meth:`PyQt5.QtWebEngineWidgets.QWebEngineView.load` because that
# changes the url directly and is more useful for unittests
self.html.setUrl(QtCore.QUrl(url))
def url_changed(self, url):
"""Triggered when the url is changed to update the adress line"""
try:
url = url.toString()
except AttributeError:
pass
logger.debug('url changed to %s', url)
try:
self.tb_url.setCurrentText(url)
except AttributeError: # Qt4
self.tb_url.setEditText(url)
self.tb_url.add_text_on_top(url, block=True)
def update_url_lock_from_rc(self, online):
if (online and self.bt_url_lock.isChecked()
or not online and not self.bt_url_lock.isChecked()):
self.bt_url_lock.click()
def toogle_url_lock(self):
"""Disable (or enable) the loading of web pages in www"""
bt = self.bt_url_lock
offline = bt.isChecked()
bt.setIcon(QIcon(
get_icon('world_red.png' if offline else 'world.png')))
online_message = "Go online"
if not with_qt5:
online_message += ("\nWARNING: This mode is unstable under Qt4 "
"and might result in a complete program crash!")
bt.setToolTip(online_message if offline else "Offline mode")
if rcParams['help_explorer.online'] is offline:
rcParams['help_explorer.online'] = not offline
def toogle_lock(self):
"""Disable (or enable) the changing of the current webpage"""
bt = self.bt_lock
bt.setIcon(
QIcon(get_icon('lock.png' if bt.isChecked() else 'lock_open.png')))
bt.setToolTip("Unlock" if bt.isChecked() else "Lock to current page")
def __init__(self, *args, **kwargs):
super(FuncArgParser, self).__init__(*args, **kwargs)
self.__arguments = OrderedDict()
self.__funcs = []
self.__main = None
class FuncArgParser(ArgumentParser):
"""Subclass of an argument parser that get's parts of the information
from a given function"""
def __init__(self, *args, **kwargs):
super(FuncArgParser, self).__init__(*args, **kwargs)
self.__arguments = OrderedDict()
self.__funcs = []
self.__main = None
def setup_args(self, func):
"""Add the parameters from the given `func` to the parameter settings
"""
self.__funcs.append(func)
args_dict = self.__arguments
args, varargs, varkw, defaults = inspect.getargspec(func)
full_doc = inspect.getdoc(func)
doc = docstrings._get_section(full_doc, 'Parameters') + '\n'
doc += docstrings._get_section(full_doc, 'Other Parameters')
doc = doc.rstrip()
default_min = len(args) - len(defaults)
for i, arg in enumerate(args):
if arg == 'self' or arg in args_dict:
continue
arg_doc = docstrings._keep_params(doc, [arg]) or \
docstrings._keep_types(doc, [arg])
args_dict[arg] = d = {'dest': arg, 'short': arg, 'long': arg}
if arg_doc:
lines = arg_doc.splitlines()
d['help'] = '\n'.join(lines[1:])
metavar = lines[0].split(':', 1)
if i >= default_min:
d['default'] = defaults[i - default_min]
if len(metavar) > 1:
dtype = metavar[1].strip()
if dtype == 'bool' and 'default' in d:
d['action'] = 'store_false' if d['default'] else \
'store_true'
else:
d['metavar'] = metavar[1].strip()
else:
d['positional'] = True
def update_arg(self, arg, if_existent=True, **kwargs):
"""Update the `add_argument` data for the given parameter
"""
if not if_existent:
self.__arguments.setdefault(arg, kwargs)
self.__arguments[arg].update(kwargs)
def pop_key(self, arg, key, *args, **kwargs):
"""Delete a previously defined key for the `add_argument`
"""
return self.__arguments[arg].pop(key, *args, **kwargs)
def create_arguments(self):
"""Create and add the arguments"""
ret = []
for arg, d in self.__arguments.items():
try:
is_positional = d.pop('positional', False)
short = d.pop('short')
long_name = d.pop('long', None)
if short == long_name:
long_name = None
args = [short, long_name] if long_name else [short]
if not is_positional:
for i, arg in enumerate(args):
args[i] = '-' * (i + 1) + arg
else:
d.pop('dest', None)
group = d.pop('group', self)
ret.append(group.add_argument(*args, **d))
except Exception:
print('Error while creating argument %s' % arg)
raise
return ret
@docstrings.get_sectionsf('FuncArgParser.parse_to_func')
@docstrings.dedent
def parse_to_func(self, args=None):
"""
Parse the given arguments to the main function
Parameters
----------
args: list
The list of arguments given to the
:meth:`ArgumentParser.parse_args` function. If None, the sys.argv
is used."""
if args is not None:
(self.__main or self.__funcs[0])(**vars(self.parse_args(args)))
else:
(self.__main or self.__funcs[0])(**vars(self.parse_args()))
@docstrings.dedent
def parse_known_to_func(self, args=None):
"""
Parse the known arguments from the given to the main function
Parameters
----------
%(FuncArgParser.parse_to_func.parameters)s"""
if args is not None:
(self.__main or self.__funcs[0])(
**vars(self.parse_known_args(args)[0]))
else:
(self.__main or self.__funcs[0])(
**vars(self.parse_known_args()[0]))
def set_main(self, func):
"""Set the function that is called by the :meth:`parse_to_func` and
:meth:`parse_known_to_func` function"""
self.__main = func
class QuantileEvaluation(Evaluator):
"""Evaluator to evaluate specific quantiles"""
name = 'quants'
summary = 'Compare the quantiles of simulation and observation'
names = OrderedDict([('prcp', {
'long_name': 'Precipitation',
'units': 'mm'
}), ('tmin', {
'long_name': 'Min. Temperature',
'units': 'degC'
}), ('tmax', {
'long_name': 'Max. Temperature',
'units': 'degC'
}), ('mean_cloud', {
'long_name': 'Cloud fraction',
'units': '-'
}), ('wind', {
'long_name': 'Wind Speed',
'units': 'm/s'
})])
@property
def all_variables(self):
return [[v + '_ref', v + '_sim'] for v in self.names]
setup_requires = ['prepare', 'output']
has_run = True
_datafile = 'quantile_evaluation.csv'
dbname = 'quantile_evaluation'
#: default formatoptions for the
#: :class:`psyplot.plotter.linreg.DensityRegPlotter` plotter
fmt = kwargs = dict(
legend={'loc': 'upper left'},
cmap='w_Reds',
title='%(pctl)sth percentile',
xlabel='%(type)s {desc}',
ylabel='%(type)s {desc}',
xrange=(['minmax', 1], ['minmax', 99]),
yrange=(['minmax', 1], ['minmax', 99]),
legendlabels=['$R^2$ = %(rsquared)s'],
bounds=['minmax', 11, 0, 99],
cbar='',
bins=10,
ideal=[0, 1],
id_color='r',
sym_lims='max',
)
@property
def default_config(self):
return default_quantile_config()._replace(
**super(QuantileEvaluation, self).default_config._asdict())
@property
def ds(self):
"""The dataset of the quantiles"""
import xarray as xr
data = self.data.reset_index()
ds = xr.Dataset.from_dataframe(
data.set_index('pctl', append=True).swaplevel())
full = xr.Dataset.from_dataframe(data).drop(
list(chain(self.data.index.names, ['pctl'])))
idx_name = full[next(v for v in self.names) + '_sim'].dims[-1]
full.rename({idx_name: 'full_index'}, inplace=True)
for vref, vsim in self.all_variables:
full.rename({
vref: 'all_' + vref,
vsim: 'all_' + vsim
},
inplace=True)
ds.merge(full, inplace=True)
for orig, attrs, (vref, vsim) in zip(self.names, self.names.values(),
self.all_variables):
for prefix in ['', 'all_']:
ds[prefix + vsim].attrs.update(attrs)
ds[prefix + vref].attrs.update(attrs)
ds[prefix + vsim].attrs['standard_name'] = orig
ds[prefix + vref].attrs['standard_name'] = orig
ds[prefix + vref].attrs['type'] = 'observed'
ds[prefix + vsim].attrs['type'] = 'simulated'
ds['all_' + vsim].attrs['pctl'] = 'All'
ds['all_' + vsim].attrs['pctl'] = 'All'
ds.pctl.attrs['long_name'] = 'Percentile'
return ds
def __init__(self, *args, **kwargs):
super(QuantileEvaluation, self).__init__(*args, **kwargs)
names = self.task_config.names
if names is not None:
self.names = OrderedDict(t for t in self.names.items()
if t[0] in names)
def setup_from_file(self, *args, **kwargs):
kwargs['index_col'] = ['id', 'year']
return super(QuantileEvaluation, self).setup_from_file(*args, **kwargs)
def setup_from_db(self, *args, **kwargs):
kwargs['index_col'] = ['id', 'year']
return super(QuantileEvaluation, self).setup_from_db(*args, **kwargs)
def setup_from_scratch(self):
df_ref = self.prepare.reference_data
dfi = self.prepare.input_data.reset_index(['lon', 'lat'])
# create simulation dataframe
df_sim = self.output.data
if len(df_ref) == 0 or len(df_sim) == 0:
self.logger.debug(
'Skipping %s because reference data contains no information!',
self.name)
return
names = self.names
# load observed precision
if self.task_config.no_rounding:
for name in names:
df_sim[name].values[:] = self.round_to_ref_prec(
df_ref[name].values, df_sim[name].values)
# merge reference and simulation into one single dataframe
df = df_ref.merge(df_sim,
left_index=True,
right_index=True,
suffixes=['_ref', '_sim'])
if {'mean_cloud', 'wind'}.intersection(names):
df.reset_index('day', inplace=True)
df = df.merge(dfi[['mean_cloud', 'wind']],
left_index=True,
right_index=True)
# mask out non-complete months for cloud validation and months with
# 0 or 1 cloud fraction
if 'mean_cloud' in names:
df.ix[df['mean_cloud_ref'].isnull().values |
(df['mean_cloud'] == 0.0) | (df['mean_cloud'] == 1.0),
['mean_cloud_sim', 'mean_cloud_ref']] = np.nan
# mask out non-complete wind for wind validation and months with
# a mean wind speed of 0
if 'wind' in names:
df.ix[df['wind_ref'].isnull().values | (df['wind'] == 0.0),
['wind_sim', 'wind_ref']] = np.nan
df.drop(['mean_cloud', 'wind'], 1, inplace=True)
df.set_index('day', append=True, inplace=True)
# transform wind
if self.task_config.transform_wind and 'wind' in names:
df['wind_ref'] **= 0.5
df['wind_sim'] **= 0.5
# calculate the percentiles for each station and month
g = df.sort_index().groupby(level=['id', 'year'])
self.logger.debug('Done with basic setup')
data = g.apply(self.calc)
if len(data):
data.index = data.index.droplevel(2)
self.data = data
@classmethod
def _modify_parser(cls, parser):
parser.setup_args(default_ks_config)
parser.setup_args(default_quantile_config)
parser, setup_grp, run_grp = super(QuantileEvaluation,
cls)._modify_parser(parser)
parser.update_arg('quantiles',
short='q',
group=run_grp,
type=utils.str_ranges,
metavar='f1[,f21[-f22[-f23]]]',
help=docstrings.dedents("""
The quantiles to use for calculating the percentiles.
%(str_ranges.s_help)s."""))
parser.pop_key('quantiles', 'nargs', None)
parser.update_arg('no_rounding', short='nr', group=run_grp)
parser.update_arg('names',
short='n',
group=setup_grp,
nargs='+',
metavar='variable',
choices=list(cls.names))
parser.update_arg('transform_wind', short='tw', group=setup_grp)
return parser, setup_grp, run_grp
def create_project(self, ds):
import psyplot.project as psy
import seaborn as sns
sns.set_style('white')
for name, (vref, vsim) in zip(self.names, self.all_variables):
self.logger.debug('Creating plots of %s', vsim)
kwargs = dict(precision=0.1) if vref.startswith('prcp') else {}
psy.plot.densityreg(ds,
name='all_' + vsim,
coord='all_' + vref,
fmt=self.fmt,
title='All percentiles',
arr_names=['%s_all' % name],
**kwargs)
arr_names = ['%s_%1.2f' % (name, p) for p in ds.pctl.values]
psy.plot.densityreg(ds,
name=vsim,
coord=vref,
fmt=self.fmt,
arr_names=arr_names,
pctl=range(ds.pctl.size),
**kwargs)
return psy.gcp(True)[:]
def make_run_config(self, sp, info):
for orig in self.names:
info[orig] = d = OrderedDict()
for plotter in sp(standard_name=orig).plotters:
d[plotter.data.pctl if plotter.data.name.
startswith('all') else int(plotter.data.pctl.values
)] = pctl_d = OrderedDict()
for key in ['rsquared', 'slope', 'intercept']:
val = plotter.plot_data[1].attrs.get(key)
if val is not None:
pctl_d[key] = float(val)
return info
def calc(self, group):
def calc_percentiles(vname):
arr = group[vname].values
arr = arr[~np.isnan(arr)]
if vname.startswith('prcp'):
arr = arr[arr > 0]
if len(arr) == 0:
return np.array([np.nan] * len(quantiles))
else:
return np.percentile(arr, quantiles)
quantiles = self.task_config.quantiles
df = pd.DataFrame.from_dict(
dict(
zip(chain(*self.all_variables),
map(calc_percentiles, chain(*self.all_variables)))))
df['pctl'] = quantiles
df.set_index('pctl')
return df
@staticmethod
def round_to_ref_prec(ref, sim, func=np.ceil):
"""Round one array to the precision of another
Parameters
----------
ref: np.ndarray
The reference array to get the precision from
sim: np.ndarray
The simulated array to round
func: function
The rounding function to use
Returns
-------
np.ndarray
Rounded `sim`"""
ref_sorted = np.unique(ref)
if len(ref_sorted) < 2:
return sim
precision = (ref_sorted[1:] - ref_sorted[:-1]).min()
return func((sim / precision) * precision)
