Esempio n. 1
0
class ProjectContent(QToolBox):
    """Display the content in the current project

    This toolbox contains several :class:`PlotterList` that show the content
    of the current main and subproject"""

    #: :class:`OrderedDict` containing the :class:`PlotterList` instances
    #: of the different selection attributes
    lists = OrderedDict()

    @property
    def current_names(self):
        return [self.itemText(i) for i in range(self.count())]

    def __init__(self, *args, **kwargs):
        super(ProjectContent, self).__init__(*args, **kwargs)
        self.lists = OrderedDict()
        for attr in chain(['All'], sorted(Project._registered_plotters)):
            item = self.add_plotterlist(attr, force=(attr == 'All'))
            self.lists[attr] = item
        self.currentChanged.connect(self.update_current_list)
        Project.oncpchange.connect(self.update_lists)

    def enable_list(self, list_widget):
        """Enable a given list widget based upon whether it is empty or not"""
        i = self.indexOf(list_widget)
        if i != -1:
            self.setItemEnabled(i, not list_widget.is_empty)

    def add_plotterlist(self, identifier, force=False):
        """Create a :class:`PlotterList` from an identifier from the
        :class:`psyplot.project.Project` class"""
        attr = identifier if identifier != 'All' else None
        item = PlotterList(attr)
        if not item.can_import_plotter:
            return item
        if force or not item.is_empty:
            item.setParent(self)
            item.updated_from_project.connect(self.enable_list)
            self.addItem(item, identifier)
            i = self.indexOf(item)
            self.setItemEnabled(i, not item.is_empty)
        return item

    def update_current_list(self):
        """Update the current list from the current main and sub project"""
        self.currentWidget().update_from_project(gcp(True))
        self.currentWidget().update_from_project(gcp())

    def update_lists(self, p):
        # check new lists
        current_items = self.current_names
        for name, l in self.lists.items():
            if not p.is_main:
                l.update_from_project(p.main)
            l.update_from_project(p)
            if l.is_empty:
                l.disconnect_items()
            if name != 'All' and l.is_empty:
                i = self.indexOf(l)
                self.removeItem(i)
            elif not l.is_empty and name not in current_items:
                self.addItem(l, name)
Esempio n. 2
0
        ('regular', QtGui.QFont.Normal),
        ('book', QtGui.QFont.Normal),
        ('medium', QtGui.QFont.Normal),
        ('roman', QtGui.QFont.Normal),
        ('semibold', QtGui.QFont.DemiBold),
        ('demibold', QtGui.QFont.DemiBold),
        ('demi', QtGui.QFont.DemiBold),
        ('bold', QtGui.QFont.Bold),
        ('heavy', QtGui.QFont.Bold),
        ('extra bold', QtGui.QFont.Black),
        ('black', QtGui.QFont.Black),
    ])

weights_qt2mpl = OrderedDict(
    map(reversed,
        utils.unique_everseen(weights_mpl2qt.items(), key=lambda t: t[1])))


def mpl_weight2qt(weight):
    """Convert a weight from matplotlib definition to a Qt weight

    Parameters
    ----------
    weight: int or string
        Either an integer between 1 and 1000 or a string out of
        :attr:`weights_mpl2qt`

    Returns
    -------
    int
        One type of the PyQt5.QtGui.QFont.Weight"""
Esempio n. 3
0
class FuncArgParser(ArgumentParser):
    """Subclass of an argument parser that get's parts of the information
    from a given function"""

    def __init__(self, *args, **kwargs):
        super(FuncArgParser, self).__init__(*args, **kwargs)
        self.__arguments = OrderedDict()
        self.__funcs = []
        self.__main = None

    def setup_args(self, func):
        """Add the parameters from the given `func` to the parameter settings
        """
        self.__funcs.append(func)
        args_dict = self.__arguments
        args, varargs, varkw, defaults = inspect.getargspec(func)
        full_doc = inspect.getdoc(func)
        doc = docstrings._get_section(full_doc, 'Parameters') + '\n'
        doc += docstrings._get_section(full_doc, 'Other Parameters')
        doc = doc.rstrip()
        default_min = len(args) - len(defaults)
        for i, arg in enumerate(args):
            if arg == 'self' or arg in args_dict:
                continue
            arg_doc = docstrings._keep_params(doc, [arg]) or \
                docstrings._keep_types(doc, [arg])
            args_dict[arg] = d = {'dest': arg, 'short': arg, 'long': arg}
            if arg_doc:
                lines = arg_doc.splitlines()
                d['help'] = '\n'.join(lines[1:])
                metavar = lines[0].split(':', 1)
                if i >= default_min:
                    d['default'] = defaults[i - default_min]
                if len(metavar) > 1:
                    dtype = metavar[1].strip()
                    if dtype == 'bool' and 'default' in d:
                        d['action'] = 'store_false' if d['default'] else \
                            'store_true'
                    else:
                        d['metavar'] = metavar[1].strip()
                else:
                    d['positional'] = True

    def update_arg(self, arg, if_existent=True, **kwargs):
        """Update the `add_argument` data for the given parameter
        """
        if not if_existent:
            self.__arguments.setdefault(arg, kwargs)
        self.__arguments[arg].update(kwargs)

    def pop_key(self, arg, key, *args, **kwargs):
        """Delete a previously defined key for the `add_argument`
        """
        return self.__arguments[arg].pop(key, *args, **kwargs)

    def create_arguments(self):
        """Create and add the arguments"""
        ret = []
        for arg, d in self.__arguments.items():
            try:
                is_positional = d.pop('positional', False)
                short = d.pop('short')
                long_name = d.pop('long', None)
                if short == long_name:
                    long_name = None
                args = [short, long_name] if long_name else [short]
                if not is_positional:
                    for i, arg in enumerate(args):
                        args[i] = '-' * (i + 1) + arg
                else:
                    d.pop('dest', None)
                group = d.pop('group', self)
                ret.append(group.add_argument(*args, **d))
            except Exception:
                print('Error while creating argument %s' % arg)
                raise
        return ret

    @docstrings.get_sectionsf('FuncArgParser.parse_to_func')
    @docstrings.dedent
    def parse_to_func(self, args=None):
        """
        Parse the given arguments to the main function

        Parameters
        ----------
        args: list
            The list of arguments given to the
            :meth:`ArgumentParser.parse_args` function. If None, the sys.argv
            is used."""
        if args is not None:
            (self.__main or self.__funcs[0])(**vars(self.parse_args(args)))
        else:
            (self.__main or self.__funcs[0])(**vars(self.parse_args()))

    @docstrings.dedent
    def parse_known_to_func(self, args=None):
        """
        Parse the known arguments from the given to the main function

        Parameters
        ----------
        %(FuncArgParser.parse_to_func.parameters)s"""
        if args is not None:
            (self.__main or self.__funcs[0])(
                **vars(self.parse_known_args(args)[0]))
        else:
            (self.__main or self.__funcs[0])(
                **vars(self.parse_known_args()[0]))

    def set_main(self, func):
        """Set the function that is called by the :meth:`parse_to_func` and
        :meth:`parse_known_to_func` function"""
        self.__main = func
Esempio n. 4
0
    def recursive_processing(self, base_dir, target_dir, it):
        """Method to recursivly process the notebooks in the `base_dir`

        Parameters
        ----------
        base_dir: str
            Path to the base example directory (see the `examples_dir`
            parameter for the :class:`Gallery` class)
        target_dir: str
            Path to the output directory for the rst files (see the
            `gallery_dirs` parameter for the :class:`Gallery` class)
        it: iterable
            The iterator over the subdirectories and files in `base_dir`
            generated by the :func:`os.walk` function"""
        try:
            file_dir, dirs, files = next(it)
        except StopIteration:
            return '', []
        readme_files = {'README.md', 'README.rst', 'README.txt'}
        if readme_files.intersection(files):
            foutdir = file_dir.replace(base_dir, target_dir)
            create_dirs(foutdir)
            this_nbps = [
                NotebookProcessor(
                    infile=f,
                    outfile=os.path.join(foutdir, os.path.basename(f)),
                    disable_warnings=self.disable_warnings,
                    preprocess=(
                        (self.preprocess is True or f in self.preprocess) and
                        not (self.dont_preprocess is True or
                             f in self.dont_preprocess)),
                    clear=((self.clear is True or f in self.clear) and not
                           (self.dont_clear is True or f in self.dont_clear)))
                for f in map(lambda f: os.path.join(file_dir, f),
                             filter(self.pattern.match, files))]
            readme_file = next(iter(readme_files.intersection(files)))
        else:
            return '', []
        labels = OrderedDict()
        this_label = 'gallery_' + foutdir.replace(os.path.sep, '_')
        if this_label.endswith('_'):
            this_label = this_label[:-1]
        for d in dirs:
            label, nbps = self.recursive_processing(
                base_dir, target_dir, it)
            if label:
                labels[label] = nbps
        s = ".. _%s:\n\n" % this_label
        with open(os.path.join(file_dir, readme_file)) as f:
            s += f.read().rstrip() + '\n\n'

        s += "\n\n.. toctree::\n\n"
        s += ''.join('    %s\n' % os.path.splitext(os.path.basename(
            nbp.get_out_file()))[0] for nbp in this_nbps)
        for d in dirs:
            findex = os.path.join(d, 'index.rst')
            if os.path.exists(os.path.join(foutdir, findex)):
                s += '    %s\n' % os.path.splitext(findex)[0]

        s += '\n'

        for nbp in this_nbps:
                s += nbp.thumbnail_div + '\n'
        s += "\n.. raw:: html\n\n    <div style='clear:both'></div>\n"
        for label, nbps in labels.items():
            s += '\n.. only:: html\n\n    .. rubric:: :ref:`%s`\n\n' % (
                label)
            for nbp in nbps:
                s += nbp.thumbnail_div + '\n'
            s += "\n.. raw:: html\n\n    <div style='clear:both'></div>\n"

        s += '\n'

        with open(os.path.join(foutdir, 'index.rst'), 'w') as f:
            f.write(s)
        return this_label, list(chain(this_nbps, *labels.values()))
Esempio n. 5
0
class HelpExplorer(QWidget, DockMixin):
    """A widget for showing the documentation. It behaves somewhat similar
    to spyders object inspector plugin and can show restructured text either
    as html (if sphinx is installed) or as plain text. It furthermore has a
    browser to show html content

    Warnings
    --------
    The :class:`HelpBrowser` class is known to crash under PyQt4 when new web
    page domains are loaded. Hence you should disable the browsing to different
    remote websites and even disable intersphinx"""

    #: The viewer classes used by the help explorer. :class:`HelpExplorer`
    #: instances replace this attribute with the corresponding HelpMixin
    #: instance
    viewers = OrderedDict([('HTML help', UrlHelp), ('Plain text', TextHelp)])

    def __init__(self, *args, **kwargs):
        super(HelpExplorer, self).__init__(*args, **kwargs)
        self.vbox = vbox = QVBoxLayout()
        self.combo = QComboBox(parent=self)
        vbox.addWidget(self.combo)
        if _viewers:
            self.viewers = _viewers.copy()
            for w in self.viewers.values():
                w.setParent(self)
        else:
            self.viewers = OrderedDict(
                [(key, cls(parent=self)) for key, cls in six.iteritems(
                    self.viewers)])
            # save the UrlHelp because QWebEngineView creates child processes
            # that are not properly closed by PyQt and as such use too much
            # memory
            if is_running_tests():
                for key, val in self.viewers.items():
                    _viewers[key] = val
        for key, ini in six.iteritems(self.viewers):
            self.combo.addItem(key)
            ini.hide()
            vbox.addWidget(ini)
        self.viewer = next(six.itervalues(self.viewers))
        self.viewer.show()
        self.combo.currentIndexChanged[str].connect(self.set_viewer)
        self.setLayout(vbox)

    def set_viewer(self, name):
        """Sets the current documentation viewer

        Parameters
        ----------
        name: str or object
            A string must be one of the :attr:`viewers` attribute. An object
            can be one of the values in the :attr:`viewers` attribute"""
        if isstring(name) and asstring(name) not in self.viewers:
            raise ValueError("Don't have a viewer named %s" % (name, ))
        elif not isstring(name):
            viewer = name
        else:
            name = asstring(name)
            viewer = self.viewers[name]
        self.viewer.hide()
        self.viewer = viewer
        self.viewer.show()
        if (isstring(name) and
                not self.combo.currentText() == name):
            self.combo.setCurrentIndex(list(self.viewers).index(name))

    @docstrings.dedent
    def show_help(self, obj, oname='', files=None):
        """
        Show the documentaion of the given object

        We first try to use the current viewer based upon it's
        :attr:`HelpMixin.can_document_object` attribute. If this does not work,
        we check the other viewers

        Parameters
        ----------
        %(HelpMixin.show_help.parameters)s"""
        oname = asstring(oname)
        ret = None
        if self.viewer.can_document_object:
            try:
                ret = self.viewer.show_help(obj, oname=oname, files=files)
            except Exception:
                logger.debug("Could not document %s with %s viewer!",
                             oname, self.combo.currentText(), exc_info=True)
        else:
            curr_i = self.combo.currentIndex()
            for i, (viewername, viewer) in enumerate(
                    six.iteritems(self.viewers)):
                if i != curr_i and viewer.can_document_object:
                    self.set_viewer(viewername)
                    self.combo.blockSignals(True)
                    self.combo.setCurrentIndex(i)
                    self.combo.blockSignals(False)
                    try:
                        ret = viewer.show_help(obj, oname=oname, files=files)
                    except Exception:
                        logger.debug("Could not document %s with %s viewer!",
                                     oname, viewername, exc_info=True)
        if ret:
            self.parent().raise_()
        return ret

    @docstrings.dedent
    def show_rst(self, text, oname='', files=None):
        """
        Show restructured text

        We first try to use the current viewer based upon it's
        :attr:`HelpMixin.can_show_rst` attribute. If this does not work,
        we check the other viewers

        Parameters
        ----------
        %(HelpMixin.show_rst.parameters)s"""
        ret = None
        if self.viewer.can_show_rst:
            ret = self.viewer.show_rst(text, oname=oname, files=files)
        else:
            for viewer in six.itervalues(self.viewers):
                if viewer.can_show_rst:
                    self.set_viewer(viewer)
                    ret = viewer.show_rst(text, oname=oname, files=files)
                    break
        if ret:
            self.parent().raise_()
        return ret

    @docstrings.dedent
    def show_intro(self, text=''):
        """
        Show an intro text

        We first try to use the current viewer based upon it's
        :attr:`HelpMixin.can_show_rst` attribute. If this does not work,
        we check the other viewers

        Parameters
        ----------
        %(HelpMixin.show_intro.parameters)s"""
        found = False
        for i, viewer in enumerate(six.itervalues(self.viewers)):
            viewer.show_intro(text)
            if not found and viewer.can_show_rst:
                if i:
                    self.set_viewer(viewer)
                found = True

    def close(self, *args, **kwargs):
        try:
            self.viewers['HTML help'].close(*args, **kwargs)
        except AttributeError:
            pass
        return super(HelpExplorer, self).close(*args, **kwargs)
Esempio n. 6
0
class QuantileEvaluation(Evaluator):
    """Evaluator to evaluate specific quantiles"""

    name = 'quants'

    summary = 'Compare the quantiles of simulation and observation'

    names = OrderedDict([('prcp', {
        'long_name': 'Precipitation',
        'units': 'mm'
    }), ('tmin', {
        'long_name': 'Min. Temperature',
        'units': 'degC'
    }), ('tmax', {
        'long_name': 'Max. Temperature',
        'units': 'degC'
    }), ('mean_cloud', {
        'long_name': 'Cloud fraction',
        'units': '-'
    }), ('wind', {
        'long_name': 'Wind Speed',
        'units': 'm/s'
    })])

    @property
    def all_variables(self):
        return [[v + '_ref', v + '_sim'] for v in self.names]

    setup_requires = ['prepare', 'output']

    has_run = True

    _datafile = 'quantile_evaluation.csv'

    dbname = 'quantile_evaluation'

    #: default formatoptions for the
    #: :class:`psyplot.plotter.linreg.DensityRegPlotter` plotter
    fmt = kwargs = dict(
        legend={'loc': 'upper left'},
        cmap='w_Reds',
        title='%(pctl)sth percentile',
        xlabel='%(type)s {desc}',
        ylabel='%(type)s {desc}',
        xrange=(['minmax', 1], ['minmax', 99]),
        yrange=(['minmax', 1], ['minmax', 99]),
        legendlabels=['$R^2$ = %(rsquared)s'],
        bounds=['minmax', 11, 0, 99],
        cbar='',
        bins=10,
        ideal=[0, 1],
        id_color='r',
        sym_lims='max',
    )

    @property
    def default_config(self):
        return default_quantile_config()._replace(
            **super(QuantileEvaluation, self).default_config._asdict())

    @property
    def ds(self):
        """The dataset of the quantiles"""
        import xarray as xr
        data = self.data.reset_index()
        ds = xr.Dataset.from_dataframe(
            data.set_index('pctl', append=True).swaplevel())
        full = xr.Dataset.from_dataframe(data).drop(
            list(chain(self.data.index.names, ['pctl'])))
        idx_name = full[next(v for v in self.names) + '_sim'].dims[-1]
        full.rename({idx_name: 'full_index'}, inplace=True)
        for vref, vsim in self.all_variables:
            full.rename({
                vref: 'all_' + vref,
                vsim: 'all_' + vsim
            },
                        inplace=True)
        ds.merge(full, inplace=True)
        for orig, attrs, (vref, vsim) in zip(self.names, self.names.values(),
                                             self.all_variables):
            for prefix in ['', 'all_']:
                ds[prefix + vsim].attrs.update(attrs)
                ds[prefix + vref].attrs.update(attrs)
                ds[prefix + vsim].attrs['standard_name'] = orig
                ds[prefix + vref].attrs['standard_name'] = orig
                ds[prefix + vref].attrs['type'] = 'observed'
                ds[prefix + vsim].attrs['type'] = 'simulated'
            ds['all_' + vsim].attrs['pctl'] = 'All'
            ds['all_' + vsim].attrs['pctl'] = 'All'
        ds.pctl.attrs['long_name'] = 'Percentile'
        return ds

    def __init__(self, *args, **kwargs):
        super(QuantileEvaluation, self).__init__(*args, **kwargs)
        names = self.task_config.names
        if names is not None:
            self.names = OrderedDict(t for t in self.names.items()
                                     if t[0] in names)

    def setup_from_file(self, *args, **kwargs):
        kwargs['index_col'] = ['id', 'year']
        return super(QuantileEvaluation, self).setup_from_file(*args, **kwargs)

    def setup_from_db(self, *args, **kwargs):
        kwargs['index_col'] = ['id', 'year']
        return super(QuantileEvaluation, self).setup_from_db(*args, **kwargs)

    def setup_from_scratch(self):
        df_ref = self.prepare.reference_data
        dfi = self.prepare.input_data.reset_index(['lon', 'lat'])
        # create simulation dataframe
        df_sim = self.output.data
        if len(df_ref) == 0 or len(df_sim) == 0:
            self.logger.debug(
                'Skipping %s because reference data contains no information!',
                self.name)
            return
        names = self.names
        # load observed precision
        if self.task_config.no_rounding:
            for name in names:
                df_sim[name].values[:] = self.round_to_ref_prec(
                    df_ref[name].values, df_sim[name].values)
        # merge reference and simulation into one single dataframe
        df = df_ref.merge(df_sim,
                          left_index=True,
                          right_index=True,
                          suffixes=['_ref', '_sim'])
        if {'mean_cloud', 'wind'}.intersection(names):
            df.reset_index('day', inplace=True)
            df = df.merge(dfi[['mean_cloud', 'wind']],
                          left_index=True,
                          right_index=True)
            # mask out non-complete months for cloud validation and months with
            # 0 or 1 cloud fraction
            if 'mean_cloud' in names:
                df.ix[df['mean_cloud_ref'].isnull().values |
                      (df['mean_cloud'] == 0.0) | (df['mean_cloud'] == 1.0),
                      ['mean_cloud_sim', 'mean_cloud_ref']] = np.nan
            # mask out non-complete wind for wind validation and months with
            # a mean wind speed of 0
            if 'wind' in names:
                df.ix[df['wind_ref'].isnull().values | (df['wind'] == 0.0),
                      ['wind_sim', 'wind_ref']] = np.nan
            df.drop(['mean_cloud', 'wind'], 1, inplace=True)
            df.set_index('day', append=True, inplace=True)

        # transform wind
        if self.task_config.transform_wind and 'wind' in names:
            df['wind_ref'] **= 0.5
            df['wind_sim'] **= 0.5
        # calculate the percentiles for each station and month
        g = df.sort_index().groupby(level=['id', 'year'])
        self.logger.debug('Done with basic setup')
        data = g.apply(self.calc)
        if len(data):
            data.index = data.index.droplevel(2)
        self.data = data

    @classmethod
    def _modify_parser(cls, parser):
        parser.setup_args(default_ks_config)
        parser.setup_args(default_quantile_config)
        parser, setup_grp, run_grp = super(QuantileEvaluation,
                                           cls)._modify_parser(parser)
        parser.update_arg('quantiles',
                          short='q',
                          group=run_grp,
                          type=utils.str_ranges,
                          metavar='f1[,f21[-f22[-f23]]]',
                          help=docstrings.dedents("""
                The quantiles to use for calculating the percentiles.
                %(str_ranges.s_help)s."""))
        parser.pop_key('quantiles', 'nargs', None)
        parser.update_arg('no_rounding', short='nr', group=run_grp)
        parser.update_arg('names',
                          short='n',
                          group=setup_grp,
                          nargs='+',
                          metavar='variable',
                          choices=list(cls.names))
        parser.update_arg('transform_wind', short='tw', group=setup_grp)
        return parser, setup_grp, run_grp

    def create_project(self, ds):
        import psyplot.project as psy
        import seaborn as sns
        sns.set_style('white')
        for name, (vref, vsim) in zip(self.names, self.all_variables):
            self.logger.debug('Creating plots of %s', vsim)
            kwargs = dict(precision=0.1) if vref.startswith('prcp') else {}
            psy.plot.densityreg(ds,
                                name='all_' + vsim,
                                coord='all_' + vref,
                                fmt=self.fmt,
                                title='All percentiles',
                                arr_names=['%s_all' % name],
                                **kwargs)
            arr_names = ['%s_%1.2f' % (name, p) for p in ds.pctl.values]
            psy.plot.densityreg(ds,
                                name=vsim,
                                coord=vref,
                                fmt=self.fmt,
                                arr_names=arr_names,
                                pctl=range(ds.pctl.size),
                                **kwargs)
        return psy.gcp(True)[:]

    def make_run_config(self, sp, info):
        for orig in self.names:
            info[orig] = d = OrderedDict()
            for plotter in sp(standard_name=orig).plotters:
                d[plotter.data.pctl if plotter.data.name.
                  startswith('all') else int(plotter.data.pctl.values
                                             )] = pctl_d = OrderedDict()
                for key in ['rsquared', 'slope', 'intercept']:
                    val = plotter.plot_data[1].attrs.get(key)
                    if val is not None:
                        pctl_d[key] = float(val)
        return info

    def calc(self, group):
        def calc_percentiles(vname):
            arr = group[vname].values
            arr = arr[~np.isnan(arr)]
            if vname.startswith('prcp'):
                arr = arr[arr > 0]
            if len(arr) == 0:
                return np.array([np.nan] * len(quantiles))
            else:
                return np.percentile(arr, quantiles)

        quantiles = self.task_config.quantiles
        df = pd.DataFrame.from_dict(
            dict(
                zip(chain(*self.all_variables),
                    map(calc_percentiles, chain(*self.all_variables)))))
        df['pctl'] = quantiles
        df.set_index('pctl')
        return df

    @staticmethod
    def round_to_ref_prec(ref, sim, func=np.ceil):
        """Round one array to the precision of another

        Parameters
        ----------
        ref: np.ndarray
            The reference array to get the precision from
        sim: np.ndarray
            The simulated array to round
        func: function
            The rounding function to use

        Returns
        -------
        np.ndarray
            Rounded `sim`"""
        ref_sorted = np.unique(ref)
        if len(ref_sorted) < 2:
            return sim
        precision = (ref_sorted[1:] - ref_sorted[:-1]).min()
        return func((sim / precision) * precision)