Exemple #1
0
    def __init__(self, model_run_dict):
        """
        Take a Calliope model_run and convert it into an xarray Dataset, ready for
        constraint generation. Timeseries data is also extracted from file at this
        point, and the time dimension added to the data

        Parameters
        ----------
        model_run_dict : AttrDict
            preprocessed model_run dictionary, as produced by
            Calliope.preprocess.preprocess_model

        Returns
        -------
        data : xarray Dataset
            Dataset with optimisation param_dict as variables, optimisation sets as
            coordinates, and other information in attributes.
        data_pre_time : xarray Dataset, only returned if debug = True
            Dataset, prior to time dimension addition, with optimisation param_dict
            as variables, optimisation sets as coordinates, and other information
            in attributes.

        """
        self.node_dict = model_run_dict.nodes.as_dict_flat()
        self.tech_dict = model_run_dict.techs.as_dict_flat()
        self.model_run = model_run_dict
        self.model_data = xr.Dataset(
            coords={"timesteps": model_run_dict.timeseries_data.index})
        self._add_attributes(model_run_dict)
        self.template_config = AttrDict.from_yaml(
            os.path.join(os.path.dirname(calliope.__file__), "config",
                         "model_data_lookup.yaml"))
        self._strip_unwanted_keys()
        self._add_node_tech_sets()
Exemple #2
0
def model_run_from_yaml(model_file, scenario=None, override_dict=None):
    """
    Generate processed ModelRun configuration from a
    YAML model configuration file.

    Parameters
    ----------
    model_file : str
        Path to YAML file with model configuration.
    scenario : str, optional
        Name of scenario to apply. Can either be a named scenario, or a
        comman-separated list of individual overrides to be combined
        ad-hoc, e.g. 'my_scenario_name' or 'override1,override2'.
    override_dict : dict or AttrDict, optional

    """
    config = AttrDict.from_yaml(model_file)
    config.config_path = model_file

    config_with_overrides, debug_comments, overrides, scenario = apply_overrides(
        config, scenario=scenario, override_dict=override_dict
    )

    return generate_model_run(
        config_with_overrides, debug_comments, overrides, scenario)
Exemple #3
0
    def test_model_from_dict(self):
        """
        Test loading a file from dict/AttrDict instead of from YAML
        """
        this_path = os.path.dirname(__file__)
        model_location = os.path.join(this_path, 'common', 'test_model',
                                      'model.yaml')
        model_dict = AttrDict.from_yaml(model_location)
        location_dict = AttrDict({
            'locations': {
                '0': {
                    'techs': {
                        'test_supply_elec': {},
                        'test_demand_elec': {}
                    }
                },
                '1': {
                    'techs': {
                        'test_supply_elec': {},
                        'test_demand_elec': {}
                    }
                }
            }
        })
        model_dict.union(location_dict)
        model_dict.model['timeseries_data_path'] = os.path.join(
            this_path, 'common', 'test_model',
            model_dict.model['timeseries_data_path'])
        # test as AttrDict
        calliope.Model(model_dict)

        # test as dict
        calliope.Model(model_dict.as_dict())
Exemple #4
0
def model_run_from_yaml(model_file, scenario=None, override_dict=None):
    """
    Generate processed ModelRun configuration from a
    YAML model configuration file.

    Parameters
    ----------
    model_file : str
        Path to YAML file with model configuration.
    scenario : str, optional
        Name of scenario to apply. Can either be a named scenario, or a
        comman-separated list of individual overrides to be combined
        ad-hoc, e.g. 'my_scenario_name' or 'override1,override2'.
    override_dict : dict or AttrDict, optional

    """
    config = AttrDict.from_yaml(model_file)
    config.config_path = model_file

    config_with_overrides, debug_comments, overrides, scenario = apply_overrides(
        config, scenario=scenario, override_dict=override_dict
    )

    return generate_model_run(
        config_with_overrides, debug_comments, overrides, scenario)
Exemple #5
0
 def test_get_comment(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     result = {
         'above': '# a comment about `c`\n',
         'inline': '# a comment inline with `c`\n',
         'below': None
     }
     assert d.get_comments('c') == result
Exemple #6
0
    def test_comment_roundtrip(self, yaml_file):
        d = AttrDict.from_yaml(yaml_file)
        with tempfile.TemporaryDirectory() as tempdir:
            out_file = os.path.join(tempdir, 'test.yaml')
            d.to_yaml(out_file)

            with open(out_file, 'r') as f:
                result = f.read()

            assert 'x: foo  # a comment on foo' in result
Exemple #7
0
def load_with_import_resolution(in_path):
    files = {}
    top = AttrDict.from_yaml(in_path, resolve_imports=False)
    files[in_path] = top
    base_path = os.path.dirname(in_path)
    for import_path in top.get('import', []):
        result = load_with_import_resolution(
            os.path.join(base_path, import_path))
        for k, v in result.items():
            files[k] = v
    return files
Exemple #8
0
    def test_add_comment_exists(self, yaml_file):
        d = AttrDict.from_yaml(yaml_file)

        # should work, as no inline comment exists
        d.set_comment('b', 'test comment')

        # should fail, as an above comment exists
        with pytest.raises(ValueError) as excinfo:
            d.set_comment('c', 'test comment', kind='above')

        assert check_error_or_warning(excinfo, 'Comment exists')
Exemple #9
0
    def test_add_comment_exists_but_empty(self, yaml_file):
        d = AttrDict.from_yaml(yaml_file)

        # Ensure comments were added
        assert d.c.__dict_comments__['y'][2].value == '#\n'

        # should work, as inline comment is empty
        d.set_comment('c.y', 'test comment')

        # should work, as above comment is empty
        d.set_comment('c.y', 'test comment', kind='above')
Exemple #10
0
 def test_union_preserves_comments(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     d_new = AttrDict.from_yaml_string("""
         test: 1  # And a comment
         somekey:
             bar:
                 baz: 2  # Another comment
     """)
     d.union(d_new)
     assert d.get_comments(
         'somekey.bar.baz')['inline'] == '# Another comment\n'
Exemple #11
0
def model_run_from_yaml(model_file,
                        timeseries_dataframes=None,
                        scenario=None,
                        override_dict=None):
    """
    Generate processed ModelRun configuration from a
    YAML model configuration file.

    Parameters
    ----------
    model_file : str
        Path to YAML file with model configuration.
    timeseries_dataframes : dict, optional
        Dictionary of timeseries dataframes. The keys are strings
        corresponding to the dataframe names given in the yaml files and
        the values are dataframes with time series data.
    scenario : str, optional
        Name of scenario to apply. Can either be a named scenario, or a
        comma-separated list of individual overrides to be combined
        ad-hoc, e.g. 'my_scenario_name' or 'override1,override2'.
    override_dict : dict or AttrDict, optional

    """
    config = AttrDict.from_yaml(model_file)
    config.config_path = model_file

    config_with_overrides, debug_comments, overrides, scenario = apply_overrides(
        config, scenario=scenario, override_dict=override_dict)
    subsets = AttrDict.from_yaml(
        os.path.join(os.path.dirname(calliope.__file__), "config",
                     "subsets.yaml"))

    return generate_model_run(
        config_with_overrides,
        timeseries_dataframes,
        debug_comments,
        overrides,
        scenario,
        subsets,
    )
Exemple #12
0
    def test_to_yaml(self, yaml_file):
        d = AttrDict.from_yaml(yaml_file)
        d.set_key('numpy.some_int', np.int32(10))
        d.set_key('numpy.some_float', np.float64(0.5))
        d.a_list = [0, 1, 2]
        with tempfile.TemporaryDirectory() as tempdir:
            out_file = os.path.join(tempdir, 'test.yaml')
            d.to_yaml(out_file)

            with open(out_file, 'r') as f:
                result = f.read()

            assert 'some_int: 10' in result
            assert 'some_float: 0.5' in result
            assert 'a_list:\n- 0\n- 1\n- 2' in result
    def test_to_yaml(self, yaml_file):
        d = AttrDict.from_yaml(yaml_file)
        d.set_key("numpy.some_int", np.int32(10))
        d.set_key("numpy.some_float", np.float64(0.5))
        d.a_list = [0, 1, 2]
        with tempfile.TemporaryDirectory() as tempdir:
            out_file = os.path.join(tempdir, "test.yaml")
            d.to_yaml(out_file)

            with open(out_file, "r") as f:
                result = f.read()

            assert "some_int: 10" in result
            assert "some_float: 0.5" in result
            assert "a_list:\n- 0\n- 1\n- 2" in result
    def test_model_from_dict(self):
        """
        Test loading a file from dict/AttrDict instead of from YAML
        """
        this_path = os.path.dirname(__file__)
        model_location = os.path.join(this_path, 'common', 'test_model', 'model.yaml')
        model_dict = AttrDict.from_yaml(model_location)
        location_dict = AttrDict({
            'locations': {
                '0': {'techs': {'test_supply_elec': {}, 'test_demand_elec': {}}},
                '1': {'techs': {'test_supply_elec': {}, 'test_demand_elec': {}}}
            }
        })
        model_dict.union(location_dict)
        model_dict.model['timeseries_data_path'] = os.path.join(
            this_path, 'common', 'test_model', model_dict.model['timeseries_data_path']
        )
        # test as AttrDict
        calliope.Model(model_dict)

        # test as dict
        calliope.Model(model_dict.as_dict())
Exemple #15
0
def model_run_from_yaml(model_file, override_file=None, override_dict=None):
    """
    Generate processed ModelRun configuration from a
    YAML model configuration file.

    Parameters
    ----------
    model_file : str
        Path to YAML file with model configuration.
    override_file : str, optional
        Path to YAML file with model configuration overrides and the override
        group to use, separated by ':', e.g. 'overrides.yaml:group1'.
    override_dict : dict or AttrDict, optional

    """
    config = AttrDict.from_yaml(model_file)
    config.config_path = model_file

    config_with_overrides, debug_comments = apply_overrides(
        config, override_file=override_file, override_dict=override_dict)

    return generate_model_run(config_with_overrides, debug_comments)
Exemple #16
0
def combine_overrides(override_file_path, override_groups):
    if ',' in override_groups:
        overrides = override_groups.split(',')
    else:
        overrides = [override_groups]

    override = AttrDict()
    for group in overrides:
        try:
            override_group_from_file = AttrDict.from_yaml(
                override_file_path)[group]
        except KeyError:
            raise exceptions.ModelError(
                'Override group `{}` does not exist in file `{}`.'.format(
                    group, override_file_path))
        try:
            override.union(override_group_from_file, allow_override=False)
        except KeyError as e:
            raise exceptions.ModelError(
                str(e)[1:-1] + '. Already specified but defined again in '
                'override group `{}`.'.format(group))

    return override
Exemple #17
0
 def test_del_comment(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     assert 'y' in d.c.__dict_comments__
     d.del_comments('c.y')
     assert 'y' not in d.c.__dict_comments__
import os
import calliope
import pytest  # pylint: disable=unused-import
import tempfile

from calliope.core.attrdict import AttrDict
from calliope.test.common.util import check_error_or_warning, python36_or_higher


HTML_STRINGS = AttrDict.from_yaml(
    os.path.join(os.path.dirname(__file__), 'common', 'html_strings.yaml')
)


class TestPlotting:
    @pytest.fixture(scope="module")
    def national_scale_example(self):
        model = calliope.examples.national_scale(
            override_dict={'model.subset_time': '2005-01-01'}
        )
        model.run()
        return model

    @python36_or_higher
    def test_national_scale_plotting(self, national_scale_example):
        model = national_scale_example

        plot_html_outputs = {
            'capacity': model.plot.capacity(html_only=True),
            'timeseries': model.plot.timeseries(html_only=True),
            'transmission': model.plot.transmission(html_only=True),
Exemple #19
0
def apply_overrides(config, scenario=None, override_dict=None):
    """
    Generate processed Model configuration, applying any scenarios overrides.

    Parameters
    ----------
    config : AttrDict
        a model configuration AttrDict
    scenario : str, optional
    override_dict : str or dict or AttrDict, optional
        If a YAML string, converted to AttrDict

    """
    debug_comments = AttrDict()

    base_model_config_file = os.path.join(
        os.path.dirname(calliope.__file__),
        'config', 'model.yaml'
    )
    config_model = AttrDict.from_yaml(base_model_config_file)

    # Interpret timeseries_data_path as relative
    config.model.timeseries_data_path = relative_path(
        config.config_path, config.model.timeseries_data_path
    )

    # The input files are allowed to override other model defaults
    config_model.union(config, allow_override=True)

    # First pass of applying override dict before applying scenarios,
    # so that can override scenario definitions by override_dict
    if override_dict:
        if isinstance(override_dict, str):
            override_dict = AttrDict.from_yaml_string(override_dict)
        elif not isinstance(override_dict, AttrDict):
            override_dict = AttrDict(override_dict)

        warnings = checks.check_overrides(config_model, override_dict)
        exceptions.print_warnings_and_raise_errors(warnings=warnings)

        config_model.union(
            override_dict, allow_override=True, allow_replacement=True
        )

    if scenario:
        scenarios = config_model.get('scenarios', {})

        if scenario in scenarios:
            # Manually defined scenario names cannot be the same as single
            # overrides or any combination of semicolon-delimited overrides
            if all([i in config_model.get('overrides', {})
                    for i in scenario.split(',')]):
                raise exceptions.ModelError(
                    'Manually defined scenario cannot be a combination of override names.'
                )
            if not isinstance(scenarios[scenario], str):
                raise exceptions.ModelError(
                    'Scenario definition must be string of comma-separated overrides.'
                )
            overrides = scenarios[scenario].split(',')
            logger.info(
                'Using scenario `{}` leading to the application of '
                'overrides `{}`.'.format(scenario, scenarios[scenario])
            )
        else:
            overrides = str(scenario).split(',')
            logger.info(
                'Applying overrides `{}` without a '
                'specific scenario name.'.format(scenario)
            )

        overrides_from_scenario = combine_overrides(config_model, overrides)

        warnings = checks.check_overrides(config_model, overrides_from_scenario)
        exceptions.print_warnings_and_raise_errors(warnings=warnings)

        config_model.union(
            overrides_from_scenario, allow_override=True, allow_replacement=True
        )
        for k, v in overrides_from_scenario.as_dict_flat().items():
            debug_comments.set_key(
                '{}'.format(k),
                'Applied from override')
    else:
        overrides = []

    # Second pass of applying override dict after applying scenarios,
    # so that scenario-based overrides are overridden by override_dict!
    if override_dict:
        config_model.union(
            override_dict, allow_override=True, allow_replacement=True
        )
        for k, v in override_dict.as_dict_flat().items():
            debug_comments.set_key(
                '{}'.format(k),
                'Overridden via override dictionary.')

    return config_model, debug_comments, overrides, scenario
Exemple #20
0
 def test_delete_key_deletes_comments(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     assert 'x' in d.c.__dict_comments__
     d.del_key('c.x')
     assert 'x' not in d.c.__dict_comments__
Exemple #21
0
 def test_keys_with_comments_not_filtered(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     keys = d.keys(filtered=False)
     assert '__dict_comments__' in keys
Exemple #22
0
 def test_iter_with_comments_filtered(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     keys = [k for k in d]
     assert '__dict_comments__' not in keys
Exemple #23
0
import logging

import numpy as np
import xarray as xr

import calliope
from calliope._version import __version__
from calliope.core.attrdict import AttrDict
from calliope.core.util.tools import flatten_list
from calliope.core.preprocess.util import get_all_carriers

_defaults_files = {
    k: os.path.join(os.path.dirname(calliope.__file__), 'config', k + '.yaml')
    for k in ['model', 'defaults']
}
defaults = AttrDict.from_yaml(_defaults_files['defaults'])
defaults_model = AttrDict.from_yaml(_defaults_files['model'])


def check_overrides(config_model, override):
    """
    Perform checks on the override dict and override file inputs to ensure they
    are not doing something silly.
    """
    warnings = []
    info = []
    for key in override.as_dict_flat().keys():
        if key in config_model.as_dict_flat().keys():
            info.append('Override applied to {}: {} -> {}'.format(
                key, config_model.get_key(key), override.get_key(key)))
        else:
 def test_from_yaml_fobj(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     assert d.a == 1
     assert d.c.z.II == 2
Exemple #25
0
import numpy as np
import pandas as pd

from inspect import signature

import calliope
from calliope._version import __version__
from calliope.core.attrdict import AttrDict
from calliope.preprocess.util import get_all_carriers
from calliope.core.util.tools import load_function

logger = logging.getLogger(__name__)

DEFAULTS = AttrDict.from_yaml(
    os.path.join(os.path.dirname(calliope.__file__), "config",
                 "defaults.yaml"))
POSSIBLE_COSTS = [
    i for i in DEFAULTS.techs.default_tech.costs.default_cost.keys()
]


def check_overrides(config_model, override):
    """
    Perform checks on the override dict and override file inputs to ensure they
    are not doing something silly.
    """
    model_warnings = []
    info = []
    for key in override.as_dict_flat().keys():
        if key in config_model.as_dict_flat().keys():
Exemple #26
0
import xarray as xr

from inspect import signature

import calliope
from calliope._version import __version__
from calliope.core.attrdict import AttrDict
from calliope.core.preprocess.util import get_all_carriers, flatten_list
from calliope.core.util.logging import logger
from calliope.core.util.tools import load_function

_defaults_files = {
    k: os.path.join(os.path.dirname(calliope.__file__), 'config', k + '.yaml')
    for k in ['model', 'defaults']
}
defaults = AttrDict.from_yaml(_defaults_files['defaults'])
defaults_model = AttrDict.from_yaml(_defaults_files['model'])


def check_overrides(config_model, override):
    """
    Perform checks on the override dict and override file inputs to ensure they
    are not doing something silly.
    """
    model_warnings = []
    info = []
    for key in override.as_dict_flat().keys():
        if key in config_model.as_dict_flat().keys():
            info.append(
                'Override applied to {}: {} -> {}'
                .format(key, config_model.get_key(key), override.get_key(key))
Exemple #27
0
 def test_get_comment_inexistent(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     with pytest.raises(KeyError):
         d.get_comments('b')
Exemple #28
0
 def test_get_comment_all_none(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     d.__dict_comments__['b'] = [None, None, None, None]
     result = {'above': None, 'inline': None, 'below': None}
     assert d.get_comments('b') == result
Exemple #29
0
 def test_values_with_comments_not_filtered(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     values = d.values(filtered=False)
     c_value = [i for i in values if isinstance(i, dict) and 'c' in i]
     assert c_value[0]['c'][0] is None
Exemple #30
0
 def test_keys_nested_with_comments_flat(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     dd = d.keys_nested()
     assert dd == ['a', 'b', 'c.x', 'c.y', 'c.z.I', 'c.z.II', 'd']
Exemple #31
0
 def test_keys_nested_with_comments_as_dict(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     dd = d.keys_nested(subkeys_as='dict')
     assert dd == ['a', 'b', {'c': ['x', 'y', {'z': ['I', 'II']}]}, 'd']
 def test_to_yaml_string(self, yaml_file):
     d = AttrDict.from_yaml(yaml_file)
     result = d.to_yaml()
     assert "a: 1" in result
Exemple #33
0
import os
import calliope
import pytest  # pylint: disable=unused-import
import tempfile

from calliope.core.attrdict import AttrDict
from calliope.test.common.util import check_error_or_warning, python36_or_higher

HTML_STRINGS = AttrDict.from_yaml(
    os.path.join(os.path.dirname(__file__), 'common', 'html_strings.yaml'))


class TestPlotting:
    @pytest.fixture(scope="module")
    def national_scale_example(self):
        model = calliope.examples.national_scale(
            override_dict={'model.subset_time': '2005-01-01'})
        model.run()
        return model

    @python36_or_higher
    def test_national_scale_plotting(self, national_scale_example):
        model = national_scale_example

        plot_html_outputs = {
            'capacity': model.plot.capacity(html_only=True),
            'timeseries': model.plot.timeseries(html_only=True),
            'transmission': model.plot.transmission(html_only=True),
            'flows': model.plot.flows(html_only=True),
        }
 def test_from_yaml_path(self):
     this_path = os.path.dirname(__file__)
     yaml_path = os.path.join(this_path, "common", "yaml_file.yaml")
     d = AttrDict.from_yaml(yaml_path)
     assert d.a == 1
     assert d.c.z.II == 2
Exemple #35
0
def apply_overrides(config, scenario=None, override_dict=None):
    """
    Generate processed Model configuration, applying any scenarios overrides.

    Parameters
    ----------
    config : AttrDict
        a model configuration AttrDict
    scenario : str, optional
    override_dict : str or dict or AttrDict, optional
        If a YAML string, converted to AttrDict

    """
    debug_comments = AttrDict()

    base_model_config_file = os.path.join(
        os.path.dirname(calliope.__file__),
        'config', 'model.yaml'
    )
    config_model = AttrDict.from_yaml(base_model_config_file)

    # Interpret timeseries_data_path as relative
    config.model.timeseries_data_path = relative_path(
        config.config_path, config.model.timeseries_data_path
    )

    # The input files are allowed to override other model defaults
    config_model.union(config, allow_override=True)

    # First pass of applying override dict before applying scenarios,
    # so that can override scenario definitions by override_dict
    if override_dict:
        if isinstance(override_dict, str):
            override_dict = AttrDict.from_yaml_string(override_dict)
        elif not isinstance(override_dict, AttrDict):
            override_dict = AttrDict(override_dict)

        warnings = checks.check_overrides(config_model, override_dict)
        exceptions.print_warnings_and_raise_errors(warnings=warnings)

        config_model.union(
            override_dict, allow_override=True, allow_replacement=True
        )

    if scenario:
        scenarios = config_model.get('scenarios', {})

        if scenario in scenarios.keys():
            # Manually defined scenario names cannot be the same as single
            # overrides or any combination of semicolon-delimited overrides
            if all([i in config_model.get('overrides', {})
                    for i in scenario.split(',')]):
                raise exceptions.ModelError(
                    'Manually defined scenario cannot be a combination of override names.'
                )
            if not isinstance(scenarios[scenario], list):
                raise exceptions.ModelError(
                    'Scenario definition must be a list of override names.'
                )
            overrides = [str(i) for i in scenarios[scenario]]
            logger.info(
                'Using scenario `{}` leading to the application of '
                'overrides `{}`.'.format(scenario, overrides)
            )
        else:
            overrides = str(scenario).split(',')
            logger.info(
                'Applying the following overrides without a '
                'specific scenario name: {}'.format(overrides)
            )

        overrides_from_scenario = combine_overrides(config_model, overrides)

        warnings = checks.check_overrides(config_model, overrides_from_scenario)
        exceptions.print_warnings_and_raise_errors(warnings=warnings)

        config_model.union(
            overrides_from_scenario, allow_override=True, allow_replacement=True
        )
        for k, v in overrides_from_scenario.as_dict_flat().items():
            debug_comments.set_key(
                '{}'.format(k),
                'Applied from override')
    else:
        overrides = []

    # Second pass of applying override dict after applying scenarios,
    # so that scenario-based overrides are overridden by override_dict!
    if override_dict:
        config_model.union(
            override_dict, allow_override=True, allow_replacement=True
        )
        for k, v in override_dict.as_dict_flat().items():
            debug_comments.set_key(
                '{}'.format(k),
                'Overridden via override dictionary.')

    return config_model, debug_comments, overrides, scenario