def __init__(self, model_run_dict): """ Take a Calliope model_run and convert it into an xarray Dataset, ready for constraint generation. Timeseries data is also extracted from file at this point, and the time dimension added to the data Parameters ---------- model_run_dict : AttrDict preprocessed model_run dictionary, as produced by Calliope.preprocess.preprocess_model Returns ------- data : xarray Dataset Dataset with optimisation param_dict as variables, optimisation sets as coordinates, and other information in attributes. data_pre_time : xarray Dataset, only returned if debug = True Dataset, prior to time dimension addition, with optimisation param_dict as variables, optimisation sets as coordinates, and other information in attributes. """ self.node_dict = model_run_dict.nodes.as_dict_flat() self.tech_dict = model_run_dict.techs.as_dict_flat() self.model_run = model_run_dict self.model_data = xr.Dataset( coords={"timesteps": model_run_dict.timeseries_data.index}) self._add_attributes(model_run_dict) self.template_config = AttrDict.from_yaml( os.path.join(os.path.dirname(calliope.__file__), "config", "model_data_lookup.yaml")) self._strip_unwanted_keys() self._add_node_tech_sets()
def model_run_from_yaml(model_file, scenario=None, override_dict=None): """ Generate processed ModelRun configuration from a YAML model configuration file. Parameters ---------- model_file : str Path to YAML file with model configuration. scenario : str, optional Name of scenario to apply. Can either be a named scenario, or a comman-separated list of individual overrides to be combined ad-hoc, e.g. 'my_scenario_name' or 'override1,override2'. override_dict : dict or AttrDict, optional """ config = AttrDict.from_yaml(model_file) config.config_path = model_file config_with_overrides, debug_comments, overrides, scenario = apply_overrides( config, scenario=scenario, override_dict=override_dict ) return generate_model_run( config_with_overrides, debug_comments, overrides, scenario)
def test_model_from_dict(self): """ Test loading a file from dict/AttrDict instead of from YAML """ this_path = os.path.dirname(__file__) model_location = os.path.join(this_path, 'common', 'test_model', 'model.yaml') model_dict = AttrDict.from_yaml(model_location) location_dict = AttrDict({ 'locations': { '0': { 'techs': { 'test_supply_elec': {}, 'test_demand_elec': {} } }, '1': { 'techs': { 'test_supply_elec': {}, 'test_demand_elec': {} } } } }) model_dict.union(location_dict) model_dict.model['timeseries_data_path'] = os.path.join( this_path, 'common', 'test_model', model_dict.model['timeseries_data_path']) # test as AttrDict calliope.Model(model_dict) # test as dict calliope.Model(model_dict.as_dict())
def test_get_comment(self, yaml_file): d = AttrDict.from_yaml(yaml_file) result = { 'above': '# a comment about `c`\n', 'inline': '# a comment inline with `c`\n', 'below': None } assert d.get_comments('c') == result
def test_comment_roundtrip(self, yaml_file): d = AttrDict.from_yaml(yaml_file) with tempfile.TemporaryDirectory() as tempdir: out_file = os.path.join(tempdir, 'test.yaml') d.to_yaml(out_file) with open(out_file, 'r') as f: result = f.read() assert 'x: foo # a comment on foo' in result
def load_with_import_resolution(in_path): files = {} top = AttrDict.from_yaml(in_path, resolve_imports=False) files[in_path] = top base_path = os.path.dirname(in_path) for import_path in top.get('import', []): result = load_with_import_resolution( os.path.join(base_path, import_path)) for k, v in result.items(): files[k] = v return files
def test_add_comment_exists(self, yaml_file): d = AttrDict.from_yaml(yaml_file) # should work, as no inline comment exists d.set_comment('b', 'test comment') # should fail, as an above comment exists with pytest.raises(ValueError) as excinfo: d.set_comment('c', 'test comment', kind='above') assert check_error_or_warning(excinfo, 'Comment exists')
def test_add_comment_exists_but_empty(self, yaml_file): d = AttrDict.from_yaml(yaml_file) # Ensure comments were added assert d.c.__dict_comments__['y'][2].value == '#\n' # should work, as inline comment is empty d.set_comment('c.y', 'test comment') # should work, as above comment is empty d.set_comment('c.y', 'test comment', kind='above')
def test_union_preserves_comments(self, yaml_file): d = AttrDict.from_yaml(yaml_file) d_new = AttrDict.from_yaml_string(""" test: 1 # And a comment somekey: bar: baz: 2 # Another comment """) d.union(d_new) assert d.get_comments( 'somekey.bar.baz')['inline'] == '# Another comment\n'
def model_run_from_yaml(model_file, timeseries_dataframes=None, scenario=None, override_dict=None): """ Generate processed ModelRun configuration from a YAML model configuration file. Parameters ---------- model_file : str Path to YAML file with model configuration. timeseries_dataframes : dict, optional Dictionary of timeseries dataframes. The keys are strings corresponding to the dataframe names given in the yaml files and the values are dataframes with time series data. scenario : str, optional Name of scenario to apply. Can either be a named scenario, or a comma-separated list of individual overrides to be combined ad-hoc, e.g. 'my_scenario_name' or 'override1,override2'. override_dict : dict or AttrDict, optional """ config = AttrDict.from_yaml(model_file) config.config_path = model_file config_with_overrides, debug_comments, overrides, scenario = apply_overrides( config, scenario=scenario, override_dict=override_dict) subsets = AttrDict.from_yaml( os.path.join(os.path.dirname(calliope.__file__), "config", "subsets.yaml")) return generate_model_run( config_with_overrides, timeseries_dataframes, debug_comments, overrides, scenario, subsets, )
def test_to_yaml(self, yaml_file): d = AttrDict.from_yaml(yaml_file) d.set_key('numpy.some_int', np.int32(10)) d.set_key('numpy.some_float', np.float64(0.5)) d.a_list = [0, 1, 2] with tempfile.TemporaryDirectory() as tempdir: out_file = os.path.join(tempdir, 'test.yaml') d.to_yaml(out_file) with open(out_file, 'r') as f: result = f.read() assert 'some_int: 10' in result assert 'some_float: 0.5' in result assert 'a_list:\n- 0\n- 1\n- 2' in result
def test_to_yaml(self, yaml_file): d = AttrDict.from_yaml(yaml_file) d.set_key("numpy.some_int", np.int32(10)) d.set_key("numpy.some_float", np.float64(0.5)) d.a_list = [0, 1, 2] with tempfile.TemporaryDirectory() as tempdir: out_file = os.path.join(tempdir, "test.yaml") d.to_yaml(out_file) with open(out_file, "r") as f: result = f.read() assert "some_int: 10" in result assert "some_float: 0.5" in result assert "a_list:\n- 0\n- 1\n- 2" in result
def test_model_from_dict(self): """ Test loading a file from dict/AttrDict instead of from YAML """ this_path = os.path.dirname(__file__) model_location = os.path.join(this_path, 'common', 'test_model', 'model.yaml') model_dict = AttrDict.from_yaml(model_location) location_dict = AttrDict({ 'locations': { '0': {'techs': {'test_supply_elec': {}, 'test_demand_elec': {}}}, '1': {'techs': {'test_supply_elec': {}, 'test_demand_elec': {}}} } }) model_dict.union(location_dict) model_dict.model['timeseries_data_path'] = os.path.join( this_path, 'common', 'test_model', model_dict.model['timeseries_data_path'] ) # test as AttrDict calliope.Model(model_dict) # test as dict calliope.Model(model_dict.as_dict())
def model_run_from_yaml(model_file, override_file=None, override_dict=None): """ Generate processed ModelRun configuration from a YAML model configuration file. Parameters ---------- model_file : str Path to YAML file with model configuration. override_file : str, optional Path to YAML file with model configuration overrides and the override group to use, separated by ':', e.g. 'overrides.yaml:group1'. override_dict : dict or AttrDict, optional """ config = AttrDict.from_yaml(model_file) config.config_path = model_file config_with_overrides, debug_comments = apply_overrides( config, override_file=override_file, override_dict=override_dict) return generate_model_run(config_with_overrides, debug_comments)
def combine_overrides(override_file_path, override_groups): if ',' in override_groups: overrides = override_groups.split(',') else: overrides = [override_groups] override = AttrDict() for group in overrides: try: override_group_from_file = AttrDict.from_yaml( override_file_path)[group] except KeyError: raise exceptions.ModelError( 'Override group `{}` does not exist in file `{}`.'.format( group, override_file_path)) try: override.union(override_group_from_file, allow_override=False) except KeyError as e: raise exceptions.ModelError( str(e)[1:-1] + '. Already specified but defined again in ' 'override group `{}`.'.format(group)) return override
def test_del_comment(self, yaml_file): d = AttrDict.from_yaml(yaml_file) assert 'y' in d.c.__dict_comments__ d.del_comments('c.y') assert 'y' not in d.c.__dict_comments__
import os import calliope import pytest # pylint: disable=unused-import import tempfile from calliope.core.attrdict import AttrDict from calliope.test.common.util import check_error_or_warning, python36_or_higher HTML_STRINGS = AttrDict.from_yaml( os.path.join(os.path.dirname(__file__), 'common', 'html_strings.yaml') ) class TestPlotting: @pytest.fixture(scope="module") def national_scale_example(self): model = calliope.examples.national_scale( override_dict={'model.subset_time': '2005-01-01'} ) model.run() return model @python36_or_higher def test_national_scale_plotting(self, national_scale_example): model = national_scale_example plot_html_outputs = { 'capacity': model.plot.capacity(html_only=True), 'timeseries': model.plot.timeseries(html_only=True), 'transmission': model.plot.transmission(html_only=True),
def apply_overrides(config, scenario=None, override_dict=None): """ Generate processed Model configuration, applying any scenarios overrides. Parameters ---------- config : AttrDict a model configuration AttrDict scenario : str, optional override_dict : str or dict or AttrDict, optional If a YAML string, converted to AttrDict """ debug_comments = AttrDict() base_model_config_file = os.path.join( os.path.dirname(calliope.__file__), 'config', 'model.yaml' ) config_model = AttrDict.from_yaml(base_model_config_file) # Interpret timeseries_data_path as relative config.model.timeseries_data_path = relative_path( config.config_path, config.model.timeseries_data_path ) # The input files are allowed to override other model defaults config_model.union(config, allow_override=True) # First pass of applying override dict before applying scenarios, # so that can override scenario definitions by override_dict if override_dict: if isinstance(override_dict, str): override_dict = AttrDict.from_yaml_string(override_dict) elif not isinstance(override_dict, AttrDict): override_dict = AttrDict(override_dict) warnings = checks.check_overrides(config_model, override_dict) exceptions.print_warnings_and_raise_errors(warnings=warnings) config_model.union( override_dict, allow_override=True, allow_replacement=True ) if scenario: scenarios = config_model.get('scenarios', {}) if scenario in scenarios: # Manually defined scenario names cannot be the same as single # overrides or any combination of semicolon-delimited overrides if all([i in config_model.get('overrides', {}) for i in scenario.split(',')]): raise exceptions.ModelError( 'Manually defined scenario cannot be a combination of override names.' ) if not isinstance(scenarios[scenario], str): raise exceptions.ModelError( 'Scenario definition must be string of comma-separated overrides.' ) overrides = scenarios[scenario].split(',') logger.info( 'Using scenario `{}` leading to the application of ' 'overrides `{}`.'.format(scenario, scenarios[scenario]) ) else: overrides = str(scenario).split(',') logger.info( 'Applying overrides `{}` without a ' 'specific scenario name.'.format(scenario) ) overrides_from_scenario = combine_overrides(config_model, overrides) warnings = checks.check_overrides(config_model, overrides_from_scenario) exceptions.print_warnings_and_raise_errors(warnings=warnings) config_model.union( overrides_from_scenario, allow_override=True, allow_replacement=True ) for k, v in overrides_from_scenario.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Applied from override') else: overrides = [] # Second pass of applying override dict after applying scenarios, # so that scenario-based overrides are overridden by override_dict! if override_dict: config_model.union( override_dict, allow_override=True, allow_replacement=True ) for k, v in override_dict.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Overridden via override dictionary.') return config_model, debug_comments, overrides, scenario
def test_delete_key_deletes_comments(self, yaml_file): d = AttrDict.from_yaml(yaml_file) assert 'x' in d.c.__dict_comments__ d.del_key('c.x') assert 'x' not in d.c.__dict_comments__
def test_keys_with_comments_not_filtered(self, yaml_file): d = AttrDict.from_yaml(yaml_file) keys = d.keys(filtered=False) assert '__dict_comments__' in keys
def test_iter_with_comments_filtered(self, yaml_file): d = AttrDict.from_yaml(yaml_file) keys = [k for k in d] assert '__dict_comments__' not in keys
import logging import numpy as np import xarray as xr import calliope from calliope._version import __version__ from calliope.core.attrdict import AttrDict from calliope.core.util.tools import flatten_list from calliope.core.preprocess.util import get_all_carriers _defaults_files = { k: os.path.join(os.path.dirname(calliope.__file__), 'config', k + '.yaml') for k in ['model', 'defaults'] } defaults = AttrDict.from_yaml(_defaults_files['defaults']) defaults_model = AttrDict.from_yaml(_defaults_files['model']) def check_overrides(config_model, override): """ Perform checks on the override dict and override file inputs to ensure they are not doing something silly. """ warnings = [] info = [] for key in override.as_dict_flat().keys(): if key in config_model.as_dict_flat().keys(): info.append('Override applied to {}: {} -> {}'.format( key, config_model.get_key(key), override.get_key(key))) else:
def test_from_yaml_fobj(self, yaml_file): d = AttrDict.from_yaml(yaml_file) assert d.a == 1 assert d.c.z.II == 2
import numpy as np import pandas as pd from inspect import signature import calliope from calliope._version import __version__ from calliope.core.attrdict import AttrDict from calliope.preprocess.util import get_all_carriers from calliope.core.util.tools import load_function logger = logging.getLogger(__name__) DEFAULTS = AttrDict.from_yaml( os.path.join(os.path.dirname(calliope.__file__), "config", "defaults.yaml")) POSSIBLE_COSTS = [ i for i in DEFAULTS.techs.default_tech.costs.default_cost.keys() ] def check_overrides(config_model, override): """ Perform checks on the override dict and override file inputs to ensure they are not doing something silly. """ model_warnings = [] info = [] for key in override.as_dict_flat().keys(): if key in config_model.as_dict_flat().keys():
import xarray as xr from inspect import signature import calliope from calliope._version import __version__ from calliope.core.attrdict import AttrDict from calliope.core.preprocess.util import get_all_carriers, flatten_list from calliope.core.util.logging import logger from calliope.core.util.tools import load_function _defaults_files = { k: os.path.join(os.path.dirname(calliope.__file__), 'config', k + '.yaml') for k in ['model', 'defaults'] } defaults = AttrDict.from_yaml(_defaults_files['defaults']) defaults_model = AttrDict.from_yaml(_defaults_files['model']) def check_overrides(config_model, override): """ Perform checks on the override dict and override file inputs to ensure they are not doing something silly. """ model_warnings = [] info = [] for key in override.as_dict_flat().keys(): if key in config_model.as_dict_flat().keys(): info.append( 'Override applied to {}: {} -> {}' .format(key, config_model.get_key(key), override.get_key(key))
def test_get_comment_inexistent(self, yaml_file): d = AttrDict.from_yaml(yaml_file) with pytest.raises(KeyError): d.get_comments('b')
def test_get_comment_all_none(self, yaml_file): d = AttrDict.from_yaml(yaml_file) d.__dict_comments__['b'] = [None, None, None, None] result = {'above': None, 'inline': None, 'below': None} assert d.get_comments('b') == result
def test_values_with_comments_not_filtered(self, yaml_file): d = AttrDict.from_yaml(yaml_file) values = d.values(filtered=False) c_value = [i for i in values if isinstance(i, dict) and 'c' in i] assert c_value[0]['c'][0] is None
def test_keys_nested_with_comments_flat(self, yaml_file): d = AttrDict.from_yaml(yaml_file) dd = d.keys_nested() assert dd == ['a', 'b', 'c.x', 'c.y', 'c.z.I', 'c.z.II', 'd']
def test_keys_nested_with_comments_as_dict(self, yaml_file): d = AttrDict.from_yaml(yaml_file) dd = d.keys_nested(subkeys_as='dict') assert dd == ['a', 'b', {'c': ['x', 'y', {'z': ['I', 'II']}]}, 'd']
def test_to_yaml_string(self, yaml_file): d = AttrDict.from_yaml(yaml_file) result = d.to_yaml() assert "a: 1" in result
import os import calliope import pytest # pylint: disable=unused-import import tempfile from calliope.core.attrdict import AttrDict from calliope.test.common.util import check_error_or_warning, python36_or_higher HTML_STRINGS = AttrDict.from_yaml( os.path.join(os.path.dirname(__file__), 'common', 'html_strings.yaml')) class TestPlotting: @pytest.fixture(scope="module") def national_scale_example(self): model = calliope.examples.national_scale( override_dict={'model.subset_time': '2005-01-01'}) model.run() return model @python36_or_higher def test_national_scale_plotting(self, national_scale_example): model = national_scale_example plot_html_outputs = { 'capacity': model.plot.capacity(html_only=True), 'timeseries': model.plot.timeseries(html_only=True), 'transmission': model.plot.transmission(html_only=True), 'flows': model.plot.flows(html_only=True), }
def test_from_yaml_path(self): this_path = os.path.dirname(__file__) yaml_path = os.path.join(this_path, "common", "yaml_file.yaml") d = AttrDict.from_yaml(yaml_path) assert d.a == 1 assert d.c.z.II == 2
def apply_overrides(config, scenario=None, override_dict=None): """ Generate processed Model configuration, applying any scenarios overrides. Parameters ---------- config : AttrDict a model configuration AttrDict scenario : str, optional override_dict : str or dict or AttrDict, optional If a YAML string, converted to AttrDict """ debug_comments = AttrDict() base_model_config_file = os.path.join( os.path.dirname(calliope.__file__), 'config', 'model.yaml' ) config_model = AttrDict.from_yaml(base_model_config_file) # Interpret timeseries_data_path as relative config.model.timeseries_data_path = relative_path( config.config_path, config.model.timeseries_data_path ) # The input files are allowed to override other model defaults config_model.union(config, allow_override=True) # First pass of applying override dict before applying scenarios, # so that can override scenario definitions by override_dict if override_dict: if isinstance(override_dict, str): override_dict = AttrDict.from_yaml_string(override_dict) elif not isinstance(override_dict, AttrDict): override_dict = AttrDict(override_dict) warnings = checks.check_overrides(config_model, override_dict) exceptions.print_warnings_and_raise_errors(warnings=warnings) config_model.union( override_dict, allow_override=True, allow_replacement=True ) if scenario: scenarios = config_model.get('scenarios', {}) if scenario in scenarios.keys(): # Manually defined scenario names cannot be the same as single # overrides or any combination of semicolon-delimited overrides if all([i in config_model.get('overrides', {}) for i in scenario.split(',')]): raise exceptions.ModelError( 'Manually defined scenario cannot be a combination of override names.' ) if not isinstance(scenarios[scenario], list): raise exceptions.ModelError( 'Scenario definition must be a list of override names.' ) overrides = [str(i) for i in scenarios[scenario]] logger.info( 'Using scenario `{}` leading to the application of ' 'overrides `{}`.'.format(scenario, overrides) ) else: overrides = str(scenario).split(',') logger.info( 'Applying the following overrides without a ' 'specific scenario name: {}'.format(overrides) ) overrides_from_scenario = combine_overrides(config_model, overrides) warnings = checks.check_overrides(config_model, overrides_from_scenario) exceptions.print_warnings_and_raise_errors(warnings=warnings) config_model.union( overrides_from_scenario, allow_override=True, allow_replacement=True ) for k, v in overrides_from_scenario.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Applied from override') else: overrides = [] # Second pass of applying override dict after applying scenarios, # so that scenario-based overrides are overridden by override_dict! if override_dict: config_model.union( override_dict, allow_override=True, allow_replacement=True ) for k, v in override_dict.as_dict_flat().items(): debug_comments.set_key( '{}'.format(k), 'Overridden via override dictionary.') return config_model, debug_comments, overrides, scenario