def __init__(self, model_run_dict):
"""
Take a Calliope model_run and convert it into an xarray Dataset, ready for
constraint generation. Timeseries data is also extracted from file at this
point, and the time dimension added to the data
Parameters
----------
model_run_dict : AttrDict
preprocessed model_run dictionary, as produced by
Calliope.preprocess.preprocess_model
Returns
-------
data : xarray Dataset
Dataset with optimisation param_dict as variables, optimisation sets as
coordinates, and other information in attributes.
data_pre_time : xarray Dataset, only returned if debug = True
Dataset, prior to time dimension addition, with optimisation param_dict
as variables, optimisation sets as coordinates, and other information
in attributes.
"""
self.node_dict = model_run_dict.nodes.as_dict_flat()
self.tech_dict = model_run_dict.techs.as_dict_flat()
self.model_run = model_run_dict
self.model_data = xr.Dataset(
coords={"timesteps": model_run_dict.timeseries_data.index})
self._add_attributes(model_run_dict)
self.template_config = AttrDict.from_yaml(
os.path.join(os.path.dirname(calliope.__file__), "config",
"model_data_lookup.yaml"))
self._strip_unwanted_keys()
self._add_node_tech_sets()
def model_run_from_yaml(model_file, scenario=None, override_dict=None):
"""
Generate processed ModelRun configuration from a
YAML model configuration file.
Parameters
----------
model_file : str
Path to YAML file with model configuration.
scenario : str, optional
Name of scenario to apply. Can either be a named scenario, or a
comman-separated list of individual overrides to be combined
ad-hoc, e.g. 'my_scenario_name' or 'override1,override2'.
override_dict : dict or AttrDict, optional
"""
config = AttrDict.from_yaml(model_file)
config.config_path = model_file
config_with_overrides, debug_comments, overrides, scenario = apply_overrides(
config, scenario=scenario, override_dict=override_dict
)
return generate_model_run(
config_with_overrides, debug_comments, overrides, scenario)
def test_model_from_dict(self):
"""
Test loading a file from dict/AttrDict instead of from YAML
"""
this_path = os.path.dirname(__file__)
model_location = os.path.join(this_path, 'common', 'test_model',
'model.yaml')
model_dict = AttrDict.from_yaml(model_location)
location_dict = AttrDict({
'locations': {
'0': {
'techs': {
'test_supply_elec': {},
'test_demand_elec': {}
}
},
'1': {
'techs': {
'test_supply_elec': {},
'test_demand_elec': {}
}
}
}
})
model_dict.union(location_dict)
model_dict.model['timeseries_data_path'] = os.path.join(
this_path, 'common', 'test_model',
model_dict.model['timeseries_data_path'])
# test as AttrDict
calliope.Model(model_dict)
# test as dict
calliope.Model(model_dict.as_dict())
def model_run_from_yaml(model_file, scenario=None, override_dict=None):
"""
Generate processed ModelRun configuration from a
YAML model configuration file.
Parameters
----------
model_file : str
Path to YAML file with model configuration.
scenario : str, optional
Name of scenario to apply. Can either be a named scenario, or a
comman-separated list of individual overrides to be combined
ad-hoc, e.g. 'my_scenario_name' or 'override1,override2'.
override_dict : dict or AttrDict, optional
"""
config = AttrDict.from_yaml(model_file)
config.config_path = model_file
config_with_overrides, debug_comments, overrides, scenario = apply_overrides(
config, scenario=scenario, override_dict=override_dict
)
return generate_model_run(
config_with_overrides, debug_comments, overrides, scenario)
def test_get_comment(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
result = {
'above': '# a comment about `c`\n',
'inline': '# a comment inline with `c`\n',
'below': None
}
assert d.get_comments('c') == result
def test_comment_roundtrip(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
with tempfile.TemporaryDirectory() as tempdir:
out_file = os.path.join(tempdir, 'test.yaml')
d.to_yaml(out_file)
with open(out_file, 'r') as f:
result = f.read()
assert 'x: foo # a comment on foo' in result
def load_with_import_resolution(in_path):
files = {}
top = AttrDict.from_yaml(in_path, resolve_imports=False)
files[in_path] = top
base_path = os.path.dirname(in_path)
for import_path in top.get('import', []):
result = load_with_import_resolution(
os.path.join(base_path, import_path))
for k, v in result.items():
files[k] = v
return files
def test_add_comment_exists(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
# should work, as no inline comment exists
d.set_comment('b', 'test comment')
# should fail, as an above comment exists
with pytest.raises(ValueError) as excinfo:
d.set_comment('c', 'test comment', kind='above')
assert check_error_or_warning(excinfo, 'Comment exists')
def test_add_comment_exists_but_empty(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
# Ensure comments were added
assert d.c.__dict_comments__['y'][2].value == '#\n'
# should work, as inline comment is empty
d.set_comment('c.y', 'test comment')
# should work, as above comment is empty
d.set_comment('c.y', 'test comment', kind='above')
def test_union_preserves_comments(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
d_new = AttrDict.from_yaml_string("""
test: 1 # And a comment
somekey:
bar:
baz: 2 # Another comment
""")
d.union(d_new)
assert d.get_comments(
'somekey.bar.baz')['inline'] == '# Another comment\n'
def model_run_from_yaml(model_file,
timeseries_dataframes=None,
scenario=None,
override_dict=None):
"""
Generate processed ModelRun configuration from a
YAML model configuration file.
Parameters
----------
model_file : str
Path to YAML file with model configuration.
timeseries_dataframes : dict, optional
Dictionary of timeseries dataframes. The keys are strings
corresponding to the dataframe names given in the yaml files and
the values are dataframes with time series data.
scenario : str, optional
Name of scenario to apply. Can either be a named scenario, or a
comma-separated list of individual overrides to be combined
ad-hoc, e.g. 'my_scenario_name' or 'override1,override2'.
override_dict : dict or AttrDict, optional
"""
config = AttrDict.from_yaml(model_file)
config.config_path = model_file
config_with_overrides, debug_comments, overrides, scenario = apply_overrides(
config, scenario=scenario, override_dict=override_dict)
subsets = AttrDict.from_yaml(
os.path.join(os.path.dirname(calliope.__file__), "config",
"subsets.yaml"))
return generate_model_run(
config_with_overrides,
timeseries_dataframes,
debug_comments,
overrides,
scenario,
subsets,
)
def test_to_yaml(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
d.set_key('numpy.some_int', np.int32(10))
d.set_key('numpy.some_float', np.float64(0.5))
d.a_list = [0, 1, 2]
with tempfile.TemporaryDirectory() as tempdir:
out_file = os.path.join(tempdir, 'test.yaml')
d.to_yaml(out_file)
with open(out_file, 'r') as f:
result = f.read()
assert 'some_int: 10' in result
assert 'some_float: 0.5' in result
assert 'a_list:\n- 0\n- 1\n- 2' in result
def test_to_yaml(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
d.set_key("numpy.some_int", np.int32(10))
d.set_key("numpy.some_float", np.float64(0.5))
d.a_list = [0, 1, 2]
with tempfile.TemporaryDirectory() as tempdir:
out_file = os.path.join(tempdir, "test.yaml")
d.to_yaml(out_file)
with open(out_file, "r") as f:
result = f.read()
assert "some_int: 10" in result
assert "some_float: 0.5" in result
assert "a_list:\n- 0\n- 1\n- 2" in result
def test_model_from_dict(self):
"""
Test loading a file from dict/AttrDict instead of from YAML
"""
this_path = os.path.dirname(__file__)
model_location = os.path.join(this_path, 'common', 'test_model', 'model.yaml')
model_dict = AttrDict.from_yaml(model_location)
location_dict = AttrDict({
'locations': {
'0': {'techs': {'test_supply_elec': {}, 'test_demand_elec': {}}},
'1': {'techs': {'test_supply_elec': {}, 'test_demand_elec': {}}}
}
})
model_dict.union(location_dict)
model_dict.model['timeseries_data_path'] = os.path.join(
this_path, 'common', 'test_model', model_dict.model['timeseries_data_path']
)
# test as AttrDict
calliope.Model(model_dict)
# test as dict
calliope.Model(model_dict.as_dict())
def model_run_from_yaml(model_file, override_file=None, override_dict=None):
"""
Generate processed ModelRun configuration from a
YAML model configuration file.
Parameters
----------
model_file : str
Path to YAML file with model configuration.
override_file : str, optional
Path to YAML file with model configuration overrides and the override
group to use, separated by ':', e.g. 'overrides.yaml:group1'.
override_dict : dict or AttrDict, optional
"""
config = AttrDict.from_yaml(model_file)
config.config_path = model_file
config_with_overrides, debug_comments = apply_overrides(
config, override_file=override_file, override_dict=override_dict)
return generate_model_run(config_with_overrides, debug_comments)
def combine_overrides(override_file_path, override_groups):
if ',' in override_groups:
overrides = override_groups.split(',')
else:
overrides = [override_groups]
override = AttrDict()
for group in overrides:
try:
override_group_from_file = AttrDict.from_yaml(
override_file_path)[group]
except KeyError:
raise exceptions.ModelError(
'Override group `{}` does not exist in file `{}`.'.format(
group, override_file_path))
try:
override.union(override_group_from_file, allow_override=False)
except KeyError as e:
raise exceptions.ModelError(
str(e)[1:-1] + '. Already specified but defined again in '
'override group `{}`.'.format(group))
return override
def test_del_comment(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
assert 'y' in d.c.__dict_comments__
d.del_comments('c.y')
assert 'y' not in d.c.__dict_comments__
import os
import calliope
import pytest # pylint: disable=unused-import
import tempfile
from calliope.core.attrdict import AttrDict
from calliope.test.common.util import check_error_or_warning, python36_or_higher
HTML_STRINGS = AttrDict.from_yaml(
os.path.join(os.path.dirname(__file__), 'common', 'html_strings.yaml')
)
class TestPlotting:
@pytest.fixture(scope="module")
def national_scale_example(self):
model = calliope.examples.national_scale(
override_dict={'model.subset_time': '2005-01-01'}
)
model.run()
return model
@python36_or_higher
def test_national_scale_plotting(self, national_scale_example):
model = national_scale_example
plot_html_outputs = {
'capacity': model.plot.capacity(html_only=True),
'timeseries': model.plot.timeseries(html_only=True),
'transmission': model.plot.transmission(html_only=True),
def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(
os.path.dirname(calliope.__file__),
'config', 'model.yaml'
)
config_model = AttrDict.from_yaml(base_model_config_file)
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path
)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warnings = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
if scenario:
scenarios = config_model.get('scenarios', {})
if scenario in scenarios:
# Manually defined scenario names cannot be the same as single
# overrides or any combination of semicolon-delimited overrides
if all([i in config_model.get('overrides', {})
for i in scenario.split(',')]):
raise exceptions.ModelError(
'Manually defined scenario cannot be a combination of override names.'
)
if not isinstance(scenarios[scenario], str):
raise exceptions.ModelError(
'Scenario definition must be string of comma-separated overrides.'
)
overrides = scenarios[scenario].split(',')
logger.info(
'Using scenario `{}` leading to the application of '
'overrides `{}`.'.format(scenario, scenarios[scenario])
)
else:
overrides = str(scenario).split(',')
logger.info(
'Applying overrides `{}` without a '
'specific scenario name.'.format(scenario)
)
overrides_from_scenario = combine_overrides(config_model, overrides)
warnings = checks.check_overrides(config_model, overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
overrides_from_scenario, allow_override=True, allow_replacement=True
)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Applied from override')
else:
overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments, overrides, scenario
def test_delete_key_deletes_comments(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
assert 'x' in d.c.__dict_comments__
d.del_key('c.x')
assert 'x' not in d.c.__dict_comments__
def test_keys_with_comments_not_filtered(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
keys = d.keys(filtered=False)
assert '__dict_comments__' in keys
def test_iter_with_comments_filtered(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
keys = [k for k in d]
assert '__dict_comments__' not in keys
import logging
import numpy as np
import xarray as xr
import calliope
from calliope._version import __version__
from calliope.core.attrdict import AttrDict
from calliope.core.util.tools import flatten_list
from calliope.core.preprocess.util import get_all_carriers
_defaults_files = {
k: os.path.join(os.path.dirname(calliope.__file__), 'config', k + '.yaml')
for k in ['model', 'defaults']
}
defaults = AttrDict.from_yaml(_defaults_files['defaults'])
defaults_model = AttrDict.from_yaml(_defaults_files['model'])
def check_overrides(config_model, override):
"""
Perform checks on the override dict and override file inputs to ensure they
are not doing something silly.
"""
warnings = []
info = []
for key in override.as_dict_flat().keys():
if key in config_model.as_dict_flat().keys():
info.append('Override applied to {}: {} -> {}'.format(
key, config_model.get_key(key), override.get_key(key)))
else:
def test_from_yaml_fobj(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
assert d.a == 1
assert d.c.z.II == 2
import numpy as np
import pandas as pd
from inspect import signature
import calliope
from calliope._version import __version__
from calliope.core.attrdict import AttrDict
from calliope.preprocess.util import get_all_carriers
from calliope.core.util.tools import load_function
logger = logging.getLogger(__name__)
DEFAULTS = AttrDict.from_yaml(
os.path.join(os.path.dirname(calliope.__file__), "config",
"defaults.yaml"))
POSSIBLE_COSTS = [
i for i in DEFAULTS.techs.default_tech.costs.default_cost.keys()
]
def check_overrides(config_model, override):
"""
Perform checks on the override dict and override file inputs to ensure they
are not doing something silly.
"""
model_warnings = []
info = []
for key in override.as_dict_flat().keys():
if key in config_model.as_dict_flat().keys():
import xarray as xr
from inspect import signature
import calliope
from calliope._version import __version__
from calliope.core.attrdict import AttrDict
from calliope.core.preprocess.util import get_all_carriers, flatten_list
from calliope.core.util.logging import logger
from calliope.core.util.tools import load_function
_defaults_files = {
k: os.path.join(os.path.dirname(calliope.__file__), 'config', k + '.yaml')
for k in ['model', 'defaults']
}
defaults = AttrDict.from_yaml(_defaults_files['defaults'])
defaults_model = AttrDict.from_yaml(_defaults_files['model'])
def check_overrides(config_model, override):
"""
Perform checks on the override dict and override file inputs to ensure they
are not doing something silly.
"""
model_warnings = []
info = []
for key in override.as_dict_flat().keys():
if key in config_model.as_dict_flat().keys():
info.append(
'Override applied to {}: {} -> {}'
.format(key, config_model.get_key(key), override.get_key(key))
def test_get_comment_inexistent(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
with pytest.raises(KeyError):
d.get_comments('b')
def test_get_comment_all_none(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
d.__dict_comments__['b'] = [None, None, None, None]
result = {'above': None, 'inline': None, 'below': None}
assert d.get_comments('b') == result
def test_values_with_comments_not_filtered(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
values = d.values(filtered=False)
c_value = [i for i in values if isinstance(i, dict) and 'c' in i]
assert c_value[0]['c'][0] is None
def test_keys_nested_with_comments_flat(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
dd = d.keys_nested()
assert dd == ['a', 'b', 'c.x', 'c.y', 'c.z.I', 'c.z.II', 'd']
def test_keys_nested_with_comments_as_dict(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
dd = d.keys_nested(subkeys_as='dict')
assert dd == ['a', 'b', {'c': ['x', 'y', {'z': ['I', 'II']}]}, 'd']
def test_to_yaml_string(self, yaml_file):
d = AttrDict.from_yaml(yaml_file)
result = d.to_yaml()
assert "a: 1" in result
import os
import calliope
import pytest # pylint: disable=unused-import
import tempfile
from calliope.core.attrdict import AttrDict
from calliope.test.common.util import check_error_or_warning, python36_or_higher
HTML_STRINGS = AttrDict.from_yaml(
os.path.join(os.path.dirname(__file__), 'common', 'html_strings.yaml'))
class TestPlotting:
@pytest.fixture(scope="module")
def national_scale_example(self):
model = calliope.examples.national_scale(
override_dict={'model.subset_time': '2005-01-01'})
model.run()
return model
@python36_or_higher
def test_national_scale_plotting(self, national_scale_example):
model = national_scale_example
plot_html_outputs = {
'capacity': model.plot.capacity(html_only=True),
'timeseries': model.plot.timeseries(html_only=True),
'transmission': model.plot.transmission(html_only=True),
'flows': model.plot.flows(html_only=True),
}
def test_from_yaml_path(self):
this_path = os.path.dirname(__file__)
yaml_path = os.path.join(this_path, "common", "yaml_file.yaml")
d = AttrDict.from_yaml(yaml_path)
assert d.a == 1
assert d.c.z.II == 2
def apply_overrides(config, scenario=None, override_dict=None):
"""
Generate processed Model configuration, applying any scenarios overrides.
Parameters
----------
config : AttrDict
a model configuration AttrDict
scenario : str, optional
override_dict : str or dict or AttrDict, optional
If a YAML string, converted to AttrDict
"""
debug_comments = AttrDict()
base_model_config_file = os.path.join(
os.path.dirname(calliope.__file__),
'config', 'model.yaml'
)
config_model = AttrDict.from_yaml(base_model_config_file)
# Interpret timeseries_data_path as relative
config.model.timeseries_data_path = relative_path(
config.config_path, config.model.timeseries_data_path
)
# The input files are allowed to override other model defaults
config_model.union(config, allow_override=True)
# First pass of applying override dict before applying scenarios,
# so that can override scenario definitions by override_dict
if override_dict:
if isinstance(override_dict, str):
override_dict = AttrDict.from_yaml_string(override_dict)
elif not isinstance(override_dict, AttrDict):
override_dict = AttrDict(override_dict)
warnings = checks.check_overrides(config_model, override_dict)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
if scenario:
scenarios = config_model.get('scenarios', {})
if scenario in scenarios.keys():
# Manually defined scenario names cannot be the same as single
# overrides or any combination of semicolon-delimited overrides
if all([i in config_model.get('overrides', {})
for i in scenario.split(',')]):
raise exceptions.ModelError(
'Manually defined scenario cannot be a combination of override names.'
)
if not isinstance(scenarios[scenario], list):
raise exceptions.ModelError(
'Scenario definition must be a list of override names.'
)
overrides = [str(i) for i in scenarios[scenario]]
logger.info(
'Using scenario `{}` leading to the application of '
'overrides `{}`.'.format(scenario, overrides)
)
else:
overrides = str(scenario).split(',')
logger.info(
'Applying the following overrides without a '
'specific scenario name: {}'.format(overrides)
)
overrides_from_scenario = combine_overrides(config_model, overrides)
warnings = checks.check_overrides(config_model, overrides_from_scenario)
exceptions.print_warnings_and_raise_errors(warnings=warnings)
config_model.union(
overrides_from_scenario, allow_override=True, allow_replacement=True
)
for k, v in overrides_from_scenario.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Applied from override')
else:
overrides = []
# Second pass of applying override dict after applying scenarios,
# so that scenario-based overrides are overridden by override_dict!
if override_dict:
config_model.union(
override_dict, allow_override=True, allow_replacement=True
)
for k, v in override_dict.as_dict_flat().items():
debug_comments.set_key(
'{}'.format(k),
'Overridden via override dictionary.')
return config_model, debug_comments, overrides, scenario
