def test_apply_units(data, caplog):
# Unpack
*_, x = data
# Brute-force replacement with incompatible units
with assert_logs(caplog, "Replace 'kilogram' with incompatible 'liter'"):
result = computations.apply_units(x, 'litres')
assert result.attrs['_unit'] == REGISTRY.Unit('litre')
# No change in values
assert_series_equal(result.to_series(), x.to_series())
caplog.set_level(logging.DEBUG)
# Compatible units: magnitudes are also converted
with assert_logs(caplog, "Convert 'kilogram' to 'metric_ton'"):
result = computations.apply_units(x, 'tonne')
assert result.attrs['_unit'] == REGISTRY.Unit('tonne')
assert_series_equal(result.to_series(), x.to_series() * 0.001)
# Remove unit
x.attrs['_unit'] = REGISTRY.Unit('dimensionless')
caplog.clear()
result = computations.apply_units(x, 'kg')
# Nothing logged when _unit attr is missing
assert len(caplog.messages) == 0
assert result.attrs['_unit'] == REGISTRY.Unit('kg')
assert_series_equal(result.to_series(), x.to_series())
def test_model_initialize(test_mp, caplog):
# Model.initialize runs on an empty Scenario
s = make_dantzig(test_mp)
b1 = s.par('b')
assert len(b1) == 3
# Modify a value for 'b'
s.check_out()
new_value = 301
s.add_par('b', 'chicago', new_value, 'cases')
s.commit('Overwrite b(chicago)')
# Model.initialize runs on an already-initialized Scenario, without error
DantzigModel.initialize(s, with_data=True)
# Data has the same length...
b2 = s.par('b')
assert len(b2) == 3
# ...but modified value(s) are not overwritten
assert (b2.query("j == 'chicago'")['value'] == new_value).all()
# Unrecognized Scenario(scheme=...) is initialized using the base method, a
# no-op
messages = [
"No scheme for new Scenario model-name/scenario-name",
"No initialization for None-scheme Scenario",
]
with assert_logs(caplog, messages, at_level=logging.DEBUG):
Scenario(test_mp, model='model-name', scenario='scenario-name',
version='new')
with assert_logs(caplog, "No initialization for 'foo'-scheme Scenario",
at_level=logging.DEBUG):
Scenario(test_mp, model='model-name', scenario='scenario-name',
version='new', scheme='foo')
# Keyword arguments to Scenario(...) that are not recognized by
# Model.initialize() raise an intelligible exception
with pytest.raises(TypeError,
match="unexpected keyword argument 'bad_arg1'"):
Scenario(test_mp, model='model-name', scenario='scenario-name',
version='new', scheme='unknown', bad_arg1=111)
with pytest.raises(TypeError,
match="unexpected keyword argument 'bad_arg2'"):
Scenario(test_mp, model='model-name', scenario='scenario-name',
version='new', scheme='dantzig', with_data=True,
bad_arg2=222)
# Replace b[j] with a parameter of the same name, but different indices
s.check_out()
s.remove_par('b')
s.init_par('b', idx_sets=['i'], idx_names=['i_dim'])
# Logs an error message
with assert_logs(caplog,
"Existing index sets of 'b' ['i'] do not match ['j']"):
DantzigModel.initialize(s)
def test_update_scenario(caplog, test_mp):
scen = make_dantzig(test_mp)
scen.check_out()
scen.add_set("j", "toronto")
scen.commit("Add j=toronto")
# Number of rows in the 'd' parameter
N_before = len(scen.par("d"))
assert 6 == N_before
# A Computer used as calculation engine
c = Computer()
# Target Scenario for updating data
c.add("target", scen)
# Create a pd.DataFrame suitable for Scenario.add_par()
data = dantzig_data["d"].query("j == 'chicago'").assign(j="toronto")
data["value"] += 1.0
# Add to the Reporter
c.add("input", data)
# Task to update the scenario with the data
c.add("test 1",
(partial(update_scenario, params=["d"]), "target", "input"))
# Trigger the computation that results in data being added
with assert_logs(caplog, f"'d' ← {len(data)} rows", at_level=logging.INFO):
# Returns nothing
assert c.get("test 1") is None
# Rows were added to the parameter
assert len(scen.par("d")) == N_before + len(data)
# Modify the data
data = pd.concat([dantzig_data["d"], data]).reset_index(drop=True)
data["value"] *= 2.0
# Convert to a Quantity object and re-add
q = Quantity(data.set_index(["i", "j"])["value"], name="d", units="km")
c.add("input", q)
# Revise the task; the parameter name ('demand') is read from the Quantity
c.add("test 2", (update_scenario, "target", "input"))
# Trigger the computation
with assert_logs(caplog, f"'d' ← {len(data)} rows", at_level=logging.INFO):
c.get("test 2")
# All the rows have been updated
assert_frame_equal(scen.par("d"), data)
def test_computationerror(caplog):
ce_none = ComputationError(None)
# Message ends with ',)' on Python 3.6, only ')' on Python 3.7
msg = ("Exception raised while formatting None:\nAttributeError"
"(\"'NoneType' object has no attribute '__traceback__'\"")
with assert_logs(caplog, msg):
str(ce_none)
def test_add_timeslice_duplicate(caplog, test_mp):
test_mp.add_timeslice("foo_slice", "foo_category", 0.2)
# Adding same name with different duration raises an error
msg = "timeslice `foo_slice` already defined with duration 0.2"
with raises(ValueError, match=re.escape(msg)):
test_mp.add_timeslice("foo_slice", "bar_category", 0.3)
# Re-adding with the same duration only logs a message
with assert_logs(caplog, msg, at_level=logging.INFO):
test_mp.add_timeslice("foo_slice", "bar_category", 0.2)
def test_reporter_no_solution(caplog, message_test_mp):
scen = Scenario(message_test_mp, **SCENARIO["dantzig"])
with assert_logs(
caplog,
[
'Scenario "Canning problem (MESSAGE scheme)/standard" has no solution',
"Some reporting may not function as expected",
],
):
rep = Reporter.from_scenario(scen)
# Input parameters are still available
demand = rep.full_key("demand")
result = rep.get(demand)
assert 3 == len(result)
def test_from_url(self, mp, caplog):
url = f'ixmp://{mp.name}/Douglas Adams/Hitchhiker'
# Default version is loaded
scen, mp = ixmp.Scenario.from_url(url)
assert scen.version == 1
# Giving an invalid version with errors='raise' raises an exception
expected = ("There exists no Scenario 'Douglas Adams|Hitchhiker' "
"(version: 10000) in the database!")
with pytest.raises(Exception, match=expected):
scen, mp = ixmp.Scenario.from_url(url + '#10000', errors='raise')
# Giving an invalid scenario with errors='warn' raises an exception
msg = ("ValueError: scenario='Hitchhikerfoo'\nwhen loading Scenario "
f"from url: {(url + 'foo')!r}")
with assert_logs(caplog, msg):
scen, mp = ixmp.Scenario.from_url(url + 'foo')
assert scen is None and isinstance(mp, ixmp.Platform)
def test_excel_io(self, scen, scen_empty, tmp_path, caplog):
tmp_path /= 'output.xlsx'
# FIXME remove_solution, check_out, commit, solve, commit should not
# be needed to make this small data addition.
scen.remove_solution()
scen.check_out()
# A 1-D set indexed by another set
scen.init_set('foo', 'j')
scen.add_set('foo', [['new-york'], ['topeka']])
# A scalar parameter with unusual units
scen.platform.add_unit('pounds')
scen.init_scalar('bar', 100, 'pounds')
# A parameter with no values
scen.init_par('baz_1', ['i', 'j'])
# A parameter with ambiguous index name
scen.init_par('baz_2', ['i'], ['i_dim'])
scen.add_par('baz_2', dict(value=[1.1], i_dim=['seattle']))
# A 2-D set with ambiguous index names
scen.init_set('baz_3', ['i', 'i'], ['i', 'i_also'])
scen.add_set('baz_3', [['seattle', 'seattle']])
# A set with no elements
scen.init_set('foo_2', ['j'])
scen.commit('')
scen.solve()
# Solved Scenario can be written to file
scen.to_excel(tmp_path, items=ixmp.ItemType.MODEL)
# With init_items=False, can't be read into an empty Scenario.
# Exception raised is the first index set, alphabetically
with pytest.raises(ValueError,
match="no set 'i'; "
"try init_items=True"):
scen_empty.read_excel(tmp_path)
# File can be read with init_items=True
scen_empty.read_excel(tmp_path, init_items=True, commit_steps=True)
# Contents of the Scenarios are the same, except for unreadable items
assert set(scen_empty.par_list()) | {'baz_1', 'baz_2'} \
== set(scen.par_list())
assert set(scen_empty.set_list()) | {'baz_3'} == set(scen.set_list())
assert_frame_equal(scen_empty.set('foo'), scen.set('foo'))
# NB could make a more exact comparison of the Scenarios
# Pre-initialize skipped items 'baz_2' and 'baz_3'
scen_empty.init_par('baz_2', ['i'], ['i_dim'])
scen_empty.init_set('baz_3', ['i', 'i'], ['i', 'i_also'])
# Data can be read into an existing Scenario without init_items or
# commit_steps arguments
scen_empty.read_excel(tmp_path)
# Re-initialize an item with different index names
scen_empty.remove_par('d')
scen_empty.init_par('d', idx_sets=['i', 'j'], idx_names=['I', 'J'])
# Reading now logs an error about conflicting dims
with assert_logs(caplog, "Existing par 'd' has index names(s)"):
scen_empty.read_excel(tmp_path, init_items=True)
# A new, empty Platform (different from the one under scen -> mp ->
# test_mp) that lacks all units
mp = ixmp.Platform(backend='jdbc',
driver='hsqldb',
url='jdbc:hsqldb:mem:excel_io')
# A Scenario without the 'dantzig' scheme -> no contents at all
s = ixmp.Scenario(mp,
model='foo',
scenario='bar',
scheme='empty',
version='new')
# Fails with add_units=False
with pytest.raises(ValueError,
match="The unit 'pounds' does not exist"
" in the database!"):
s.read_excel(tmp_path, init_items=True)
# Succeeds with add_units=True
s.read_excel(tmp_path, add_units=True, init_items=True)
def test_filters(test_mp, tmp_path, caplog):
"""Reporting can be filtered ex ante."""
scen = ixmp.Scenario(test_mp, 'Reporting filters', 'Reporting filters',
'new')
t, t_foo, t_bar, x = add_test_data(scen)
rep = Reporter.from_scenario(scen)
x_key = rep.full_key('x')
def assert_t_indices(labels):
assert set(rep.get(x_key).coords['t'].values) == set(labels)
# 1. Set filters directly
rep.graph['config']['filters'] = {'t': t_foo}
assert_t_indices(t_foo)
# Reporter can be re-used by changing filters
rep.graph['config']['filters'] = {'t': t_bar}
assert_t_indices(t_bar)
rep.graph['config']['filters'] = {}
assert_t_indices(t)
# 2. Set filters using a convenience method
rep = Reporter.from_scenario(scen)
rep.set_filters(t=t_foo)
assert_t_indices(t_foo)
# Clear filters using the convenience method
rep.set_filters(t=None)
assert_t_indices(t)
# Clear using the convenience method with no args
rep.set_filters(t=t_foo)
assert_t_indices(t_foo)
rep.set_filters()
assert_t_indices(t)
# 3. Set filters via configuration keys
# NB passes through from_scenario() -> __init__() -> configure()
rep = Reporter.from_scenario(scen, filters={'t': t_foo})
assert_t_indices(t_foo)
# Configuration key can also be read from file
rep = Reporter.from_scenario(scen)
# Write a temporary file containing the desired labels
config_file = tmp_path / 'config.yaml'
config_file.write_text('\n'.join([
'filters:',
' t: {!r}'.format(t_bar),
]))
rep.configure(config_file)
assert_t_indices(t_bar)
# Filtering too heavily:
# Remove one value from the database at valid coordinates
removed = {'t': t[:1], 'y': list(x.coords['y'].values)[:1]}
scen.remove_par('x', removed)
# Set filters to retrieve only this coordinate
rep.set_filters(**removed)
# A warning is logged
msg = '\n '.join([
"0 values for par 'x' using filters:",
repr(removed), 'Subsequent computations may fail.'
])
with assert_logs(caplog, msg):
rep.get(x_key)
def test_platform_units(test_mp, caplog, ureg):
"""Test handling of units from ixmp.Platform.
test_mp is loaded with some units including '-', '???', 'G$', etc. which
are not parseable with pint; and others which are not defined in a default
pint.UnitRegistry. These tests check the handling of those units.
"""
# Prepare a Scenario with test data
scen = ixmp.Scenario(test_mp, 'reporting_platform_units',
'reporting_platform_units', 'new')
t, t_foo, t_bar, x = add_test_data(scen)
rep = Reporter.from_scenario(scen)
x_key = rep.full_key('x')
# Convert 'x' to dataframe
x = x.to_series().rename('value').reset_index()
# Exception message, formatted as a regular expression
msg = r"unit '{}' cannot be parsed; contains invalid character\(s\) '{}'"
# Unit and components for the regex
bad_units = [('-', '-', '-'), ('???', r'\?\?\?', r'\?'),
('E$', r'E\$', r'\$')]
for unit, expr, chars in bad_units:
# Add the unit
test_mp.add_unit(unit)
# Overwrite the parameter
x['unit'] = unit
scen.add_par('x', x)
# Parsing units with invalid chars raises an intelligible exception
with pytest.raises(ComputationError, match=msg.format(expr, chars)):
rep.get(x_key)
# Now using parseable but unrecognized units
x['unit'] = 'USD/kWa'
scen.add_par('x', x)
# Unrecognized units are added automatically, with log messages emitted
with assert_logs(caplog, ['Add unit definition: kWa = [kWa]']):
rep.get(x_key)
# Mix of recognized/unrecognized units can be added: USD is already in the
# unit registry, so is not re-added
x['unit'] = 'USD/pkm'
test_mp.add_unit('USD/pkm')
scen.add_par('x', x)
caplog.clear()
rep.get(x_key)
assert not any('Add unit definition: USD = [USD]' in m
for m in caplog.messages)
# Mixed units are discarded
x.loc[0, 'unit'] = 'kg'
scen.add_par('x', x)
with assert_logs(caplog, "x: mixed units ['kg', 'USD/pkm'] discarded"):
rep.get(x_key)
# Configured unit substitutions are applied
rep.graph['config']['units'] = dict(apply=dict(x='USD/pkm'))
with assert_logs(caplog, "x: replace units dimensionless with USD/pkm"):
x = rep.get(x_key)
# Applied units are pint objects with the correct dimensionality
unit = x.attrs['_unit']
assert isinstance(unit, pint.Unit)
assert unit.dimensionality == {'[USD]': 1, '[km]': -1}
def test_platform_units(test_mp, caplog, ureg):
"""Test handling of units from ixmp.Platform.
test_mp is loaded with some units including '-', '???', 'G$', etc. which
are not parseable with pint; and others which are not defined in a default
pint.UnitRegistry. These tests check the handling of those units.
"""
# Prepare a Scenario with test data
scen = ixmp.Scenario(test_mp, "reporting_platform_units",
"reporting_platform_units", "new")
t, t_foo, t_bar, x = add_test_data(scen)
rep = Reporter.from_scenario(scen)
x_key = rep.full_key("x")
# Convert 'x' to dataframe
x = x.to_series().rename("value").reset_index()
# Exception message, formatted as a regular expression
msg = r"unit '{}' cannot be parsed; contains invalid character\(s\) '{}'"
# Unit and components for the regex
bad_units = [("-", "-", "-"), ("???", r"\?\?\?", r"\?"),
("E$", r"E\$", r"\$")]
for unit, expr, chars in bad_units:
# Add the unit
test_mp.add_unit(unit)
# Overwrite the parameter
x["unit"] = unit
scen.add_par("x", x)
# Parsing units with invalid chars raises an intelligible exception
with pytest.raises(ComputationError, match=msg.format(expr, chars)):
rep.get(x_key)
# Now using parseable but unrecognized units
x["unit"] = "USD/kWa"
scen.add_par("x", x)
# Unrecognized units are added automatically, with log messages emitted
caplog.clear()
rep.get(x_key)
# NB cannot use assert_logs here. reporting.utils.parse_units uses the
# pint application registry, so depending which tests are run and in
# which order, this unit may already be defined.
if len(caplog.messages):
assert "Add unit definition: kWa = [kWa]" in caplog.messages
# Mix of recognized/unrecognized units can be added: USD is already in the
# unit registry, so is not re-added
x["unit"] = "USD/pkm"
test_mp.add_unit("USD/pkm")
scen.add_par("x", x)
caplog.clear()
rep.get(x_key)
assert not any("Add unit definition: USD = [USD]" in m
for m in caplog.messages)
# Mixed units are discarded
x.loc[0, "unit"] = "kg"
scen.add_par("x", x)
with assert_logs(caplog,
"x: mixed units ['kg', 'USD/pkm'] discarded",
at_level=logging.INFO):
rep.get(x_key)
# Configured unit substitutions are applied
rep.graph["config"]["units"] = dict(apply=dict(x="USD/pkm"))
with assert_logs(caplog,
"x: replace units dimensionless with USD/pkm",
at_level=logging.INFO):
x = rep.get(x_key)
# Applied units are pint objects with the correct dimensionality
unit = x.attrs["_unit"]
assert isinstance(unit, pint.Unit)
assert unit.dimensionality == {"[USD]": 1, "[pkm]": -1}
def test_filters(test_mp, tmp_path, caplog):
"""Reporting can be filtered ex ante."""
scen = ixmp.Scenario(test_mp, "Reporting filters", "Reporting filters",
"new")
t, t_foo, t_bar, x = add_test_data(scen)
rep = Reporter.from_scenario(scen)
x_key = rep.full_key("x")
def assert_t_indices(labels):
assert set(rep.get(x_key).coords["t"].values) == set(labels)
# 1. Set filters directly
rep.graph["config"]["filters"] = {"t": t_foo}
assert_t_indices(t_foo)
# Reporter can be re-used by changing filters
rep.graph["config"]["filters"] = {"t": t_bar}
assert_t_indices(t_bar)
rep.graph["config"]["filters"] = {}
assert_t_indices(t)
# 2. Set filters using a convenience method
rep = Reporter.from_scenario(scen)
rep.set_filters(t=t_foo)
assert_t_indices(t_foo)
# Clear filters using the convenience method
rep.set_filters(t=None)
assert_t_indices(t)
# Clear using the convenience method with no args
rep.set_filters(t=t_foo)
assert_t_indices(t_foo)
rep.set_filters()
assert_t_indices(t)
# 3. Set filters via configuration keys
# NB passes through from_scenario() -> __init__() -> configure()
rep = Reporter.from_scenario(scen, filters={"t": t_foo})
assert_t_indices(t_foo)
# Configuration key can also be read from file
rep = Reporter.from_scenario(scen)
# Write a temporary file containing the desired labels
config_file = tmp_path / "config.yaml"
config_file.write_text("\n".join(["filters:", f" t: {repr(t_bar)}"]))
rep.configure(config_file)
assert_t_indices(t_bar)
# Filtering too heavily:
# Remove one value from the database at valid coordinates
removed = {"t": t[:1], "y": list(x.coords["y"].values)[:1]}
scen.remove_par("x", removed)
# Set filters to retrieve only this coordinate
rep.set_filters(**removed)
# A warning is logged
with assert_logs(
caplog,
(
f"0 values for par 'x' using filters: {repr(removed)}",
"May be the cause of subsequent errors",
),
at_level=logging.DEBUG,
):
rep.get(x_key)
def test_cached(caplog, test_context, tmp_path):
""":func:`.cached` works as expected.
.. todo:: test behaviour when :data:`.SKIP_CACHE` is :obj:`True`
"""
# Clear seen paths, so that log message below is guaranteed to occur
message_ix_models.util.cache.PATHS_SEEN.clear()
# Store in the temporary directory for this session, to avoid collisions across
# sessions
test_context.cache_path = tmp_path.joinpath("cache")
# A dummy path to be hashed as an argument
path_foo = tmp_path.joinpath("foo", "bar")
with caplog.at_level(logging.DEBUG, logger="message_ix_models"):
@cached
def func0(ctx, a, path, b=3):
"""A test function."""
log.info("func0 runs")
return f"{id(ctx)}, {a + b}"
# Docstring is modified
assert "Data returned by this function is cached" in func0.__doc__
# Message is logged
assert f"func0() will cache in {tmp_path.joinpath('cache')}" in caplog.messages
@cached
def func1(x=1, y=2, **kwargs):
# Function with defaults for all arguments
log.info("func1 runs")
return x + y
caplog.clear()
# pathlib.Path argument is serialized to JSON as part of the argument hash;
# function runs, messages logged
with assert_logs(caplog, "func0 runs"):
result0 = func0(test_context, 1, path_foo)
caplog.clear()
result1 = func0(test_context, 1, path_foo)
# Function does not run
assert "func0 runs" not in caplog.messages
assert caplog.messages[0].startswith("Cache hit for func0")
# Results identical
assert result0 == result1
# Different context object with identical contents hashes equal
ctx2 = deepcopy(test_context)
assert id(test_context) != id(ctx2)
result2 = func0(ctx2, 1, path_foo)
# Function does not run
assert "func0 runs" not in caplog.messages
# Results are identical, i.e. including the old ID
assert result0 == result2
ctx2.delete()
caplog.clear()
# Hash of no arguments is the same, function only runs once
assert 3 == func1() == func1()
assert 1 == sum(m == "func1 runs" for m in caplog.messages)
# Warnings logged for unhashables; ScenarioInfo is hashed as dict
caplog.clear()
with assert_logs(
caplog,
[
"ignores <class 'xarray.core.dataset.Dataset'>",
"ignores <class 'ixmp.core.platform.Platform'>",
],
):
func1(ds=xr.Dataset(),
mp=test_context.get_platform(),
si=ScenarioInfo())
# Unserializable type raises an exception
with pytest.raises(TypeError,
match="Object of type slice is not JSON serializable"):
func1(arg=slice(None))
