def test_reporter_file(tmp_path):
r = Reporter()
# Path to a temporary file
p = tmp_path / 'foo.txt'
# File can be added to the Reporter before it is created, because the file
# is not read until/unless required
k1 = r.add_file(p)
# File has the expected key
assert k1 == 'file:foo.txt'
# Add some contents to the file
p.write_text('Hello, world!')
# The file's contents can be read through the Reporter
assert r.get('file:foo.txt') == 'Hello, world!'
# Write the report to file
p2 = tmp_path / 'bar.txt'
r.write('file:foo.txt', p2)
# Write using a string path
r.write('file:foo.txt', str(p2))
# The Reporter produces the expected output file
assert p2.read_text() == 'Hello, world!'
def test_reporting_filters(test_mp, tmp_path):
"""Reporting can be filtered ex ante."""
scen = ixmp.Scenario(test_mp, 'Reporting filters', 'Reporting filters',
'new')
t, t_foo, t_bar, x = add_test_data(scen)
rep = Reporter.from_scenario(scen)
x_key = rep.full_key('x')
def assert_t_indices(labels):
assert set(rep.get(x_key).index.levels[0]) == set(labels)
# 1. Set filters directly
rep.graph['filters'] = {'t': t_foo}
assert_t_indices(t_foo)
# Reporter can be re-used by changing filters
rep.graph['filters'] = {'t': t_bar}
assert_t_indices(t_bar)
rep.graph['filters'] = {}
assert_t_indices(t)
# 2. Set filters using a convenience method
rep = Reporter.from_scenario(scen)
rep.set_filters(t=t_foo)
assert_t_indices(t_foo)
# Clear filters using the convenience method
rep.set_filters(t=None)
assert_t_indices(t)
# 3. Set filters via configuration keys
# NB passes through from_scenario() -> __init__() -> configure()
rep = Reporter.from_scenario(scen, filters={'t': t_foo})
assert_t_indices(t_foo)
# Configuration key can also be read from file
rep = Reporter.from_scenario(scen)
# Write a temporary file containing the desired labels
config_file = tmp_path / 'config.yaml'
config_file.write_text('\n'.join([
'filters:',
' t: {!r}'.format(t_bar),
]))
rep.configure(config_file)
assert_t_indices(t_bar)
def test_reporter_disaggregate():
r = Reporter()
foo = Key('foo', ['a', 'b', 'c'])
r.add(foo, '<foo data>')
r.add('d_shares', '<share data>')
# Disaggregation works
r.disaggregate(foo, 'd', args=['d_shares'])
assert 'foo:a-b-c-d' in r.graph
assert r.graph['foo:a-b-c-d'] == (computations.disaggregate_shares,
'foo:a-b-c', 'd_shares')
# Invalid method
with pytest.raises(ValueError):
r.disaggregate(foo, 'd', method='baz')
def test_report_size(test_mp):
"""Stress-test reporting of large, sparse quantities."""
from itertools import zip_longest
import numpy as np
# test_mp.add_unit('kg')
scen = ixmp.Scenario(test_mp, 'size test', 'base', version='new')
# Dimensions and their lengths
dims = 'abcdef'
sizes = [1, 5, 21, 21, 89, 377] # Fibonacci #s; next 1597, 6765
# commented: "377 / 73984365 elements = 0.00051% full"
# from functools import reduce
# from operator import mul
# size = reduce(mul, sizes)
# print('{} / {} elements = {:.5f}% full'
# .format(max(sizes), size, 100 * max(sizes) / size))
# Names like f_0000 ... f_1596 along each dimension
coords = []
for d, N in zip(dims, sizes):
# py3.5 compat: could use an f-string here
coords.append(['{}_{:04d}'.format(d, i) for i in range(N)])
# Add to Scenario
scen.init_set(d)
scen.add_set(d, coords[-1])
def _make_values():
"""Make a DataFrame containing each label in *coords* at least once."""
values = list(zip_longest(*coords, np.random.rand(max(sizes))))
result = pd.DataFrame(values, columns=list(dims) + ['value']) \
.ffill()
result['unit'] = 'kg'
return result
# Fill the Scenario with quantities named q_01 ... q_09
N = 10
names = []
for i in range(10):
# py3.5 compat: could use an f-string here
name = 'q_{:02d}'.format(i)
scen.init_par(name, list(dims))
scen.add_par(name, _make_values())
names.append(name)
# Create the reporter
rep = Reporter.from_scenario(scen)
# Add an operation that takes the product, i.e. requires all the q_*
keys = [rep.full_key(name) for name in names]
rep.add('bigmem', tuple([computations.product] + keys))
# One quantity fits in memory
rep.get(keys[0])
# assert False
# All quantities together trigger MemoryError
rep.get('bigmem')
def test_reporter_describe(test_mp, test_data_path, capsys):
scen = make_dantzig(test_mp)
r = Reporter.from_scenario(scen)
# hexadecimal ID of *scen*
id_ = hex(id(scen)) if os.name != 'nt' else \
'{:#018X}'.format(id(scen)).replace('X', 'x')
# Describe one key
desc1 = """'d:i':
- sum(dimensions=['j'], weights=None, ...)
- 'd:i-j':
- data_for_quantity('par', 'd', 'value', ...)
- 'scenario':
- <ixmp.core.Scenario object at {id}>
- 'filters':
- {{}}""".format(id=id_)
assert desc1 == r.describe('d:i')
# Description was also written to stdout
out1, _ = capsys.readouterr()
assert desc1 + '\n' == out1
# Description of all keys is as expected
desc2 = (test_data_path / 'report-describe.txt').read_text() \
.format(id=id_)
assert desc2 == r.describe() + '\n'
# Result was also written to stdout
out2, _ = capsys.readouterr()
assert desc2 == out2
def test_reporter_from_dantzig(test_mp, test_data_path):
scen = make_dantzig(test_mp, solve=test_data_path)
# Reporter.from_scenario can handle the Dantzig problem
rep = Reporter.from_scenario(scen)
# Partial sums are available automatically (d is defined over i and j)
d_i = rep.get('d:i')
# Units pass through summation
assert d_i.attrs['_unit'] == ureg.parse_units('km')
# Summation across all dimensions results a 1-element Quantity
d = rep.get('d:')
assert len(d) == 1
assert np.isclose(d.iloc[0], 11.7)
# Weighted sum
weights = Quantity(xr.DataArray(
[1, 2, 3],
coords=['chicago new-york topeka'.split()],
dims=['j']))
new_key = rep.aggregate('d:i-j', 'weighted', 'j', weights)
# ...produces the expected new key with the summed dimension removed and
# tag added
assert new_key == 'd:i:weighted'
# ...produces the expected new value
obs = rep.get(new_key)
exp = (rep.get('d:i-j') * weights).sum(dim=['j']) / weights.sum(dim=['j'])
# TODO: attrs has to be explicitly copied here because math is done which
# returns a pd.Series
exp = Quantity(exp, attrs=rep.get('d:i-j').attrs)
assert_series_equal(obs.sort_index(), exp.sort_index())
# Disaggregation with explicit data
# (cases of canned food 'p'acked in oil or water)
shares = xr.DataArray([0.8, 0.2], coords=[['oil', 'water']], dims=['p'])
new_key = rep.disaggregate('b:j', 'p', args=[Quantity(shares)])
# ...produces the expected key with new dimension added
assert new_key == 'b:j-p'
b_jp = rep.get('b:j-p')
# Units pass through disaggregation
assert b_jp.attrs['_unit'] == 'cases'
# Set elements are available
assert rep.get('j') == ['new-york', 'chicago', 'topeka']
# 'all' key retrieves all quantities
obs = {da.name for da in rep.get('all')}
exp = set('a b d f demand demand-margin z x'.split())
assert obs == exp
# Shorthand for retrieving a full key name
assert rep.full_key('d') == 'd:i-j' and isinstance(rep.full_key('d'), Key)
def test_reporting_platform_units(test_mp, caplog):
"""Test handling of units from ixmp.Platform.
test_mp is loaded with some units includig '-', '???', 'G$', etc. which are
not parseable with pint; and others which are not defined in a default
pint.UnitRegistry. These tests check the handling of those units.
"""
# Prepare a Scenario with test data
scen = ixmp.Scenario(test_mp, 'reporting_platform_units',
'reporting_platform_units', 'new')
t, t_foo, t_bar, x = add_test_data(scen)
rep = Reporter.from_scenario(scen)
x_key = rep.full_key('x')
# Convert 'x' to dataframe
x = x.to_series().rename('value').reset_index()
# Exception message, formatted as a regular expression
msg = r"unit '{}' cannot be parsed; contains invalid character\(s\) '{}'"
# Unit and components for the regex
bad_units = [('-', '-', '-'), ('???', r'\?\?\?', r'\?'),
('G$', r'G\$', r'\$')]
for unit, expr, chars in bad_units:
# Overwrite the parameter
x['unit'] = unit
scen.add_par('x', x)
# Parsing units with invalid chars raises an intelligible exception
with pytest.raises(ValueError, match=msg.format(expr, chars)):
rep.get(x_key)
# Now using parseable but unrecognized units
x['unit'] = 'USD/kWa'
scen.add_par('x', x)
# Unrecognized units are added automatically, with log messages emitted
caplog.clear()
rep.get(x_key)
expected = [
'Add unit definition: USD = [USD]',
'Add unit definition: kWa = [kWa]',
]
assert all(e in [rec.message for rec in caplog.records] for e in expected)
# Mix of recognized/unrecognized units can be added: USD is already in the
# unit registry, so is not re-added
x['unit'] = 'USD/pkm'
test_mp.add_unit('USD/pkm')
scen.add_par('x', x)
caplog.clear()
rep.get(x_key)
assert not any('Add unit definition: USD = [USD]' in rec.message
for rec in caplog.records)
assert any('Add unit definition: pkm = [pkm]' in rec.message
for rec in caplog.records)
def test_reporter_add_queue():
r = Reporter()
r.add('foo-0', (lambda x: x, 42))
# A computation
def _product(a, b):
return a * b
# A queue of computations to add. Only foo-1 succeeds on the first pass;
# only foo-2 on the second pass, etc.
strict = dict(strict=True)
queue = [
(('foo-4', _product, 'foo-3', 10), strict),
(('foo-3', _product, 'foo-2', 10), strict),
(('foo-2', _product, 'foo-1', 10), strict),
(('foo-1', _product, 'foo-0', 10), {}),
]
# Maximum 3 attempts → foo-4 fails on the start of the 3rd pass
with pytest.raises(MissingKeyError, match='foo-3'):
r.add(queue, max_tries=3, fail='raise')
# But foo-2 was successfully added on the second pass, and gives the
# correct result
assert r.get('foo-2') == 42 * 10 * 10
def test_reporter_read_config(test_mp, test_data_path):
scen = make_dantzig(test_mp)
rep = Reporter.from_scenario(scen)
# Configuration can be read from file
rep.configure(test_data_path / 'report-config-0.yaml')
# Data from configured file is available
assert rep.get('d_check').loc['seattle', 'chicago'] == 1.7
def test_reporter(test_mp):
scen = Scenario(test_mp,
'canning problem (MESSAGE scheme)',
'standard')
# Varies between local & CI contexts
# DEBUG may be due to reuse of test_mp in a non-deterministic order
if not scen.has_solution():
scen.solve()
# IXMPReporter can be initialized on a MESSAGE Scenario
rep_ix = ixmp_Reporter.from_scenario(scen)
# message_ix.Reporter can also be initialized
rep = Reporter.from_scenario(scen)
# Number of quantities available in a rudimentary MESSAGEix Scenario
assert len(rep.graph['all']) == 120
# Quantities have short dimension names
assert 'demand:n-c-l-y-h' in rep.graph
# Aggregates are available
assert 'demand:n-l-h' in rep.graph
# Quantities contain expected data
dims = dict(coords=['chicago new-york topeka'.split()], dims=['n'])
demand = xr.DataArray([300, 325, 275], **dims)
# NB the call to squeeze() drops the length-1 dimensions c-l-y-h
obs = rep.get('demand:n-c-l-y-h').squeeze(drop=True)
# TODO: Squeeze on AttrSeries still returns full index, whereas xarray
# drops everything except node
obs = obs.reset_index(['c', 'l', 'y', 'h'], drop=True)
# check_dtype is false because of casting in pd.Series to float
# check_attrsis false because we don't get the unit addition in bare xarray
assert_qty_equal(obs.sort_index(), demand,
check_attrs=False, check_dtype=False)
# ixmp.Reporter pre-populated with only model quantities and aggregates
assert len(rep_ix.graph) == 5088
# message_ix.Reporter pre-populated with additional, derived quantities
assert len(rep.graph) == 7975
# Derived quantities have expected dimensions
vom_key = rep.full_key('vom')
assert vom_key not in rep_ix
assert vom_key == 'vom:nl-t-yv-ya-m-h'
# …and expected values
vom = (
rep.get(rep.full_key('ACT')) * rep.get(rep.full_key('var_cost'))
).dropna()
# check_attrs false because `vom` multiply above does not add units
assert_qty_equal(vom, rep.get(vom_key), check_attrs=False)
def test_reporter_read_config(test_mp, test_data_path):
scen = make_dantzig(test_mp)
rep = Reporter.from_scenario(scen)
with pytest.warns(UserWarning,
match=r"Unrecognized sections {'notarealsection'}"):
rep.read_config(test_data_path / 'report-config-0.yaml')
# Data from configured file is available
assert rep.get('d_check').loc['seattle', 'chicago'] == 1.7
def test_reporter_from_scenario(message_test_mp):
scen = Scenario(message_test_mp, **SCENARIO["dantzig"])
# Varies between local & CI contexts
# DEBUG may be due to reuse of test_mp in a non-deterministic order
if not scen.has_solution():
scen.solve(quiet=True)
# IXMPReporter can be initialized on a MESSAGE Scenario
rep_ix = ixmp_Reporter.from_scenario(scen)
# message_ix.Reporter can also be initialized
rep = Reporter.from_scenario(scen)
# Number of quantities available in a rudimentary MESSAGEix Scenario
assert len(rep.graph["all"]) == 123
# Quantities have short dimension names
assert "demand:n-c-l-y-h" in rep
# Aggregates are available
assert "demand:n-l-h" in rep
# Quantities contain expected data
dims = dict(coords=["chicago new-york topeka".split()], dims=["n"])
demand = Quantity(xr.DataArray([300, 325, 275], **dims), name="demand")
# NB the call to squeeze() drops the length-1 dimensions c-l-y-h
obs = rep.get("demand:n-c-l-y-h").squeeze(drop=True)
# check_attrs False because we don't get the unit addition in bare xarray
assert_qty_equal(obs, demand, check_attrs=False)
# ixmp.Reporter pre-populated with only model quantities and aggregates
assert len(rep_ix.graph) == 5225
# message_ix.Reporter pre-populated with additional, derived quantities
# This is the same value as in test_tutorials.py
assert len(rep.graph) == 12690
# Derived quantities have expected dimensions
vom_key = rep.full_key("vom")
assert vom_key not in rep_ix
assert vom_key == "vom:nl-t-yv-ya-m-h"
# …and expected values
var_cost = rep.get(rep.full_key("var_cost"))
ACT = rep.get(rep.full_key("ACT"))
product = rep.get_comp("product")
vom = product(var_cost, ACT)
# check_attrs false because `vom` multiply above does not add units
assert_qty_equal(vom, rep.get(vom_key))
def report(context, config, key):
"""Run reporting for KEY."""
# Import here to avoid importing reporting dependencies when running
# other commands
from ixmp.reporting import Reporter
# Instantiate the Reporter with the Scenario loaded by main()
r = Reporter.from_scenario(context['scen'])
# Read the configuration file, if any
if config:
r.read_config(config)
# Print the target
print(r.get(key))
def test_reporter_read_config(test_mp, test_data_path, caplog):
scen = make_dantzig(test_mp)
rep = Reporter.from_scenario(scen)
caplog.clear()
# Warning is raised when reading configuration with unrecognized section(s)
rep.read_config(test_data_path / 'report-config-0.yaml')
assert ("Unrecognized sections ['notarealsection'] in reporting "
"configuration will have no effect") == caplog.records[0].message
# Data from configured file is available
assert rep.get('d_check').loc['seattle', 'chicago'] == 1.7
def test_reporting_aggregate(test_mp):
scen = ixmp.Scenario(test_mp, 'Group reporting', 'group reporting', 'new')
t, t_foo, t_bar, x = add_test_data(scen)
# Reporter
rep = Reporter.from_scenario(scen)
# Define some groups
t_groups = {'foo': t_foo, 'bar': t_bar, 'baz': ['foo1', 'bar5', 'bar6']}
# Add aggregates
key1 = rep.aggregate('x:t-y', 'agg1', {'t': t_groups}, keep=True)
# Group has expected key and contents
assert key1 == 'x:t-y:agg1'
# Aggregate is computed without error
agg1 = rep.get(key1)
# Expected set of keys along the aggregated dimension
assert set(agg1.coords['t'].values) == set(t) | set(t_groups.keys())
# Sums are as expected
# TODO: the check_dtype arg assumes Quantity backend is a AttrSeries,
# should that be made default in assert_qty_allclose?
assert_qty_allclose(agg1.sel(t='foo', drop=True),
x.sel(t=t_foo).sum('t'),
check_dtype=False)
assert_qty_allclose(agg1.sel(t='bar', drop=True),
x.sel(t=t_bar).sum('t'),
check_dtype=False)
assert_qty_allclose(agg1.sel(t='baz', drop=True),
x.sel(t=['foo1', 'bar5', 'bar6']).sum('t'),
check_dtype=False)
# Add aggregates, without keeping originals
key2 = rep.aggregate('x:t-y', 'agg2', {'t': t_groups}, keep=False)
# Distinct keys
assert key2 != key1
# Only the aggregated and no original keys along the aggregated dimension
agg2 = rep.get(key2)
assert set(agg2.coords['t'].values) == set(t_groups.keys())
with pytest.raises(NotImplementedError):
# Not yet supported; requires two separate operations
rep.aggregate('x:t-y', 'agg3', {'t': t_groups, 'y': [2000, 2010]})
def test_reporter_add_product(test_mp):
scen = ixmp.Scenario(test_mp, 'reporter_add_product',
'reporter_add_product', 'new')
*_, x = add_test_data(scen)
rep = Reporter.from_scenario(scen)
# add_product() works
key = rep.add_product('x squared', 'x', 'x', sums=True)
# Product has the expected dimensions
assert key == 'x squared:t-y'
# Product has the expected value
exp = as_quantity(x * x)
exp.attrs['_unit'] = UNITS('kilogram ** 2').units
assert_qty_equal(exp, rep.get(key))
def test_aggregate(test_mp):
scen = ixmp.Scenario(test_mp, 'Group reporting', 'group reporting', 'new')
t, t_foo, t_bar, x = add_test_data(scen)
# Reporter
rep = Reporter.from_scenario(scen)
# Define some groups
t_groups = {'foo': t_foo, 'bar': t_bar, 'baz': ['foo1', 'bar5', 'bar6']}
# Use the computation directly
agg1 = computations.aggregate(as_quantity(x), {'t': t_groups}, True)
# Expected set of keys along the aggregated dimension
assert set(agg1.coords['t'].values) == set(t) | set(t_groups.keys())
# Sums are as expected
assert_qty_allclose(agg1.sel(t='foo', drop=True), x.sel(t=t_foo).sum('t'))
assert_qty_allclose(agg1.sel(t='bar', drop=True), x.sel(t=t_bar).sum('t'))
assert_qty_allclose(agg1.sel(t='baz', drop=True),
x.sel(t=['foo1', 'bar5', 'bar6']).sum('t'))
# Use Reporter convenience method
key2 = rep.aggregate('x:t-y', 'agg2', {'t': t_groups}, keep=True)
# Group has expected key and contents
assert key2 == 'x:t-y:agg2'
# Aggregate is computed without error
agg2 = rep.get(key2)
assert_qty_equal(agg1, agg2)
# Add aggregates, without keeping originals
key3 = rep.aggregate('x:t-y', 'agg3', {'t': t_groups}, keep=False)
# Distinct keys
assert key3 != key2
# Only the aggregated and no original keys along the aggregated dimension
agg3 = rep.get(key3)
assert set(agg3.coords['t'].values) == set(t_groups.keys())
with pytest.raises(NotImplementedError):
# Not yet supported; requires two separate operations
rep.aggregate('x:t-y', 'agg3', {'t': t_groups, 'y': [2000, 2010]})
def report(context, config, key):
"""Run reporting for KEY."""
# Import here to avoid importing reporting dependencies when running
# other commands
from ixmp.reporting import Reporter
if not context:
raise click.UsageError('give either --url, --platform or --dbprops '
'before command report')
# Instantiate the Reporter with the Scenario loaded by main()
r = Reporter.from_scenario(context['scen'])
# Read the configuration file, if any
r.configure(config)
# Print the target
print(r.get(key))
def report(ctx, config, default):
# Import here to avoid importing reporting dependencies when running
# other commands
from ixmp.reporting import Reporter
# Instantiate the Reporter with the Scenario loaded by main()
r = Reporter.from_scenario(ctx.obj['scen'])
# Read the configuration file, if any
if config:
r.read_config(config)
# Process remaining configuration from command-line arguments
if default:
r.configure(default=default)
# Print the default target
print(r.get())
def test_configure(test_mp, test_data_path):
# TODO test: configuration keys 'units', 'replace_units'
# Configure globally; reads 'rename_dims' section
configure(rename_dims={'i': 'i_renamed'})
# Reporting uses the RENAME_DIMS mapping of 'i' to 'i_renamed'
scen = make_dantzig(test_mp)
rep = Reporter.from_scenario(scen)
assert 'd:i_renamed-j' in rep, rep.graph.keys()
assert ['seattle', 'san-diego'] == rep.get('i_renamed')
# Original name 'i' are not found in the reporter
assert 'd:i-j' not in rep, rep.graph.keys()
pytest.raises(KeyError, rep.get, 'i')
# Remove the configuration for renaming 'i', so that other tests work
RENAME_DIMS.pop('i')
def test_reporting_file_formats(test_data_path, tmp_path):
r = Reporter()
expected = xr.DataArray.from_series(
pd.read_csv(test_data_path / 'report-input.csv',
index_col=['i', 'j'])['value'])
# CSV file is automatically parsed to xr.DataArray
p1 = test_data_path / 'report-input.csv'
k = r.add_file(p1)
assert_xr_equal(r.get(k), expected)
# Write to CSV
p2 = tmp_path / 'report-output.csv'
r.write(k, p2)
# Output is identical to input file, except for order
assert (sorted(p1.read_text().split('\n')) == sorted(
p2.read_text().split('\n')))
# Write to Excel
p3 = tmp_path / 'report-output.xlsx'
r.write(k, p3)
def test_reporter_describe(test_mp, test_data_path):
scen = make_dantzig(test_mp)
r = Reporter.from_scenario(scen)
# hexadecimal ID of *scen*
id_ = hex(id(scen)) if os.name != 'nt' else \
'{:#018X}'.format(id(scen)).replace('X', 'x')
# Describe one key
expected = """'d:i':
- sum(dimensions=['j'], weights=None, ...)
- 'd:i-j':
- data_for_quantity('par', 'd', 'value', ...)
- 'scenario':
- <ixmp.core.Scenario object at {id}>
- 'filters':
- {{}}
""".format(id=id_)
assert r.describe('d:i') == expected
# Describe all keys
expected = (test_data_path / 'report-describe.txt').read_text() \
.format(id=id_)
assert r.describe() == expected
def test_reporter_visualize(test_mp, tmp_path):
scen = make_dantzig(test_mp)
r = Reporter.from_scenario(scen)
r.visualize(str(tmp_path / 'visualize.png'))
def test_reporting_units():
"""Test handling of units within Reporter computations."""
r = Reporter()
# Create some dummy data
dims = dict(coords=['a b c'.split()], dims=['x'])
r.add('energy:x',
Quantity(xr.DataArray([1., 3, 8], attrs={'_unit': 'MJ'}, **dims)))
r.add('time',
Quantity(xr.DataArray([5., 6, 8], attrs={'_unit': 'hour'}, **dims)))
r.add('efficiency', Quantity(xr.DataArray([0.9, 0.8, 0.95], **dims)))
# Aggregation preserves units
r.add('energy', (computations.sum, 'energy:x', None, ['x']))
assert r.get('energy').attrs['_unit'] == ureg.parse_units('MJ')
# Units are derived for a ratio of two quantities
r.add('power', (computations.ratio, 'energy:x', 'time'))
assert r.get('power').attrs['_unit'] == ureg.parse_units('MJ/hour')
# Product of dimensioned and dimensionless quantities keeps the former
r.add('energy2', (computations.product, 'energy:x', 'efficiency'))
assert r.get('energy2').attrs['_unit'] == ureg.parse_units('MJ')
def test_platform_units(test_mp, caplog, ureg):
"""Test handling of units from ixmp.Platform.
test_mp is loaded with some units including '-', '???', 'G$', etc. which
are not parseable with pint; and others which are not defined in a default
pint.UnitRegistry. These tests check the handling of those units.
"""
# Prepare a Scenario with test data
scen = ixmp.Scenario(test_mp, 'reporting_platform_units',
'reporting_platform_units', 'new')
t, t_foo, t_bar, x = add_test_data(scen)
rep = Reporter.from_scenario(scen)
x_key = rep.full_key('x')
# Convert 'x' to dataframe
x = x.to_series().rename('value').reset_index()
# Exception message, formatted as a regular expression
msg = r"unit '{}' cannot be parsed; contains invalid character\(s\) '{}'"
# Unit and components for the regex
bad_units = [('-', '-', '-'), ('???', r'\?\?\?', r'\?'),
('E$', r'E\$', r'\$')]
for unit, expr, chars in bad_units:
# Add the unit
test_mp.add_unit(unit)
# Overwrite the parameter
x['unit'] = unit
scen.add_par('x', x)
# Parsing units with invalid chars raises an intelligible exception
with pytest.raises(ComputationError, match=msg.format(expr, chars)):
rep.get(x_key)
# Now using parseable but unrecognized units
x['unit'] = 'USD/kWa'
scen.add_par('x', x)
# Unrecognized units are added automatically, with log messages emitted
with assert_logs(caplog, ['Add unit definition: kWa = [kWa]']):
rep.get(x_key)
# Mix of recognized/unrecognized units can be added: USD is already in the
# unit registry, so is not re-added
x['unit'] = 'USD/pkm'
test_mp.add_unit('USD/pkm')
scen.add_par('x', x)
caplog.clear()
rep.get(x_key)
assert not any('Add unit definition: USD = [USD]' in m
for m in caplog.messages)
# Mixed units are discarded
x.loc[0, 'unit'] = 'kg'
scen.add_par('x', x)
with assert_logs(caplog, "x: mixed units ['kg', 'USD/pkm'] discarded"):
rep.get(x_key)
# Configured unit substitutions are applied
rep.graph['config']['units'] = dict(apply=dict(x='USD/pkm'))
with assert_logs(caplog, "x: replace units dimensionless with USD/pkm"):
x = rep.get(x_key)
# Applied units are pint objects with the correct dimensionality
unit = x.attrs['_unit']
assert isinstance(unit, pint.Unit)
assert unit.dimensionality == {'[USD]': 1, '[km]': -1}
def test_reporter_apply():
# Reporter with two scalar values
r = Reporter()
r.add('foo', (lambda x: x, 42))
r.add('bar', (lambda x: x, 11))
N = len(r.keys())
# A computation
def _product(a, b):
return a * b
# A generator method that yields keys and computations
def baz_qux(key):
yield key + ':baz', (_product, key, 0.5)
yield key + ':qux', (_product, key, 1.1)
# Apply the generator to two targets
r.apply(baz_qux, 'foo')
r.apply(baz_qux, 'bar')
# Four computations were added to the reporter
N += 4
assert len(r.keys()) == N
assert r.get('foo:baz') == 42 * 0.5
assert r.get('foo:qux') == 42 * 1.1
assert r.get('bar:baz') == 11 * 0.5
assert r.get('bar:qux') == 11 * 1.1
# A generator that takes two arguments
def twoarg(key1, key2):
yield key1 + '__' + key2, (_product, key1, key2)
r.apply(twoarg, 'foo:baz', 'bar:qux')
# One computation added to the reporter
N += 1
assert len(r.keys()) == N
assert r.get('foo:baz__bar:qux') == 42 * 0.5 * 11 * 1.1
# A useless generator that does nothing
def useless(key):
return
r.apply(useless, 'foo:baz__bar:qux')
# Nothing added to the reporter
assert len(r.keys()) == N
def test_reporter(scenario):
r = Reporter.from_scenario(scenario)
r.finalize(scenario)
assert 'scenario' in r.graph
def test_reporter_add():
"""Adding computations that refer to missing keys raises KeyError."""
r = Reporter()
r.add('a', 3)
r.add('d', 4)
# Adding an existing key with strict=True
with pytest.raises(KeyExistsError, match=r"key 'a' already exists"):
r.add('a', 5, strict=True)
def gen(other):
"""A generator for apply()."""
return (lambda a, b: a * b, 'a', other)
def msg(*keys):
"""Return a regex for str(MissingKeyError(*keys))."""
return 'required keys {!r} not defined'.format(tuple(keys)) \
.replace('(', '\\(') \
.replace(')', '\\)')
# One missing key
with pytest.raises(MissingKeyError, match=msg('b')):
r.add_product('ab', 'a', 'b')
# Two missing keys
with pytest.raises(MissingKeyError, match=msg('c', 'b')):
r.add_product('abc', 'c', 'a', 'b')
# Using apply() targeted at non-existent keys also raises an Exception
with pytest.raises(MissingKeyError, match=msg('e', 'f')):
r.apply(gen, 'd', 'e', 'f')
# add(..., strict=True) checks str or Key arguments
g = Key('g', 'hi')
with pytest.raises(MissingKeyError, match=msg('b', g)):
r.add('foo', (computations.product, 'a', 'b', g), strict=True)
# aggregate() and disaggregate() call add(), which raises the exception
with pytest.raises(MissingKeyError, match=msg(g)):
r.aggregate(g, 'tag', 'i')
with pytest.raises(MissingKeyError, match=msg(g)):
r.disaggregate(g, 'j')
# add(..., sums=True) also adds partial sums
r.add('foo:a-b-c', [], sums=True)
assert 'foo:b' in r
# add(name, ...) where name is the name of a computation
r.add('select', 'bar', 'a', indexers={'dim': ['d0', 'd1', 'd2']})
# add(name, ...) with keyword arguments not recognized by the computation
# raises an exception
msg = "unexpected keyword argument 'bad_kwarg'"
with pytest.raises(TypeError, match=msg):
r.add('select', 'bar', 'a', bad_kwarg='foo', index=True)
def data(test_mp, request):
scen = ixmp.Scenario(test_mp, request.node.name, request.node.name, 'new')
rep = Reporter.from_scenario(scen)
yield [scen, rep] + list(add_test_data(scen))
def test_filters(test_mp, tmp_path, caplog):
"""Reporting can be filtered ex ante."""
scen = ixmp.Scenario(test_mp, 'Reporting filters', 'Reporting filters',
'new')
t, t_foo, t_bar, x = add_test_data(scen)
rep = Reporter.from_scenario(scen)
x_key = rep.full_key('x')
def assert_t_indices(labels):
assert set(rep.get(x_key).coords['t'].values) == set(labels)
# 1. Set filters directly
rep.graph['config']['filters'] = {'t': t_foo}
assert_t_indices(t_foo)
# Reporter can be re-used by changing filters
rep.graph['config']['filters'] = {'t': t_bar}
assert_t_indices(t_bar)
rep.graph['config']['filters'] = {}
assert_t_indices(t)
# 2. Set filters using a convenience method
rep = Reporter.from_scenario(scen)
rep.set_filters(t=t_foo)
assert_t_indices(t_foo)
# Clear filters using the convenience method
rep.set_filters(t=None)
assert_t_indices(t)
# Clear using the convenience method with no args
rep.set_filters(t=t_foo)
assert_t_indices(t_foo)
rep.set_filters()
assert_t_indices(t)
# 3. Set filters via configuration keys
# NB passes through from_scenario() -> __init__() -> configure()
rep = Reporter.from_scenario(scen, filters={'t': t_foo})
assert_t_indices(t_foo)
# Configuration key can also be read from file
rep = Reporter.from_scenario(scen)
# Write a temporary file containing the desired labels
config_file = tmp_path / 'config.yaml'
config_file.write_text('\n'.join([
'filters:',
' t: {!r}'.format(t_bar),
]))
rep.configure(config_file)
assert_t_indices(t_bar)
# Filtering too heavily:
# Remove one value from the database at valid coordinates
removed = {'t': t[:1], 'y': list(x.coords['y'].values)[:1]}
scen.remove_par('x', removed)
# Set filters to retrieve only this coordinate
rep.set_filters(**removed)
# A warning is logged
msg = '\n '.join([
"0 values for par 'x' using filters:",
repr(removed), 'Subsequent computations may fail.'
])
with assert_logs(caplog, msg):
rep.get(x_key)
