def test_reporter(test_mp):
scen = Scenario(test_mp,
'canning problem (MESSAGE scheme)',
'standard')
# Varies between local & CI contexts
# DEBUG may be due to reuse of test_mp in a non-deterministic order
if not scen.has_solution():
scen.solve()
# IXMPReporter can be initialized on a MESSAGE Scenario
rep_ix = ixmp_Reporter.from_scenario(scen)
# message_ix.Reporter can also be initialized
rep = Reporter.from_scenario(scen)
# Number of quantities available in a rudimentary MESSAGEix Scenario
assert len(rep.graph['all']) == 120
# Quantities have short dimension names
assert 'demand:n-c-l-y-h' in rep.graph
# Aggregates are available
assert 'demand:n-l-h' in rep.graph
# Quantities contain expected data
dims = dict(coords=['chicago new-york topeka'.split()], dims=['n'])
demand = xr.DataArray([300, 325, 275], **dims)
# NB the call to squeeze() drops the length-1 dimensions c-l-y-h
obs = rep.get('demand:n-c-l-y-h').squeeze(drop=True)
# TODO: Squeeze on AttrSeries still returns full index, whereas xarray
# drops everything except node
obs = obs.reset_index(['c', 'l', 'y', 'h'], drop=True)
# check_dtype is false because of casting in pd.Series to float
# check_attrsis false because we don't get the unit addition in bare xarray
assert_qty_equal(obs.sort_index(), demand,
check_attrs=False, check_dtype=False)
# ixmp.Reporter pre-populated with only model quantities and aggregates
assert len(rep_ix.graph) == 5088
# message_ix.Reporter pre-populated with additional, derived quantities
assert len(rep.graph) == 7975
# Derived quantities have expected dimensions
vom_key = rep.full_key('vom')
assert vom_key not in rep_ix
assert vom_key == 'vom:nl-t-yv-ya-m-h'
# …and expected values
vom = (
rep.get(rep.full_key('ACT')) * rep.get(rep.full_key('var_cost'))
).dropna()
# check_attrs false because `vom` multiply above does not add units
assert_qty_equal(vom, rep.get(vom_key), check_attrs=False)
def test_reporter(message_test_mp):
scen = Scenario(message_test_mp, **SCENARIO["dantzig"])
# Varies between local & CI contexts
# DEBUG may be due to reuse of test_mp in a non-deterministic order
if not scen.has_solution():
scen.solve()
# IXMPReporter can be initialized on a MESSAGE Scenario
rep_ix = ixmp_Reporter.from_scenario(scen)
# message_ix.Reporter can also be initialized
rep = Reporter.from_scenario(scen)
# Number of quantities available in a rudimentary MESSAGEix Scenario
assert len(rep.graph["all"]) == 123
# Quantities have short dimension names
assert "demand:n-c-l-y-h" in rep.graph
# Aggregates are available
assert "demand:n-l-h" in rep.graph
# Quantities contain expected data
dims = dict(coords=["chicago new-york topeka".split()], dims=["n"])
demand = Quantity(xr.DataArray([300, 325, 275], **dims), name="demand")
# NB the call to squeeze() drops the length-1 dimensions c-l-y-h
obs = rep.get("demand:n-c-l-y-h").squeeze(drop=True)
# check_attrs False because we don't get the unit addition in bare xarray
assert_qty_equal(obs, demand, check_attrs=False)
# ixmp.Reporter pre-populated with only model quantities and aggregates
assert len(rep_ix.graph) == 5223
# message_ix.Reporter pre-populated with additional, derived quantities
# This is the same value as in test_tutorials.py
assert len(rep.graph) == 12688
# Derived quantities have expected dimensions
vom_key = rep.full_key("vom")
assert vom_key not in rep_ix
assert vom_key == "vom:nl-t-yv-ya-m-h"
# …and expected values
var_cost = rep.get(rep.full_key("var_cost"))
ACT = rep.get(rep.full_key("ACT"))
vom = computations.product(var_cost, ACT)
# check_attrs false because `vom` multiply above does not add units
assert_qty_equal(vom, rep.get(vom_key))
def test_as_quantity():
"""Test conversion to sparse.COO-backed xr.DataArray in as_quantity()."""
x_series = pd.Series(
data=[1, 2, 3, 4],
index=pd.MultiIndex.from_product([['a', 'b'], ['c', 'd']],
names=['foo', 'bar']),
)
y_series = pd.Series(data=[5, 6], index=pd.Index(['e', 'f'], name='baz'))
x = xr.DataArray.from_series(x_series, sparse=True)
y = xr.DataArray.from_series(y_series, sparse=True)
x_dense = xr.DataArray.from_series(x_series)
y_dense = xr.DataArray.from_series(y_series)
with pytest.raises(ValueError, match='make sure that the broadcast shape'):
x_dense * y
z1 = as_quantity(x_dense) * y
z2 = x * as_quantity(y_dense)
assert z1.dims == ('foo', 'bar', 'baz')
assert_qty_equal(z1, z2)
z3 = as_quantity(x) * as_quantity(y)
assert_qty_equal(z1, z3)
z4 = as_quantity(x) * y
assert_qty_equal(z1, z4)
z5 = as_quantity(x_series) * y
assert_qty_equal(z1, z5)
def test_reporter_add_product(test_mp):
scen = ixmp.Scenario(test_mp, 'reporter_add_product',
'reporter_add_product', 'new')
*_, x = add_test_data(scen)
rep = Reporter.from_scenario(scen)
# add_product() works
key = rep.add_product('x squared', 'x', 'x', sums=True)
# Product has the expected dimensions
assert key == 'x squared:t-y'
# Product has the expected value
exp = as_quantity(x * x)
exp.attrs['_unit'] = UNITS('kilogram ** 2').units
assert_qty_equal(exp, rep.get(key))
def test_aggregate(test_mp):
scen = ixmp.Scenario(test_mp, 'Group reporting', 'group reporting', 'new')
t, t_foo, t_bar, x = add_test_data(scen)
# Reporter
rep = Reporter.from_scenario(scen)
# Define some groups
t_groups = {'foo': t_foo, 'bar': t_bar, 'baz': ['foo1', 'bar5', 'bar6']}
# Use the computation directly
agg1 = computations.aggregate(as_quantity(x), {'t': t_groups}, True)
# Expected set of keys along the aggregated dimension
assert set(agg1.coords['t'].values) == set(t) | set(t_groups.keys())
# Sums are as expected
assert_qty_allclose(agg1.sel(t='foo', drop=True), x.sel(t=t_foo).sum('t'))
assert_qty_allclose(agg1.sel(t='bar', drop=True), x.sel(t=t_bar).sum('t'))
assert_qty_allclose(agg1.sel(t='baz', drop=True),
x.sel(t=['foo1', 'bar5', 'bar6']).sum('t'))
# Use Reporter convenience method
key2 = rep.aggregate('x:t-y', 'agg2', {'t': t_groups}, keep=True)
# Group has expected key and contents
assert key2 == 'x:t-y:agg2'
# Aggregate is computed without error
agg2 = rep.get(key2)
assert_qty_equal(agg1, agg2)
# Add aggregates, without keeping originals
key3 = rep.aggregate('x:t-y', 'agg3', {'t': t_groups}, keep=False)
# Distinct keys
assert key3 != key2
# Only the aggregated and no original keys along the aggregated dimension
agg3 = rep.get(key3)
assert set(agg3.coords['t'].values) == set(t_groups.keys())
with pytest.raises(NotImplementedError):
# Not yet supported; requires two separate operations
rep.aggregate('x:t-y', 'agg3', {'t': t_groups, 'y': [2000, 2010]})
def test_product0():
A = Quantity(
xr.DataArray([1, 2], coords=[["a0", "a1"]], dims=["a"])
)
B = Quantity(
xr.DataArray([3, 4], coords=[["b0", "b1"]], dims=["b"])
)
exp = Quantity(
xr.DataArray(
[[3, 4], [6, 8]],
coords=[["a0", "a1"], ["b0", "b1"]],
dims=["a", "b"],
),
units="1",
)
assert_qty_equal(exp, computations.product(A, B))
computations.product(exp, B)
def test_assert_qty():
# tests without `attr` property, in which case direct pd.Series and
# xr.DataArray comparisons are possible
a = xr.DataArray([0.8, 0.2], coords=[['oil', 'water']], dims=['p'])
b = a.to_series()
assert_qty_equal(a, b)
assert_qty_equal(b, a)
assert_qty_allclose(a, b)
assert_qty_allclose(b, a)
c = Quantity(a)
assert_qty_equal(a, c)
assert_qty_equal(c, a)
assert_qty_allclose(a, c)
assert_qty_allclose(c, a)
def test_reporting_file_formats(test_data_path, tmp_path):
r = Reporter()
expected = xr.DataArray.from_series(
pd.read_csv(test_data_path / 'report-input.csv',
index_col=['i', 'j'])['value'])
# CSV file is automatically parsed to xr.DataArray
p1 = test_data_path / 'report-input.csv'
k = r.add_file(p1)
assert_qty_equal(r.get(k), expected)
# Write to CSV
p2 = tmp_path / 'report-output.csv'
r.write(k, p2)
# Output is identical to input file, except for order
assert (sorted(p1.read_text().split('\n')) == sorted(
p2.read_text().split('\n')))
# Write to Excel
p3 = tmp_path / 'report-output.xlsx'
r.write(k, p3)
def test_file_formats(test_data_path, tmp_path):
r = Reporter()
expected = as_quantity(pd.read_csv(test_data_path / 'report-input0.csv',
index_col=['i', 'j'])['value'],
units='km')
# CSV file is automatically parsed to xr.DataArray
p1 = test_data_path / 'report-input0.csv'
k = r.add_file(p1, units=pint.Unit('km'))
assert_qty_equal(r.get(k), expected)
# Dimensions can be specified
p2 = test_data_path / 'report-input1.csv'
k2 = r.add_file(p2, dims=dict(i='i', j_dim='j'))
assert_qty_equal(r.get(k), r.get(k2))
# Units are loaded from a column
assert r.get(k2).attrs['_unit'] == pint.Unit('km')
# Specifying units that do not match file contents → ComputationError
r.add_file(p2, key='bad', dims=dict(i='i', j_dim='j'), units='kg')
with pytest.raises(ComputationError):
r.get('bad')
# Write to CSV
p3 = tmp_path / 'report-output.csv'
r.write(k, p3)
# Output is identical to input file, except for order
assert (sorted(p1.read_text().split('\n')) == sorted(
p3.read_text().split('\n')))
# Write to Excel
p4 = tmp_path / 'report-output.xlsx'
r.write(k, p4)
def test_assert(self, a):
"""Test assertions about Quantity.
These are tests without `attr` property, in which case direct pd.Series
and xr.DataArray comparisons are possible.
"""
# Convert to pd.Series
b = a.to_series()
assert_qty_equal(a, b)
assert_qty_equal(b, a)
assert_qty_allclose(a, b)
assert_qty_allclose(b, a)
c = Quantity(a)
assert_qty_equal(a, c)
assert_qty_equal(c, a)
assert_qty_allclose(a, c)
assert_qty_allclose(c, a)
def test_assert_qty_attrs():
# tests *with* `attr` property, in which case direct pd.Series and
# xr.DataArray comparisons *not* are possible
a = xr.DataArray([0.8, 0.2], coords=[['oil', 'water']], dims=['p'])
attrs = {'foo': 'bar'}
a.attrs = attrs
b = Quantity(a)
# make sure it has the correct property
assert a.attrs == attrs
assert b.attrs == attrs
assert_qty_equal(a, b)
assert_qty_equal(b, a)
assert_qty_allclose(a, b)
assert_qty_allclose(b, a)
a.attrs = {'bar': 'foo'}
assert_qty_equal(a, b, check_attrs=False)
def test_assert_with_attrs(self, a):
"""Test assertions about Quantity with attrs.
Here direct pd.Series and xr.DataArray comparisons are *not* possible.
"""
attrs = {'foo': 'bar'}
a.attrs = attrs
b = Quantity(a)
# make sure it has the correct property
assert a.attrs == attrs
assert b.attrs == attrs
assert_qty_equal(a, b)
assert_qty_equal(b, a)
assert_qty_allclose(a, b)
assert_qty_allclose(b, a)
# check_attrs=False allows a successful equals assertion even when the
# attrs are different
a.attrs = {'bar': 'foo'}
assert_qty_equal(a, b, check_attrs=False)
def test_reporter_from_dantzig(test_mp, ureg):
scen = make_dantzig(test_mp, solve=True)
# Reporter.from_scenario can handle the Dantzig problem
rep = Reporter.from_scenario(scen)
# Partial sums are available automatically (d is defined over i and j)
d_i = rep.get('d:i')
# Units pass through summation
assert d_i.attrs['_unit'] == ureg.parse_units('km')
# Summation across all dimensions results a 1-element Quantity
d = rep.get('d:')
assert d.shape == ((1, ) if Quantity is AttrSeries else tuple())
assert d.size == 1
assert np.isclose(d.values, 11.7)
# Weighted sum
weights = Quantity(
xr.DataArray([1, 2, 3],
coords=['chicago new-york topeka'.split()],
dims=['j']))
new_key = rep.aggregate('d:i-j', 'weighted', 'j', weights)
# ...produces the expected new key with the summed dimension removed and
# tag added
assert new_key == 'd:i:weighted'
# ...produces the expected new value
obs = rep.get(new_key)
d_ij = rep.get('d:i-j')
exp = (d_ij * weights).sum(dim=['j']) / weights.sum(dim=['j'])
# FIXME attrs has to be explicitly copied here because math is done which
# returns a pd.Series
exp.attrs = d_ij.attrs
assert_qty_equal(exp, obs)
# Disaggregation with explicit data
# (cases of canned food 'p'acked in oil or water)
shares = xr.DataArray([0.8, 0.2], coords=[['oil', 'water']], dims=['p'])
new_key = rep.disaggregate('b:j', 'p', args=[as_quantity(shares)])
# ...produces the expected key with new dimension added
assert new_key == 'b:j-p'
b_jp = rep.get('b:j-p')
# Units pass through disaggregation
assert b_jp.attrs['_unit'] == 'cases'
# Set elements are available
assert rep.get('j') == ['new-york', 'chicago', 'topeka']
# 'all' key retrieves all quantities
obs = {da.name for da in rep.get('all')}
exp = set(('a b d f x z cost cost-margin demand demand-margin supply '
'supply-margin').split())
assert obs == exp
# Shorthand for retrieving a full key name
assert rep.full_key('d') == 'd:i-j' and isinstance(rep.full_key('d'), Key)
