def test_reporter(test_mp): scen = Scenario(test_mp, 'canning problem (MESSAGE scheme)', 'standard') # Varies between local & CI contexts # DEBUG may be due to reuse of test_mp in a non-deterministic order if not scen.has_solution(): scen.solve() # IXMPReporter can be initialized on a MESSAGE Scenario rep_ix = ixmp_Reporter.from_scenario(scen) # message_ix.Reporter can also be initialized rep = Reporter.from_scenario(scen) # Number of quantities available in a rudimentary MESSAGEix Scenario assert len(rep.graph['all']) == 120 # Quantities have short dimension names assert 'demand:n-c-l-y-h' in rep.graph # Aggregates are available assert 'demand:n-l-h' in rep.graph # Quantities contain expected data dims = dict(coords=['chicago new-york topeka'.split()], dims=['n']) demand = xr.DataArray([300, 325, 275], **dims) # NB the call to squeeze() drops the length-1 dimensions c-l-y-h obs = rep.get('demand:n-c-l-y-h').squeeze(drop=True) # TODO: Squeeze on AttrSeries still returns full index, whereas xarray # drops everything except node obs = obs.reset_index(['c', 'l', 'y', 'h'], drop=True) # check_dtype is false because of casting in pd.Series to float # check_attrsis false because we don't get the unit addition in bare xarray assert_qty_equal(obs.sort_index(), demand, check_attrs=False, check_dtype=False) # ixmp.Reporter pre-populated with only model quantities and aggregates assert len(rep_ix.graph) == 5088 # message_ix.Reporter pre-populated with additional, derived quantities assert len(rep.graph) == 7975 # Derived quantities have expected dimensions vom_key = rep.full_key('vom') assert vom_key not in rep_ix assert vom_key == 'vom:nl-t-yv-ya-m-h' # …and expected values vom = ( rep.get(rep.full_key('ACT')) * rep.get(rep.full_key('var_cost')) ).dropna() # check_attrs false because `vom` multiply above does not add units assert_qty_equal(vom, rep.get(vom_key), check_attrs=False)
def test_reporter(message_test_mp): scen = Scenario(message_test_mp, **SCENARIO["dantzig"]) # Varies between local & CI contexts # DEBUG may be due to reuse of test_mp in a non-deterministic order if not scen.has_solution(): scen.solve() # IXMPReporter can be initialized on a MESSAGE Scenario rep_ix = ixmp_Reporter.from_scenario(scen) # message_ix.Reporter can also be initialized rep = Reporter.from_scenario(scen) # Number of quantities available in a rudimentary MESSAGEix Scenario assert len(rep.graph["all"]) == 123 # Quantities have short dimension names assert "demand:n-c-l-y-h" in rep.graph # Aggregates are available assert "demand:n-l-h" in rep.graph # Quantities contain expected data dims = dict(coords=["chicago new-york topeka".split()], dims=["n"]) demand = Quantity(xr.DataArray([300, 325, 275], **dims), name="demand") # NB the call to squeeze() drops the length-1 dimensions c-l-y-h obs = rep.get("demand:n-c-l-y-h").squeeze(drop=True) # check_attrs False because we don't get the unit addition in bare xarray assert_qty_equal(obs, demand, check_attrs=False) # ixmp.Reporter pre-populated with only model quantities and aggregates assert len(rep_ix.graph) == 5223 # message_ix.Reporter pre-populated with additional, derived quantities # This is the same value as in test_tutorials.py assert len(rep.graph) == 12688 # Derived quantities have expected dimensions vom_key = rep.full_key("vom") assert vom_key not in rep_ix assert vom_key == "vom:nl-t-yv-ya-m-h" # …and expected values var_cost = rep.get(rep.full_key("var_cost")) ACT = rep.get(rep.full_key("ACT")) vom = computations.product(var_cost, ACT) # check_attrs false because `vom` multiply above does not add units assert_qty_equal(vom, rep.get(vom_key))
def test_as_quantity(): """Test conversion to sparse.COO-backed xr.DataArray in as_quantity().""" x_series = pd.Series( data=[1, 2, 3, 4], index=pd.MultiIndex.from_product([['a', 'b'], ['c', 'd']], names=['foo', 'bar']), ) y_series = pd.Series(data=[5, 6], index=pd.Index(['e', 'f'], name='baz')) x = xr.DataArray.from_series(x_series, sparse=True) y = xr.DataArray.from_series(y_series, sparse=True) x_dense = xr.DataArray.from_series(x_series) y_dense = xr.DataArray.from_series(y_series) with pytest.raises(ValueError, match='make sure that the broadcast shape'): x_dense * y z1 = as_quantity(x_dense) * y z2 = x * as_quantity(y_dense) assert z1.dims == ('foo', 'bar', 'baz') assert_qty_equal(z1, z2) z3 = as_quantity(x) * as_quantity(y) assert_qty_equal(z1, z3) z4 = as_quantity(x) * y assert_qty_equal(z1, z4) z5 = as_quantity(x_series) * y assert_qty_equal(z1, z5)
def test_reporter_add_product(test_mp): scen = ixmp.Scenario(test_mp, 'reporter_add_product', 'reporter_add_product', 'new') *_, x = add_test_data(scen) rep = Reporter.from_scenario(scen) # add_product() works key = rep.add_product('x squared', 'x', 'x', sums=True) # Product has the expected dimensions assert key == 'x squared:t-y' # Product has the expected value exp = as_quantity(x * x) exp.attrs['_unit'] = UNITS('kilogram ** 2').units assert_qty_equal(exp, rep.get(key))
def test_aggregate(test_mp): scen = ixmp.Scenario(test_mp, 'Group reporting', 'group reporting', 'new') t, t_foo, t_bar, x = add_test_data(scen) # Reporter rep = Reporter.from_scenario(scen) # Define some groups t_groups = {'foo': t_foo, 'bar': t_bar, 'baz': ['foo1', 'bar5', 'bar6']} # Use the computation directly agg1 = computations.aggregate(as_quantity(x), {'t': t_groups}, True) # Expected set of keys along the aggregated dimension assert set(agg1.coords['t'].values) == set(t) | set(t_groups.keys()) # Sums are as expected assert_qty_allclose(agg1.sel(t='foo', drop=True), x.sel(t=t_foo).sum('t')) assert_qty_allclose(agg1.sel(t='bar', drop=True), x.sel(t=t_bar).sum('t')) assert_qty_allclose(agg1.sel(t='baz', drop=True), x.sel(t=['foo1', 'bar5', 'bar6']).sum('t')) # Use Reporter convenience method key2 = rep.aggregate('x:t-y', 'agg2', {'t': t_groups}, keep=True) # Group has expected key and contents assert key2 == 'x:t-y:agg2' # Aggregate is computed without error agg2 = rep.get(key2) assert_qty_equal(agg1, agg2) # Add aggregates, without keeping originals key3 = rep.aggregate('x:t-y', 'agg3', {'t': t_groups}, keep=False) # Distinct keys assert key3 != key2 # Only the aggregated and no original keys along the aggregated dimension agg3 = rep.get(key3) assert set(agg3.coords['t'].values) == set(t_groups.keys()) with pytest.raises(NotImplementedError): # Not yet supported; requires two separate operations rep.aggregate('x:t-y', 'agg3', {'t': t_groups, 'y': [2000, 2010]})
def test_product0(): A = Quantity( xr.DataArray([1, 2], coords=[["a0", "a1"]], dims=["a"]) ) B = Quantity( xr.DataArray([3, 4], coords=[["b0", "b1"]], dims=["b"]) ) exp = Quantity( xr.DataArray( [[3, 4], [6, 8]], coords=[["a0", "a1"], ["b0", "b1"]], dims=["a", "b"], ), units="1", ) assert_qty_equal(exp, computations.product(A, B)) computations.product(exp, B)
def test_assert_qty(): # tests without `attr` property, in which case direct pd.Series and # xr.DataArray comparisons are possible a = xr.DataArray([0.8, 0.2], coords=[['oil', 'water']], dims=['p']) b = a.to_series() assert_qty_equal(a, b) assert_qty_equal(b, a) assert_qty_allclose(a, b) assert_qty_allclose(b, a) c = Quantity(a) assert_qty_equal(a, c) assert_qty_equal(c, a) assert_qty_allclose(a, c) assert_qty_allclose(c, a)
def test_reporting_file_formats(test_data_path, tmp_path): r = Reporter() expected = xr.DataArray.from_series( pd.read_csv(test_data_path / 'report-input.csv', index_col=['i', 'j'])['value']) # CSV file is automatically parsed to xr.DataArray p1 = test_data_path / 'report-input.csv' k = r.add_file(p1) assert_qty_equal(r.get(k), expected) # Write to CSV p2 = tmp_path / 'report-output.csv' r.write(k, p2) # Output is identical to input file, except for order assert (sorted(p1.read_text().split('\n')) == sorted( p2.read_text().split('\n'))) # Write to Excel p3 = tmp_path / 'report-output.xlsx' r.write(k, p3)
def test_file_formats(test_data_path, tmp_path): r = Reporter() expected = as_quantity(pd.read_csv(test_data_path / 'report-input0.csv', index_col=['i', 'j'])['value'], units='km') # CSV file is automatically parsed to xr.DataArray p1 = test_data_path / 'report-input0.csv' k = r.add_file(p1, units=pint.Unit('km')) assert_qty_equal(r.get(k), expected) # Dimensions can be specified p2 = test_data_path / 'report-input1.csv' k2 = r.add_file(p2, dims=dict(i='i', j_dim='j')) assert_qty_equal(r.get(k), r.get(k2)) # Units are loaded from a column assert r.get(k2).attrs['_unit'] == pint.Unit('km') # Specifying units that do not match file contents → ComputationError r.add_file(p2, key='bad', dims=dict(i='i', j_dim='j'), units='kg') with pytest.raises(ComputationError): r.get('bad') # Write to CSV p3 = tmp_path / 'report-output.csv' r.write(k, p3) # Output is identical to input file, except for order assert (sorted(p1.read_text().split('\n')) == sorted( p3.read_text().split('\n'))) # Write to Excel p4 = tmp_path / 'report-output.xlsx' r.write(k, p4)
def test_assert(self, a): """Test assertions about Quantity. These are tests without `attr` property, in which case direct pd.Series and xr.DataArray comparisons are possible. """ # Convert to pd.Series b = a.to_series() assert_qty_equal(a, b) assert_qty_equal(b, a) assert_qty_allclose(a, b) assert_qty_allclose(b, a) c = Quantity(a) assert_qty_equal(a, c) assert_qty_equal(c, a) assert_qty_allclose(a, c) assert_qty_allclose(c, a)
def test_assert_qty_attrs(): # tests *with* `attr` property, in which case direct pd.Series and # xr.DataArray comparisons *not* are possible a = xr.DataArray([0.8, 0.2], coords=[['oil', 'water']], dims=['p']) attrs = {'foo': 'bar'} a.attrs = attrs b = Quantity(a) # make sure it has the correct property assert a.attrs == attrs assert b.attrs == attrs assert_qty_equal(a, b) assert_qty_equal(b, a) assert_qty_allclose(a, b) assert_qty_allclose(b, a) a.attrs = {'bar': 'foo'} assert_qty_equal(a, b, check_attrs=False)
def test_assert_with_attrs(self, a): """Test assertions about Quantity with attrs. Here direct pd.Series and xr.DataArray comparisons are *not* possible. """ attrs = {'foo': 'bar'} a.attrs = attrs b = Quantity(a) # make sure it has the correct property assert a.attrs == attrs assert b.attrs == attrs assert_qty_equal(a, b) assert_qty_equal(b, a) assert_qty_allclose(a, b) assert_qty_allclose(b, a) # check_attrs=False allows a successful equals assertion even when the # attrs are different a.attrs = {'bar': 'foo'} assert_qty_equal(a, b, check_attrs=False)
def test_reporter_from_dantzig(test_mp, ureg): scen = make_dantzig(test_mp, solve=True) # Reporter.from_scenario can handle the Dantzig problem rep = Reporter.from_scenario(scen) # Partial sums are available automatically (d is defined over i and j) d_i = rep.get('d:i') # Units pass through summation assert d_i.attrs['_unit'] == ureg.parse_units('km') # Summation across all dimensions results a 1-element Quantity d = rep.get('d:') assert d.shape == ((1, ) if Quantity is AttrSeries else tuple()) assert d.size == 1 assert np.isclose(d.values, 11.7) # Weighted sum weights = Quantity( xr.DataArray([1, 2, 3], coords=['chicago new-york topeka'.split()], dims=['j'])) new_key = rep.aggregate('d:i-j', 'weighted', 'j', weights) # ...produces the expected new key with the summed dimension removed and # tag added assert new_key == 'd:i:weighted' # ...produces the expected new value obs = rep.get(new_key) d_ij = rep.get('d:i-j') exp = (d_ij * weights).sum(dim=['j']) / weights.sum(dim=['j']) # FIXME attrs has to be explicitly copied here because math is done which # returns a pd.Series exp.attrs = d_ij.attrs assert_qty_equal(exp, obs) # Disaggregation with explicit data # (cases of canned food 'p'acked in oil or water) shares = xr.DataArray([0.8, 0.2], coords=[['oil', 'water']], dims=['p']) new_key = rep.disaggregate('b:j', 'p', args=[as_quantity(shares)]) # ...produces the expected key with new dimension added assert new_key == 'b:j-p' b_jp = rep.get('b:j-p') # Units pass through disaggregation assert b_jp.attrs['_unit'] == 'cases' # Set elements are available assert rep.get('j') == ['new-york', 'chicago', 'topeka'] # 'all' key retrieves all quantities obs = {da.name for da in rep.get('all')} exp = set(('a b d f x z cost cost-margin demand demand-margin supply ' 'supply-margin').split()) assert obs == exp # Shorthand for retrieving a full key name assert rep.full_key('d') == 'd:i-j' and isinstance(rep.full_key('d'), Key)