def test_reporter_file(tmp_path): r = Reporter() # Path to a temporary file p = tmp_path / 'foo.txt' # File can be added to the Reporter before it is created, because the file # is not read until/unless required k1 = r.add_file(p) # File has the expected key assert k1 == 'file:foo.txt' # Add some contents to the file p.write_text('Hello, world!') # The file's contents can be read through the Reporter assert r.get('file:foo.txt') == 'Hello, world!' # Write the report to file p2 = tmp_path / 'bar.txt' r.write('file:foo.txt', p2) # Write using a string path r.write('file:foo.txt', str(p2)) # The Reporter produces the expected output file assert p2.read_text() == 'Hello, world!'
def test_reporting_filters(test_mp, tmp_path): """Reporting can be filtered ex ante.""" scen = ixmp.Scenario(test_mp, 'Reporting filters', 'Reporting filters', 'new') t, t_foo, t_bar, x = add_test_data(scen) rep = Reporter.from_scenario(scen) x_key = rep.full_key('x') def assert_t_indices(labels): assert set(rep.get(x_key).index.levels[0]) == set(labels) # 1. Set filters directly rep.graph['filters'] = {'t': t_foo} assert_t_indices(t_foo) # Reporter can be re-used by changing filters rep.graph['filters'] = {'t': t_bar} assert_t_indices(t_bar) rep.graph['filters'] = {} assert_t_indices(t) # 2. Set filters using a convenience method rep = Reporter.from_scenario(scen) rep.set_filters(t=t_foo) assert_t_indices(t_foo) # Clear filters using the convenience method rep.set_filters(t=None) assert_t_indices(t) # 3. Set filters via configuration keys # NB passes through from_scenario() -> __init__() -> configure() rep = Reporter.from_scenario(scen, filters={'t': t_foo}) assert_t_indices(t_foo) # Configuration key can also be read from file rep = Reporter.from_scenario(scen) # Write a temporary file containing the desired labels config_file = tmp_path / 'config.yaml' config_file.write_text('\n'.join([ 'filters:', ' t: {!r}'.format(t_bar), ])) rep.configure(config_file) assert_t_indices(t_bar)
def test_reporter_disaggregate(): r = Reporter() foo = Key('foo', ['a', 'b', 'c']) r.add(foo, '<foo data>') r.add('d_shares', '<share data>') # Disaggregation works r.disaggregate(foo, 'd', args=['d_shares']) assert 'foo:a-b-c-d' in r.graph assert r.graph['foo:a-b-c-d'] == (computations.disaggregate_shares, 'foo:a-b-c', 'd_shares') # Invalid method with pytest.raises(ValueError): r.disaggregate(foo, 'd', method='baz')
def test_report_size(test_mp): """Stress-test reporting of large, sparse quantities.""" from itertools import zip_longest import numpy as np # test_mp.add_unit('kg') scen = ixmp.Scenario(test_mp, 'size test', 'base', version='new') # Dimensions and their lengths dims = 'abcdef' sizes = [1, 5, 21, 21, 89, 377] # Fibonacci #s; next 1597, 6765 # commented: "377 / 73984365 elements = 0.00051% full" # from functools import reduce # from operator import mul # size = reduce(mul, sizes) # print('{} / {} elements = {:.5f}% full' # .format(max(sizes), size, 100 * max(sizes) / size)) # Names like f_0000 ... f_1596 along each dimension coords = [] for d, N in zip(dims, sizes): # py3.5 compat: could use an f-string here coords.append(['{}_{:04d}'.format(d, i) for i in range(N)]) # Add to Scenario scen.init_set(d) scen.add_set(d, coords[-1]) def _make_values(): """Make a DataFrame containing each label in *coords* at least once.""" values = list(zip_longest(*coords, np.random.rand(max(sizes)))) result = pd.DataFrame(values, columns=list(dims) + ['value']) \ .ffill() result['unit'] = 'kg' return result # Fill the Scenario with quantities named q_01 ... q_09 N = 10 names = [] for i in range(10): # py3.5 compat: could use an f-string here name = 'q_{:02d}'.format(i) scen.init_par(name, list(dims)) scen.add_par(name, _make_values()) names.append(name) # Create the reporter rep = Reporter.from_scenario(scen) # Add an operation that takes the product, i.e. requires all the q_* keys = [rep.full_key(name) for name in names] rep.add('bigmem', tuple([computations.product] + keys)) # One quantity fits in memory rep.get(keys[0]) # assert False # All quantities together trigger MemoryError rep.get('bigmem')
def test_reporter_describe(test_mp, test_data_path, capsys): scen = make_dantzig(test_mp) r = Reporter.from_scenario(scen) # hexadecimal ID of *scen* id_ = hex(id(scen)) if os.name != 'nt' else \ '{:#018X}'.format(id(scen)).replace('X', 'x') # Describe one key desc1 = """'d:i': - sum(dimensions=['j'], weights=None, ...) - 'd:i-j': - data_for_quantity('par', 'd', 'value', ...) - 'scenario': - <ixmp.core.Scenario object at {id}> - 'filters': - {{}}""".format(id=id_) assert desc1 == r.describe('d:i') # Description was also written to stdout out1, _ = capsys.readouterr() assert desc1 + '\n' == out1 # Description of all keys is as expected desc2 = (test_data_path / 'report-describe.txt').read_text() \ .format(id=id_) assert desc2 == r.describe() + '\n' # Result was also written to stdout out2, _ = capsys.readouterr() assert desc2 == out2
def test_reporter_from_dantzig(test_mp, test_data_path): scen = make_dantzig(test_mp, solve=test_data_path) # Reporter.from_scenario can handle the Dantzig problem rep = Reporter.from_scenario(scen) # Partial sums are available automatically (d is defined over i and j) d_i = rep.get('d:i') # Units pass through summation assert d_i.attrs['_unit'] == ureg.parse_units('km') # Summation across all dimensions results a 1-element Quantity d = rep.get('d:') assert len(d) == 1 assert np.isclose(d.iloc[0], 11.7) # Weighted sum weights = Quantity(xr.DataArray( [1, 2, 3], coords=['chicago new-york topeka'.split()], dims=['j'])) new_key = rep.aggregate('d:i-j', 'weighted', 'j', weights) # ...produces the expected new key with the summed dimension removed and # tag added assert new_key == 'd:i:weighted' # ...produces the expected new value obs = rep.get(new_key) exp = (rep.get('d:i-j') * weights).sum(dim=['j']) / weights.sum(dim=['j']) # TODO: attrs has to be explicitly copied here because math is done which # returns a pd.Series exp = Quantity(exp, attrs=rep.get('d:i-j').attrs) assert_series_equal(obs.sort_index(), exp.sort_index()) # Disaggregation with explicit data # (cases of canned food 'p'acked in oil or water) shares = xr.DataArray([0.8, 0.2], coords=[['oil', 'water']], dims=['p']) new_key = rep.disaggregate('b:j', 'p', args=[Quantity(shares)]) # ...produces the expected key with new dimension added assert new_key == 'b:j-p' b_jp = rep.get('b:j-p') # Units pass through disaggregation assert b_jp.attrs['_unit'] == 'cases' # Set elements are available assert rep.get('j') == ['new-york', 'chicago', 'topeka'] # 'all' key retrieves all quantities obs = {da.name for da in rep.get('all')} exp = set('a b d f demand demand-margin z x'.split()) assert obs == exp # Shorthand for retrieving a full key name assert rep.full_key('d') == 'd:i-j' and isinstance(rep.full_key('d'), Key)
def test_reporting_platform_units(test_mp, caplog): """Test handling of units from ixmp.Platform. test_mp is loaded with some units includig '-', '???', 'G$', etc. which are not parseable with pint; and others which are not defined in a default pint.UnitRegistry. These tests check the handling of those units. """ # Prepare a Scenario with test data scen = ixmp.Scenario(test_mp, 'reporting_platform_units', 'reporting_platform_units', 'new') t, t_foo, t_bar, x = add_test_data(scen) rep = Reporter.from_scenario(scen) x_key = rep.full_key('x') # Convert 'x' to dataframe x = x.to_series().rename('value').reset_index() # Exception message, formatted as a regular expression msg = r"unit '{}' cannot be parsed; contains invalid character\(s\) '{}'" # Unit and components for the regex bad_units = [('-', '-', '-'), ('???', r'\?\?\?', r'\?'), ('G$', r'G\$', r'\$')] for unit, expr, chars in bad_units: # Overwrite the parameter x['unit'] = unit scen.add_par('x', x) # Parsing units with invalid chars raises an intelligible exception with pytest.raises(ValueError, match=msg.format(expr, chars)): rep.get(x_key) # Now using parseable but unrecognized units x['unit'] = 'USD/kWa' scen.add_par('x', x) # Unrecognized units are added automatically, with log messages emitted caplog.clear() rep.get(x_key) expected = [ 'Add unit definition: USD = [USD]', 'Add unit definition: kWa = [kWa]', ] assert all(e in [rec.message for rec in caplog.records] for e in expected) # Mix of recognized/unrecognized units can be added: USD is already in the # unit registry, so is not re-added x['unit'] = 'USD/pkm' test_mp.add_unit('USD/pkm') scen.add_par('x', x) caplog.clear() rep.get(x_key) assert not any('Add unit definition: USD = [USD]' in rec.message for rec in caplog.records) assert any('Add unit definition: pkm = [pkm]' in rec.message for rec in caplog.records)
def test_reporter_add_queue(): r = Reporter() r.add('foo-0', (lambda x: x, 42)) # A computation def _product(a, b): return a * b # A queue of computations to add. Only foo-1 succeeds on the first pass; # only foo-2 on the second pass, etc. strict = dict(strict=True) queue = [ (('foo-4', _product, 'foo-3', 10), strict), (('foo-3', _product, 'foo-2', 10), strict), (('foo-2', _product, 'foo-1', 10), strict), (('foo-1', _product, 'foo-0', 10), {}), ] # Maximum 3 attempts → foo-4 fails on the start of the 3rd pass with pytest.raises(MissingKeyError, match='foo-3'): r.add(queue, max_tries=3, fail='raise') # But foo-2 was successfully added on the second pass, and gives the # correct result assert r.get('foo-2') == 42 * 10 * 10
def test_reporter_read_config(test_mp, test_data_path): scen = make_dantzig(test_mp) rep = Reporter.from_scenario(scen) # Configuration can be read from file rep.configure(test_data_path / 'report-config-0.yaml') # Data from configured file is available assert rep.get('d_check').loc['seattle', 'chicago'] == 1.7
def test_reporter(test_mp): scen = Scenario(test_mp, 'canning problem (MESSAGE scheme)', 'standard') # Varies between local & CI contexts # DEBUG may be due to reuse of test_mp in a non-deterministic order if not scen.has_solution(): scen.solve() # IXMPReporter can be initialized on a MESSAGE Scenario rep_ix = ixmp_Reporter.from_scenario(scen) # message_ix.Reporter can also be initialized rep = Reporter.from_scenario(scen) # Number of quantities available in a rudimentary MESSAGEix Scenario assert len(rep.graph['all']) == 120 # Quantities have short dimension names assert 'demand:n-c-l-y-h' in rep.graph # Aggregates are available assert 'demand:n-l-h' in rep.graph # Quantities contain expected data dims = dict(coords=['chicago new-york topeka'.split()], dims=['n']) demand = xr.DataArray([300, 325, 275], **dims) # NB the call to squeeze() drops the length-1 dimensions c-l-y-h obs = rep.get('demand:n-c-l-y-h').squeeze(drop=True) # TODO: Squeeze on AttrSeries still returns full index, whereas xarray # drops everything except node obs = obs.reset_index(['c', 'l', 'y', 'h'], drop=True) # check_dtype is false because of casting in pd.Series to float # check_attrsis false because we don't get the unit addition in bare xarray assert_qty_equal(obs.sort_index(), demand, check_attrs=False, check_dtype=False) # ixmp.Reporter pre-populated with only model quantities and aggregates assert len(rep_ix.graph) == 5088 # message_ix.Reporter pre-populated with additional, derived quantities assert len(rep.graph) == 7975 # Derived quantities have expected dimensions vom_key = rep.full_key('vom') assert vom_key not in rep_ix assert vom_key == 'vom:nl-t-yv-ya-m-h' # …and expected values vom = ( rep.get(rep.full_key('ACT')) * rep.get(rep.full_key('var_cost')) ).dropna() # check_attrs false because `vom` multiply above does not add units assert_qty_equal(vom, rep.get(vom_key), check_attrs=False)
def test_reporter_read_config(test_mp, test_data_path): scen = make_dantzig(test_mp) rep = Reporter.from_scenario(scen) with pytest.warns(UserWarning, match=r"Unrecognized sections {'notarealsection'}"): rep.read_config(test_data_path / 'report-config-0.yaml') # Data from configured file is available assert rep.get('d_check').loc['seattle', 'chicago'] == 1.7
def test_reporter_from_scenario(message_test_mp): scen = Scenario(message_test_mp, **SCENARIO["dantzig"]) # Varies between local & CI contexts # DEBUG may be due to reuse of test_mp in a non-deterministic order if not scen.has_solution(): scen.solve(quiet=True) # IXMPReporter can be initialized on a MESSAGE Scenario rep_ix = ixmp_Reporter.from_scenario(scen) # message_ix.Reporter can also be initialized rep = Reporter.from_scenario(scen) # Number of quantities available in a rudimentary MESSAGEix Scenario assert len(rep.graph["all"]) == 123 # Quantities have short dimension names assert "demand:n-c-l-y-h" in rep # Aggregates are available assert "demand:n-l-h" in rep # Quantities contain expected data dims = dict(coords=["chicago new-york topeka".split()], dims=["n"]) demand = Quantity(xr.DataArray([300, 325, 275], **dims), name="demand") # NB the call to squeeze() drops the length-1 dimensions c-l-y-h obs = rep.get("demand:n-c-l-y-h").squeeze(drop=True) # check_attrs False because we don't get the unit addition in bare xarray assert_qty_equal(obs, demand, check_attrs=False) # ixmp.Reporter pre-populated with only model quantities and aggregates assert len(rep_ix.graph) == 5225 # message_ix.Reporter pre-populated with additional, derived quantities # This is the same value as in test_tutorials.py assert len(rep.graph) == 12690 # Derived quantities have expected dimensions vom_key = rep.full_key("vom") assert vom_key not in rep_ix assert vom_key == "vom:nl-t-yv-ya-m-h" # …and expected values var_cost = rep.get(rep.full_key("var_cost")) ACT = rep.get(rep.full_key("ACT")) product = rep.get_comp("product") vom = product(var_cost, ACT) # check_attrs false because `vom` multiply above does not add units assert_qty_equal(vom, rep.get(vom_key))
def report(context, config, key): """Run reporting for KEY.""" # Import here to avoid importing reporting dependencies when running # other commands from ixmp.reporting import Reporter # Instantiate the Reporter with the Scenario loaded by main() r = Reporter.from_scenario(context['scen']) # Read the configuration file, if any if config: r.read_config(config) # Print the target print(r.get(key))
def test_reporter_read_config(test_mp, test_data_path, caplog): scen = make_dantzig(test_mp) rep = Reporter.from_scenario(scen) caplog.clear() # Warning is raised when reading configuration with unrecognized section(s) rep.read_config(test_data_path / 'report-config-0.yaml') assert ("Unrecognized sections ['notarealsection'] in reporting " "configuration will have no effect") == caplog.records[0].message # Data from configured file is available assert rep.get('d_check').loc['seattle', 'chicago'] == 1.7
def test_reporting_aggregate(test_mp): scen = ixmp.Scenario(test_mp, 'Group reporting', 'group reporting', 'new') t, t_foo, t_bar, x = add_test_data(scen) # Reporter rep = Reporter.from_scenario(scen) # Define some groups t_groups = {'foo': t_foo, 'bar': t_bar, 'baz': ['foo1', 'bar5', 'bar6']} # Add aggregates key1 = rep.aggregate('x:t-y', 'agg1', {'t': t_groups}, keep=True) # Group has expected key and contents assert key1 == 'x:t-y:agg1' # Aggregate is computed without error agg1 = rep.get(key1) # Expected set of keys along the aggregated dimension assert set(agg1.coords['t'].values) == set(t) | set(t_groups.keys()) # Sums are as expected # TODO: the check_dtype arg assumes Quantity backend is a AttrSeries, # should that be made default in assert_qty_allclose? assert_qty_allclose(agg1.sel(t='foo', drop=True), x.sel(t=t_foo).sum('t'), check_dtype=False) assert_qty_allclose(agg1.sel(t='bar', drop=True), x.sel(t=t_bar).sum('t'), check_dtype=False) assert_qty_allclose(agg1.sel(t='baz', drop=True), x.sel(t=['foo1', 'bar5', 'bar6']).sum('t'), check_dtype=False) # Add aggregates, without keeping originals key2 = rep.aggregate('x:t-y', 'agg2', {'t': t_groups}, keep=False) # Distinct keys assert key2 != key1 # Only the aggregated and no original keys along the aggregated dimension agg2 = rep.get(key2) assert set(agg2.coords['t'].values) == set(t_groups.keys()) with pytest.raises(NotImplementedError): # Not yet supported; requires two separate operations rep.aggregate('x:t-y', 'agg3', {'t': t_groups, 'y': [2000, 2010]})
def test_reporter_add_product(test_mp): scen = ixmp.Scenario(test_mp, 'reporter_add_product', 'reporter_add_product', 'new') *_, x = add_test_data(scen) rep = Reporter.from_scenario(scen) # add_product() works key = rep.add_product('x squared', 'x', 'x', sums=True) # Product has the expected dimensions assert key == 'x squared:t-y' # Product has the expected value exp = as_quantity(x * x) exp.attrs['_unit'] = UNITS('kilogram ** 2').units assert_qty_equal(exp, rep.get(key))
def test_aggregate(test_mp): scen = ixmp.Scenario(test_mp, 'Group reporting', 'group reporting', 'new') t, t_foo, t_bar, x = add_test_data(scen) # Reporter rep = Reporter.from_scenario(scen) # Define some groups t_groups = {'foo': t_foo, 'bar': t_bar, 'baz': ['foo1', 'bar5', 'bar6']} # Use the computation directly agg1 = computations.aggregate(as_quantity(x), {'t': t_groups}, True) # Expected set of keys along the aggregated dimension assert set(agg1.coords['t'].values) == set(t) | set(t_groups.keys()) # Sums are as expected assert_qty_allclose(agg1.sel(t='foo', drop=True), x.sel(t=t_foo).sum('t')) assert_qty_allclose(agg1.sel(t='bar', drop=True), x.sel(t=t_bar).sum('t')) assert_qty_allclose(agg1.sel(t='baz', drop=True), x.sel(t=['foo1', 'bar5', 'bar6']).sum('t')) # Use Reporter convenience method key2 = rep.aggregate('x:t-y', 'agg2', {'t': t_groups}, keep=True) # Group has expected key and contents assert key2 == 'x:t-y:agg2' # Aggregate is computed without error agg2 = rep.get(key2) assert_qty_equal(agg1, agg2) # Add aggregates, without keeping originals key3 = rep.aggregate('x:t-y', 'agg3', {'t': t_groups}, keep=False) # Distinct keys assert key3 != key2 # Only the aggregated and no original keys along the aggregated dimension agg3 = rep.get(key3) assert set(agg3.coords['t'].values) == set(t_groups.keys()) with pytest.raises(NotImplementedError): # Not yet supported; requires two separate operations rep.aggregate('x:t-y', 'agg3', {'t': t_groups, 'y': [2000, 2010]})
def report(context, config, key): """Run reporting for KEY.""" # Import here to avoid importing reporting dependencies when running # other commands from ixmp.reporting import Reporter if not context: raise click.UsageError('give either --url, --platform or --dbprops ' 'before command report') # Instantiate the Reporter with the Scenario loaded by main() r = Reporter.from_scenario(context['scen']) # Read the configuration file, if any r.configure(config) # Print the target print(r.get(key))
def report(ctx, config, default): # Import here to avoid importing reporting dependencies when running # other commands from ixmp.reporting import Reporter # Instantiate the Reporter with the Scenario loaded by main() r = Reporter.from_scenario(ctx.obj['scen']) # Read the configuration file, if any if config: r.read_config(config) # Process remaining configuration from command-line arguments if default: r.configure(default=default) # Print the default target print(r.get())
def test_configure(test_mp, test_data_path): # TODO test: configuration keys 'units', 'replace_units' # Configure globally; reads 'rename_dims' section configure(rename_dims={'i': 'i_renamed'}) # Reporting uses the RENAME_DIMS mapping of 'i' to 'i_renamed' scen = make_dantzig(test_mp) rep = Reporter.from_scenario(scen) assert 'd:i_renamed-j' in rep, rep.graph.keys() assert ['seattle', 'san-diego'] == rep.get('i_renamed') # Original name 'i' are not found in the reporter assert 'd:i-j' not in rep, rep.graph.keys() pytest.raises(KeyError, rep.get, 'i') # Remove the configuration for renaming 'i', so that other tests work RENAME_DIMS.pop('i')
def test_reporting_file_formats(test_data_path, tmp_path): r = Reporter() expected = xr.DataArray.from_series( pd.read_csv(test_data_path / 'report-input.csv', index_col=['i', 'j'])['value']) # CSV file is automatically parsed to xr.DataArray p1 = test_data_path / 'report-input.csv' k = r.add_file(p1) assert_xr_equal(r.get(k), expected) # Write to CSV p2 = tmp_path / 'report-output.csv' r.write(k, p2) # Output is identical to input file, except for order assert (sorted(p1.read_text().split('\n')) == sorted( p2.read_text().split('\n'))) # Write to Excel p3 = tmp_path / 'report-output.xlsx' r.write(k, p3)
def test_reporter_describe(test_mp, test_data_path): scen = make_dantzig(test_mp) r = Reporter.from_scenario(scen) # hexadecimal ID of *scen* id_ = hex(id(scen)) if os.name != 'nt' else \ '{:#018X}'.format(id(scen)).replace('X', 'x') # Describe one key expected = """'d:i': - sum(dimensions=['j'], weights=None, ...) - 'd:i-j': - data_for_quantity('par', 'd', 'value', ...) - 'scenario': - <ixmp.core.Scenario object at {id}> - 'filters': - {{}} """.format(id=id_) assert r.describe('d:i') == expected # Describe all keys expected = (test_data_path / 'report-describe.txt').read_text() \ .format(id=id_) assert r.describe() == expected
def test_reporter_visualize(test_mp, tmp_path): scen = make_dantzig(test_mp) r = Reporter.from_scenario(scen) r.visualize(str(tmp_path / 'visualize.png'))
def test_reporting_units(): """Test handling of units within Reporter computations.""" r = Reporter() # Create some dummy data dims = dict(coords=['a b c'.split()], dims=['x']) r.add('energy:x', Quantity(xr.DataArray([1., 3, 8], attrs={'_unit': 'MJ'}, **dims))) r.add('time', Quantity(xr.DataArray([5., 6, 8], attrs={'_unit': 'hour'}, **dims))) r.add('efficiency', Quantity(xr.DataArray([0.9, 0.8, 0.95], **dims))) # Aggregation preserves units r.add('energy', (computations.sum, 'energy:x', None, ['x'])) assert r.get('energy').attrs['_unit'] == ureg.parse_units('MJ') # Units are derived for a ratio of two quantities r.add('power', (computations.ratio, 'energy:x', 'time')) assert r.get('power').attrs['_unit'] == ureg.parse_units('MJ/hour') # Product of dimensioned and dimensionless quantities keeps the former r.add('energy2', (computations.product, 'energy:x', 'efficiency')) assert r.get('energy2').attrs['_unit'] == ureg.parse_units('MJ')
def test_platform_units(test_mp, caplog, ureg): """Test handling of units from ixmp.Platform. test_mp is loaded with some units including '-', '???', 'G$', etc. which are not parseable with pint; and others which are not defined in a default pint.UnitRegistry. These tests check the handling of those units. """ # Prepare a Scenario with test data scen = ixmp.Scenario(test_mp, 'reporting_platform_units', 'reporting_platform_units', 'new') t, t_foo, t_bar, x = add_test_data(scen) rep = Reporter.from_scenario(scen) x_key = rep.full_key('x') # Convert 'x' to dataframe x = x.to_series().rename('value').reset_index() # Exception message, formatted as a regular expression msg = r"unit '{}' cannot be parsed; contains invalid character\(s\) '{}'" # Unit and components for the regex bad_units = [('-', '-', '-'), ('???', r'\?\?\?', r'\?'), ('E$', r'E\$', r'\$')] for unit, expr, chars in bad_units: # Add the unit test_mp.add_unit(unit) # Overwrite the parameter x['unit'] = unit scen.add_par('x', x) # Parsing units with invalid chars raises an intelligible exception with pytest.raises(ComputationError, match=msg.format(expr, chars)): rep.get(x_key) # Now using parseable but unrecognized units x['unit'] = 'USD/kWa' scen.add_par('x', x) # Unrecognized units are added automatically, with log messages emitted with assert_logs(caplog, ['Add unit definition: kWa = [kWa]']): rep.get(x_key) # Mix of recognized/unrecognized units can be added: USD is already in the # unit registry, so is not re-added x['unit'] = 'USD/pkm' test_mp.add_unit('USD/pkm') scen.add_par('x', x) caplog.clear() rep.get(x_key) assert not any('Add unit definition: USD = [USD]' in m for m in caplog.messages) # Mixed units are discarded x.loc[0, 'unit'] = 'kg' scen.add_par('x', x) with assert_logs(caplog, "x: mixed units ['kg', 'USD/pkm'] discarded"): rep.get(x_key) # Configured unit substitutions are applied rep.graph['config']['units'] = dict(apply=dict(x='USD/pkm')) with assert_logs(caplog, "x: replace units dimensionless with USD/pkm"): x = rep.get(x_key) # Applied units are pint objects with the correct dimensionality unit = x.attrs['_unit'] assert isinstance(unit, pint.Unit) assert unit.dimensionality == {'[USD]': 1, '[km]': -1}
def test_reporter_apply(): # Reporter with two scalar values r = Reporter() r.add('foo', (lambda x: x, 42)) r.add('bar', (lambda x: x, 11)) N = len(r.keys()) # A computation def _product(a, b): return a * b # A generator method that yields keys and computations def baz_qux(key): yield key + ':baz', (_product, key, 0.5) yield key + ':qux', (_product, key, 1.1) # Apply the generator to two targets r.apply(baz_qux, 'foo') r.apply(baz_qux, 'bar') # Four computations were added to the reporter N += 4 assert len(r.keys()) == N assert r.get('foo:baz') == 42 * 0.5 assert r.get('foo:qux') == 42 * 1.1 assert r.get('bar:baz') == 11 * 0.5 assert r.get('bar:qux') == 11 * 1.1 # A generator that takes two arguments def twoarg(key1, key2): yield key1 + '__' + key2, (_product, key1, key2) r.apply(twoarg, 'foo:baz', 'bar:qux') # One computation added to the reporter N += 1 assert len(r.keys()) == N assert r.get('foo:baz__bar:qux') == 42 * 0.5 * 11 * 1.1 # A useless generator that does nothing def useless(key): return r.apply(useless, 'foo:baz__bar:qux') # Nothing added to the reporter assert len(r.keys()) == N
def test_reporter(scenario): r = Reporter.from_scenario(scenario) r.finalize(scenario) assert 'scenario' in r.graph
def test_reporter_add(): """Adding computations that refer to missing keys raises KeyError.""" r = Reporter() r.add('a', 3) r.add('d', 4) # Adding an existing key with strict=True with pytest.raises(KeyExistsError, match=r"key 'a' already exists"): r.add('a', 5, strict=True) def gen(other): """A generator for apply().""" return (lambda a, b: a * b, 'a', other) def msg(*keys): """Return a regex for str(MissingKeyError(*keys)).""" return 'required keys {!r} not defined'.format(tuple(keys)) \ .replace('(', '\\(') \ .replace(')', '\\)') # One missing key with pytest.raises(MissingKeyError, match=msg('b')): r.add_product('ab', 'a', 'b') # Two missing keys with pytest.raises(MissingKeyError, match=msg('c', 'b')): r.add_product('abc', 'c', 'a', 'b') # Using apply() targeted at non-existent keys also raises an Exception with pytest.raises(MissingKeyError, match=msg('e', 'f')): r.apply(gen, 'd', 'e', 'f') # add(..., strict=True) checks str or Key arguments g = Key('g', 'hi') with pytest.raises(MissingKeyError, match=msg('b', g)): r.add('foo', (computations.product, 'a', 'b', g), strict=True) # aggregate() and disaggregate() call add(), which raises the exception with pytest.raises(MissingKeyError, match=msg(g)): r.aggregate(g, 'tag', 'i') with pytest.raises(MissingKeyError, match=msg(g)): r.disaggregate(g, 'j') # add(..., sums=True) also adds partial sums r.add('foo:a-b-c', [], sums=True) assert 'foo:b' in r # add(name, ...) where name is the name of a computation r.add('select', 'bar', 'a', indexers={'dim': ['d0', 'd1', 'd2']}) # add(name, ...) with keyword arguments not recognized by the computation # raises an exception msg = "unexpected keyword argument 'bad_kwarg'" with pytest.raises(TypeError, match=msg): r.add('select', 'bar', 'a', bad_kwarg='foo', index=True)
def data(test_mp, request): scen = ixmp.Scenario(test_mp, request.node.name, request.node.name, 'new') rep = Reporter.from_scenario(scen) yield [scen, rep] + list(add_test_data(scen))
def test_filters(test_mp, tmp_path, caplog): """Reporting can be filtered ex ante.""" scen = ixmp.Scenario(test_mp, 'Reporting filters', 'Reporting filters', 'new') t, t_foo, t_bar, x = add_test_data(scen) rep = Reporter.from_scenario(scen) x_key = rep.full_key('x') def assert_t_indices(labels): assert set(rep.get(x_key).coords['t'].values) == set(labels) # 1. Set filters directly rep.graph['config']['filters'] = {'t': t_foo} assert_t_indices(t_foo) # Reporter can be re-used by changing filters rep.graph['config']['filters'] = {'t': t_bar} assert_t_indices(t_bar) rep.graph['config']['filters'] = {} assert_t_indices(t) # 2. Set filters using a convenience method rep = Reporter.from_scenario(scen) rep.set_filters(t=t_foo) assert_t_indices(t_foo) # Clear filters using the convenience method rep.set_filters(t=None) assert_t_indices(t) # Clear using the convenience method with no args rep.set_filters(t=t_foo) assert_t_indices(t_foo) rep.set_filters() assert_t_indices(t) # 3. Set filters via configuration keys # NB passes through from_scenario() -> __init__() -> configure() rep = Reporter.from_scenario(scen, filters={'t': t_foo}) assert_t_indices(t_foo) # Configuration key can also be read from file rep = Reporter.from_scenario(scen) # Write a temporary file containing the desired labels config_file = tmp_path / 'config.yaml' config_file.write_text('\n'.join([ 'filters:', ' t: {!r}'.format(t_bar), ])) rep.configure(config_file) assert_t_indices(t_bar) # Filtering too heavily: # Remove one value from the database at valid coordinates removed = {'t': t[:1], 'y': list(x.coords['y'].values)[:1]} scen.remove_par('x', removed) # Set filters to retrieve only this coordinate rep.set_filters(**removed) # A warning is logged msg = '\n '.join([ "0 values for par 'x' using filters:", repr(removed), 'Subsequent computations may fail.' ]) with assert_logs(caplog, msg): rep.get(x_key)