def test_apply_units(data, caplog): # Unpack *_, x = data # Brute-force replacement with incompatible units with assert_logs(caplog, "Replace 'kilogram' with incompatible 'liter'"): result = computations.apply_units(x, 'litres') assert result.attrs['_unit'] == REGISTRY.Unit('litre') # No change in values assert_series_equal(result.to_series(), x.to_series()) caplog.set_level(logging.DEBUG) # Compatible units: magnitudes are also converted with assert_logs(caplog, "Convert 'kilogram' to 'metric_ton'"): result = computations.apply_units(x, 'tonne') assert result.attrs['_unit'] == REGISTRY.Unit('tonne') assert_series_equal(result.to_series(), x.to_series() * 0.001) # Remove unit x.attrs['_unit'] = REGISTRY.Unit('dimensionless') caplog.clear() result = computations.apply_units(x, 'kg') # Nothing logged when _unit attr is missing assert len(caplog.messages) == 0 assert result.attrs['_unit'] == REGISTRY.Unit('kg') assert_series_equal(result.to_series(), x.to_series())
def test_model_initialize(test_mp, caplog): # Model.initialize runs on an empty Scenario s = make_dantzig(test_mp) b1 = s.par('b') assert len(b1) == 3 # Modify a value for 'b' s.check_out() new_value = 301 s.add_par('b', 'chicago', new_value, 'cases') s.commit('Overwrite b(chicago)') # Model.initialize runs on an already-initialized Scenario, without error DantzigModel.initialize(s, with_data=True) # Data has the same length... b2 = s.par('b') assert len(b2) == 3 # ...but modified value(s) are not overwritten assert (b2.query("j == 'chicago'")['value'] == new_value).all() # Unrecognized Scenario(scheme=...) is initialized using the base method, a # no-op messages = [ "No scheme for new Scenario model-name/scenario-name", "No initialization for None-scheme Scenario", ] with assert_logs(caplog, messages, at_level=logging.DEBUG): Scenario(test_mp, model='model-name', scenario='scenario-name', version='new') with assert_logs(caplog, "No initialization for 'foo'-scheme Scenario", at_level=logging.DEBUG): Scenario(test_mp, model='model-name', scenario='scenario-name', version='new', scheme='foo') # Keyword arguments to Scenario(...) that are not recognized by # Model.initialize() raise an intelligible exception with pytest.raises(TypeError, match="unexpected keyword argument 'bad_arg1'"): Scenario(test_mp, model='model-name', scenario='scenario-name', version='new', scheme='unknown', bad_arg1=111) with pytest.raises(TypeError, match="unexpected keyword argument 'bad_arg2'"): Scenario(test_mp, model='model-name', scenario='scenario-name', version='new', scheme='dantzig', with_data=True, bad_arg2=222) # Replace b[j] with a parameter of the same name, but different indices s.check_out() s.remove_par('b') s.init_par('b', idx_sets=['i'], idx_names=['i_dim']) # Logs an error message with assert_logs(caplog, "Existing index sets of 'b' ['i'] do not match ['j']"): DantzigModel.initialize(s)
def test_update_scenario(caplog, test_mp): scen = make_dantzig(test_mp) scen.check_out() scen.add_set("j", "toronto") scen.commit("Add j=toronto") # Number of rows in the 'd' parameter N_before = len(scen.par("d")) assert 6 == N_before # A Computer used as calculation engine c = Computer() # Target Scenario for updating data c.add("target", scen) # Create a pd.DataFrame suitable for Scenario.add_par() data = dantzig_data["d"].query("j == 'chicago'").assign(j="toronto") data["value"] += 1.0 # Add to the Reporter c.add("input", data) # Task to update the scenario with the data c.add("test 1", (partial(update_scenario, params=["d"]), "target", "input")) # Trigger the computation that results in data being added with assert_logs(caplog, f"'d' ← {len(data)} rows", at_level=logging.INFO): # Returns nothing assert c.get("test 1") is None # Rows were added to the parameter assert len(scen.par("d")) == N_before + len(data) # Modify the data data = pd.concat([dantzig_data["d"], data]).reset_index(drop=True) data["value"] *= 2.0 # Convert to a Quantity object and re-add q = Quantity(data.set_index(["i", "j"])["value"], name="d", units="km") c.add("input", q) # Revise the task; the parameter name ('demand') is read from the Quantity c.add("test 2", (update_scenario, "target", "input")) # Trigger the computation with assert_logs(caplog, f"'d' ← {len(data)} rows", at_level=logging.INFO): c.get("test 2") # All the rows have been updated assert_frame_equal(scen.par("d"), data)
def test_computationerror(caplog): ce_none = ComputationError(None) # Message ends with ',)' on Python 3.6, only ')' on Python 3.7 msg = ("Exception raised while formatting None:\nAttributeError" "(\"'NoneType' object has no attribute '__traceback__'\"") with assert_logs(caplog, msg): str(ce_none)
def test_add_timeslice_duplicate(caplog, test_mp): test_mp.add_timeslice("foo_slice", "foo_category", 0.2) # Adding same name with different duration raises an error msg = "timeslice `foo_slice` already defined with duration 0.2" with raises(ValueError, match=re.escape(msg)): test_mp.add_timeslice("foo_slice", "bar_category", 0.3) # Re-adding with the same duration only logs a message with assert_logs(caplog, msg, at_level=logging.INFO): test_mp.add_timeslice("foo_slice", "bar_category", 0.2)
def test_reporter_no_solution(caplog, message_test_mp): scen = Scenario(message_test_mp, **SCENARIO["dantzig"]) with assert_logs( caplog, [ 'Scenario "Canning problem (MESSAGE scheme)/standard" has no solution', "Some reporting may not function as expected", ], ): rep = Reporter.from_scenario(scen) # Input parameters are still available demand = rep.full_key("demand") result = rep.get(demand) assert 3 == len(result)
def test_from_url(self, mp, caplog): url = f'ixmp://{mp.name}/Douglas Adams/Hitchhiker' # Default version is loaded scen, mp = ixmp.Scenario.from_url(url) assert scen.version == 1 # Giving an invalid version with errors='raise' raises an exception expected = ("There exists no Scenario 'Douglas Adams|Hitchhiker' " "(version: 10000) in the database!") with pytest.raises(Exception, match=expected): scen, mp = ixmp.Scenario.from_url(url + '#10000', errors='raise') # Giving an invalid scenario with errors='warn' raises an exception msg = ("ValueError: scenario='Hitchhikerfoo'\nwhen loading Scenario " f"from url: {(url + 'foo')!r}") with assert_logs(caplog, msg): scen, mp = ixmp.Scenario.from_url(url + 'foo') assert scen is None and isinstance(mp, ixmp.Platform)
def test_excel_io(self, scen, scen_empty, tmp_path, caplog): tmp_path /= 'output.xlsx' # FIXME remove_solution, check_out, commit, solve, commit should not # be needed to make this small data addition. scen.remove_solution() scen.check_out() # A 1-D set indexed by another set scen.init_set('foo', 'j') scen.add_set('foo', [['new-york'], ['topeka']]) # A scalar parameter with unusual units scen.platform.add_unit('pounds') scen.init_scalar('bar', 100, 'pounds') # A parameter with no values scen.init_par('baz_1', ['i', 'j']) # A parameter with ambiguous index name scen.init_par('baz_2', ['i'], ['i_dim']) scen.add_par('baz_2', dict(value=[1.1], i_dim=['seattle'])) # A 2-D set with ambiguous index names scen.init_set('baz_3', ['i', 'i'], ['i', 'i_also']) scen.add_set('baz_3', [['seattle', 'seattle']]) # A set with no elements scen.init_set('foo_2', ['j']) scen.commit('') scen.solve() # Solved Scenario can be written to file scen.to_excel(tmp_path, items=ixmp.ItemType.MODEL) # With init_items=False, can't be read into an empty Scenario. # Exception raised is the first index set, alphabetically with pytest.raises(ValueError, match="no set 'i'; " "try init_items=True"): scen_empty.read_excel(tmp_path) # File can be read with init_items=True scen_empty.read_excel(tmp_path, init_items=True, commit_steps=True) # Contents of the Scenarios are the same, except for unreadable items assert set(scen_empty.par_list()) | {'baz_1', 'baz_2'} \ == set(scen.par_list()) assert set(scen_empty.set_list()) | {'baz_3'} == set(scen.set_list()) assert_frame_equal(scen_empty.set('foo'), scen.set('foo')) # NB could make a more exact comparison of the Scenarios # Pre-initialize skipped items 'baz_2' and 'baz_3' scen_empty.init_par('baz_2', ['i'], ['i_dim']) scen_empty.init_set('baz_3', ['i', 'i'], ['i', 'i_also']) # Data can be read into an existing Scenario without init_items or # commit_steps arguments scen_empty.read_excel(tmp_path) # Re-initialize an item with different index names scen_empty.remove_par('d') scen_empty.init_par('d', idx_sets=['i', 'j'], idx_names=['I', 'J']) # Reading now logs an error about conflicting dims with assert_logs(caplog, "Existing par 'd' has index names(s)"): scen_empty.read_excel(tmp_path, init_items=True) # A new, empty Platform (different from the one under scen -> mp -> # test_mp) that lacks all units mp = ixmp.Platform(backend='jdbc', driver='hsqldb', url='jdbc:hsqldb:mem:excel_io') # A Scenario without the 'dantzig' scheme -> no contents at all s = ixmp.Scenario(mp, model='foo', scenario='bar', scheme='empty', version='new') # Fails with add_units=False with pytest.raises(ValueError, match="The unit 'pounds' does not exist" " in the database!"): s.read_excel(tmp_path, init_items=True) # Succeeds with add_units=True s.read_excel(tmp_path, add_units=True, init_items=True)
def test_filters(test_mp, tmp_path, caplog): """Reporting can be filtered ex ante.""" scen = ixmp.Scenario(test_mp, 'Reporting filters', 'Reporting filters', 'new') t, t_foo, t_bar, x = add_test_data(scen) rep = Reporter.from_scenario(scen) x_key = rep.full_key('x') def assert_t_indices(labels): assert set(rep.get(x_key).coords['t'].values) == set(labels) # 1. Set filters directly rep.graph['config']['filters'] = {'t': t_foo} assert_t_indices(t_foo) # Reporter can be re-used by changing filters rep.graph['config']['filters'] = {'t': t_bar} assert_t_indices(t_bar) rep.graph['config']['filters'] = {} assert_t_indices(t) # 2. Set filters using a convenience method rep = Reporter.from_scenario(scen) rep.set_filters(t=t_foo) assert_t_indices(t_foo) # Clear filters using the convenience method rep.set_filters(t=None) assert_t_indices(t) # Clear using the convenience method with no args rep.set_filters(t=t_foo) assert_t_indices(t_foo) rep.set_filters() assert_t_indices(t) # 3. Set filters via configuration keys # NB passes through from_scenario() -> __init__() -> configure() rep = Reporter.from_scenario(scen, filters={'t': t_foo}) assert_t_indices(t_foo) # Configuration key can also be read from file rep = Reporter.from_scenario(scen) # Write a temporary file containing the desired labels config_file = tmp_path / 'config.yaml' config_file.write_text('\n'.join([ 'filters:', ' t: {!r}'.format(t_bar), ])) rep.configure(config_file) assert_t_indices(t_bar) # Filtering too heavily: # Remove one value from the database at valid coordinates removed = {'t': t[:1], 'y': list(x.coords['y'].values)[:1]} scen.remove_par('x', removed) # Set filters to retrieve only this coordinate rep.set_filters(**removed) # A warning is logged msg = '\n '.join([ "0 values for par 'x' using filters:", repr(removed), 'Subsequent computations may fail.' ]) with assert_logs(caplog, msg): rep.get(x_key)
def test_platform_units(test_mp, caplog, ureg): """Test handling of units from ixmp.Platform. test_mp is loaded with some units including '-', '???', 'G$', etc. which are not parseable with pint; and others which are not defined in a default pint.UnitRegistry. These tests check the handling of those units. """ # Prepare a Scenario with test data scen = ixmp.Scenario(test_mp, 'reporting_platform_units', 'reporting_platform_units', 'new') t, t_foo, t_bar, x = add_test_data(scen) rep = Reporter.from_scenario(scen) x_key = rep.full_key('x') # Convert 'x' to dataframe x = x.to_series().rename('value').reset_index() # Exception message, formatted as a regular expression msg = r"unit '{}' cannot be parsed; contains invalid character\(s\) '{}'" # Unit and components for the regex bad_units = [('-', '-', '-'), ('???', r'\?\?\?', r'\?'), ('E$', r'E\$', r'\$')] for unit, expr, chars in bad_units: # Add the unit test_mp.add_unit(unit) # Overwrite the parameter x['unit'] = unit scen.add_par('x', x) # Parsing units with invalid chars raises an intelligible exception with pytest.raises(ComputationError, match=msg.format(expr, chars)): rep.get(x_key) # Now using parseable but unrecognized units x['unit'] = 'USD/kWa' scen.add_par('x', x) # Unrecognized units are added automatically, with log messages emitted with assert_logs(caplog, ['Add unit definition: kWa = [kWa]']): rep.get(x_key) # Mix of recognized/unrecognized units can be added: USD is already in the # unit registry, so is not re-added x['unit'] = 'USD/pkm' test_mp.add_unit('USD/pkm') scen.add_par('x', x) caplog.clear() rep.get(x_key) assert not any('Add unit definition: USD = [USD]' in m for m in caplog.messages) # Mixed units are discarded x.loc[0, 'unit'] = 'kg' scen.add_par('x', x) with assert_logs(caplog, "x: mixed units ['kg', 'USD/pkm'] discarded"): rep.get(x_key) # Configured unit substitutions are applied rep.graph['config']['units'] = dict(apply=dict(x='USD/pkm')) with assert_logs(caplog, "x: replace units dimensionless with USD/pkm"): x = rep.get(x_key) # Applied units are pint objects with the correct dimensionality unit = x.attrs['_unit'] assert isinstance(unit, pint.Unit) assert unit.dimensionality == {'[USD]': 1, '[km]': -1}
def test_platform_units(test_mp, caplog, ureg): """Test handling of units from ixmp.Platform. test_mp is loaded with some units including '-', '???', 'G$', etc. which are not parseable with pint; and others which are not defined in a default pint.UnitRegistry. These tests check the handling of those units. """ # Prepare a Scenario with test data scen = ixmp.Scenario(test_mp, "reporting_platform_units", "reporting_platform_units", "new") t, t_foo, t_bar, x = add_test_data(scen) rep = Reporter.from_scenario(scen) x_key = rep.full_key("x") # Convert 'x' to dataframe x = x.to_series().rename("value").reset_index() # Exception message, formatted as a regular expression msg = r"unit '{}' cannot be parsed; contains invalid character\(s\) '{}'" # Unit and components for the regex bad_units = [("-", "-", "-"), ("???", r"\?\?\?", r"\?"), ("E$", r"E\$", r"\$")] for unit, expr, chars in bad_units: # Add the unit test_mp.add_unit(unit) # Overwrite the parameter x["unit"] = unit scen.add_par("x", x) # Parsing units with invalid chars raises an intelligible exception with pytest.raises(ComputationError, match=msg.format(expr, chars)): rep.get(x_key) # Now using parseable but unrecognized units x["unit"] = "USD/kWa" scen.add_par("x", x) # Unrecognized units are added automatically, with log messages emitted caplog.clear() rep.get(x_key) # NB cannot use assert_logs here. reporting.utils.parse_units uses the # pint application registry, so depending which tests are run and in # which order, this unit may already be defined. if len(caplog.messages): assert "Add unit definition: kWa = [kWa]" in caplog.messages # Mix of recognized/unrecognized units can be added: USD is already in the # unit registry, so is not re-added x["unit"] = "USD/pkm" test_mp.add_unit("USD/pkm") scen.add_par("x", x) caplog.clear() rep.get(x_key) assert not any("Add unit definition: USD = [USD]" in m for m in caplog.messages) # Mixed units are discarded x.loc[0, "unit"] = "kg" scen.add_par("x", x) with assert_logs(caplog, "x: mixed units ['kg', 'USD/pkm'] discarded", at_level=logging.INFO): rep.get(x_key) # Configured unit substitutions are applied rep.graph["config"]["units"] = dict(apply=dict(x="USD/pkm")) with assert_logs(caplog, "x: replace units dimensionless with USD/pkm", at_level=logging.INFO): x = rep.get(x_key) # Applied units are pint objects with the correct dimensionality unit = x.attrs["_unit"] assert isinstance(unit, pint.Unit) assert unit.dimensionality == {"[USD]": 1, "[pkm]": -1}
def test_filters(test_mp, tmp_path, caplog): """Reporting can be filtered ex ante.""" scen = ixmp.Scenario(test_mp, "Reporting filters", "Reporting filters", "new") t, t_foo, t_bar, x = add_test_data(scen) rep = Reporter.from_scenario(scen) x_key = rep.full_key("x") def assert_t_indices(labels): assert set(rep.get(x_key).coords["t"].values) == set(labels) # 1. Set filters directly rep.graph["config"]["filters"] = {"t": t_foo} assert_t_indices(t_foo) # Reporter can be re-used by changing filters rep.graph["config"]["filters"] = {"t": t_bar} assert_t_indices(t_bar) rep.graph["config"]["filters"] = {} assert_t_indices(t) # 2. Set filters using a convenience method rep = Reporter.from_scenario(scen) rep.set_filters(t=t_foo) assert_t_indices(t_foo) # Clear filters using the convenience method rep.set_filters(t=None) assert_t_indices(t) # Clear using the convenience method with no args rep.set_filters(t=t_foo) assert_t_indices(t_foo) rep.set_filters() assert_t_indices(t) # 3. Set filters via configuration keys # NB passes through from_scenario() -> __init__() -> configure() rep = Reporter.from_scenario(scen, filters={"t": t_foo}) assert_t_indices(t_foo) # Configuration key can also be read from file rep = Reporter.from_scenario(scen) # Write a temporary file containing the desired labels config_file = tmp_path / "config.yaml" config_file.write_text("\n".join(["filters:", f" t: {repr(t_bar)}"])) rep.configure(config_file) assert_t_indices(t_bar) # Filtering too heavily: # Remove one value from the database at valid coordinates removed = {"t": t[:1], "y": list(x.coords["y"].values)[:1]} scen.remove_par("x", removed) # Set filters to retrieve only this coordinate rep.set_filters(**removed) # A warning is logged with assert_logs( caplog, ( f"0 values for par 'x' using filters: {repr(removed)}", "May be the cause of subsequent errors", ), at_level=logging.DEBUG, ): rep.get(x_key)
def test_cached(caplog, test_context, tmp_path): """:func:`.cached` works as expected. .. todo:: test behaviour when :data:`.SKIP_CACHE` is :obj:`True` """ # Clear seen paths, so that log message below is guaranteed to occur message_ix_models.util.cache.PATHS_SEEN.clear() # Store in the temporary directory for this session, to avoid collisions across # sessions test_context.cache_path = tmp_path.joinpath("cache") # A dummy path to be hashed as an argument path_foo = tmp_path.joinpath("foo", "bar") with caplog.at_level(logging.DEBUG, logger="message_ix_models"): @cached def func0(ctx, a, path, b=3): """A test function.""" log.info("func0 runs") return f"{id(ctx)}, {a + b}" # Docstring is modified assert "Data returned by this function is cached" in func0.__doc__ # Message is logged assert f"func0() will cache in {tmp_path.joinpath('cache')}" in caplog.messages @cached def func1(x=1, y=2, **kwargs): # Function with defaults for all arguments log.info("func1 runs") return x + y caplog.clear() # pathlib.Path argument is serialized to JSON as part of the argument hash; # function runs, messages logged with assert_logs(caplog, "func0 runs"): result0 = func0(test_context, 1, path_foo) caplog.clear() result1 = func0(test_context, 1, path_foo) # Function does not run assert "func0 runs" not in caplog.messages assert caplog.messages[0].startswith("Cache hit for func0") # Results identical assert result0 == result1 # Different context object with identical contents hashes equal ctx2 = deepcopy(test_context) assert id(test_context) != id(ctx2) result2 = func0(ctx2, 1, path_foo) # Function does not run assert "func0 runs" not in caplog.messages # Results are identical, i.e. including the old ID assert result0 == result2 ctx2.delete() caplog.clear() # Hash of no arguments is the same, function only runs once assert 3 == func1() == func1() assert 1 == sum(m == "func1 runs" for m in caplog.messages) # Warnings logged for unhashables; ScenarioInfo is hashed as dict caplog.clear() with assert_logs( caplog, [ "ignores <class 'xarray.core.dataset.Dataset'>", "ignores <class 'ixmp.core.platform.Platform'>", ], ): func1(ds=xr.Dataset(), mp=test_context.get_platform(), si=ScenarioInfo()) # Unserializable type raises an exception with pytest.raises(TypeError, match="Object of type slice is not JSON serializable"): func1(arg=slice(None))