def test_multi_db_edit_target(tmpdir): mp1 = ixmp.Platform(tmpdir / 'mp1', dbtype='HSQLDB') scen1 = dantzig_transport(mp1) mp2 = ixmp.Platform(tmpdir / 'mp2', dbtype='HSQLDB') scen2 = scen1.clone(platform=mp2) pdt.assert_frame_equal(scen1.par('d'), scen2.par('d')) scen2.check_out() scen2.add_par('d', ['san-diego', 'topeka'], 1.5, 'km') scen2.commit('foo') obs = (scen2 .par('d') .set_index(['i', 'j']) .loc['san-diego', 'topeka'] ['value'] ) exp = 1.5 assert np.isclose(obs, exp) obs = (scen1 .par('d') .set_index(['i', 'j']) .loc['san-diego', 'topeka'] ['value'] ) exp = 1.4 assert np.isclose(obs, exp) assert_multi_db(mp1, mp2)
def test_multi_db_run(tmpdir): # create a new instance of the transport problem and solve it mp1 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp1") scen1 = make_dantzig(mp1, solve=True, quiet=True) mp2 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp2") # add other unit to make sure that the mapping is correct during clone mp2.add_unit("wrong_unit") mp2.add_region("wrong_region", "country") # check that cloning across platforms must copy the full solution pytest.raises(NotImplementedError, scen1.clone, platform=mp2, keep_solution=False) # clone solved model across platforms (with default settings) scen1.clone(platform=mp2, keep_solution=True) # close the db to ensure that data and solution of the clone are saved mp2.close_db() del mp2 # reopen the connection to the second platform and reload scenario _mp2 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp2") assert_multi_db(mp1, _mp2) scen2 = ixmp.Scenario(_mp2, **models["dantzig"]) # check that sets, variables and parameter were copied correctly assert_array_equal(scen1.set("i"), scen2.set("i")) assert_frame_equal(scen1.par("d"), scen2.par("d")) assert np.isclose(scen2.var("z")["lvl"], 153.675) assert_frame_equal(scen1.var("x"), scen2.var("x")) # check that custom unit, region and timeseries are migrated correctly assert scen2.par("f")["value"] == 90.0 assert scen2.par("f")["unit"] == "USD/km" assert_frame_equal(scen2.timeseries(iamc=True), TS_DF)
def test_timeseries_edit(test_mp_props): mp = ixmp.Platform(test_mp_props) scen = mp.TimeSeries(*test_args) df = { 'region': ['World', 'World'], 'variable': ['Testing', 'Testing'], 'unit': ['???', '???'], 'year': [2010, 2020], 'value': [23.7, 23.8] } exp = pd.DataFrame.from_dict(df) obs = scen.timeseries() npt.assert_array_equal(exp[cols_str], obs[cols_str]) npt.assert_array_almost_equal(exp['value'], obs['value']) scen.check_out(timeseries_only=True) df = { 'region': ['World', 'World', 'World'], 'variable': ['Testing', 'Testing', 'Testing2'], 'unit': ['???', '???', '???'], 'year': [2020, 2030, 2030], 'value': [24.8, 24.9, 25.1] } df = pd.DataFrame.from_dict(df) scen.add_timeseries(df) scen.commit('testing of editing timeseries') mp.close_db() mp = ixmp.Platform(test_mp_props) scen = mp.TimeSeries(*test_args) obs = scen.timeseries().sort_values(by=['year']) df = df.append(exp.loc[0]).sort_values(by=['year']) npt.assert_array_equal(df[cols_str], obs[cols_str]) npt.assert_array_almost_equal(df['value'], obs['value'])
def main(ctx, url, platform, dbprops, model, scenario, version): # Load the indicated Platform if url: if dbprops or platform or model or scenario or version: raise click.BadOptionUsage('--platform --model --scenario and/or' '--version redundant with --url') scen, mp = ixmp.Scenario.from_url(url) ctx.obj = dict(scen=scen, mp=mp) return if dbprops and platform: raise click.BadOptionUsage('give either --platform or --dbprops') elif platform: mp = ixmp.Platform(name=platform) elif dbprops: mp = ixmp.Platform(backend='jdbc', dbprops=dbprops) try: ctx.obj = dict(mp=mp) except NameError: return # With a Platform, load the indicated Scenario if model and scenario: scen = ixmp.Scenario(mp, model, scenario, version=version) ctx.obj['scen'] = scen
def test_multi_db_run(tmpdir, test_data_path): # create a new instance of the transport problem and solve it mp1 = ixmp.Platform(tmpdir / 'mp1', dbtype='HSQLDB') scen1 = make_dantzig(mp1, solve=test_data_path) mp2 = ixmp.Platform(tmpdir / 'mp2', dbtype='HSQLDB') # add other unit to make sure that the mapping is correct during clone mp2.add_unit('wrong_unit') mp2.add_region('wrong_region', 'country') # check that cloning across platforms must copy the full solution pytest.raises(ValueError, scen1.clone, platform=mp2, keep_solution=False) # clone solved model across platforms (with default settings) scen1.clone(platform=mp2, keep_solution=True) # close the db to ensure that data and solution of the clone are saved mp2.close_db() del mp2 # reopen the connection to the second platform and reload scenario _mp2 = ixmp.Platform(tmpdir / 'mp2', dbtype='HSQLDB') assert_multi_db(mp1, _mp2) scen2 = ixmp.Scenario(_mp2, **models['dantzig']) # check that sets, variables and parameter were copied correctly npt.assert_array_equal(scen1.set('i'), scen2.set('i')) pdt.assert_frame_equal(scen1.par('d'), scen2.par('d')) assert np.isclose(scen2.var('z')['lvl'], 153.675) pdt.assert_frame_equal(scen1.var('x'), scen2.var('x')) # check that custom unit, region and timeseries are migrated correctly assert scen2.par('f')['value'] == 90.0 assert scen2.par('f')['unit'] == 'USD_per_km' pdt.assert_frame_equal(scen2.timeseries(iamc=True), TS_DF)
def test_init(self): with pytest.raises( ValueError, match=re.escape("backend class 'foo' not among ['jdbc']")): ixmp.Platform(backend="foo") # name="default" is used, referring to "local" mp = ixmp.Platform() assert "local" == mp.name
def test_multi_db_run(): mp1 = ixmp.Platform(tempdir(), dbtype='HSQLDB') scen1 = make_scenario(mp1) solve_scenario(scen1) mp2 = ixmp.Platform(tempdir(), dbtype='HSQLDB') scen2 = scen1.clone(platform=mp2, keep_solution=False) solve_scenario(scen2) assert scen1.var('z') == scen2.var('z')
def get_platform(self, reload=False) -> ixmp.Platform: """Return a :class:`ixmp.Platform` from :attr:`platform_info`. When used through the CLI, :attr:`platform_info` is a 'base' platform as indicated by the --url or --platform options. If a Platform has previously been instantiated with :meth:`get_platform`, the same object is returned unless `reload=True`. """ if not reload: # Return an existing Platform, if any try: return self["_mp"] except KeyError: pass # Close any existing Platform, e.g. to reload it try: self["_mp"].close_db() del self["_mp"] except KeyError: pass # Create a Platform self["_mp"] = ixmp.Platform(**self.platform_info) return self["_mp"]
def results_to_xlsx(model, scenario, database, shale_costs, carbon_costs): # Launch data base and load baseline mp = ixmp.Platform(dbprops=f'db/{database}', dbtype='HSQLDB') base = message_ix.Scenario(mp, model=model, scenario=scenario) folder = 'results/' all_ts = pd.DataFrame() # retrieve data from scenarios ################################## for s, c in [('none', 0)] + list(product(shale_costs, carbon_costs)): if s == 'none': scen = base else: scenario = f'{s}USDpGJ-{c}USDtCO2' scen = message_ix.Scenario(mp, model=model, scenario=scenario) scenario_ts = results_to_iamc(scen, model, scenario, 'South Africa', l_year=2050) all_ts = all_ts.append(scenario_ts, sort=True) if not os.path.exists(folder): os.makedirs(folder) all_ts.to_excel(folder + 'timeseries.xlsx', index=False) df = get_capacity_factor(all_ts) df.to_excel(folder + 'capacity_factors.xlsx', index=False)
def test_import(ixmp_cli, test_mp, test_data_path): fname = test_data_path / 'timeseries_canning.csv' platform_name = test_mp.name del test_mp result = ixmp_cli.invoke([ '--platform', platform_name, '--model', 'canning problem', '--scenario', 'standard', '--version', '1', 'import', '--firstyear', '2020', str(fname), ]) assert result.exit_code == 0 # Check that the TimeSeries now contains the expected content mp = ixmp.Platform(name=platform_name) scen = ixmp.Scenario(mp, 'canning problem', 'standard', 1) exp = pd.DataFrame.from_dict({ 'region': ['World'], 'variable': ['Testing'], 'unit': ['???'], 'year': [2020], 'value': [28.3], 'model': ['canning problem'], 'scenario': ['standard'], }) pdt.assert_frame_equal(exp, scen.timeseries())
def test_run_clone(tmpdir, test_data_path): # this test is designed to cover the full functionality of the GAMS API # - initialize a new platform instance # - creates a new scenario and exports a gdx file # - runs the tutorial transport model # - reads back the solution from the output # - performs the test on the objective value and the timeseries data mp = ixmp.Platform(tmpdir, dbtype='HSQLDB') scen = make_dantzig(mp, solve=test_data_path) assert np.isclose(scen.var('z')['lvl'], 153.675) pdt.assert_frame_equal(scen.timeseries(iamc=True), TS_DF) # cloning with `keep_solution=True` keeps all timeseries and the solution scen2 = scen.clone(keep_solution=True) assert np.isclose(scen2.var('z')['lvl'], 153.675) pdt.assert_frame_equal(scen2.timeseries(iamc=True), TS_DF) # cloning with `keep_solution=True` and `first_model_year` raises an error pytest.raises(ValueError, scen.clone, first_model_year=2005) # cloning with `keep_solution=False` drops the solution and only keeps # timeseries set as `meta=True` scen3 = scen.clone(keep_solution=False) assert np.isnan(scen3.var('z')['lvl']) pdt.assert_frame_equal(scen3.timeseries(iamc=True), HIST_DF) # cloning with `keep_solution=False` and `first_model_year` # drops the solution and removes all timeseries not marked `meta=True` # in the model horizon (i.e, `year >= first_model_year`) scen4 = scen.clone(keep_solution=False, first_model_year=2005) assert np.isnan(scen4.var('z')['lvl']) pdt.assert_frame_equal(scen4.timeseries(iamc=True), TS_DF_CLEARED)
def test_reporting_cli(test_mp_props, test_data_path): # Put something in the database mp = ixmp.Platform(dbprops=test_mp_props) make_dantzig(mp) mp.close_db() del mp cmd = [ 'ixmp', '--dbprops', str(test_mp_props), '--model', 'canning problem', '--scenario', 'standard', 'report', '--config', str(test_data_path / 'report-config-0.yaml'), '--default', 'd_check', ] out = subprocess.check_output(cmd, encoding='utf-8') # Reporting produces the expected command-line output assert out.endswith(""" <xarray.DataArray 'value' (i: 2, j: 3)> array([[1.8, 2.5, 1.4], [1.7, 2.5, 1.8]]) Coordinates: * i (i) object 'san-diego' 'seattle' * j (j) object 'chicago' 'new-york' 'topeka' """)
def reload_cycle_scenario(request, tmp_path_factory, rc_data_size): """Set up a Platform with *rc_data_size* of random data.""" # Command-line option for the JVM memory limit kwarg = dict() jvm_mem_limit = int(request.config.getoption("--jvm-mem-limit")) if jvm_mem_limit > 0: kwarg["jvmargs"] = f"-Xmx{jvm_mem_limit}M" # Path for this database path = tmp_path_factory.mktemp("reload_cycle") / "base" # Create the Platform. This should be the first in the process, so the # jvmargs are used in :func:`.jdbc.start_jvm`. mp = ixmp.Platform(backend="jdbc", driver="hsqldb", path=path, **kwarg) s0 = ixmp.Scenario(mp, model="foo", scenario="bar 0", version="new") # Add data # currently omitted: time series data with *rc_data_size* elements # s0.add_timeseries(random_ts_data(rc_data_size)) # A set named 'random_set' and parameter 'random_par' with *rc_data_size* # elements add_random_model_data(s0, rc_data_size) s0.commit("") yield s0
def reload_cycle_scenario(request, tmp_path_factory, rc_data_size): """Set up a Platform with *rc_data_size* of random data.""" # Command-line option for the JVM memory limit kwarg = dict() jvm_mem_limit = int(request.config.getoption('--jvm-mem-limit')) if jvm_mem_limit > 0: kwarg['jvmargs'] = f'-Xmx{jvm_mem_limit}M' # Path for this database path = tmp_path_factory.mktemp('reload_cycle') / 'base' # Create the Platform. This should be the first in the process, so the # jvmargs are used in :func:`.jdbc.start_jvm`. mp = ixmp.Platform(backend='jdbc', driver='hsqldb', path=path, **kwarg) s0 = ixmp.Scenario(mp, model='foo', scenario='bar 0', version='new') # Add data # currently omitted: time series data with *rc_data_size* elements # s0.add_timeseries(random_ts_data(rc_data_size)) # A set named 'random_set' and parameter 'random_par' with *rc_data_size* # elements add_random_model_data(s0, rc_data_size) s0.commit('') yield s0
def test_check_single_model_access(tmpdir, test_data_path): mock = HTTPMock('localhost', 8000) mock.when('POST /login').reply( '"security-token"', headers={'Content-Type': 'application/json'}, times=FOREVER) mock.when('POST /access/list', body=".+\"test_user\".+", headers={ 'Authorization': 'Bearer security-token' }).reply('[true]', headers={'Content-Type': 'application/json'}, times=FOREVER) mock.when('POST /access/list', body=".+\"non_granted_user\".+", headers={ 'Authorization': 'Bearer security-token' }).reply('[false]', headers={'Content-Type': 'application/json'}, times=FOREVER) test_props = create_local_testdb(db_path=tmpdir, data_path=test_data_path / 'testdb', auth_url=mock.pretend_url) mp = ixmp.Platform(dbprops=test_props) mp.set_log_level('DEBUG') granted = mp.check_access('test_user', 'test_model') assert granted granted = mp.check_access('non_granted_user', 'test_model') assert not granted granted = mp.check_access('non_existing_user', 'test_model') assert not granted
def test_deprecated_warns(tmp_env): # Both warns AND raises with pytest.raises(FileNotFoundError): with pytest.warns(DeprecationWarning, match="positional arguments to " "Platform(…) for JDBCBackend"): ixmp.Platform('nonexistent.properties', name='default')
def test_run_remove_solution(tmpdir, test_data_path): # create a new instance of the transport problem and solve it mp = ixmp.Platform(tmpdir, dbtype='HSQLDB') scen = make_dantzig(mp, solve=test_data_path) assert np.isclose(scen.var('z')['lvl'], 153.675) # check that re-solving the model will raise an error if a solution exists pytest.raises(ValueError, scen.solve, model=str(test_data_path / 'transport_ixmp'), case='fail') # remove the solution, check that variables are empty # and timeseries not marked `meta=True` are removed scen2 = scen.clone() scen2.remove_solution() assert not scen2.has_solution() assert np.isnan(scen2.var('z')['lvl']) pdt.assert_frame_equal(scen2.timeseries(iamc=True), HIST_DF) # remove the solution with a specific year as first model year, check that # variables are empty and timeseries not marked `meta=True` are removed scen3 = scen.clone() scen3.remove_solution(first_model_year=2005) assert not scen3.has_solution() assert np.isnan(scen3.var('z')['lvl']) pdt.assert_frame_equal(scen3.timeseries(iamc=True), TS_DF_CLEARED)
def test_check_single_model_access(mock, tmp_path, test_data_path): mock.when( "POST /access/list", body='.+"test_user".+', headers={ "Authorization": "Bearer security-token" }, ).reply("[true]", headers={"Content-Type": "application/json"}, times=FOREVER) mock.when( "POST /access/list", body='.+"non_granted_user".+', headers={ "Authorization": "Bearer security-token" }, ).reply("[false]", headers={"Content-Type": "application/json"}, times=FOREVER) test_props = create_test_platform(tmp_path, test_data_path, "access", auth_url=mock.pretend_url) mp = ixmp.Platform(backend="jdbc", dbprops=test_props) granted = mp.check_access("test_user", "test_model") assert granted granted = mp.check_access("non_granted_user", "test_model") assert not granted granted = mp.check_access("non_existing_user", "test_model") assert not granted
def test_cli_export_test_data(monkeypatch, session_context, mix_models_cli, tmp_path): """The :command:`export-test-data` command can be invoked.""" # Create an empty scenario in the temporary local file database platform = "local" mp = ixmp.Platform(platform) scen = make_dantzig(mp) # URL url = f"ixmp://{platform}/{scen.model}/{scen.scenario}#{scen.version}" # Monkeypatch MESSAGE_DATA_PATH in case tests are being performed on a system # without message_data installed monkeypatch.setattr(util.common, "MESSAGE_DATA_PATH", tmp_path) tmp_path.joinpath("data", "tests").mkdir(exist_ok=True, parents=True) # File that will be created technology = ["coal_ppl"] dest_file = util.private_data_path( "tests", f"{scen.model}_{scen.scenario}_{'_'.join(technology)}.xlsx") # Release the database lock mp.close_db() try: # Export works result = mix_models_cli.assert_exit_0( [f"--url={url}", "export-test-data"]) # The file is created in the expected location assert str(dest_file) in result.output assert dest_file.exists() finally: # Remove this temporary file dest_file.unlink()
def test_mp_use_db_config_path(): assert not os.path.exists(CONFIG_PATH) test_props = create_local_testdb() dirname = os.path.dirname(test_props) basename = os.path.basename(test_props) # configure cmd = 'ixmp-config --db_config_path {}'.format(dirname) subprocess.check_call(cmd.split()) # start jvm ixmp.start_jvm() # launch Platform and connect to testdb (reconnect if closed) try: mp = ixmp.Platform(basename) mp.open_db() except: os.remove(CONFIG_PATH) raise yield mp os.remove(CONFIG_PATH)
def test_invalid_properties_file(test_data_path): # HyperSQL creates a file with a .properties suffix for every file-based # database, but these files do not contain the information needed to # instantiate a database connection with pytest.raises(ValueError, match='Config file contains no database URL'): ixmp.Platform(dbprops=test_data_path / 'hsqldb.properties')
def _define_platform(self) -> None: _config = MessageInterface(self.run_config) db_config = _config.config['db'] self.mp = ixmp.Platform(dbprops=db_config.get('dbprops'), dbtype=db_config.get('dbtype')) self.mp.set_log_level(self.log_level) if not db_config.get('dbtype'): self.db_server = True
def test_mp(): # start jvm ixmp.start_jvm() # launch Platform and connect to testdb (reconnect if closed) mp = ixmp.Platform(tempdir(), dbtype='HSQLDB') mp.open_db() yield mp
def test_scenario(downloaded_scenarios, model, scenario, solve, solve_opts, cases): mp = ixmp.Platform(**downloaded_scenarios) scen = message_ix.Scenario(mp, model, scenario) scen.solve(model=solve, solve_options=solve_opts) for case in cases: exp = eval(case['exp']) obs = eval(case['obs']) assert eval(case['test'])(exp, obs)
def main(ctx, url, platform, dbprops, model, scenario, version): ctx.obj = dict() # Load the indicated Platform mp = None if url: if dbprops or platform or model or scenario or version: raise click.UsageError( "--platform --model --scenario and/or --version redundant with --url" ) scen, mp = ScenarioClass.from_url(url) ctx.obj = dict(scen=scen, mp=mp) return elif dbprops and platform: raise click.UsageError("give either --platform or --dbprops") elif platform: mp = ixmp.Platform(name=platform) elif dbprops: mp = ixmp.Platform(backend="jdbc", dbprops=dbprops) if not mp: return ctx.obj.update(mp=mp) # Store the model and scenario name from arguments if model: ctx.obj["model name"] = model if scenario: ctx.obj["scenario name"] = scenario try: # Load the indicated Scenario if model and scenario: ctx.obj["scen"] = ScenarioClass(mp, model, scenario, version=version) except Exception as e: # pragma: no cover raise click.ClickException(e.args[0])
def test_connect_message(capfd, caplog): msg = "connected to database 'jdbc:hsqldb:mem://ixmptest' (user: ixmp)..." ixmp.Platform(backend="jdbc", driver="hsqldb", url="jdbc:hsqldb:mem://ixmptest") # Java code via JPype does not log to the standard Python logger assert not any(msg in m for m in caplog.messages) # Instead, log messages are printed to stdout captured = capfd.readouterr() assert msg in captured.out
def test_cache_arg(arg): """Test 'cache' argument, passed to CachingBackend.""" mp = ixmp.Platform(backend='jdbc', driver='hsqldb', url='jdbc:hsqldb:mem://test_cache_false', cache=arg) scen = make_dantzig(mp) # Maybe put something in the cache scen.par('a') assert len(mp._backend._cache) == (1 if arg else 0)
def test_mp(): test_props = create_local_testdb() # start jvm ixmp.start_jvm() # launch Platform and connect to testdb (reconnect if closed) mp = ixmp.Platform(test_props) mp.open_db() yield mp
def check_local_model(model, notebook, shell=False): mp = ixmp.Platform(dbtype='HSQLDB') if model in mp.scenario_list().model.unique(): mp.close_db() return mp.close_db() pyversion = sys.version_info[0] cmd = "jupyter nbconvert {} --ExecutePreprocessor.kernel_name='python{}' --execute" cmd = cmd.format(notebook, pyversion) subprocess.check_call(cmd.split(), shell=shell)
def test_multi_db_run(tmpdir, test_data_path): mp1 = ixmp.Platform(tmpdir / 'mp1', dbtype='HSQLDB') scen1 = dantzig_transport(mp1, solve=test_data_path) mp2 = ixmp.Platform(tmpdir / 'mp2', dbtype='HSQLDB') # add other unit to make sure that the mapping is correct during clone mp2.add_unit('wrong_unit') mp2.add_region('wrong_region', 'country') scen2 = scen1.clone(platform=mp2, keep_solution=False) assert np.isnan(scen2.var('z')['lvl']) scen2.solve(model=str(test_data_path / 'transport_ixmp')) assert scen1.var('z') == scen2.var('z') assert_multi_db(mp1, mp2) # check that custom unit and region are migrated correctly assert scen2.par('f')['value'] == 90.0 assert scen2.par('f')['unit'] == 'USD_per_km' obs = scen2.timeseries(iamc=True) pdt.assert_frame_equal(obs, TS_DF, check_dtype=False)