def test_multi_db_edit_target(tmpdir):
mp1 = ixmp.Platform(tmpdir / 'mp1', dbtype='HSQLDB')
scen1 = dantzig_transport(mp1)
mp2 = ixmp.Platform(tmpdir / 'mp2', dbtype='HSQLDB')
scen2 = scen1.clone(platform=mp2)
pdt.assert_frame_equal(scen1.par('d'), scen2.par('d'))
scen2.check_out()
scen2.add_par('d', ['san-diego', 'topeka'], 1.5, 'km')
scen2.commit('foo')
obs = (scen2
.par('d')
.set_index(['i', 'j'])
.loc['san-diego', 'topeka']
['value']
)
exp = 1.5
assert np.isclose(obs, exp)
obs = (scen1
.par('d')
.set_index(['i', 'j'])
.loc['san-diego', 'topeka']
['value']
)
exp = 1.4
assert np.isclose(obs, exp)
assert_multi_db(mp1, mp2)
def test_multi_db_run(tmpdir):
# create a new instance of the transport problem and solve it
mp1 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp1")
scen1 = make_dantzig(mp1, solve=True, quiet=True)
mp2 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp2")
# add other unit to make sure that the mapping is correct during clone
mp2.add_unit("wrong_unit")
mp2.add_region("wrong_region", "country")
# check that cloning across platforms must copy the full solution
pytest.raises(NotImplementedError, scen1.clone, platform=mp2, keep_solution=False)
# clone solved model across platforms (with default settings)
scen1.clone(platform=mp2, keep_solution=True)
# close the db to ensure that data and solution of the clone are saved
mp2.close_db()
del mp2
# reopen the connection to the second platform and reload scenario
_mp2 = ixmp.Platform(backend="jdbc", driver="hsqldb", path=tmpdir / "mp2")
assert_multi_db(mp1, _mp2)
scen2 = ixmp.Scenario(_mp2, **models["dantzig"])
# check that sets, variables and parameter were copied correctly
assert_array_equal(scen1.set("i"), scen2.set("i"))
assert_frame_equal(scen1.par("d"), scen2.par("d"))
assert np.isclose(scen2.var("z")["lvl"], 153.675)
assert_frame_equal(scen1.var("x"), scen2.var("x"))
# check that custom unit, region and timeseries are migrated correctly
assert scen2.par("f")["value"] == 90.0
assert scen2.par("f")["unit"] == "USD/km"
assert_frame_equal(scen2.timeseries(iamc=True), TS_DF)
def test_timeseries_edit(test_mp_props):
mp = ixmp.Platform(test_mp_props)
scen = mp.TimeSeries(*test_args)
df = {
'region': ['World', 'World'],
'variable': ['Testing', 'Testing'],
'unit': ['???', '???'],
'year': [2010, 2020],
'value': [23.7, 23.8]
}
exp = pd.DataFrame.from_dict(df)
obs = scen.timeseries()
npt.assert_array_equal(exp[cols_str], obs[cols_str])
npt.assert_array_almost_equal(exp['value'], obs['value'])
scen.check_out(timeseries_only=True)
df = {
'region': ['World', 'World', 'World'],
'variable': ['Testing', 'Testing', 'Testing2'],
'unit': ['???', '???', '???'],
'year': [2020, 2030, 2030],
'value': [24.8, 24.9, 25.1]
}
df = pd.DataFrame.from_dict(df)
scen.add_timeseries(df)
scen.commit('testing of editing timeseries')
mp.close_db()
mp = ixmp.Platform(test_mp_props)
scen = mp.TimeSeries(*test_args)
obs = scen.timeseries().sort_values(by=['year'])
df = df.append(exp.loc[0]).sort_values(by=['year'])
npt.assert_array_equal(df[cols_str], obs[cols_str])
npt.assert_array_almost_equal(df['value'], obs['value'])
def main(ctx, url, platform, dbprops, model, scenario, version):
# Load the indicated Platform
if url:
if dbprops or platform or model or scenario or version:
raise click.BadOptionUsage('--platform --model --scenario and/or'
'--version redundant with --url')
scen, mp = ixmp.Scenario.from_url(url)
ctx.obj = dict(scen=scen, mp=mp)
return
if dbprops and platform:
raise click.BadOptionUsage('give either --platform or --dbprops')
elif platform:
mp = ixmp.Platform(name=platform)
elif dbprops:
mp = ixmp.Platform(backend='jdbc', dbprops=dbprops)
try:
ctx.obj = dict(mp=mp)
except NameError:
return
# With a Platform, load the indicated Scenario
if model and scenario:
scen = ixmp.Scenario(mp, model, scenario, version=version)
ctx.obj['scen'] = scen
def test_multi_db_run(tmpdir, test_data_path):
# create a new instance of the transport problem and solve it
mp1 = ixmp.Platform(tmpdir / 'mp1', dbtype='HSQLDB')
scen1 = make_dantzig(mp1, solve=test_data_path)
mp2 = ixmp.Platform(tmpdir / 'mp2', dbtype='HSQLDB')
# add other unit to make sure that the mapping is correct during clone
mp2.add_unit('wrong_unit')
mp2.add_region('wrong_region', 'country')
# check that cloning across platforms must copy the full solution
pytest.raises(ValueError, scen1.clone, platform=mp2, keep_solution=False)
# clone solved model across platforms (with default settings)
scen1.clone(platform=mp2, keep_solution=True)
# close the db to ensure that data and solution of the clone are saved
mp2.close_db()
del mp2
# reopen the connection to the second platform and reload scenario
_mp2 = ixmp.Platform(tmpdir / 'mp2', dbtype='HSQLDB')
assert_multi_db(mp1, _mp2)
scen2 = ixmp.Scenario(_mp2, **models['dantzig'])
# check that sets, variables and parameter were copied correctly
npt.assert_array_equal(scen1.set('i'), scen2.set('i'))
pdt.assert_frame_equal(scen1.par('d'), scen2.par('d'))
assert np.isclose(scen2.var('z')['lvl'], 153.675)
pdt.assert_frame_equal(scen1.var('x'), scen2.var('x'))
# check that custom unit, region and timeseries are migrated correctly
assert scen2.par('f')['value'] == 90.0
assert scen2.par('f')['unit'] == 'USD_per_km'
pdt.assert_frame_equal(scen2.timeseries(iamc=True), TS_DF)
def test_init(self):
with pytest.raises(
ValueError,
match=re.escape("backend class 'foo' not among ['jdbc']")):
ixmp.Platform(backend="foo")
# name="default" is used, referring to "local"
mp = ixmp.Platform()
assert "local" == mp.name
def test_multi_db_run():
mp1 = ixmp.Platform(tempdir(), dbtype='HSQLDB')
scen1 = make_scenario(mp1)
solve_scenario(scen1)
mp2 = ixmp.Platform(tempdir(), dbtype='HSQLDB')
scen2 = scen1.clone(platform=mp2, keep_solution=False)
solve_scenario(scen2)
assert scen1.var('z') == scen2.var('z')
def get_platform(self, reload=False) -> ixmp.Platform:
"""Return a :class:`ixmp.Platform` from :attr:`platform_info`.
When used through the CLI, :attr:`platform_info` is a 'base' platform
as indicated by the --url or --platform options.
If a Platform has previously been instantiated with
:meth:`get_platform`, the same object is returned unless `reload=True`.
"""
if not reload:
# Return an existing Platform, if any
try:
return self["_mp"]
except KeyError:
pass
# Close any existing Platform, e.g. to reload it
try:
self["_mp"].close_db()
del self["_mp"]
except KeyError:
pass
# Create a Platform
self["_mp"] = ixmp.Platform(**self.platform_info)
return self["_mp"]
def results_to_xlsx(model, scenario, database, shale_costs, carbon_costs):
# Launch data base and load baseline
mp = ixmp.Platform(dbprops=f'db/{database}', dbtype='HSQLDB')
base = message_ix.Scenario(mp, model=model, scenario=scenario)
folder = 'results/'
all_ts = pd.DataFrame()
# retrieve data from scenarios
##################################
for s, c in [('none', 0)] + list(product(shale_costs, carbon_costs)):
if s == 'none':
scen = base
else:
scenario = f'{s}USDpGJ-{c}USDtCO2'
scen = message_ix.Scenario(mp, model=model, scenario=scenario)
scenario_ts = results_to_iamc(scen,
model,
scenario,
'South Africa',
l_year=2050)
all_ts = all_ts.append(scenario_ts, sort=True)
if not os.path.exists(folder):
os.makedirs(folder)
all_ts.to_excel(folder + 'timeseries.xlsx', index=False)
df = get_capacity_factor(all_ts)
df.to_excel(folder + 'capacity_factors.xlsx', index=False)
def test_import(ixmp_cli, test_mp, test_data_path):
fname = test_data_path / 'timeseries_canning.csv'
platform_name = test_mp.name
del test_mp
result = ixmp_cli.invoke([
'--platform',
platform_name,
'--model',
'canning problem',
'--scenario',
'standard',
'--version',
'1',
'import',
'--firstyear',
'2020',
str(fname),
])
assert result.exit_code == 0
# Check that the TimeSeries now contains the expected content
mp = ixmp.Platform(name=platform_name)
scen = ixmp.Scenario(mp, 'canning problem', 'standard', 1)
exp = pd.DataFrame.from_dict({
'region': ['World'],
'variable': ['Testing'],
'unit': ['???'],
'year': [2020],
'value': [28.3],
'model': ['canning problem'],
'scenario': ['standard'],
})
pdt.assert_frame_equal(exp, scen.timeseries())
def test_run_clone(tmpdir, test_data_path):
# this test is designed to cover the full functionality of the GAMS API
# - initialize a new platform instance
# - creates a new scenario and exports a gdx file
# - runs the tutorial transport model
# - reads back the solution from the output
# - performs the test on the objective value and the timeseries data
mp = ixmp.Platform(tmpdir, dbtype='HSQLDB')
scen = make_dantzig(mp, solve=test_data_path)
assert np.isclose(scen.var('z')['lvl'], 153.675)
pdt.assert_frame_equal(scen.timeseries(iamc=True), TS_DF)
# cloning with `keep_solution=True` keeps all timeseries and the solution
scen2 = scen.clone(keep_solution=True)
assert np.isclose(scen2.var('z')['lvl'], 153.675)
pdt.assert_frame_equal(scen2.timeseries(iamc=True), TS_DF)
# cloning with `keep_solution=True` and `first_model_year` raises an error
pytest.raises(ValueError, scen.clone, first_model_year=2005)
# cloning with `keep_solution=False` drops the solution and only keeps
# timeseries set as `meta=True`
scen3 = scen.clone(keep_solution=False)
assert np.isnan(scen3.var('z')['lvl'])
pdt.assert_frame_equal(scen3.timeseries(iamc=True), HIST_DF)
# cloning with `keep_solution=False` and `first_model_year`
# drops the solution and removes all timeseries not marked `meta=True`
# in the model horizon (i.e, `year >= first_model_year`)
scen4 = scen.clone(keep_solution=False, first_model_year=2005)
assert np.isnan(scen4.var('z')['lvl'])
pdt.assert_frame_equal(scen4.timeseries(iamc=True), TS_DF_CLEARED)
def test_reporting_cli(test_mp_props, test_data_path):
# Put something in the database
mp = ixmp.Platform(dbprops=test_mp_props)
make_dantzig(mp)
mp.close_db()
del mp
cmd = [
'ixmp',
'--dbprops',
str(test_mp_props),
'--model',
'canning problem',
'--scenario',
'standard',
'report',
'--config',
str(test_data_path / 'report-config-0.yaml'),
'--default',
'd_check',
]
out = subprocess.check_output(cmd, encoding='utf-8')
# Reporting produces the expected command-line output
assert out.endswith("""
<xarray.DataArray 'value' (i: 2, j: 3)>
array([[1.8, 2.5, 1.4],
[1.7, 2.5, 1.8]])
Coordinates:
* i (i) object 'san-diego' 'seattle'
* j (j) object 'chicago' 'new-york' 'topeka'
""")
def reload_cycle_scenario(request, tmp_path_factory, rc_data_size):
"""Set up a Platform with *rc_data_size* of random data."""
# Command-line option for the JVM memory limit
kwarg = dict()
jvm_mem_limit = int(request.config.getoption("--jvm-mem-limit"))
if jvm_mem_limit > 0:
kwarg["jvmargs"] = f"-Xmx{jvm_mem_limit}M"
# Path for this database
path = tmp_path_factory.mktemp("reload_cycle") / "base"
# Create the Platform. This should be the first in the process, so the
# jvmargs are used in :func:`.jdbc.start_jvm`.
mp = ixmp.Platform(backend="jdbc", driver="hsqldb", path=path, **kwarg)
s0 = ixmp.Scenario(mp, model="foo", scenario="bar 0", version="new")
# Add data
# currently omitted: time series data with *rc_data_size* elements
# s0.add_timeseries(random_ts_data(rc_data_size))
# A set named 'random_set' and parameter 'random_par' with *rc_data_size*
# elements
add_random_model_data(s0, rc_data_size)
s0.commit("")
yield s0
def reload_cycle_scenario(request, tmp_path_factory, rc_data_size):
"""Set up a Platform with *rc_data_size* of random data."""
# Command-line option for the JVM memory limit
kwarg = dict()
jvm_mem_limit = int(request.config.getoption('--jvm-mem-limit'))
if jvm_mem_limit > 0:
kwarg['jvmargs'] = f'-Xmx{jvm_mem_limit}M'
# Path for this database
path = tmp_path_factory.mktemp('reload_cycle') / 'base'
# Create the Platform. This should be the first in the process, so the
# jvmargs are used in :func:`.jdbc.start_jvm`.
mp = ixmp.Platform(backend='jdbc', driver='hsqldb', path=path, **kwarg)
s0 = ixmp.Scenario(mp, model='foo', scenario='bar 0', version='new')
# Add data
# currently omitted: time series data with *rc_data_size* elements
# s0.add_timeseries(random_ts_data(rc_data_size))
# A set named 'random_set' and parameter 'random_par' with *rc_data_size*
# elements
add_random_model_data(s0, rc_data_size)
s0.commit('')
yield s0
def test_check_single_model_access(tmpdir, test_data_path):
mock = HTTPMock('localhost', 8000)
mock.when('POST /login').reply(
'"security-token"',
headers={'Content-Type': 'application/json'},
times=FOREVER)
mock.when('POST /access/list',
body=".+\"test_user\".+",
headers={
'Authorization': 'Bearer security-token'
}).reply('[true]',
headers={'Content-Type': 'application/json'},
times=FOREVER)
mock.when('POST /access/list',
body=".+\"non_granted_user\".+",
headers={
'Authorization': 'Bearer security-token'
}).reply('[false]',
headers={'Content-Type': 'application/json'},
times=FOREVER)
test_props = create_local_testdb(db_path=tmpdir,
data_path=test_data_path / 'testdb',
auth_url=mock.pretend_url)
mp = ixmp.Platform(dbprops=test_props)
mp.set_log_level('DEBUG')
granted = mp.check_access('test_user', 'test_model')
assert granted
granted = mp.check_access('non_granted_user', 'test_model')
assert not granted
granted = mp.check_access('non_existing_user', 'test_model')
assert not granted
def test_deprecated_warns(tmp_env):
# Both warns AND raises
with pytest.raises(FileNotFoundError):
with pytest.warns(DeprecationWarning,
match="positional arguments to "
"Platform(…) for JDBCBackend"):
ixmp.Platform('nonexistent.properties', name='default')
def test_run_remove_solution(tmpdir, test_data_path):
# create a new instance of the transport problem and solve it
mp = ixmp.Platform(tmpdir, dbtype='HSQLDB')
scen = make_dantzig(mp, solve=test_data_path)
assert np.isclose(scen.var('z')['lvl'], 153.675)
# check that re-solving the model will raise an error if a solution exists
pytest.raises(ValueError,
scen.solve,
model=str(test_data_path / 'transport_ixmp'),
case='fail')
# remove the solution, check that variables are empty
# and timeseries not marked `meta=True` are removed
scen2 = scen.clone()
scen2.remove_solution()
assert not scen2.has_solution()
assert np.isnan(scen2.var('z')['lvl'])
pdt.assert_frame_equal(scen2.timeseries(iamc=True), HIST_DF)
# remove the solution with a specific year as first model year, check that
# variables are empty and timeseries not marked `meta=True` are removed
scen3 = scen.clone()
scen3.remove_solution(first_model_year=2005)
assert not scen3.has_solution()
assert np.isnan(scen3.var('z')['lvl'])
pdt.assert_frame_equal(scen3.timeseries(iamc=True), TS_DF_CLEARED)
def test_check_single_model_access(mock, tmp_path, test_data_path):
mock.when(
"POST /access/list",
body='.+"test_user".+',
headers={
"Authorization": "Bearer security-token"
},
).reply("[true]",
headers={"Content-Type": "application/json"},
times=FOREVER)
mock.when(
"POST /access/list",
body='.+"non_granted_user".+',
headers={
"Authorization": "Bearer security-token"
},
).reply("[false]",
headers={"Content-Type": "application/json"},
times=FOREVER)
test_props = create_test_platform(tmp_path,
test_data_path,
"access",
auth_url=mock.pretend_url)
mp = ixmp.Platform(backend="jdbc", dbprops=test_props)
granted = mp.check_access("test_user", "test_model")
assert granted
granted = mp.check_access("non_granted_user", "test_model")
assert not granted
granted = mp.check_access("non_existing_user", "test_model")
assert not granted
def test_cli_export_test_data(monkeypatch, session_context, mix_models_cli,
tmp_path):
"""The :command:`export-test-data` command can be invoked."""
# Create an empty scenario in the temporary local file database
platform = "local"
mp = ixmp.Platform(platform)
scen = make_dantzig(mp)
# URL
url = f"ixmp://{platform}/{scen.model}/{scen.scenario}#{scen.version}"
# Monkeypatch MESSAGE_DATA_PATH in case tests are being performed on a system
# without message_data installed
monkeypatch.setattr(util.common, "MESSAGE_DATA_PATH", tmp_path)
tmp_path.joinpath("data", "tests").mkdir(exist_ok=True, parents=True)
# File that will be created
technology = ["coal_ppl"]
dest_file = util.private_data_path(
"tests", f"{scen.model}_{scen.scenario}_{'_'.join(technology)}.xlsx")
# Release the database lock
mp.close_db()
try:
# Export works
result = mix_models_cli.assert_exit_0(
[f"--url={url}", "export-test-data"])
# The file is created in the expected location
assert str(dest_file) in result.output
assert dest_file.exists()
finally:
# Remove this temporary file
dest_file.unlink()
def test_mp_use_db_config_path():
assert not os.path.exists(CONFIG_PATH)
test_props = create_local_testdb()
dirname = os.path.dirname(test_props)
basename = os.path.basename(test_props)
# configure
cmd = 'ixmp-config --db_config_path {}'.format(dirname)
subprocess.check_call(cmd.split())
# start jvm
ixmp.start_jvm()
# launch Platform and connect to testdb (reconnect if closed)
try:
mp = ixmp.Platform(basename)
mp.open_db()
except:
os.remove(CONFIG_PATH)
raise
yield mp
os.remove(CONFIG_PATH)
def test_invalid_properties_file(test_data_path):
# HyperSQL creates a file with a .properties suffix for every file-based
# database, but these files do not contain the information needed to
# instantiate a database connection
with pytest.raises(ValueError,
match='Config file contains no database URL'):
ixmp.Platform(dbprops=test_data_path / 'hsqldb.properties')
def _define_platform(self) -> None:
_config = MessageInterface(self.run_config)
db_config = _config.config['db']
self.mp = ixmp.Platform(dbprops=db_config.get('dbprops'),
dbtype=db_config.get('dbtype'))
self.mp.set_log_level(self.log_level)
if not db_config.get('dbtype'):
self.db_server = True
def test_mp():
# start jvm
ixmp.start_jvm()
# launch Platform and connect to testdb (reconnect if closed)
mp = ixmp.Platform(tempdir(), dbtype='HSQLDB')
mp.open_db()
yield mp
def test_scenario(downloaded_scenarios, model, scenario, solve, solve_opts,
cases):
mp = ixmp.Platform(**downloaded_scenarios)
scen = message_ix.Scenario(mp, model, scenario)
scen.solve(model=solve, solve_options=solve_opts)
for case in cases:
exp = eval(case['exp'])
obs = eval(case['obs'])
assert eval(case['test'])(exp, obs)
def main(ctx, url, platform, dbprops, model, scenario, version):
ctx.obj = dict()
# Load the indicated Platform
mp = None
if url:
if dbprops or platform or model or scenario or version:
raise click.UsageError(
"--platform --model --scenario and/or --version redundant with --url"
)
scen, mp = ScenarioClass.from_url(url)
ctx.obj = dict(scen=scen, mp=mp)
return
elif dbprops and platform:
raise click.UsageError("give either --platform or --dbprops")
elif platform:
mp = ixmp.Platform(name=platform)
elif dbprops:
mp = ixmp.Platform(backend="jdbc", dbprops=dbprops)
if not mp:
return
ctx.obj.update(mp=mp)
# Store the model and scenario name from arguments
if model:
ctx.obj["model name"] = model
if scenario:
ctx.obj["scenario name"] = scenario
try:
# Load the indicated Scenario
if model and scenario:
ctx.obj["scen"] = ScenarioClass(mp,
model,
scenario,
version=version)
except Exception as e: # pragma: no cover
raise click.ClickException(e.args[0])
def test_connect_message(capfd, caplog):
msg = "connected to database 'jdbc:hsqldb:mem://ixmptest' (user: ixmp)..."
ixmp.Platform(backend="jdbc", driver="hsqldb", url="jdbc:hsqldb:mem://ixmptest")
# Java code via JPype does not log to the standard Python logger
assert not any(msg in m for m in caplog.messages)
# Instead, log messages are printed to stdout
captured = capfd.readouterr()
assert msg in captured.out
def test_cache_arg(arg):
"""Test 'cache' argument, passed to CachingBackend."""
mp = ixmp.Platform(backend='jdbc', driver='hsqldb',
url='jdbc:hsqldb:mem://test_cache_false',
cache=arg)
scen = make_dantzig(mp)
# Maybe put something in the cache
scen.par('a')
assert len(mp._backend._cache) == (1 if arg else 0)
def test_mp():
test_props = create_local_testdb()
# start jvm
ixmp.start_jvm()
# launch Platform and connect to testdb (reconnect if closed)
mp = ixmp.Platform(test_props)
mp.open_db()
yield mp
def check_local_model(model, notebook, shell=False):
mp = ixmp.Platform(dbtype='HSQLDB')
if model in mp.scenario_list().model.unique():
mp.close_db()
return
mp.close_db()
pyversion = sys.version_info[0]
cmd = "jupyter nbconvert {} --ExecutePreprocessor.kernel_name='python{}' --execute"
cmd = cmd.format(notebook, pyversion)
subprocess.check_call(cmd.split(), shell=shell)
def test_multi_db_run(tmpdir, test_data_path):
mp1 = ixmp.Platform(tmpdir / 'mp1', dbtype='HSQLDB')
scen1 = dantzig_transport(mp1, solve=test_data_path)
mp2 = ixmp.Platform(tmpdir / 'mp2', dbtype='HSQLDB')
# add other unit to make sure that the mapping is correct during clone
mp2.add_unit('wrong_unit')
mp2.add_region('wrong_region', 'country')
scen2 = scen1.clone(platform=mp2, keep_solution=False)
assert np.isnan(scen2.var('z')['lvl'])
scen2.solve(model=str(test_data_path / 'transport_ixmp'))
assert scen1.var('z') == scen2.var('z')
assert_multi_db(mp1, mp2)
# check that custom unit and region are migrated correctly
assert scen2.par('f')['value'] == 90.0
assert scen2.par('f')['unit'] == 'USD_per_km'
obs = scen2.timeseries(iamc=True)
pdt.assert_frame_equal(obs, TS_DF, check_dtype=False)
