def test_reporter_from_westeros(test_mp):
scen = make_westeros(test_mp, emissions=True, solve=True)
# Reporter.from_scenario can handle Westeros example model
rep = Reporter.from_scenario(scen)
# Westeros-specific configuration: '-' is a reserved character in pint
configure(units={'replace': {'-': ''}})
# Default target can be calculated
rep.get('all')
# message default target can be calculated
# TODO if df is empty, year is cast to float
obs = rep.get('message:default')
# all expected reporting exists
assert len(obs.data) == 69
# custom values are correct
obs = obs.filter(variable='total om*')
assert len(obs.data) == 9
assert all(
obs['variable'] == # noqa: W504
['total om cost|coal_ppl'] * 3 + # noqa: W504
['total om cost|grid'] * 3 + # noqa: W504
['total om cost|wind_ppl'] * 3
)
assert all(obs['year'] == [700, 710, 720] * 3)
obs = obs['value'].values
exp = [4832.177734, 8786.515625, 12666.666016, 5555.555664, 8333.333984,
10555.555664, 305.748138, 202.247391, 0.]
assert len(obs) == len(exp)
assert_allclose(obs, exp)
def test_reporter_from_dantzig(test_mp):
scen = make_dantzig(test_mp, solve=True)
# Reporter.from_scenario can handle Dantzig example model
rep = Reporter.from_scenario(scen)
# Default target can be calculated
rep.get("all")
def test_reporter(test_mp):
scen = Scenario(test_mp,
'canning problem (MESSAGE scheme)',
'standard')
# Varies between local & CI contexts
# DEBUG may be due to reuse of test_mp in a non-deterministic order
if not scen.has_solution():
scen.solve()
# IXMPReporter can be initialized on a MESSAGE Scenario
rep_ix = ixmp_Reporter.from_scenario(scen)
# message_ix.Reporter can also be initialized
rep = Reporter.from_scenario(scen)
# Number of quantities available in a rudimentary MESSAGEix Scenario
assert len(rep.graph['all']) == 120
# Quantities have short dimension names
assert 'demand:n-c-l-y-h' in rep.graph
# Aggregates are available
assert 'demand:n-l-h' in rep.graph
# Quantities contain expected data
dims = dict(coords=['chicago new-york topeka'.split()], dims=['n'])
demand = xr.DataArray([300, 325, 275], **dims)
# NB the call to squeeze() drops the length-1 dimensions c-l-y-h
obs = rep.get('demand:n-c-l-y-h').squeeze(drop=True)
# TODO: Squeeze on AttrSeries still returns full index, whereas xarray
# drops everything except node
obs = obs.reset_index(['c', 'l', 'y', 'h'], drop=True)
# check_dtype is false because of casting in pd.Series to float
# check_attrsis false because we don't get the unit addition in bare xarray
assert_qty_equal(obs.sort_index(), demand,
check_attrs=False, check_dtype=False)
# ixmp.Reporter pre-populated with only model quantities and aggregates
assert len(rep_ix.graph) == 5088
# message_ix.Reporter pre-populated with additional, derived quantities
assert len(rep.graph) == 7975
# Derived quantities have expected dimensions
vom_key = rep.full_key('vom')
assert vom_key not in rep_ix
assert vom_key == 'vom:nl-t-yv-ya-m-h'
# …and expected values
vom = (
rep.get(rep.full_key('ACT')) * rep.get(rep.full_key('var_cost'))
).dropna()
# check_attrs false because `vom` multiply above does not add units
assert_qty_equal(vom, rep.get(vom_key), check_attrs=False)
def test_as_pyam(message_test_mp):
scen = Scenario(message_test_mp, **SCENARIO['dantzig'])
if not scen.has_solution():
scen.solve()
rep = Reporter.from_scenario(scen)
# Quantities for 'ACT' variable at full resolution
qty = rep.get(rep.full_key('ACT'))
# Call as_pyam() with an empty quantity
p = computations.as_pyam(scen, qty[0:0], year_time_dim='ya')
assert isinstance(p, pyam.IamDataFrame)
def test_as_pyam(message_test_mp):
scen = Scenario(message_test_mp, **SCENARIO["dantzig"])
if not scen.has_solution():
scen.solve()
rep = Reporter.from_scenario(scen)
# Quantities for 'ACT' variable at full resolution
qty = rep.get(rep.full_key("ACT"))
# Call as_pyam() with an empty quantity
as_pyam = rep.get_comp("as_pyam")
p = as_pyam(scen, qty[0:0], rename=dict(nl="region", ya="year"))
assert isinstance(p, pyam.IamDataFrame)
def test_reporter_from_scenario(message_test_mp):
scen = Scenario(message_test_mp, **SCENARIO["dantzig"])
# Varies between local & CI contexts
# DEBUG may be due to reuse of test_mp in a non-deterministic order
if not scen.has_solution():
scen.solve(quiet=True)
# IXMPReporter can be initialized on a MESSAGE Scenario
rep_ix = ixmp_Reporter.from_scenario(scen)
# message_ix.Reporter can also be initialized
rep = Reporter.from_scenario(scen)
# Number of quantities available in a rudimentary MESSAGEix Scenario
assert len(rep.graph["all"]) == 123
# Quantities have short dimension names
assert "demand:n-c-l-y-h" in rep
# Aggregates are available
assert "demand:n-l-h" in rep
# Quantities contain expected data
dims = dict(coords=["chicago new-york topeka".split()], dims=["n"])
demand = Quantity(xr.DataArray([300, 325, 275], **dims), name="demand")
# NB the call to squeeze() drops the length-1 dimensions c-l-y-h
obs = rep.get("demand:n-c-l-y-h").squeeze(drop=True)
# check_attrs False because we don't get the unit addition in bare xarray
assert_qty_equal(obs, demand, check_attrs=False)
# ixmp.Reporter pre-populated with only model quantities and aggregates
assert len(rep_ix.graph) == 5225
# message_ix.Reporter pre-populated with additional, derived quantities
# This is the same value as in test_tutorials.py
assert len(rep.graph) == 12690
# Derived quantities have expected dimensions
vom_key = rep.full_key("vom")
assert vom_key not in rep_ix
assert vom_key == "vom:nl-t-yv-ya-m-h"
# …and expected values
var_cost = rep.get(rep.full_key("var_cost"))
ACT = rep.get(rep.full_key("ACT"))
product = rep.get_comp("product")
vom = product(var_cost, ACT)
# check_attrs false because `vom` multiply above does not add units
assert_qty_equal(vom, rep.get(vom_key))
def prepare_plots(rep: Reporter, input_costs="$/GWa") -> None:
"""Prepare `rep` to generate plots for tutorial energy models.
Makes available several keys:
- ``plot activity``
- ``plot demand``
- ``plot extraction``
- ``plot fossil supply curve``
- ``plot capacity``
- ``plot new capacity``
- ``plot prices``
To control the contents of each plot, use :meth:`.set_filters` on `rep`.
"""
# Conversion factors between input units and plotting units
# TODO use exact units in all tutorials
# TODO allow the correct units to pass through reporting
cost_unit_conv = {
"$/GWa": 1.0,
"$/MWa": 1e3,
"$/kWa": 1e6,
}.get(input_costs, 1.0)
# Basic setup of the reporter
rep.configure(units={"replace": {"-": ""}})
# Add one node to the reporter for each plot
for title, func, key_str, units in PLOTS:
# Convert the string to a Key object so as to reference its .dims
key = Key.from_str_or_key(key_str)
# Operation for the reporter
comp = partial(
# The function to use, e.g. stacked_bar()
func,
# Other keyword arguments to the plotting function
dims=key.dims,
units=units,
title=f"Energy System {title.title()}",
cf=1.0 if title != "prices" else (cost_unit_conv * 100 / 8760),
stacked=title != "prices",
)
# Add the computation under a key like "plot activity"
rep.add(f"plot {title}", (comp, key))
rep.add(
"plot fossil supply curve",
(
partial(
computations.plot_cumulative,
labels=("Fossil supply", "Resource volume", "Cost"),
),
"resource_volume:n-g",
"resource_cost:n-g-y",
),
)
def test_reporter_no_solution(caplog, message_test_mp):
scen = Scenario(message_test_mp, **SCENARIO["dantzig"])
with assert_logs(
caplog,
[
'Scenario "Canning problem (MESSAGE scheme)/standard" has no solution',
"Some reporting may not function as expected",
],
):
rep = Reporter.from_scenario(scen)
# Input parameters are still available
demand = rep.full_key("demand")
result = rep.get(demand)
assert 3 == len(result)
def test_reporter_from_westeros(test_mp):
scen = make_westeros(test_mp, emissions=True, solve=True, quiet=True)
# Reporter.from_scenario can handle Westeros example model
rep = Reporter.from_scenario(scen)
# Westeros-specific configuration: '-' is a reserved character in pint
configure(units={"replace": {"-": ""}})
# Default target can be calculated
rep.get("all")
# message default target can be calculated
# TODO if df is empty, year is cast to float
obs = rep.get("message:default")
# all expected reporting exists
assert len(obs.data) == 69
# custom values are correct
obs = obs.filter(variable="total om*")
assert len(obs.data) == 9
assert all(obs["variable"] == ["total om cost|coal_ppl"] * 3 # noqa: W504
+ ["total om cost|grid"] * 3 # noqa: W504
+ ["total om cost|wind_ppl"] * 3 # noqa: W504
)
assert all(obs["year"] == [700, 710, 720] * 3)
obs = obs.data["value"].values
exp = [
2842.4574905,
5373.0510978,
6933.3333333,
3055.5555555,
4555.5555555,
5777.7777777,
381.57832228,
43.340541129,
0.0,
]
assert len(obs) == len(exp)
assert_allclose(obs, exp)
a = rep.convert_pyam('PRICE_COMMODITY:n-c-y', 'y', collapse=collapse_N)
rep.write(a[0], Path('price_commodity_'+model+'_'+scen+'.xlsx'))
# 5. Carbon price
if scen!="baseline":
rep.set_filters()
a = rep.convert_pyam('PRICE_EMISSION', 'y')
rep.write(a[0], Path('price_emission_'+model+'_'+scen+'.xlsx'))
# Generate individual xlsx
for sc in scen_names:
Sc_ref = message_ix.Scenario(mp, model_name, sc)
repo = Reporter.from_scenario(Sc_ref)
GenerateOutput(model_name, sc, repo)
# Combine xlsx per each output variable
for cases in ['nf_demand', 'nf_emissions_CO2', 'nf_input', 'price_commodity', 'price_emission']:
infiles = []
for sc in scen_names:
if sc=="baseline" and cases=='price_emission':
continue
infiles.append(pd.read_excel(cases + "_"+ model_name +'_' + sc + ".xlsx"))
appended_df = pd.concat(infiles, join='outer', sort=False)
appended_df.to_excel(cases+"-"+model_name+".xlsx", index=False)
#%% Generate plots
def dantzig_reporter(message_test_mp):
scen = Scenario(message_test_mp, **SCENARIO['dantzig'])
if not scen.has_solution():
scen.solve()
yield Reporter.from_scenario(scen)
def test_reporter_convert_pyam(test_mp, caplog, tmp_path):
scen = Scenario(test_mp,
'canning problem (MESSAGE scheme)',
'standard')
if not scen.has_solution():
scen.solve()
rep = Reporter.from_scenario(scen)
# Key for 'ACT' variable at full resolution
ACT = rep.full_key('ACT')
# Add a computation that converts ACT to a pyam.IamDataFrame
rep.add('ACT IAMC', (partial(computations.as_pyam, drop=['yv'],
year_time_dim='ya'),
'scenario', ACT))
# Result is an IamDataFrame
idf1 = rep.get('ACT IAMC')
assert isinstance(idf1, pyam.IamDataFrame)
# …of expected length
assert len(idf1) == 8
# …in which variables are not renamed
assert idf1['variable'].unique() == 'ACT'
# Warning was logged because of extra columns
w = "Extra columns ['h', 'm', 't'] when converting ['ACT'] to IAMC format"
assert ('message_ix.reporting.pyam', WARNING, w) in caplog.record_tuples
# Repeat, using the message_ix.Reporter convenience function
def m_t(df):
"""Callback for collapsing ACT columns."""
# .pop() removes the named column from the returned row
df['variable'] = 'Activity|' + df['t'] + '|' + df['m']
df.drop(['t', 'm'], axis=1, inplace=True)
return df
# Use the convenience function to add the node
keys = rep.convert_pyam(ACT, 'ya', collapse=m_t)
# Keys of added node(s) are returned
assert len(keys) == 1
key2, *_ = keys
assert key2 == ACT.name + ':iamc'
caplog.clear()
# Result
idf2 = rep.get(key2)
df2 = idf2.as_pandas()
# Extra columns have been removed:
# - m and t by the collapse callback.
# - h automatically, because 'ya' was used for the year index.
assert not any(c in df2.columns for c in ['h', 'm', 't'])
# Variable names were formatted by the callback
reg_var = pd.DataFrame([
['san-diego', 'Activity|canning_plant|production'],
['san-diego', 'Activity|transport_from_san-diego|to_chicago'],
['san-diego', 'Activity|transport_from_san-diego|to_new-york'],
['san-diego', 'Activity|transport_from_san-diego|to_topeka'],
['seattle', 'Activity|canning_plant|production'],
['seattle', 'Activity|transport_from_seattle|to_chicago'],
['seattle', 'Activity|transport_from_seattle|to_new-york'],
['seattle', 'Activity|transport_from_seattle|to_topeka'],
], columns=['region', 'variable'])
assert_frame_equal(df2[['region', 'variable']], reg_var)
# message_ix.Reporter uses pyam.IamDataFrame.to_csv() to write to file
path = tmp_path / 'activity.csv'
rep.write(key2, path)
# File contents are as expected
expected = Path(__file__).parent / 'data' / 'report-pyam-write.csv'
assert path.read_text() == expected.read_text()