def test_time_only_conversion(self, months, seasons):
"""Aggregate from months to seasons, summing groups of months
"""
adaptor = IntervalAdaptor('test-month-season')
from_spec = Spec(name='test-var',
dtype='float',
dims=['months'],
coords={'months': months})
adaptor.add_input(from_spec)
to_spec = Spec(name='test-var',
dtype='float',
dims=['seasons'],
coords={'seasons': seasons})
adaptor.add_output(to_spec)
actual_coefficients = adaptor.generate_coefficients(from_spec, to_spec)
data = np.array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
data_array = DataArray(from_spec, data)
data_handle = Mock()
data_handle.get_data = Mock(return_value=data_array)
data_handle.read_coefficients = Mock(return_value=actual_coefficients)
adaptor.simulate(data_handle)
actual = data_handle.set_results.call_args[0][1]
expected = np.array([3, 3, 3, 3])
np.testing.assert_array_equal(actual, expected)
def test_aggregate_from_hour_to_day(self, twenty_four_hours, one_day):
"""Aggregate hours to a single value for a day
"""
data = np.ones((24, ))
adaptor = IntervalAdaptor('test-hourly-day')
from_spec = Spec(name='test-var',
dtype='float',
dims=['hourly_day'],
coords={'hourly_day': twenty_four_hours})
adaptor.add_input(from_spec)
to_spec = Spec(name='test-var',
dtype='float',
dims=['one_day'],
coords={'one_day': one_day})
adaptor.add_output(to_spec)
actual_coefficients = adaptor.generate_coefficients(from_spec, to_spec)
data_array = DataArray(from_spec, data)
data_handle = Mock()
data_handle.get_data = Mock(return_value=data_array)
data_handle.read_coefficients = Mock(return_value=actual_coefficients)
adaptor.simulate(data_handle)
actual = data_handle.set_results.call_args[0][1]
expected = np.array([24])
assert np.allclose(actual, expected, rtol=1e-05, atol=1e-08)
def test_one_region_convert_from_hour_to_day(self, regions_rect,
twenty_four_hours, one_day):
"""One region, time aggregation required
"""
data = np.ones((1, 24)) # area a, hours 0-23
expected = np.array([[24]]) # area a, day 0
adaptor = IntervalAdaptor('test-hours-day')
from_spec = Spec(name='test-var',
dtype='float',
dims=['rect', 'twenty_four_hours'],
coords={
'twenty_four_hours': twenty_four_hours,
'rect': regions_rect
})
adaptor.add_input(from_spec)
to_spec = Spec(name='test-var',
dtype='float',
dims=['rect', 'one_day'],
coords={
'one_day': one_day,
'rect': regions_rect
})
adaptor.add_output(to_spec)
data_array = DataArray(from_spec, data)
data_handle = Mock()
data_handle.get_data = Mock(return_value=data_array)
data_handle.read_coefficients = Mock(side_effect=SmifDataNotFoundError)
adaptor.simulate(data_handle)
actual = data_handle.set_results.call_args[0][1]
assert np.allclose(actual, expected)
def test_aggregate_from_month_to_seasons(self, months, seasons,
monthly_data,
monthly_data_as_seasons):
"""Aggregate months to values for each season
"""
adaptor = IntervalAdaptor('test-month-season')
from_spec = Spec(name='test-var',
dtype='float',
dims=['months'],
coords={'months': months})
adaptor.add_input(from_spec)
to_spec = Spec(name='test-var',
dtype='float',
dims=['seasons'],
coords={'seasons': seasons})
adaptor.add_output(to_spec)
actual_coefficients = adaptor.generate_coefficients(from_spec, to_spec)
data_array = DataArray(from_spec, monthly_data)
data_handle = Mock()
data_handle.get_data = Mock(return_value=data_array)
data_handle.read_coefficients = Mock(return_value=actual_coefficients)
adaptor.simulate(data_handle)
actual = data_handle.set_results.call_args[0][1]
expected = monthly_data_as_seasons
assert np.allclose(actual, expected, rtol=1e-05, atol=1e-08)
def dep():
"""Dependency with mocked models
"""
source_model = Mock()
source_model.name = 'source_model'
source_spec = Spec(
name='source',
dtype='float',
dims=['x', 'y'],
coords={
'x': [0, 1],
'y': [0, 1]
},
)
sink_model = Mock()
sink_model.name = 'sink_model'
sink_spec = Spec(
name='sink',
dtype='float',
dims=['x', 'y'],
coords={
'x': [0, 1],
'y': [0, 1]
},
)
return Dependency(source_model, source_spec, sink_model, sink_spec)
def scenario_model():
"""ScenarioModel providing precipitation
"""
model = ScenarioModel('climate')
model.add_output(
Spec.from_dict({
'name': 'precipitation',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'ml'
}))
model.add_output(
Spec.from_dict({
'name': 'reservoir_level',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'ml'
}))
model.scenario = 'UKCP09 High emissions'
return model
def test_coords_from_dict_error(self):
"""A Spec constructed with a dict must have dims
"""
with raises(ValueError) as ex:
Spec(name='test',
dtype='int',
coords={'countries': ["England", "Wales"]})
assert "dims must be specified" in str(ex.value)
with raises(ValueError) as ex:
Spec(name='test',
dtype='int',
dims=['countries', 'age'],
coords={'countries': ["England", "Wales"]})
assert "dims must match the keys in coords" in str(ex.value)
with raises(ValueError) as ex:
Spec(name='test',
dtype='int',
dims=['countries'],
coords={
'countries': ["England", "Wales"],
'age': [">30", "<30"]
})
assert "dims must match the keys in coords" in str(ex.value)
def energy_model():
class EnergyModel(SectorModel):
"""
electricity_demand_input -> fluffiness
"""
def simulate(self, data_handle):
"""Mimics the running of a sector model
"""
fluff = data_handle['electricity_demand_input']
data_handle['fluffiness'] = fluff * 0.819
return data_handle
energy_model = EnergyModel('energy_model')
energy_model.add_input(
Spec(
name='electricity_demand_input',
dims=['LSOA'],
coords={'LSOA': ['E090001', 'E090002']},
dtype='float',
))
energy_model.add_output(
Spec(
name='fluffiness',
dims=['LSOA'],
coords={'LSOA': ['E090001', 'E090002']},
dtype='float',
))
return energy_model
def test_create():
"""Create with models and specs, access properties
"""
source_model = Mock()
source_spec = Spec(name='source',
dtype='float',
dims=['x', 'y'],
coords={
'x': [0, 1],
'y': [0, 1]
})
sink_model = Mock()
sink_spec = Spec(
name='sink',
dtype='float',
dims=['x', 'y'],
coords={
'x': [0, 1],
'y': [0, 1]
},
)
dep = Dependency(source_model, source_spec, sink_model, sink_spec,
RelativeTimestep.PREVIOUS)
assert dep.source_model == source_model
assert dep.source == source_spec
assert dep.sink_model == sink_model
assert dep.sink == sink_spec
assert dep.timestep == RelativeTimestep.PREVIOUS
# default to current timestep
dep = Dependency(source_model, source_spec, sink_model, sink_spec)
assert dep.timestep == RelativeTimestep.CURRENT
def test_aggregate_region(self, regions_rect, regions_half_squares):
"""Two regions aggregated to one, one interval
"""
adaptor = RegionAdaptor('test-square-half')
from_spec = Spec(name='test-var',
dtype='float',
dims=['half_squares'],
coords={'half_squares': regions_half_squares})
adaptor.add_input(from_spec)
to_spec = Spec(name='test-var',
dtype='float',
dims=['rect'],
coords={'rect': regions_rect})
adaptor.add_output(to_spec)
actual_coefficients = adaptor.generate_coefficients(from_spec, to_spec)
expected = np.ones((2, 1)) # aggregating coefficients
np.testing.assert_allclose(actual_coefficients, expected, rtol=1e-3)
data = np.array([24, 24]) # area a,b
data_array = DataArray(from_spec, data)
data_handle = Mock()
data_handle.get_data = Mock(return_value=data_array)
data_handle.read_coefficients = Mock(return_value=actual_coefficients)
adaptor.simulate(data_handle)
actual = data_handle.set_results.call_args[0][1]
expected = np.array([48]) # area zero
assert np.allclose(actual, expected)
def test_time_only_conversion_disagg(self, months, seasons):
"""Disaggregate from seasons to months based on duration of each month/season
"""
adaptor = IntervalAdaptor('test-season-month')
from_spec = Spec(name='test-var',
dtype='float',
dims=['seasons'],
coords={'seasons': seasons})
adaptor.add_input(from_spec)
to_spec = Spec(name='test-var',
dtype='float',
dims=['months'],
coords={'months': months})
adaptor.add_output(to_spec)
actual_coefficients = adaptor.generate_coefficients(from_spec, to_spec)
data = np.array([3, 3, 3, 3])
data_array = DataArray(from_spec, data)
data_handle = Mock()
data_handle.get_data = Mock(return_value=data_array)
data_handle.read_coefficients = Mock(return_value=actual_coefficients)
adaptor.simulate(data_handle)
actual = data_handle.set_results.call_args[0][1]
expected = np.array([
1.033333, 0.933333, 1.01087, 0.978261, 1.01087, 0.978261, 1.01087,
1.01087, 0.989011, 1.021978, 0.989011, 1.033333
])
np.testing.assert_allclose(actual, expected, rtol=1e-3)
def test_jobgraph_multiple_timesteps_with_dep(self, mock_model_run):
"""
a[before]
| |
v V
a[sim] a[sim]
t=1 ---> t=2
"""
model_a = EmptySectorModel('model_a')
model_a.add_input(Spec('input', dtype='float'))
model_a.add_output(Spec('output', dtype='float'))
mock_model_run.sos_model.add_model(model_a)
mock_model_run.sos_model.add_dependency(model_a, 'output', model_a,
'input',
RelativeTimestep.PREVIOUS)
mock_model_run.model_horizon = [1, 2]
runner = ModelRunner()
bundle = {'decision_iterations': [0], 'timesteps': [1, 2]}
job_graph = runner.build_job_graph(mock_model_run, bundle)
actual = list(job_graph.predecessors('test_simulate_1_0_model_a'))
expected = ['test_before_model_run_model_a']
assert actual == expected
actual = list(job_graph.predecessors('test_simulate_2_0_model_a'))
expected = [
'test_before_model_run_model_a', 'test_simulate_1_0_model_a'
]
assert sorted(actual) == sorted(expected)
def test_eq(self):
"""Equality based on equivalent dtype, dims, coords, unit
"""
a = Spec(name='population',
coords=[Coordinates('countries', ["England", "Wales"])],
dtype='int',
unit='people')
b = Spec(name='pop',
coords=[Coordinates('countries', ["England", "Wales"])],
dtype='int',
unit='people')
c = Spec(name='population',
coords=[
Coordinates('countries', ["England", "Scotland", "Wales"])
],
dtype='int',
unit='people')
d = Spec(name='population',
coords=[Coordinates('countries', ["England", "Wales"])],
dtype='float',
unit='people')
e = Spec(name='population',
coords=[Coordinates('countries', ["England", "Wales"])],
dtype='int',
unit='thousand people')
assert a == b
assert a != c
assert a != d
assert a != e
def test_jobgraph_interdependency(self, mock_model_run):
"""
a[before] b[before]
| |
v V
a[sim] ---> b[sim]
<---
"""
model_a = EmptySectorModel('model_a')
model_a.add_input(Spec('b', dtype='float'))
model_a.add_output(Spec('a', dtype='float'))
model_b = EmptySectorModel('model_b')
model_b.add_input(Spec('a', dtype='float'))
model_b.add_output(Spec('b', dtype='float'))
mock_model_run.sos_model.add_model(model_a)
mock_model_run.sos_model.add_model(model_b)
mock_model_run.sos_model.add_dependency(model_a, 'a', model_b, 'a')
mock_model_run.sos_model.add_dependency(model_b, 'b', model_a, 'b')
runner = ModelRunner()
bundle = {'decision_iterations': [0], 'timesteps': [1]}
with raises(NotImplementedError):
runner.build_job_graph(mock_model_run, bundle)
def test_convert_custom():
"""Convert custom units
"""
data_handle = Mock()
data = np.array([0.18346346], dtype=float)
from_spec = Spec(name='test_variable', dtype='float', unit='mcm')
to_spec = Spec(name='test_variable', dtype='float', unit='GW')
data_array = DataArray(from_spec, data)
data_handle.get_data = Mock(return_value=data_array)
data_handle.read_unit_definitions = Mock(
return_value=['mcm = 10.901353 * GW'])
adaptor = UnitAdaptor('test-mcm-GW')
adaptor.add_input(from_spec)
adaptor.add_output(to_spec)
adaptor.before_model_run(
data_handle) # must have run before_model_run to register units
adaptor.simulate(data_handle)
actual = data_handle.set_results.call_args[0][1]
expected = np.array([2], dtype=float)
np.testing.assert_allclose(actual, expected)
def test_jobgraph_multiple_models(self, mock_model_run):
"""
a[before] b[before] c[before]
| | |
v V V
a[sim] ---> b[sim] ---> c[sim]
|------------------>
"""
model_a = EmptySectorModel('model_a')
model_a.add_output(Spec('a', dtype='float'))
model_b = EmptySectorModel('model_b')
model_b.add_input(Spec('a', dtype='float'))
model_b.add_output(Spec('b', dtype='float'))
model_c = EmptySectorModel('model_c')
model_c.add_input(Spec('a', dtype='float'))
model_c.add_input(Spec('b', dtype='float'))
mock_model_run.sos_model.add_model(model_a)
mock_model_run.sos_model.add_model(model_b)
mock_model_run.sos_model.add_model(model_c)
mock_model_run.sos_model.add_dependency(model_a, 'a', model_b, 'a')
mock_model_run.sos_model.add_dependency(model_a, 'a', model_c, 'a')
mock_model_run.sos_model.add_dependency(model_b, 'b', model_c, 'b')
runner = ModelRunner()
bundle = {'decision_iterations': [0], 'timesteps': [1]}
job_graph = runner.build_job_graph(mock_model_run, bundle)
actual = list(job_graph.predecessors('test_simulate_1_0_model_a'))
expected = ['test_before_model_run_model_a']
assert actual == expected
actual = list(job_graph.successors('test_simulate_1_0_model_a'))
expected = ['test_simulate_1_0_model_b', 'test_simulate_1_0_model_c']
assert sorted(actual) == sorted(expected)
actual = list(job_graph.predecessors('test_simulate_1_0_model_b'))
expected = [
'test_before_model_run_model_b', 'test_simulate_1_0_model_a'
]
assert sorted(actual) == sorted(expected)
actual = list(job_graph.successors('test_simulate_1_0_model_b'))
expected = ['test_simulate_1_0_model_c']
assert actual == expected
actual = list(job_graph.predecessors('test_simulate_1_0_model_c'))
expected = [
'test_before_model_run_model_c', 'test_simulate_1_0_model_a',
'test_simulate_1_0_model_b'
]
assert sorted(actual) == sorted(expected)
actual = list(job_graph.successors('test_simulate_1_0_model_c'))
expected = []
assert actual == expected
def test_ranges_must_be_min_max(self):
"""Absolute and expected ranges must be in order
"""
with raises(ValueError) as ex:
Spec(dtype='int', abs_range=[2, 1])
assert "min value must be smaller than max value" in str(ex.value)
with raises(ValueError) as ex:
Spec(dtype='int', exp_range=[2, 1])
assert "min value must be smaller than max value" in str(ex.value)
def test_ranges_must_be_list_like(self):
"""Absolute and expected ranges must be list or tuple
"""
with raises(TypeError) as ex:
Spec(dtype='int', abs_range='string should fail')
assert "range must be a list or tuple" in str(ex.value)
with raises(TypeError) as ex:
Spec(dtype='int', exp_range='string should fail')
assert "range must be a list or tuple" in str(ex.value)
def test_ranges_must_be_len_two(self):
"""Absolute and expected ranges must be length two (min and max)
"""
with raises(ValueError) as ex:
Spec(dtype='int', abs_range=[0])
assert "range must have min and max values only" in str(ex.value)
with raises(ValueError) as ex:
Spec(dtype='int', exp_range=[0, 1, 2])
assert "range must have min and max values only" in str(ex.value)
def from_dict(cls, config):
"""Create object from dictionary serialisation
"""
model = cls(config['name'])
model.description = config['description']
for input_ in config['inputs']:
model.add_input(Spec.from_dict(input_))
for output in config['outputs']:
model.add_output(Spec.from_dict(output))
for param in config['parameters']:
model.add_parameter(Spec.from_dict(param))
return model
def test_construct(self, sector_model):
assert sector_model.description == 'a description'
assert sector_model.inputs == {
'raininess':
Spec.from_dict({
'name': 'raininess',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'milliliter'
})
}
spec = Spec.from_dict({
'name': 'assump_diff_floorarea_pp',
'description': 'Difference in floor area per person',
'dims': ['national'],
'coords': {
'national': ['GB']
},
'abs_range': (0.5, 2),
'exp_range': (0.5, 2),
'dtype': 'float',
'unit': '%'
})
assert sector_model.parameters == {'assump_diff_floorarea_pp': spec}
assert sector_model.outputs == {
'cost':
Spec.from_dict({
'name': 'cost',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'million GBP'
}),
'water':
Spec.from_dict({
'name': 'water',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'megaliter'
})
}
def test_narrative_data(self, setup_folder_structure, config_handler,
get_narrative):
""" Test to dump a narrative (yml) data-file and then read the file
using the datafile interface. Finally check the data shows up in the
returned dictionary.
"""
basefolder = setup_folder_structure
filepath = os.path.join(str(basefolder), 'data', 'narratives',
'central_planning.csv')
with open(filepath, 'w') as csvfile:
writer = csv.DictWriter(csvfile,
fieldnames=['homogeneity_coefficient'])
writer.writeheader()
writer.writerow({'homogeneity_coefficient': 8})
spec = Spec.from_dict({
'name': 'homogeneity_coefficient',
'description': "How homegenous the centralisation process is",
'absolute_range': [0, 1],
'expected_range': [0, 1],
'unit': 'percentage',
'dtype': 'float'
})
actual = config_handler.read_narrative_variant_data(
'central_planning', spec)
assert actual == DataArray(spec, np.array(8, dtype=float))
def test_error_duplicate_rows_single_index(self, config_handler):
spec = Spec(name='test', dims=['a'], coords={'a': [1, 2]}, dtype='int')
path = os.path.join(config_handler.data_folders['parameters'],
'default.csv')
with open(path, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=['test', 'a'])
writer.writeheader()
writer.writerows([
{
'a': 1,
'test': 0
},
{
'a': 2,
'test': 1
},
{
'a': 1,
'test': 2
},
])
with raises(SmifDataMismatchError) as ex:
config_handler.read_model_parameter_default('default', spec)
msg = "Data for 'test' contains duplicate values at [{'a': 1}]"
assert msg in str(ex.value)
def test_single_dim_order(self):
spec = Spec(name='test',
dims=['technology_type'],
coords={
'technology_type':
['water_meter', 'electricity_meter', 'other', 'aaa']
},
dtype='float')
df = pd.DataFrame([
{
'technology_type': 'water_meter',
'test': 5
},
{
'technology_type': 'electricity_meter',
'test': 6
},
{
'technology_type': 'other',
'test': 7
},
{
'technology_type': 'aaa',
'test': 8
},
])
da = DataArray(spec, numpy.array([5., 6., 7., 8.]))
da_from_df = DataArray.from_df(spec, df)
da_from_df_2 = DataArray.from_df(spec, df)
assert da == da_from_df
assert da == da_from_df_2
def test_from_dict(self):
"""classmethod to construct from serialisation
"""
spec = Spec.from_dict({
'name': 'population',
'description': 'Population by age class',
'dims': ['countries', 'age'],
'coords': {
'age': [">30", "<30"],
'countries': ["England", "Wales"]
},
'dtype': 'int',
'abs_range': (0, float('inf')),
'exp_range': (10e6, 10e9),
'unit': 'people'
})
assert spec.name == 'population'
assert spec.description == 'Population by age class'
assert spec.unit == 'people'
assert spec.abs_range == (0, float('inf'))
assert spec.exp_range == (10e6, 10e9)
assert spec.dtype == 'int'
assert spec.shape == (2, 2)
assert spec.ndim == 2
assert spec.dims == ['countries', 'age']
assert spec.coords == [
Coordinates('countries', ["England", "Wales"]),
Coordinates('age', [">30", "<30"])
]
def test_empty_dtype_error(self):
"""A Spec must be constructed with a dtype
"""
with raises(ValueError) as ex:
Spec(name='test',
coords=[Coordinates('countries', ["England", "Wales"])])
assert "dtype must be provided" in str(ex.value)
def test_coords_type_error(self):
"""A Spec must be constructed with a list of Coordinates
"""
with raises(ValueError) as ex:
Spec(name='test', dtype='int', coords=["England", "Wales"])
assert "coords may be a dict[str,list] or a list[Coordinates]" in str(
ex.value)
def test_equality(dep):
a = dep
b = copy(dep)
assert a == b
c = copy(dep)
c.source_model = Mock()
assert a != c
another_spec = Spec(name='another',
dtype='float',
dims=['z'],
coords={'z': [0, 1]})
d = copy(dep)
d.source = another_spec
assert a != d
e = copy(dep)
e.sink = another_spec
assert a != e
f = copy(dep)
f.sink_model = Mock()
assert a != f
g = copy(dep)
g.timestep = RelativeTimestep.BASE
assert a != g
def test_canonical_missing_results(self, store, sample_dimensions,
get_sos_model, get_sector_model,
energy_supply_sector_model, model_run):
for dim in sample_dimensions:
store.write_dimension(dim)
store.write_sos_model(get_sos_model)
store.write_model_run(model_run)
store.write_model(get_sector_model)
store.write_model(energy_supply_sector_model)
# All the results are missing
missing_results = set()
missing_results.add((2015, 0, 'energy_demand', 'gas_demand'))
missing_results.add((2020, 0, 'energy_demand', 'gas_demand'))
missing_results.add((2025, 0, 'energy_demand', 'gas_demand'))
assert (store.canonical_missing_results(
model_run['name']) == missing_results)
spec = Spec(name='gas_demand', dtype='float')
data = np.array(1, dtype=float)
fake_data = DataArray(spec, data)
store.write_results(fake_data, model_run['name'], 'energy_demand',
2015, 0)
missing_results.remove((2015, 0, 'energy_demand', 'gas_demand'))
assert (store.canonical_missing_results(
model_run['name']) == missing_results)
def sample_narrative_data(sample_narratives, get_sector_model,
energy_supply_sector_model,
water_supply_sector_model):
narrative_data = {}
sos_model_name = 'energy'
sector_models = {}
sector_models[get_sector_model['name']] = get_sector_model
sector_models[
energy_supply_sector_model['name']] = energy_supply_sector_model
sector_models[
water_supply_sector_model['name']] = water_supply_sector_model
for narrative in sample_narratives:
for sector_model_name, param_names in narrative['provides'].items():
sector_model = sector_models[sector_model_name]
for param_name in param_names:
param = _pick_from_list(sector_model['parameters'], param_name)
for variant in narrative['variants']:
spec = Spec.from_dict(param)
nda = np.random.random(spec.shape)
da = DataArray(spec, nda)
key = (sos_model_name, narrative['name'], variant['name'],
param_name)
narrative_data[key] = da
return narrative_data
