def scenario_model():
"""ScenarioModel providing precipitation
"""
model = ScenarioModel('climate')
model.add_output(
Spec.from_dict({
'name': 'precipitation',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'ml'
}))
model.add_output(
Spec.from_dict({
'name': 'reservoir_level',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'ml'
}))
model.scenario = 'UKCP09 High emissions'
return model
def from_dict(cls, config):
"""Create object from dictionary serialisation
"""
model = cls(config['name'])
model.description = config['description']
for input_ in config['inputs']:
model.add_input(Spec.from_dict(input_))
for output in config['outputs']:
model.add_output(Spec.from_dict(output))
for param in config['parameters']:
model.add_parameter(Spec.from_dict(param))
return model
def test_construct(self, sector_model):
assert sector_model.description == 'a description'
assert sector_model.inputs == {
'raininess':
Spec.from_dict({
'name': 'raininess',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'milliliter'
})
}
spec = Spec.from_dict({
'name': 'assump_diff_floorarea_pp',
'description': 'Difference in floor area per person',
'dims': ['national'],
'coords': {
'national': ['GB']
},
'abs_range': (0.5, 2),
'exp_range': (0.5, 2),
'dtype': 'float',
'unit': '%'
})
assert sector_model.parameters == {'assump_diff_floorarea_pp': spec}
assert sector_model.outputs == {
'cost':
Spec.from_dict({
'name': 'cost',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'million GBP'
}),
'water':
Spec.from_dict({
'name': 'water',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'megaliter'
})
}
def test_as_dict_multi_output(self, scenario_model, scenario_model_dict):
# with additional output
scenario_model.add_output(
Spec.from_dict({
'name': 'population_density',
'dims': ['LSOA'],
'coords': {
'LSOA': ['a', 'b', 'c']
},
'dtype': 'int',
'unit': 'people / kilometer ** 2'
}))
scenario_model_dict['outputs'].append({
'name': 'population_density',
'dims': ['LSOA'],
'coords': {
'LSOA': ['a', 'b', 'c']
},
'dtype': 'int',
'unit': 'people / kilometer ** 2',
'abs_range': None,
'exp_range': None,
'description': None
})
actual = scenario_model.as_dict()
expected = scenario_model_dict
# equivalent up to order of outputs
actual['outputs'].sort(key=lambda p: p['name'])
expected['outputs'].sort(key=lambda p: p['name'])
assert actual == expected
def sample_narrative_data(sample_narratives, get_sector_model,
energy_supply_sector_model,
water_supply_sector_model):
narrative_data = {}
sos_model_name = 'energy'
sector_models = {}
sector_models[get_sector_model['name']] = get_sector_model
sector_models[
energy_supply_sector_model['name']] = energy_supply_sector_model
sector_models[
water_supply_sector_model['name']] = water_supply_sector_model
for narrative in sample_narratives:
for sector_model_name, param_names in narrative['provides'].items():
sector_model = sector_models[sector_model_name]
for param_name in param_names:
param = _pick_from_list(sector_model['parameters'], param_name)
for variant in narrative['variants']:
spec = Spec.from_dict(param)
nda = np.random.random(spec.shape)
da = DataArray(spec, nda)
key = (sos_model_name, narrative['name'], variant['name'],
param_name)
narrative_data[key] = da
return narrative_data
def test_from_dict(self):
"""classmethod to construct from serialisation
"""
spec = Spec.from_dict({
'name': 'population',
'description': 'Population by age class',
'dims': ['countries', 'age'],
'coords': {
'age': [">30", "<30"],
'countries': ["England", "Wales"]
},
'dtype': 'int',
'abs_range': (0, float('inf')),
'exp_range': (10e6, 10e9),
'unit': 'people'
})
assert spec.name == 'population'
assert spec.description == 'Population by age class'
assert spec.unit == 'people'
assert spec.abs_range == (0, float('inf'))
assert spec.exp_range == (10e6, 10e9)
assert spec.dtype == 'int'
assert spec.shape == (2, 2)
assert spec.ndim == 2
assert spec.dims == ['countries', 'age']
assert spec.coords == [
Coordinates('countries', ["England", "Wales"]),
Coordinates('age', [">30", "<30"])
]
def test_narrative_data(self, setup_folder_structure, config_handler,
get_narrative):
""" Test to dump a narrative (yml) data-file and then read the file
using the datafile interface. Finally check the data shows up in the
returned dictionary.
"""
basefolder = setup_folder_structure
filepath = os.path.join(str(basefolder), 'data', 'narratives',
'central_planning.csv')
with open(filepath, 'w') as csvfile:
writer = csv.DictWriter(csvfile,
fieldnames=['homogeneity_coefficient'])
writer.writeheader()
writer.writerow({'homogeneity_coefficient': 8})
spec = Spec.from_dict({
'name': 'homogeneity_coefficient',
'description': "How homegenous the centralisation process is",
'absolute_range': [0, 1],
'expected_range': [0, 1],
'unit': 'percentage',
'dtype': 'float'
})
actual = config_handler.read_narrative_variant_data(
'central_planning', spec)
assert actual == DataArray(spec, np.array(8, dtype=float))
def sector_model():
model = WaterSupplySectorModel('water_supply')
model.description = 'a description'
model.add_input(
Spec.from_dict({
'name': 'raininess',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'milliliter'
}))
model.add_parameter(
Spec.from_dict({
'name': 'assump_diff_floorarea_pp',
'description': 'Difference in floor area per person',
'dims': ['national'],
'coords': {
'national': ['GB']
},
'abs_range': (0.5, 2),
'exp_range': (0.5, 2),
'dtype': 'float',
'unit': '%'
}))
model.add_output(
Spec.from_dict({
'name': 'cost',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'million GBP'
}))
model.add_output(
Spec.from_dict({
'name': 'water',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'megaliter'
}))
return model
def test_narrative_data_missing(self, config_handler):
"""Should raise a SmifDataNotFoundError if narrative has no data
"""
spec = Spec.from_dict({
'name': 'homogeneity_coefficient',
'unit': 'percentage',
'dtype': 'float'
})
with raises(SmifDataNotFoundError):
config_handler.read_narrative_variant_data('does not exist', spec)
def test_read_write_data_array(self, handler, scenario):
spec_config = deepcopy(scenario['provides'][0])
spec_config['dims'] = ['timestep'] + spec_config['dims']
spec_config['coords']['timestep'] = [{'name': 2010}]
spec = Spec.from_dict(spec_config)
data = np.array([[0, 1]], dtype='float')
da = DataArray(spec, data)
handler.write_scenario_variant_data('mortality.csv', da)
spec_config = deepcopy(scenario['provides'][0])
spec = Spec.from_dict(spec_config)
data = np.array([0, 1], dtype='float')
expected = DataArray(spec, data)
actual = handler.read_scenario_variant_data('mortality.csv', spec,
2010)
assert actual == expected
np.testing.assert_array_equal(actual.as_ndarray(),
expected.as_ndarray())
def from_dict(cls, data):
"""Create ScenarioModel from dict serialisation
"""
scenario = cls(data['name'])
scenario.scenario = data['scenario']
if 'description' in data:
scenario.description = data['description']
for output in data['outputs']:
spec = Spec.from_dict(output)
scenario.add_output(spec)
return scenario
def get_sector_model_parameter_defaults(get_sector_model):
"""DataArray for each parameter default
"""
data = {
'smart_meter_savings': np.array(0.5),
'homogeneity_coefficient': np.array(0.1)
}
for param in get_sector_model['parameters']:
nda = data[param['name']]
spec = Spec.from_dict(param)
data[param['name']] = DataArray(spec, nda)
return data
def sample_gas_demand_results(lad, hourly):
spec = Spec.from_dict({
'name': 'gas_demand',
'dtype': 'float',
'dims': ['lad', 'hourly'],
'coords': {
'lad': lad,
'hourly': hourly
}
})
data = np.zeros(spec.shape, dtype=float)
return DataArray(spec, data)
def test_df_round_trip_2d(self):
spec = Spec.from_dict({
'name': 'two_d',
'dims': ['a', 'z'],
'coords': {
'a': ['q', 'p'],
'z': ['a', 'c', 'b'],
},
'dtype': 'float'
})
da = DataArray(spec, numpy.array([
[5., 6., 7.],
[8., 9., 0.],
]))
df = pd.DataFrame([
{
'z': 'a',
'a': 'p',
'two_d': 8.
},
{
'z': 'c',
'a': 'q',
'two_d': 6.
},
{
'z': 'a',
'a': 'q',
'two_d': 5.
},
{
'z': 'b',
'a': 'q',
'two_d': 7.
},
{
'z': 'b',
'a': 'p',
'two_d': 0.
},
{
'z': 'c',
'a': 'p',
'two_d': 9.
},
])
df = df.set_index(spec.dims)
df_from_da = da.as_df()
da_from_df = DataArray.from_df(spec, df_from_da)
assert_array_equal(da.data, da_from_df.data)
def test_read_data_array_missing_timestep(self, handler, scenario):
data = np.array([[0, 1]], dtype=float)
spec_config = deepcopy(scenario['provides'][0])
spec_config['dims'] = ['timestep'] + spec_config['dims']
spec_config['coords']['timestep'] = [{'name': 2010}]
spec = Spec.from_dict(spec_config)
da = DataArray(spec, data)
handler.write_scenario_variant_data('mortality.csv', da)
msg = "not found for timestep 2011"
with raises(SmifDataNotFoundError) as ex:
handler.read_scenario_variant_data('mortality.csv', spec, 2011)
assert msg in str(ex.value)
def test_from_dict_defaults(self):
"""classmethod to construct from serialisation
"""
spec = Spec.from_dict({'dtype': 'int'})
assert spec.name is None
assert spec.description is None
assert spec.unit is None
assert spec.abs_range is None
assert spec.exp_range is None
assert spec.dtype == 'int'
assert spec.shape == ()
assert spec.ndim == 0
assert spec.dims == []
assert spec.coords == []
def test_read_write_data_array_all(self, handler, scenario):
spec = Spec.from_dict(deepcopy(scenario['provides'][0]))
spec_with_t = scenario['provides'][0]
spec_with_t['dims'].insert(0, 'timestep')
spec_with_t['coords']['timestep'] = [2010, 2015]
spec_with_t = Spec.from_dict(spec_with_t)
da = DataArray(spec_with_t, np.array([[0, 1], [2, 3]], dtype='float'))
handler.write_scenario_variant_data('mortality.csv', da)
actual = handler.read_scenario_variant_data('mortality.csv',
spec_with_t)
expected = np.array([[0, 1], [2, 3]], dtype='float')
np.testing.assert_array_equal(actual.as_ndarray(), expected)
da_2010 = handler.read_scenario_variant_data('mortality.csv', spec,
2010)
expected = np.array([0, 1], dtype='float')
np.testing.assert_array_equal(da_2010.as_ndarray(), expected)
da_2015 = handler.read_scenario_variant_data('mortality.csv', spec,
2015)
expected = np.array([2, 3], dtype='float')
np.testing.assert_array_equal(da_2015.as_ndarray(), expected)
def economic_model():
"""SectorModel requiring precipitation and cost, providing water
"""
model = EmptySectorModel('economic_model')
model.add_output(
Spec.from_dict({
'name': 'gva',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'million GBP'
}))
return model
def test_construct(self, scenario_model):
assert scenario_model.name == 'population'
assert scenario_model.scenario == 'High Population (ONS)'
assert scenario_model.description == \
'The High ONS Forecast for UK population out to 2050'
assert scenario_model.outputs == {
'population_count':
Spec.from_dict({
'name': 'population_count',
'dims': ['LSOA'],
'coords': {
'LSOA': ['a', 'b', 'c']
},
'dtype': 'int',
'unit': 'people'
})
}
def sample_scenario_data(scenario_with_timestep, get_sector_model,
energy_supply_sector_model,
water_supply_sector_model):
scenario_data = {}
for scenario in [scenario_with_timestep]:
for variant in scenario['variants']:
for data_key, data_value in variant['data'].items():
spec_dict = [
provides for provides in scenario['provides']
if provides['name'] == data_key
][0]
spec = Spec.from_dict(spec_dict)
nda = np.random.random(spec.shape)
da = DataArray(spec, nda)
key = (scenario['name'], variant['name'], data_key)
scenario_data[key] = da
return scenario_data
def test_add_parameter(self, empty_sector_model):
"""Adding a parameter adds a reference to the parameter list entry to
the model that contains it.
"""
spec = Spec.from_dict({
'name': 'smart_meter_savings',
'description': 'The savings from smart meters',
'abs_range': (0, 100),
'exp_range': (3, 10),
'dims': ['national'],
'coords': {
'national': ['GB']
},
'dtype': 'float',
'unit': '%'
})
empty_sector_model.add_parameter(spec)
expected = spec
actual = empty_sector_model.parameters['smart_meter_savings']
assert actual == expected
def test_df_round_trip(self):
spec = Spec.from_dict({
'name': 'multi_savings',
'description': 'The savings from various technologies',
'dims': ['technology_type'],
'coords': {
'technology_type':
['water_meter', 'electricity_meter', 'other', 'aaa']
},
'dtype': 'float',
'abs_range': (0, 100),
'exp_range': (3, 10),
'unit': '%'
})
da = DataArray(spec, numpy.array([5., 6., 7., 8.]))
df = pd.DataFrame([
{
'technology_type': 'water_meter',
'multi_savings': 5.
},
{
'technology_type': 'electricity_meter',
'multi_savings': 6.
},
{
'technology_type': 'other',
'multi_savings': 7.
},
{
'technology_type': 'aaa',
'multi_savings': 8.
},
])
df = df.set_index(spec.dims)
df_from_da = da.as_df()
da_from_df = DataArray.from_df(spec, df_from_da)
assert_array_equal(da.data, da_from_df.data)
def results_with_results(results_no_results):
sample_output = {
'name': 'sample_output',
'dtype': 'int',
'dims': ['sample_dim', 'sample_dim_colour'],
'coords': {
'sample_dim': [{'name': 'a'}, {'name': 'b'}],
'sample_dim_colour': [{'name': 'red'}, {'name': 'green'}, {'name': 'blue'}],
},
'unit': 'm'
}
spec = Spec.from_dict(sample_output)
data = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int32)
sample_results = DataArray(spec, data)
results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2010, 0)
results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2015, 0)
results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2020, 0)
results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2015, 1)
results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2020, 1)
results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2015, 2)
results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2020, 2)
results_no_results._store.write_results(sample_results, 'model_run_1', 'b_model', 2010, 0)
results_no_results._store.write_results(sample_results, 'model_run_1', 'b_model', 2015, 0)
results_no_results._store.write_results(sample_results, 'model_run_1', 'b_model', 2020, 0)
results_no_results._store.write_results(sample_results, 'model_run_1', 'b_model', 2025, 0)
results_no_results._store.write_results(sample_results, 'model_run_1', 'b_model', 2030, 0)
results_no_results._store.write_results(sample_results, 'model_run_2', 'b_model', 2010, 0)
results_no_results._store.write_results(sample_results, 'model_run_2', 'b_model', 2015, 0)
results_no_results._store.write_results(sample_results, 'model_run_2', 'b_model', 2020, 0)
results_no_results._store.write_results(sample_results, 'model_run_2', 'b_model', 2025, 0)
results_no_results._store.write_results(sample_results, 'model_run_2', 'b_model', 2030, 0)
return results_no_results
def get_multidimensional_param():
spec = Spec.from_dict({
'name': 'ss_t_base_heating',
'description': 'Industrial base temperature',
'default': '../energy_demand/parameters/ss_t_base_heating.csv',
'unit': '',
'dims': ['interpolation_params', 'end_yr'],
'coords': {
'interpolation_params': ['diffusion_choice', 'value_ey'],
'end_yr': [2030, 2050]
},
'dtype': 'float'
})
dataframe = pd.DataFrame([
{
'interpolation_params': 'diffusion_choice',
'end_yr': 2030,
'ss_t_base_heating': 0
},
{
'interpolation_params': 'diffusion_choice',
'end_yr': 2050,
'ss_t_base_heating': 0
},
{
'interpolation_params': 'value_ey',
'end_yr': 2030,
'ss_t_base_heating': 15.5
},
{
'interpolation_params': 'value_ey',
'end_yr': 2050,
'ss_t_base_heating': 15.5
},
]).set_index(['interpolation_params', 'end_yr'])
return DataArray.from_df(spec, dataframe)
def sector_model():
"""SectorModel requiring precipitation and cost, providing water
"""
model = EmptySectorModel('water_supply')
model.add_input(
Spec.from_dict({
'name': 'precipitation',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'ml'
}))
model.add_input(
Spec.from_dict({
'name': 'reservoir_level',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'ml'
}))
model.add_input(
Spec.from_dict({
'name': 'rGVA',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'million GBP'
}))
model.add_output(
Spec.from_dict({
'name': 'water',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'Ml'
}))
model.add_output(
Spec.from_dict({
'name': 'reservoir_level',
'dims': ['LSOA'],
'coords': {
'LSOA': [1, 2, 3]
},
'dtype': 'float',
'unit': 'ml'
}))
model.add_parameter(
Spec.from_dict({
'name': 'test_parameter',
'description': 'a dummy parameter to test narratives',
'dims': ['national'],
'coords': {
'national': ['GB']
},
'abs_range': (0.5, 2),
'exp_range': (0.5, 2),
'dtype': 'float',
'default': [[1.]],
'unit': '%'
}))
return model
def _set_spec_timesteps(spec, timesteps):
spec_config = spec.as_dict()
if 'timestep' not in spec_config['dims']:
spec_config['dims'] = ['timestep'] + spec_config['dims']
spec_config['coords']['timestep'] = timesteps
return Spec.from_dict(spec_config)
