def scenario_model(): """ScenarioModel providing precipitation """ model = ScenarioModel('climate') model.add_output( Spec.from_dict({ 'name': 'precipitation', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'ml' })) model.add_output( Spec.from_dict({ 'name': 'reservoir_level', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'ml' })) model.scenario = 'UKCP09 High emissions' return model
def from_dict(cls, config): """Create object from dictionary serialisation """ model = cls(config['name']) model.description = config['description'] for input_ in config['inputs']: model.add_input(Spec.from_dict(input_)) for output in config['outputs']: model.add_output(Spec.from_dict(output)) for param in config['parameters']: model.add_parameter(Spec.from_dict(param)) return model
def test_construct(self, sector_model): assert sector_model.description == 'a description' assert sector_model.inputs == { 'raininess': Spec.from_dict({ 'name': 'raininess', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'milliliter' }) } spec = Spec.from_dict({ 'name': 'assump_diff_floorarea_pp', 'description': 'Difference in floor area per person', 'dims': ['national'], 'coords': { 'national': ['GB'] }, 'abs_range': (0.5, 2), 'exp_range': (0.5, 2), 'dtype': 'float', 'unit': '%' }) assert sector_model.parameters == {'assump_diff_floorarea_pp': spec} assert sector_model.outputs == { 'cost': Spec.from_dict({ 'name': 'cost', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'million GBP' }), 'water': Spec.from_dict({ 'name': 'water', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'megaliter' }) }
def test_as_dict_multi_output(self, scenario_model, scenario_model_dict): # with additional output scenario_model.add_output( Spec.from_dict({ 'name': 'population_density', 'dims': ['LSOA'], 'coords': { 'LSOA': ['a', 'b', 'c'] }, 'dtype': 'int', 'unit': 'people / kilometer ** 2' })) scenario_model_dict['outputs'].append({ 'name': 'population_density', 'dims': ['LSOA'], 'coords': { 'LSOA': ['a', 'b', 'c'] }, 'dtype': 'int', 'unit': 'people / kilometer ** 2', 'abs_range': None, 'exp_range': None, 'description': None }) actual = scenario_model.as_dict() expected = scenario_model_dict # equivalent up to order of outputs actual['outputs'].sort(key=lambda p: p['name']) expected['outputs'].sort(key=lambda p: p['name']) assert actual == expected
def sample_narrative_data(sample_narratives, get_sector_model, energy_supply_sector_model, water_supply_sector_model): narrative_data = {} sos_model_name = 'energy' sector_models = {} sector_models[get_sector_model['name']] = get_sector_model sector_models[ energy_supply_sector_model['name']] = energy_supply_sector_model sector_models[ water_supply_sector_model['name']] = water_supply_sector_model for narrative in sample_narratives: for sector_model_name, param_names in narrative['provides'].items(): sector_model = sector_models[sector_model_name] for param_name in param_names: param = _pick_from_list(sector_model['parameters'], param_name) for variant in narrative['variants']: spec = Spec.from_dict(param) nda = np.random.random(spec.shape) da = DataArray(spec, nda) key = (sos_model_name, narrative['name'], variant['name'], param_name) narrative_data[key] = da return narrative_data
def test_from_dict(self): """classmethod to construct from serialisation """ spec = Spec.from_dict({ 'name': 'population', 'description': 'Population by age class', 'dims': ['countries', 'age'], 'coords': { 'age': [">30", "<30"], 'countries': ["England", "Wales"] }, 'dtype': 'int', 'abs_range': (0, float('inf')), 'exp_range': (10e6, 10e9), 'unit': 'people' }) assert spec.name == 'population' assert spec.description == 'Population by age class' assert spec.unit == 'people' assert spec.abs_range == (0, float('inf')) assert spec.exp_range == (10e6, 10e9) assert spec.dtype == 'int' assert spec.shape == (2, 2) assert spec.ndim == 2 assert spec.dims == ['countries', 'age'] assert spec.coords == [ Coordinates('countries', ["England", "Wales"]), Coordinates('age', [">30", "<30"]) ]
def test_narrative_data(self, setup_folder_structure, config_handler, get_narrative): """ Test to dump a narrative (yml) data-file and then read the file using the datafile interface. Finally check the data shows up in the returned dictionary. """ basefolder = setup_folder_structure filepath = os.path.join(str(basefolder), 'data', 'narratives', 'central_planning.csv') with open(filepath, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=['homogeneity_coefficient']) writer.writeheader() writer.writerow({'homogeneity_coefficient': 8}) spec = Spec.from_dict({ 'name': 'homogeneity_coefficient', 'description': "How homegenous the centralisation process is", 'absolute_range': [0, 1], 'expected_range': [0, 1], 'unit': 'percentage', 'dtype': 'float' }) actual = config_handler.read_narrative_variant_data( 'central_planning', spec) assert actual == DataArray(spec, np.array(8, dtype=float))
def sector_model(): model = WaterSupplySectorModel('water_supply') model.description = 'a description' model.add_input( Spec.from_dict({ 'name': 'raininess', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'milliliter' })) model.add_parameter( Spec.from_dict({ 'name': 'assump_diff_floorarea_pp', 'description': 'Difference in floor area per person', 'dims': ['national'], 'coords': { 'national': ['GB'] }, 'abs_range': (0.5, 2), 'exp_range': (0.5, 2), 'dtype': 'float', 'unit': '%' })) model.add_output( Spec.from_dict({ 'name': 'cost', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'million GBP' })) model.add_output( Spec.from_dict({ 'name': 'water', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'megaliter' })) return model
def test_narrative_data_missing(self, config_handler): """Should raise a SmifDataNotFoundError if narrative has no data """ spec = Spec.from_dict({ 'name': 'homogeneity_coefficient', 'unit': 'percentage', 'dtype': 'float' }) with raises(SmifDataNotFoundError): config_handler.read_narrative_variant_data('does not exist', spec)
def test_read_write_data_array(self, handler, scenario): spec_config = deepcopy(scenario['provides'][0]) spec_config['dims'] = ['timestep'] + spec_config['dims'] spec_config['coords']['timestep'] = [{'name': 2010}] spec = Spec.from_dict(spec_config) data = np.array([[0, 1]], dtype='float') da = DataArray(spec, data) handler.write_scenario_variant_data('mortality.csv', da) spec_config = deepcopy(scenario['provides'][0]) spec = Spec.from_dict(spec_config) data = np.array([0, 1], dtype='float') expected = DataArray(spec, data) actual = handler.read_scenario_variant_data('mortality.csv', spec, 2010) assert actual == expected np.testing.assert_array_equal(actual.as_ndarray(), expected.as_ndarray())
def from_dict(cls, data): """Create ScenarioModel from dict serialisation """ scenario = cls(data['name']) scenario.scenario = data['scenario'] if 'description' in data: scenario.description = data['description'] for output in data['outputs']: spec = Spec.from_dict(output) scenario.add_output(spec) return scenario
def get_sector_model_parameter_defaults(get_sector_model): """DataArray for each parameter default """ data = { 'smart_meter_savings': np.array(0.5), 'homogeneity_coefficient': np.array(0.1) } for param in get_sector_model['parameters']: nda = data[param['name']] spec = Spec.from_dict(param) data[param['name']] = DataArray(spec, nda) return data
def sample_gas_demand_results(lad, hourly): spec = Spec.from_dict({ 'name': 'gas_demand', 'dtype': 'float', 'dims': ['lad', 'hourly'], 'coords': { 'lad': lad, 'hourly': hourly } }) data = np.zeros(spec.shape, dtype=float) return DataArray(spec, data)
def test_df_round_trip_2d(self): spec = Spec.from_dict({ 'name': 'two_d', 'dims': ['a', 'z'], 'coords': { 'a': ['q', 'p'], 'z': ['a', 'c', 'b'], }, 'dtype': 'float' }) da = DataArray(spec, numpy.array([ [5., 6., 7.], [8., 9., 0.], ])) df = pd.DataFrame([ { 'z': 'a', 'a': 'p', 'two_d': 8. }, { 'z': 'c', 'a': 'q', 'two_d': 6. }, { 'z': 'a', 'a': 'q', 'two_d': 5. }, { 'z': 'b', 'a': 'q', 'two_d': 7. }, { 'z': 'b', 'a': 'p', 'two_d': 0. }, { 'z': 'c', 'a': 'p', 'two_d': 9. }, ]) df = df.set_index(spec.dims) df_from_da = da.as_df() da_from_df = DataArray.from_df(spec, df_from_da) assert_array_equal(da.data, da_from_df.data)
def test_read_data_array_missing_timestep(self, handler, scenario): data = np.array([[0, 1]], dtype=float) spec_config = deepcopy(scenario['provides'][0]) spec_config['dims'] = ['timestep'] + spec_config['dims'] spec_config['coords']['timestep'] = [{'name': 2010}] spec = Spec.from_dict(spec_config) da = DataArray(spec, data) handler.write_scenario_variant_data('mortality.csv', da) msg = "not found for timestep 2011" with raises(SmifDataNotFoundError) as ex: handler.read_scenario_variant_data('mortality.csv', spec, 2011) assert msg in str(ex.value)
def test_from_dict_defaults(self): """classmethod to construct from serialisation """ spec = Spec.from_dict({'dtype': 'int'}) assert spec.name is None assert spec.description is None assert spec.unit is None assert spec.abs_range is None assert spec.exp_range is None assert spec.dtype == 'int' assert spec.shape == () assert spec.ndim == 0 assert spec.dims == [] assert spec.coords == []
def test_read_write_data_array_all(self, handler, scenario): spec = Spec.from_dict(deepcopy(scenario['provides'][0])) spec_with_t = scenario['provides'][0] spec_with_t['dims'].insert(0, 'timestep') spec_with_t['coords']['timestep'] = [2010, 2015] spec_with_t = Spec.from_dict(spec_with_t) da = DataArray(spec_with_t, np.array([[0, 1], [2, 3]], dtype='float')) handler.write_scenario_variant_data('mortality.csv', da) actual = handler.read_scenario_variant_data('mortality.csv', spec_with_t) expected = np.array([[0, 1], [2, 3]], dtype='float') np.testing.assert_array_equal(actual.as_ndarray(), expected) da_2010 = handler.read_scenario_variant_data('mortality.csv', spec, 2010) expected = np.array([0, 1], dtype='float') np.testing.assert_array_equal(da_2010.as_ndarray(), expected) da_2015 = handler.read_scenario_variant_data('mortality.csv', spec, 2015) expected = np.array([2, 3], dtype='float') np.testing.assert_array_equal(da_2015.as_ndarray(), expected)
def economic_model(): """SectorModel requiring precipitation and cost, providing water """ model = EmptySectorModel('economic_model') model.add_output( Spec.from_dict({ 'name': 'gva', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'million GBP' })) return model
def test_construct(self, scenario_model): assert scenario_model.name == 'population' assert scenario_model.scenario == 'High Population (ONS)' assert scenario_model.description == \ 'The High ONS Forecast for UK population out to 2050' assert scenario_model.outputs == { 'population_count': Spec.from_dict({ 'name': 'population_count', 'dims': ['LSOA'], 'coords': { 'LSOA': ['a', 'b', 'c'] }, 'dtype': 'int', 'unit': 'people' }) }
def sample_scenario_data(scenario_with_timestep, get_sector_model, energy_supply_sector_model, water_supply_sector_model): scenario_data = {} for scenario in [scenario_with_timestep]: for variant in scenario['variants']: for data_key, data_value in variant['data'].items(): spec_dict = [ provides for provides in scenario['provides'] if provides['name'] == data_key ][0] spec = Spec.from_dict(spec_dict) nda = np.random.random(spec.shape) da = DataArray(spec, nda) key = (scenario['name'], variant['name'], data_key) scenario_data[key] = da return scenario_data
def test_add_parameter(self, empty_sector_model): """Adding a parameter adds a reference to the parameter list entry to the model that contains it. """ spec = Spec.from_dict({ 'name': 'smart_meter_savings', 'description': 'The savings from smart meters', 'abs_range': (0, 100), 'exp_range': (3, 10), 'dims': ['national'], 'coords': { 'national': ['GB'] }, 'dtype': 'float', 'unit': '%' }) empty_sector_model.add_parameter(spec) expected = spec actual = empty_sector_model.parameters['smart_meter_savings'] assert actual == expected
def test_df_round_trip(self): spec = Spec.from_dict({ 'name': 'multi_savings', 'description': 'The savings from various technologies', 'dims': ['technology_type'], 'coords': { 'technology_type': ['water_meter', 'electricity_meter', 'other', 'aaa'] }, 'dtype': 'float', 'abs_range': (0, 100), 'exp_range': (3, 10), 'unit': '%' }) da = DataArray(spec, numpy.array([5., 6., 7., 8.])) df = pd.DataFrame([ { 'technology_type': 'water_meter', 'multi_savings': 5. }, { 'technology_type': 'electricity_meter', 'multi_savings': 6. }, { 'technology_type': 'other', 'multi_savings': 7. }, { 'technology_type': 'aaa', 'multi_savings': 8. }, ]) df = df.set_index(spec.dims) df_from_da = da.as_df() da_from_df = DataArray.from_df(spec, df_from_da) assert_array_equal(da.data, da_from_df.data)
def results_with_results(results_no_results): sample_output = { 'name': 'sample_output', 'dtype': 'int', 'dims': ['sample_dim', 'sample_dim_colour'], 'coords': { 'sample_dim': [{'name': 'a'}, {'name': 'b'}], 'sample_dim_colour': [{'name': 'red'}, {'name': 'green'}, {'name': 'blue'}], }, 'unit': 'm' } spec = Spec.from_dict(sample_output) data = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int32) sample_results = DataArray(spec, data) results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2010, 0) results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2015, 0) results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2020, 0) results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2015, 1) results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2020, 1) results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2015, 2) results_no_results._store.write_results(sample_results, 'model_run_1', 'a_model', 2020, 2) results_no_results._store.write_results(sample_results, 'model_run_1', 'b_model', 2010, 0) results_no_results._store.write_results(sample_results, 'model_run_1', 'b_model', 2015, 0) results_no_results._store.write_results(sample_results, 'model_run_1', 'b_model', 2020, 0) results_no_results._store.write_results(sample_results, 'model_run_1', 'b_model', 2025, 0) results_no_results._store.write_results(sample_results, 'model_run_1', 'b_model', 2030, 0) results_no_results._store.write_results(sample_results, 'model_run_2', 'b_model', 2010, 0) results_no_results._store.write_results(sample_results, 'model_run_2', 'b_model', 2015, 0) results_no_results._store.write_results(sample_results, 'model_run_2', 'b_model', 2020, 0) results_no_results._store.write_results(sample_results, 'model_run_2', 'b_model', 2025, 0) results_no_results._store.write_results(sample_results, 'model_run_2', 'b_model', 2030, 0) return results_no_results
def get_multidimensional_param(): spec = Spec.from_dict({ 'name': 'ss_t_base_heating', 'description': 'Industrial base temperature', 'default': '../energy_demand/parameters/ss_t_base_heating.csv', 'unit': '', 'dims': ['interpolation_params', 'end_yr'], 'coords': { 'interpolation_params': ['diffusion_choice', 'value_ey'], 'end_yr': [2030, 2050] }, 'dtype': 'float' }) dataframe = pd.DataFrame([ { 'interpolation_params': 'diffusion_choice', 'end_yr': 2030, 'ss_t_base_heating': 0 }, { 'interpolation_params': 'diffusion_choice', 'end_yr': 2050, 'ss_t_base_heating': 0 }, { 'interpolation_params': 'value_ey', 'end_yr': 2030, 'ss_t_base_heating': 15.5 }, { 'interpolation_params': 'value_ey', 'end_yr': 2050, 'ss_t_base_heating': 15.5 }, ]).set_index(['interpolation_params', 'end_yr']) return DataArray.from_df(spec, dataframe)
def sector_model(): """SectorModel requiring precipitation and cost, providing water """ model = EmptySectorModel('water_supply') model.add_input( Spec.from_dict({ 'name': 'precipitation', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'ml' })) model.add_input( Spec.from_dict({ 'name': 'reservoir_level', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'ml' })) model.add_input( Spec.from_dict({ 'name': 'rGVA', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'million GBP' })) model.add_output( Spec.from_dict({ 'name': 'water', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'Ml' })) model.add_output( Spec.from_dict({ 'name': 'reservoir_level', 'dims': ['LSOA'], 'coords': { 'LSOA': [1, 2, 3] }, 'dtype': 'float', 'unit': 'ml' })) model.add_parameter( Spec.from_dict({ 'name': 'test_parameter', 'description': 'a dummy parameter to test narratives', 'dims': ['national'], 'coords': { 'national': ['GB'] }, 'abs_range': (0.5, 2), 'exp_range': (0.5, 2), 'dtype': 'float', 'default': [[1.]], 'unit': '%' })) return model
def _set_spec_timesteps(spec, timesteps): spec_config = spec.as_dict() if 'timestep' not in spec_config['dims']: spec_config['dims'] = ['timestep'] + spec_config['dims'] spec_config['coords']['timestep'] = timesteps return Spec.from_dict(spec_config)