def test_mean_flow_recorder(solver): model = Model(solver=solver) model.timestepper.start = pandas.to_datetime("2016-01-01") model.timestepper.end = pandas.to_datetime("2016-01-04") inpt = Input(model, "input") otpt = Output(model, "output") inpt.connect(otpt) rec_flow = NumpyArrayNodeRecorder(model, inpt) rec_mean = MeanFlowRecorder(model, node=inpt, timesteps=3) scenario = Scenario(model, "dummy", size=2) inpt.max_flow = inpt.min_flow = FunctionParameter(model, inpt, lambda model, t, si: 2 + t.index) model.run() expected = [ 2.0, (2.0 + 3.0) / 2, (2.0 + 3.0 + 4.0) / 3, (3.0 + 4.0 + 5.0) / 3, # zeroth day forgotten ] for value, expected_value in zip(rec_mean.data[:, 0], expected): assert_allclose(value, expected_value)
def test_scenario_storage(): """Test the behaviour of Storage nodes with multiple scenarios The model defined has two inflow scenarios: 5 and 10. It is expected that the volume in the storage node should increase at different rates in the two scenarios. """ model = Model() i = Input(model, 'input', max_flow=999) s = Storage(model, 'storage', num_inputs=1, num_outputs=1, max_volume=1000, initial_volume=500) o = Output(model, 'output', max_flow=999) scenario_input = Scenario(model, 'Inflow', size=2) i.min_flow = ConstantScenarioParameter(model, scenario_input, [5.0, 10.0]) i.connect(s) s.connect(o) s_rec = NumpyArrayStorageRecorder(model, s) model.run() assert_allclose(i.flow, [5, 10]) assert_allclose(s_rec.data[0], [505, 510]) assert_allclose(s_rec.data[1], [510, 520])
def test_mean_flow_recorder_days(solver): model = Model(solver=solver) model.timestepper.delta = 7 inpt = Input(model, "input") otpt = Output(model, "output") inpt.connect(otpt) rec_mean = MeanFlowRecorder(model, node=inpt, days=31) model.run() assert(rec_mean.timesteps == 4)
def test_scenario_collection(): """ Basic test of Scenario and ScenarioCollection API """ model = Model() # There is 1 combination when there are no Scenarios model.scenarios.setup() assert(len(model.scenarios.combinations) == 1) assert(len(model.scenarios) == 0) scA = Scenario(model, 'Scenario A', size=3) model.scenarios.setup() assert(len(model.scenarios.combinations) == 3) assert(len(model.scenarios) == 1) scA = Scenario(model, 'Scenario B', size=2) model.scenarios.setup() assert(len(model.scenarios.combinations) == 6) assert(len(model.scenarios) == 2) assert_equal([comb.indices for comb in model.scenarios.combinations], [[0, 0], [0, 1], [1, 0], [1, 1], [2, 0], [2, 1]]) names = model.scenarios.combination_names for n, (ia, ib) in zip(names, [[0, 0], [0, 1], [1, 0], [1, 1], [2, 0], [2, 1]]): assert n == 'Scenario A.{:03d}-Scenario B.{:03d}'.format(ia, ib) index = model.scenarios.multiindex assert_equal(index.tolist(), [[0, 0], [0, 1], [1, 0], [1, 1], [2, 0], [2, 1]]) assert_equal(index.names, ['Scenario A', 'Scenario B'])
def model(solver): model = Model( solver=solver, start=pandas.to_datetime("2016-01-01"), end=pandas.to_datetime("2016-01-10"), ) return model
def test_demand_saving_with_indexed_array(self, solver, tmpdir): """Test recording various items from demand saving example. This time the TablesRecorder is defined in JSON. """ import os, json, tables filename = "demand_saving_with_tables_recorder.json" # This is a bit horrible, but need to edit the JSON dynamically # so that the output.h5 is written in the temporary directory path = os.path.join(os.path.dirname(__file__), 'models') with open(os.path.join(path, filename), 'r') as f: data = f.read() data = json.loads(data) # Make an absolute, but temporary, path for the recorder url = data['recorders']['database']['url'] data['recorders']['database']['url'] = str(tmpdir.join(url)) model = Model.load(data, path=path, solver=solver) model.timestepper.end = "2016-01-31" model.check() # run model model.run() # run model again (to test reset behaviour) model.run() max_volume = model.nodes["Reservoir"].max_volume h5file = tmpdir.join('output.h5') with tables.open_file(str(h5file), 'r') as h5f: assert model.metadata['title'] == h5f.title # Check metadata on root node assert h5f.root._v_attrs.author == 'pytest' assert h5f.root._v_attrs.run_number == 0 rec_demand = h5f.get_node('/outputs/demand').read() rec_storage = h5f.get_node('/storage/reservoir').read() # model starts with no demand saving demand_baseline = 50.0 demand_factor = 0.9 # jan-apr demand_saving = 1.0 assert_allclose(rec_demand[0, 0], demand_baseline * demand_factor * demand_saving) # first control curve breached demand_saving = 0.95 assert (rec_storage[4, 0] < (0.8 * max_volume)) assert_allclose(rec_demand[5, 0], demand_baseline * demand_factor * demand_saving) # second control curve breached demand_saving = 0.5 assert (rec_storage[11, 0] < (0.5 * max_volume)) assert_allclose(rec_demand[12, 0], demand_baseline * demand_factor * demand_saving)
def load_model(self, root_dir, model_path, bucket=None, network_key=None, check_graph=False): os.chdir(root_dir) # needed when loading JSON file root_path = 's3://{}/{}/'.format(bucket, network_key) os.environ['ROOT_S3_PATH'] = root_path # Step 1: Load and register policies sys.path.insert(0, os.getcwd()) policy_folder = '_parameters' for filename in os.listdir(policy_folder): if '__init__' in filename: continue policy_name = os.path.splitext(filename)[0] policy_module = '.{policy_name}'.format(policy_name=policy_name) # package = '.{}'.format(policy_folder) import_module(policy_module, policy_folder) # from domains import Hydropower, InstreamFlowRequirement modules = [('.IFRS', 'policies'), ('.domains', 'domains')] for name, package in modules: try: import_module(name, package) except Exception as err: logger.warning("""{name} could not be imported from {package} Here's the real error: {error_type} {err} """.format(name=name, package=package, error_type=type(err), err=err)) # Step 2: Load and run model self.model = Model.load(model_path, path=model_path) # check network graph if check_graph: try: self.model.check_graph() except Exception as err: raise Exception('Pywr error: {}'.format(err)) self.setup() return
def create_planning_model(model_path): root, filename = os.path.split(model_path) base, ext = os.path.splitext(filename) new_filename = '{}_monthly'.format(base) + ext monthly_model_path = os.path.join(root, new_filename) prepare_planning_model(model_path, monthly_model_path) # monthly_model = load_model(root_dir, monthly_model_path, bucket=bucket, network_key=network_key, mode='planning') monthly_model = Model.load(monthly_model_path, path=monthly_model_path) setattr(monthly_model, 'mode', 'planning') monthly_model.setup() print('Monthly model setup complete') return monthly_model
def load_model(filename=None, data=None, solver=None): '''Load a test model and check it''' if data is None: path = os.path.join(os.path.dirname(__file__), 'models') with open(os.path.join(path, filename), 'r') as f: data = f.read() else: path = None model = Model.loads(data, path=path) model.check() return model
def create_model(): # create a model model = Model(start="2016-01-01", end="2019-12-31", timestep=7) # create three nodes (an input, a link, and an output) A = Input(model, name="A", max_flow=10.0) B = Link(model, name="B", cost=1.0) C = Output(model, name="C", max_flow=5.0, cost=-2.0) # connect nodes A.connect(B) B.connect(C) return model
def simple_linear_model(request, solver): """ Make a simple model with a single Input and Output. Input -> Link -> Output """ model = Model(solver=solver) inpt = Input(model, name="Input") lnk = Link(model, name="Link", cost=1.0) inpt.connect(lnk) otpt = Output(model, name="Output") lnk.connect(otpt) return model
def test_invalid_parameter_values(): """ Test that `load_parameter_values` returns a ValueError rather than KeyError. This is useful to catch and give useful messages when no valid reference to a data location is given. Regression test for Issue #247 (https://github.com/pywr/pywr/issues/247) """ from pywr.parameters._parameters import load_parameter_values m = Model() data = {'name': 'my_parameter', 'type': 'AParameterThatShouldHaveValues'} with pytest.raises(ValueError): load_parameter_values(model, data)
def test_catchment_many_successors(solver): """Test if node with fixed flow can have multiple successors. See #225""" model = Model(solver=solver) catchment = Catchment(model, "catchment", flow=100) out1 = Output(model, "out1", max_flow=10, cost=-100) out2 = Output(model, "out2", max_flow=15, cost=-50) out3 = Output(model, "out3") catchment.connect(out1) catchment.connect(out2) catchment.connect(out3) model.check() model.run() assert_allclose(out1.flow, 10) assert_allclose(out2.flow, 15) assert_allclose(out3.flow, 75)
def test_reset_timestepper_recorder(solver): model = Model( solver=solver, start=pandas.to_datetime('2016-01-01'), end=pandas.to_datetime('2016-01-01') ) inpt = Input(model, "input", max_flow=10) otpt = Output(model, "output", max_flow=50, cost=-10) inpt.connect(otpt) rec = NumpyArrayNodeRecorder(model, otpt) model.run() model.timestepper.end = pandas.to_datetime("2016-01-02") model.run()
def test_loading_csv_recorder_from_json(solver, tmpdir): """ Test the CSV Recorder which is loaded from json """ filename = 'csv_recorder.json' # This is a bit horrible, but need to edit the JSON dynamically # so that the output.h5 is written in the temporary directory path = os.path.join(os.path.dirname(__file__), 'models') with open(os.path.join(path, filename), 'r') as f: data = f.read() data = json.loads(data) # Make an absolute, but temporary, path for the recorder url = data['recorders']['model_out']['url'] data['recorders']['model_out']['url'] = str(tmpdir.join(url)) model = Model.load(data, path=path, solver=solver) csvfile = tmpdir.join('output.csv') model.run() import csv with open(str(csvfile), 'r') as fh: dialect = csv.Sniffer().sniff(fh.read(1024)) fh.seek(0) reader = csv.reader(fh, dialect) for irow, row in enumerate(reader): if irow == 0: expected = ['Datetime', 'inpt', 'otpt'] actual = row else: dt = model.timestepper.start+(irow-1)*model.timestepper.delta expected = [dt.isoformat()] actual = [row[0]] assert np.all((np.array([float(v) for v in row[1:]]) - 10.0) < 1e-12) assert expected == actual
from pywr.core import Model, Input, Output model = Model() a = Input(model, 'a', max_flow=10.0) b = Output(model, 'b', max_flow=5.0, cost=-1) a.connect(b) model.run() assert(b.flow[0] == 5.0)
def model(request): model = Model() return model
A.connect(Z) B.connect(Z) agg = AggregatedNode(model, "agg", [A, B]) agg.flow_weights = flow_weights agg.max_flow = 30.0 model.run() assert_allclose(agg.flow, expected_agg_flow) assert_allclose(A.flow, expected_A_flow) assert_allclose(B.flow, expected_B_flow) @pytest.mark.skipif(Model().solver.name == "lpsolve", reason="Not supported in lpsolve.") def test_aggregated_node_max_flow_parameter(model): """Nodes constrained by the max_flow of their AggregatedNode using a Parameter """ A = Input(model, "A", max_flow=20.0, cost=1) B = Input(model, "B", max_flow=20.0, cost=2) Z = Output(model, "Z", max_flow=100, cost=-10) A.connect(Z) B.connect(Z) agg = AggregatedNode(model, "agg", [A, B]) agg.max_flow = ConstantParameter(model, 30.0) model.run() assert_allclose(agg.flow, 30.0)
def create_model(self, network, template, initial_volumes=None): model = Model(solver='glpk-edge') # -----------------GENERATE NETWORK STRUCTURE ----------------------- output_ids = [] input_ids = [] non_storage_types = list(output_types.keys()) + list( input_types.keys()) + list(node_types.keys()) # create node dictionaries by name and id node_lookup = {} for node in network['nodes']: name = '{} (node)'.format(node['name']) types = [ t for t in node['types'] if t['template_id'] == template['id'] ] if not types: continue if len(types) > 1: msg = "Type is ambiguous for {}. Please remove extra types.".format( name) raise Exception(msg) type_name = types[-1]['name'] node_lookup[node.get("id")] = { 'type': type_name, 'name': name, 'connect_in': 0, 'connect_out': 0, } if type_name in output_types: output_ids.append(node['id']) elif type_name in input_types: input_ids.append(node['id']) # create link lookups and pywr links link_lookup = {} for link in network['links']: name = '{} (link)'.format(link['name']) types = [ t for t in link['types'] if t['template_id'] == template['id'] ] if not types: continue type_name = types[-1]['name'] link_id = link['id'] node_1_id = link['node_1_id'] node_2_id = link['node_2_id'] node_lookup[node_1_id]['connect_out'] += 1 node_lookup[node_2_id]['connect_in'] += 1 link_lookup[link_id] = { 'name': name, 'type': type_name, 'node_1_id': node_1_id, 'node_2_id': node_2_id, 'from_slot': node_lookup[node_1_id]['connect_out'] - 1, 'to_slot': node_lookup[node_2_id]['connect_in'] - 1, } if node_1_id in output_ids: node = node_lookup[node_1_id] msg = 'Topology error: Output {} appears to be upstream of {}'.format( node['name'], name) raise Exception(msg) elif node_2_id in input_ids: node = node_lookup[node_2_id] msg = 'Topology error: Input {} appears to be downstream of {}'.format( node['name'], name) raise Exception(msg) LinkType = link_types.get(type_name, Link) self.non_storage[('link', link_id)] = LinkType(model, name=name) # Q/C # remove unconnected links d = [] for link_id, link in link_lookup.items(): if link['node_1_id'] not in node_lookup or link[ 'node_2_id'] not in node_lookup: d.append(link_id) for link_id in d: del link_lookup[link_id] connected_nodes = [] for link_id, link in link_lookup.items(): connected_nodes.append(link['node_1_id']) connected_nodes.append(link['node_2_id']) # remove unconnected nodes d = [] for node_id in node_lookup: if node_id not in connected_nodes: d.append(node_id) for node_id in d: del node_lookup[node_id] # create pywr nodes dictionary with format ["name" = pywr type + 'name'] # for storage and non storage # TODO: change looping variable notation for node_id, node in node_lookup.items(): type_name = node['type'] name = node['name'] connect_in = node.get('connect_in', 0) connect_out = node.get('connect_out', 0) if (type_name in storage_types or connect_out > 1) and type_name not in non_storage_types: initial_volume = initial_volumes.get( node_id, 0.0) if initial_volumes is not None else 0.0 self.storage[node_id] = Storage(model, name=name, num_outputs=connect_in, num_inputs=connect_out, initial_volume=initial_volume) if type_name not in storage_types: self.storage[node_id].max_volume = 0.0 else: if type_name in input_types: NodeType = input_types[type_name] elif type_name in output_types: NodeType = output_types[type_name] elif type_name in node_types: NodeType = node_types[type_name] elif connect_in > 1: NodeType = River else: NodeType = Link self.non_storage[('node', node_id)] = NodeType(model, name=name) # create network connections # must assign connection slots for storage # TODO: change looping variable notation for link_id, link in link_lookup.items(): node_1_id = link['node_1_id'] node_2_id = link['node_2_id'] _link = self.non_storage[('link', link_id)] up_storage = self.storage.get(node_1_id) up_node = self.non_storage.get(('node', node_1_id)) down_storage = self.storage.get(node_2_id) down_node = self.non_storage.get(('node', node_2_id)) if up_storage: up_storage.connect(_link, from_slot=link['from_slot']) else: up_node.connect(_link) if down_storage: _link.connect(down_storage, to_slot=link['to_slot']) else: _link.connect(down_node) self.model = model
def model(request, solver): model = Model(solver=solver) return model
def run_model(basin, network_key): # ======================== # Set up model environment # ======================== root_dir = os.path.join(os.getcwd(), basin) bucket = 'openagua-networks' model_path = os.path.join(root_dir, 'pywr_model.json') # setup_model(root_dir, model_path, bucket=bucket, network_key=network_key) os.chdir(root_dir) # needed when loading JSON file # root_path = 's3://{}/{}/'.format(bucket, network_key) root_path = '../data' os.environ['ROOT_S3_PATH'] = root_path # ========================================= # Load and register custom model parameters # ========================================= sys.path.insert(0, os.getcwd()) policy_folder = '_parameters' for filename in os.listdir(policy_folder): if '__init__' in filename: continue policy_name = os.path.splitext(filename)[0] policy_module = '.{policy_name}'.format(policy_name=policy_name) # package = '.{}'.format(policy_folder) import_module(policy_module, policy_folder) modules = [('.IFRS', 'policies'), ('.domains', 'domains')] for name, package in modules: try: import_module(name, package) except Exception as err: print(' [-] WARNING: {} could not be imported from {}'.format( name, package)) print(type(err)) print(err) # ================== # Create daily model # ================== include_monthly = False daily_model = Model.load(model_path, path=model_path) print('Daily model loaded') daily_model.setup() print('Daily model setup completed') # ===================== # Create planning model # ===================== # create and initialize monthly model if include_monthly: monthly_model = create_planning_model(model_path) timesteps = range(len(daily_model.timestepper)) # run model # note that tqdm + step adds a little bit of overhead. # use model.run() instead if seeing progress is not important for step in tqdm(timesteps, ncols=80): today = daily_model.timestepper.current if step else daily_model.timestepper.start try: # Step 1: run planning model & update daily model if include_monthly and today.day == 1: # Step 1a: update planning model # ...update time steps monthly_model.timestepper.start = today monthly_model.timestepper.end = today # ...update initial conditions (not needed for the first step) if step > 0: for node in monthly_model.nodes: if node['type'] != 'Storage': continue node['initial_volume'] = daily_model.node[ node['name']].volume # Step 1b: run planning model print('Running planning model') monthly_model.step( ) # redundant with run, since only one timestep # Step 1c: update daily model with planning model results print('Updating daily model') # Step 3: run daily model daily_model.step() except Exception as err: print('\nFailed at step {}'.format(today)) print(err) # continue break # save results to CSV results = daily_model.to_dataframe() results_path = './results' results.columns = results.columns.droplevel(1) if not os.path.exists(results_path): os.makedirs(results_path) results.to_csv(os.path.join(results_path, 'system.csv')) attributes = {} for c in results.columns: attribute = c.split('/')[-1] if attribute in attributes: attributes[attribute].append(c) else: attributes[attribute] = [c] for attribute in attributes: path = os.path.join(results_path, '{}.csv'.format(attribute)) df = results[attributes[attribute]] df.columns = [c.split('/')[-2] for c in df.columns] df.to_csv(path) if attribute == 'flow': df2 = df[[c for c in df.columns if c[-3:] == ' PH']] path2 = os.path.join(results_path, 'powerhouse flow.csv') df2.to_csv(path2)
def model(): return Model()
def model(solver): return Model(solver=solver)
sc_index = self.model.scenarios.multiindex return pandas.DataFrame(data=np.array(self._data), index=index, columns=sc_index) @classmethod def load(cls, model, data): node = model._get_node_from_ref(model, data.pop("node")) if "storage_node" in data: storage_node = model._get_node_from_ref(model, data.pop("storage_node")) else: storage_node = None return cls(model, node, storage_node=storage_node, **data) HydroPowerRecorder.register() if __name__ == '__main__': m = Model.load('hydropower_example.json') stats = m.run() print(stats) print(m.recorders["turbine1_energy"].values()) df = m.to_dataframe() print(df.head()) from matplotlib import pyplot as plt df.plot(subplots=True) plt.show()
def test_keating_aquifer(solver): model = Model( solver=solver, start=pandas.to_datetime('2016-01-01'), end=pandas.to_datetime('2016-01-01'), ) aqfer = KeatingAquifer( model, 'keating', num_streams, num_additional_inputs, stream_flow_levels, transmissivity, coefficient, levels, area=area, storativity=storativity, ) catchment = Input(model, 'catchment', max_flow=0) stream = Output(model, 'stream', max_flow=np.inf, cost=0) abstraction = Output(model, 'abstraction', max_flow=15, cost=-999) catchment.connect(aqfer) aqfer.connect(stream, from_slot=0) aqfer.connect(abstraction, from_slot=1) rec_level = NumpyArrayLevelRecorder(model, aqfer) rec_volume = NumpyArrayStorageRecorder(model, aqfer) rec_stream = NumpyArrayNodeRecorder(model, stream) rec_abstraction = NumpyArrayNodeRecorder(model, abstraction) model.check() assert(len(aqfer.inputs) == (num_streams + num_additional_inputs)) for initial_level in (50, 100, 110, 150): # set the inital aquifer level and therefor the initial volume aqfer.initial_level = initial_level initial_volume = aqfer.initial_volume assert(initial_volume == (area * storativity[0] * initial_level * 0.001)) # run the model (for one timestep only) model.run() # manually calculate keating streamflow and check model flows are OK Qp = 2 * transmissivity[0] * max(initial_level - stream_flow_levels[0][0], 0) * coefficient Qe = 2 * transmissivity[1] * max(initial_level - stream_flow_levels[0][1], 0) * coefficient delta_storage = initial_volume - rec_volume.data[0, 0] abs_flow = rec_abstraction.data[0, 0] stream_flow = rec_stream.data[0, 0] assert(delta_storage == (stream_flow + abs_flow)) assert(stream_flow == (Qp+Qe)) A_VERY_LARGE_NUMBER = 9999999999999 model.timestepper.end = pandas.to_datetime('2016-01-02') # fill the aquifer completely # there is no spill for the storage so it should find no feasible solution with pytest.raises(RuntimeError): catchment.max_flow = A_VERY_LARGE_NUMBER catchment.min_flow = A_VERY_LARGE_NUMBER model.run() # drain the aquifer completely catchment.min_flow = 0 catchment.max_flow = 0 abstraction.max_flow = A_VERY_LARGE_NUMBER model.run() assert(rec_volume.data[1, 0] == 0) abs_flow = rec_abstraction.data[1, 0] stream_flow = rec_stream.data[1, 0] assert(stream_flow == 0) assert(abs_flow == 0)
results_folder = 'C:/Users/Josh Soper/Documents/Mexico/Summer 2018/Analysis' # ----------------- CREATE MODEL ----------------------- # TODO: End time needs to be last day of final model year, not first day of next year for scenarios in data['network']['scenarios']: if scenarios['name'] == option: meta = scenarios start = datetime.strptime(meta['start_time'], '%Y-%m-%d %H:%M:%S').strftime('%Y-%m-%d') # end = datetime.strptime(meta['end_time'], '%Y-%m-%d %H:%M:%S').strftime('%Y-%m-%d') end = '2015-12-31' if meta['time_step'] == 'day': ts = 1 model = Model(start=start, end=end, timestep=ts, solver='glpk') # -----------------GENERATE NETWORK STRUCTURE ----------------------- # create node dictionaries by name and id node_lookup_name = {} node_lookup_id = {} for node in nodes_list: types = [ t for t in node['types'] if abs(t['template_id']) == abs(template['id']) ] node_lookup_name[node.get('name')] = { 'type': types[0]['name'] if types else None,
def create_model(self, network, template, constants=None, variables=None, policies=None, initial_volumes=None): model = Model(solver='glpk-edge') # -----------------GENERATE NETWORK STRUCTURE ----------------------- # ...and add initial parameter values output_ids = [] input_ids = [] non_storage_types = list(output_types.keys()) + list(input_types.keys()) + list(node_types.keys()) def add_value_to_node(res_attr_idx, type_name, attr_name): pywr_param = None constant = constants.pop(res_attr_idx, None) if constant: pywr_param = ConstantParameter(model, constant) elif variables: variable = variables.pop(res_attr_idx, None) if variable: values = list(variable['values'].values()) pywr_param = ArrayIndexedParameter(model, values) elif policies: policy = policies.pop(res_attr_idx, None) if policy: pywr_param = self.create_register_policy(policy) if pywr_param is not None: type_name = type_name.lower() attr_name = attr_name.lower() (resource_type, resource_id, attr_id) = res_attr_idx try: self.update_param(resource_type, resource_id, type_name, attr_name, value=pywr_param) except: raise # create node dictionaries by name and id node_lookup = {} for node in network['nodes']: name = '{} (node)'.format(node['name']) types = [t for t in node['types'] if t['template_id'] == template['id']] if not types: continue if len(types) > 1: msg = "Type is ambiguous for {}. Please remove extra types.".format(name) raise Exception(msg) type_name = types[-1]['name'] node_lookup[node.get("id")] = { 'type': type_name, 'name': name, 'connect_in': 0, 'connect_out': 0, 'attributes': node['attributes'] } if type_name in output_types: output_ids.append(node['id']) elif type_name in input_types: input_ids.append(node['id']) # create link lookups and pywr links link_lookup = {} for link in network['links']: residx = ('link', link['id']) name = '{} (link)'.format(link['name']) types = [t for t in link['types'] if t['template_id'] == template['id']] if not types: continue type_name = types[-1]['name'] link_id = link['id'] node_1_id = link['node_1_id'] node_2_id = link['node_2_id'] try: node_lookup[node_1_id]['connect_out'] += 1 except: raise Exception('Topology error for {}. Upstream node ID {} not found.'.format(name, node_1_id)) try: node_lookup[node_2_id]['connect_in'] += 1 except: raise Exception('Topology error for {}. Downstream node ID {} not found.'.format(name, node_2_id)) link_lookup[link_id] = { 'name': name, 'type': type_name, 'node_1_id': node_1_id, 'node_2_id': node_2_id, 'from_slot': node_lookup[node_1_id]['connect_out'] - 1, 'to_slot': node_lookup[node_2_id]['connect_in'] - 1, } if node_1_id in output_ids: node = node_lookup[node_1_id] msg = 'Topology error: Output {} appears to be upstream of {}'.format(node['name'], name) raise Exception(msg) elif node_2_id in input_ids: node = node_lookup[node_2_id] msg = 'Topology error: Input {} appears to be downstream of {}'.format(node['name'], name) raise Exception(msg) LinkType = link_types.get(type_name, Link) self.non_storage[residx] = LinkType(model, name=name) for ra in link['attributes']: res_attr_idx = ('link', link['id'], ra['attr_id']) add_value_to_node(res_attr_idx, type_name, ra['attr_name']) # Q/C # remove unconnected links d = [] for link_id, link in link_lookup.items(): if link['node_1_id'] not in node_lookup or link['node_2_id'] not in node_lookup: d.append(link_id) for link_id in d: del link_lookup[link_id] connected_nodes = [] for link_id, link in link_lookup.items(): connected_nodes.append(link['node_1_id']) connected_nodes.append(link['node_2_id']) # remove unconnected nodes d = [] for node_id in node_lookup: if node_id not in connected_nodes: d.append(node_id) for node_id in d: del node_lookup[node_id] # create pywr nodes dictionary with format ["name" = pywr type + 'name'] # for storage and non storage for node_id, node in node_lookup.items(): residx = ('node', node_id) type_name = node['type'] name = node['name'] connect_in = node.get('connect_in', 0) connect_out = node.get('connect_out', 0) if (type_name in storage_types or connect_out > 1) and type_name not in non_storage_types: self.storage[node_id] = Storage( model, name=name, num_outputs=connect_in, num_inputs=connect_out, initial_volume=initial_volumes.get(node_id, 0.0) if initial_volumes is not None else 0.0 ) if type_name not in storage_types: self.storage[node_id].max_volume = 0.0 else: if type_name in input_types: NodeType = input_types[type_name] elif type_name in output_types: NodeType = output_types[type_name] elif type_name in node_types: NodeType = node_types[type_name] elif connect_in > 1: NodeType = River else: NodeType = Link self.non_storage[residx] = NodeType(model, name=name) for ra in node['attributes']: res_attr_idx = ('node', node_id, ra['attr_id']) try: add_value_to_node(res_attr_idx, type_name, ra['attr_name']) except Exception as err: print(err) raise # create network connections # must assign connection slots for storage # TODO: change looping variable notation for link_id, link in link_lookup.items(): node_1_id = link['node_1_id'] node_2_id = link['node_2_id'] _link = self.non_storage[('link', link_id)] up_storage = self.storage.get(node_1_id) up_node = self.non_storage.get(('node', node_1_id)) down_storage = self.storage.get(node_2_id) down_node = self.non_storage.get(('node', node_2_id)) if up_storage: up_storage.connect(_link, from_slot=link['from_slot']) else: up_node.connect(_link) if down_storage: _link.connect(down_storage, to_slot=link['to_slot']) else: _link.connect(down_node) self.model = model
from pywr.core import Model if __name__ == '__main__': import sys m = Model.load(sys.argv[1]) m.run() for node in m.nodes: print(node, node.flow)
def _run_model(climate, basin, start=None, end=None, years=None, run_name="default", include_planning=False, simplify=True, use_multiprocessing=False, debug=False, planning_months=12, scenarios=None, show_progress=False, data_path=None, file_suffix=None): logger.info("Running \"{}\" scenario for {} basin, {} climate".format( run_name, basin.upper(), climate.upper())) climate_set, climate_scenario = climate.split('/') if debug: from sierra.utilities import check_nan basin_path = os.path.join(data_path, basin.replace('_', ' ').title() + ' River') total_nan = check_nan(basin_path, climate) try: assert (total_nan == 0) logger.info('No NaNs found in data files') except AssertionError: logger.warning('{} NaNs found in data files.'.format(total_nan)) # if debug: # from sierra import create_schematic # Some adjustments if basin in ['merced', 'tuolumne']: include_planning = False # Set up dates if start is None or end is None: # TODO: get start and end years from outside, not hard coded if climate_scenario == 'Livneh': start_year = 1950 end_year = 2012 elif climate_set == 'gcms': start_year = 2030 end_year = 2060 elif climate_set == 'sequences': # name format is N01_S01, where N01 refers to the number of drought years # the total number of years is 1 + N + 2 (1 year at the end as a buffer) N = int(climate_scenario.split('Y')[1].split('_')[0]) start_year = 2000 end_year = start_year + N else: raise Exception("Climate scenario unknown") start = '{}-10-01'.format(start_year) end = '{}-09-30'.format(end_year) # ======================== # Set up model environment # ======================== here = os.path.dirname(os.path.realpath(__file__)) os.chdir(here) root_dir = os.path.join(here, 'models', basin) temp_dir = os.path.join(root_dir, 'temp') if not os.path.exists(temp_dir): os.makedirs(temp_dir) bucket = 'openagua-networks' base_filename = 'pywr_model.json' model_filename_base = 'pywr_model_{}'.format(climate_scenario) model_filename = model_filename_base + '.json' base_path = os.path.join(root_dir, base_filename) model_path = os.path.join(temp_dir, model_filename) # first order of business: update file paths in json file with open(base_path) as f: base_model = json.load(f) # update model with scenarios, if any def update_model(scenario_path): if os.path.exists(scenario_path): with open(scenario_path) as f: scenario_model = json.load(f) for key, scenario_items in scenario_model.items(): if key in base_model: if type(scenario_items) == dict: base_model[key].update(scenario_items) else: base_model[key].extend(scenario_items) elif key in ['scenarios', 'nodes']: items = { item['name']: item for item in base_model.get(key, []) } new_items = {item['name']: item for item in scenario_items} items.update(new_items) base_model[key] = list(items.values()) else: raise Exception( 'Scenario path {} does not exist.'.format(scenario_path)) if scenarios is not None: for s in scenarios: # update from scenarios folder scenario_path = os.path.join(data_path, 'metadata', 'scenario_definitions', '{}.json'.format(s)) update_model(scenario_path) new_model_parts = {} for model_part in ['tables', 'parameters']: if model_part not in base_model: continue new_model_parts[model_part] = {} for pname, param in base_model[model_part].items(): if 'observed' in pname.lower(): continue url = param.get('url') if url: if data_path: url = url.replace('../data', data_path) url = url.replace('historical/Livneh', climate) param['url'] = url new_model_parts[model_part][pname] = param base_model.update(new_model_parts) base_model['timestepper']['start'] = start base_model['timestepper']['end'] = end with open(model_path, 'w') as f: json.dump(base_model, f, indent=4) # ========================================= # Load and register global model parameters # ========================================= # sys.path.insert(0, os.getcwd()) policy_folder = 'parameters' for filename in os.listdir(policy_folder): if '__init__' in filename: continue policy_name = os.path.splitext(filename)[0] policy_module = 'sierra.parameters.{policy_name}'.format( policy_name=policy_name) import_module(policy_module, policy_folder) # ========================================= # Load and register custom model parameters # ========================================= sys.path.insert(0, os.getcwd()) policy_folder = os.path.join('models', basin, '_parameters') for filename in os.listdir(policy_folder): if '__init__' in filename: continue policy_name = os.path.splitext(filename)[0] policy_module = 'models.{basin}._parameters.{policy_name}'.format( basin=basin, policy_name=policy_name) import_module(policy_module, policy_folder) # import domains import_module('.domains', 'domains') if debug: logger.info("Domains imported") # import custom policies try: import_module('{}.policies'.format(basin)) except: pass # ========================================= # Load and register custom model recorders # ========================================= from recorders.hydropower import HydropowerEnergyRecorder HydropowerEnergyRecorder.register() # prepare the model files if simplify or include_planning: with open(model_path, 'r') as f: model_json = json.load(f) if simplify: # simplify model simplified_filename = model_filename_base + '_simplified.json' simplified_model_path = os.path.join(temp_dir, simplified_filename) model_json = simplify_network(model_json, basin=basin, climate=climate, delete_gauges=True, delete_observed=True) with open(simplified_model_path, 'w') as f: f.write(json.dumps(model_json, indent=4)) if debug: try: create_schematic(basin, 'simplified') except FileNotFoundError as err: logger.warning('Could not create schematic from Livneh model.') except ExecutableNotFound: logger.warning( 'Could not create daily schematic from Livneh model.') model_path = simplified_model_path # Area for testing monthly model save_results = debug planning_model = None df_planning = None if include_planning: logger.info('Creating planning model (this may take a minute or two)') # create filenames, etc. monthly_filename = model_filename_base + '_monthly.json' planning_model_path = os.path.join(temp_dir, monthly_filename) prepare_planning_model(model_json, basin, climate, planning_model_path, steps=planning_months, debug=debug, remove_rim_dams=True) if debug: try: create_schematic(basin, 'monthly') except ExecutableNotFound: logger.warning( 'Graphviz executable not found. Monthly schematic not created.' ) # create pywr model try: planning_model = Model.load(planning_model_path, path=planning_model_path) except Exception as err: logger.error("Planning model failed to load") # logger.error(err) raise # set model mode to planning planning_model.mode = 'planning' planning_model.blocks = {} # set time steps # start = planning_model.timestepper.start end = planning_model.timestepper.end end -= relativedelta(months=planning_months) planning_model.setup() # if debug == 'm': # test_planning_model(planning_model, months=planning_months, save_results=save_results) # return # ================== # Create daily model # ================== logger.info('Loading daily model') try: model = Model.load(model_path, path=model_path) except Exception as err: logger.error(err) raise model.blocks = {} model.setup() # run model # note that tqdm + step adds a little bit of overhead. # use model.run() instead if seeing progress is not important # IMPORTANT: The following can be embedded into the scheduling model via # the 'before' and 'after' functions. days_to_omit = 0 if include_planning: end = model.timestepper.end new_end = end + relativedelta(months=-planning_months) model.timestepper.end = new_end step = -1 now = datetime.now() monthly_seconds = 0 model.mode = 'scheduling' model.planning = None if include_planning: model.planning = planning_model model.planning.scheduling = model disable_progress_bar = not debug and not show_progress n_timesteps = len(model.timestepper.datetime_index) for date in tqdm(model.timestepper.datetime_index, ncols=60, disable=disable_progress_bar): step += 1 if disable_progress_bar and date.month == 9 and date.day == 30: logger.info('{}% complete (finsished year {})'.format( round(step / n_timesteps * 100), date.year)) try: # Step 1: run planning model if include_planning and date.day == 1: # update planning model model.planning.reset(start=date.to_timestamp()) # run planning model (intial conditions are set within the model step) model.planning.step() if debug and save_results: df_month = get_planning_dataframe(model.planning) if df_planning is None: df_planning = df_month else: df_planning = pd.concat([df_planning, df_month]) # Step 2: run daily model model.step() except Exception as err: traceback.print_exc() logger.error('Failed at step {}'.format(date)) raise if debug: total_seconds = (datetime.now() - now).total_seconds() logger.debug('Total run: {} seconds'.format(total_seconds)) monthly_pct = monthly_seconds / total_seconds * 100 logger.debug('Monthly overhead: {} seconds ({:02}% of total)'.format( monthly_seconds, monthly_pct)) # save results to CSV # results_path = os.path.join('./results', run_name, basin, climate) if debug: base_results_path = '../results' else: base_results_path = os.environ.get('SIERRA_RESULTS_PATH', '../results') suffix = ' - {}'.format(file_suffix) if file_suffix else '' run_folder = run_name + suffix results_path = os.path.join(base_results_path, run_folder, basin, climate) save_model_results(model, results_path, file_suffix, debug=debug)
def model(solver): model = Model(solver=solver) model.timestepper.start = Timestamp("2016-01-01") model.timestepper.end = Timestamp("2016-01-02") return model
A.connect(Z) B.connect(Z) agg = AggregatedNode(model, "agg", [A, B]) agg.flow_weights = flow_weights agg.max_flow = 30.0 model.run() assert_allclose(agg.flow, expected_agg_flow) assert_allclose(A.flow, expected_A_flow) assert_allclose(B.flow, expected_B_flow) @pytest.mark.skipif(Model().solver.name == "lpsolve", reason="Not supported in lpsolve.") def test_aggregated_node_max_flow_parameter(model): """Nodes constrained by the max_flow of their AggregatedNode using a Parameter """ A = Input(model, "A", max_flow=20.0, cost=1) B = Input(model, "B", max_flow=20.0, cost=2) Z = Output(model, "Z", max_flow=100, cost=-10) A.connect(Z) B.connect(Z) agg = AggregatedNode(model, "agg", [A, B]) agg.max_flow = ConstantParameter(model, 30.0) model.run()