import random
# In[3]:
inp_file = '../Code/c-town_true_network_simplified_controls.inp'
# # Setting up multiple simulations for AI training
# In[4]:
# ::: Running multiple simulations
for i in range(1): # Number of simulations
print(i)
# ::: Resetting initial water networks settings
ctown = twm(inp_file)
ctown.wn.reset_initial_values()
# ::: Initializing random seed
random.seed(i)
# ::: Setting up time for simulation
nDaysSim = 1
nHourDay = 24
simTimeSteps = 10 # nDaysSim*nHourDay*4 # Sampling frequency of 15 min
controlTvary = 8 # Number of time steps for varying controls (if set to 8 -> controls are stable for 2 hours)
# ::: Getting tank elevations
tankEl = []
for tank, name in ctown.wn.tanks():
def get_qual_data(file_list, narx_horizon, cluster_labels, pressure_factor, narx_input=True, narx_output=False, return_lists=False, inp_file=None, shift_x=False):
"""
--------------------------------------------------
Get network informations
--------------------------------------------------
"""
if not inp_file:
inp_file = '../Code/c-town_true_network_simplified_controls.inp'
ctown = twm(inp_file)
nw_node_df = pd.DataFrame(ctown.wn.nodes.todict())
nw_link_df = pd.DataFrame(ctown.wn.links.todict())
node_names = ctown.getNodeName()
link_names = ctown.getLinkName()
"""
--------------------------------------------------
Data Pre-Processing: 01
--------------------------------------------------
"""
n_clusters = 30
nn_input_list = []
nn_output_list = []
for file in file_list:
# Get results
with open(file, 'rb') as f:
results = pickle.load(f)
""" Junctions """
junction_qual = results.node['quality'][node_names[2]]
jun_cl_qual = junction_qual.groupby(cluster_labels.loc['quality'], axis=1)
jun_cl_qual_max_95 = jun_cl_qual.aggregate(lambda arr: np.percentile(arr, axis=1, q=95))
jun_cl_demand = results.node['demand'][node_names[2]].groupby(cluster_labels.loc['quality'], axis=1)
jun_cl_demand_sum = jun_cl_demand.sum()
""" Tanks """
tank_press = results.node['pressure'][node_names[0]]
tank_qual = results.node['quality'][node_names[0]]
""" Pumps """
# Overview over all pumps in nw: nw_link_df[nw_link_df.keys()[nw_link_df.loc['link_type'] == 'Pump']]
# https://wntr.readthedocs.io/en/latest/apidoc/wntr.network.elements.html#wntr.network.elements.HeadPump
# The setting for head pump is alias for normalized speed (usually in the range of 0-1)
head_pump_speed = results.link['setting'][nw_link_df.keys()[nw_link_df.loc['link_type'] == 'Pump']]
head_pump_status = results.link['status'][nw_link_df.keys()[nw_link_df.loc['link_type'] == 'Pump']]
head_pump_speed_corr = head_pump_speed*head_pump_status
""" Valves """
# Overview over all valves in nw: nw_link_df[nw_link_df.keys()[nw_link_df.loc['link_type'] == 'Valve']]
# https://wntr.readthedocs.io/en/latest/apidoc/wntr.network.elements.html#wntr.network.elements.TCValve
# TODO: Only one valve type in the future?
PRValve_dp = results.link['setting'][nw_link_df.keys()[nw_link_df.loc['valve_type'] == 'PRV']]
TCValve_throttle = results.link['setting'][nw_link_df.keys()[nw_link_df.loc['valve_type'] == 'TCV']]
"""
--------------------------------------------------
Data Pre-Processing: 02 - Create states + inputs
--------------------------------------------------
"""
# TODO: Reservoir?
state_dict = { # 'jun_cl_press_mean': jun_cl_press_mean,
'jun_cl_qual_max_95': jun_cl_qual_max_95,
'tank_qual': tank_qual,
}
sys_states = pd.concat(state_dict.values(), axis=1, keys=state_dict.keys())
if shift_x:
sys_states = sys_states.shift(1, axis=0)
input_dict = { # 'head_pump_speed': head_pump_speed,
'tank_press': tank_press,
'head_pump_speed': head_pump_speed_corr,
'PRValve_dp': PRValve_dp,
'TCValve_throttle': TCValve_throttle,
'jun_cl_demand_sum': jun_cl_demand_sum}
sys_inputs = pd.concat(input_dict.values(), axis=1, keys=input_dict.keys())
"""
--------------------------------------------------
Data Pre-Processing: 03 - Neural Network I/O
--------------------------------------------------
"""
nn_input_dict = {'sys_states': sys_states,
'sys_inputs': sys_inputs}
nn_input = pd.concat(nn_input_dict.values(), axis=1, keys=nn_input_dict.keys(), names=['type', 'name', 'index'])
if narx_input:
arx_input = []
for i in range(narx_horizon):
arx_input.append(nn_input.shift(i, axis=0))
arx_input = pd.concat(arx_input, keys=np.arange(narx_horizon), names=['NARX', 'type', 'name', 'index'], axis=1)
nn_input = arx_input
if narx_output:
nn_output = nn_input.shift(-1, axis=0)
else:
sys_states_next = sys_states.shift(-1, axis=0)
dsys_states = sys_states.diff(axis=0)
dsys_states_next = dsys_states.shift(-1, axis=0)
nn_output_dict = { # 'sys_states': sys_states_next,
'sys_states': dsys_states_next}
nn_output = pd.concat(nn_output_dict.values(), axis=1, keys=nn_output_dict.keys())
# Filter nan:
output_filter = nn_output.isnull().any(axis=1)
if output_filter.any():
nn_input = nn_input[~output_filter]
nn_output = nn_output[~output_filter]
input_filter = nn_input.isnull().any(axis=1)
if input_filter.any():
nn_input = nn_input[~input_filter]
nn_output = nn_output[~input_filter]
nn_input_list.append(nn_input)
nn_output_list.append(nn_output)
nn_input_concat = pd.concat(nn_input_list, axis=0)
nn_output_concat = pd.concat(nn_output_list, axis=0)
if return_lists:
return nn_input_list, nn_output_list
else:
return nn_input_concat, nn_output_concat
#import sklearn import numpy as np import time from testWN import testWN as twm import wntr import wntr.network.controls as controls import wntr.metrics.economic as economics import pdb import pickle import random # In[3]: inp_file = '../Code/c-town_true_network_simplified_controls.inp' ctown = twm(inp_file) # # Setting up multiple simulations for AI training # In[4]: # ::: Setting up time for simulation nDaysSim = 7 nHourDay = 24 simTimeSteps = nDaysSim * nHourDay * 4 # Sampling frequency of 15 min controlTvary = 8 # Number of time steps for varying controls (if set to 8 -> controls are stable for 2 hours) # ::: Getting tank elevations tankEl = [] for tank, name in ctown.wn.tanks():
def get_data(file_list, narx_horizon, cluster_labels, pressure_factor, narx_input=True, narx_output=False, return_lists=False, inp_file=None):
"""
This file is an outsourced implementation from the code in the jupyter notebook "dnn_surrogate_prototyping.ipynb".
In this notebook we developed and tested the pre-processing pipeline on a single result file from WNTR/Epanet.
We tested the training process on this limited data source but didn't use the resulting model for control.
In the jupyter notebook "dnn_surrogate_full_model.ipynb" we use this outsourced implementation to pre-process multiple result files
and create a large training data base. We furthermore trained the model in "dnn_surrogate_full_model.ipynb"
and used the resulting models for the control task.
"""
"""
--------------------------------------------------
Get network informations
--------------------------------------------------
"""
if not inp_file:
# For legacy reasons.
inp_file = '../../WNTR_Model/c-town_true_network_simplified_controls.inp'
ctown = twm(inp_file)
nw_node_df = pd.DataFrame(ctown.wn.nodes.todict())
nw_link_df = pd.DataFrame(ctown.wn.links.todict())
node_names = ctown.getNodeName()
link_names = ctown.getLinkName()
"""
--------------------------------------------------
Data Pre-Processing: 01
--------------------------------------------------
"""
# For Legacy reasons: Check the name of the pressure clusters in cluster_labels:
if 'pressure' in cluster_labels.index.tolist():
cl_ind_press = 'pressure'
else:
cl_ind_press = 'pressure_cluster'
nn_input_list = []
nn_output_list = []
for i, file in enumerate(file_list):
progress = (i+1)/len(file_list)
print('{:.2f}% complete'.format(100*progress), end="\r")
# Get results
with open(file, 'rb') as f:
results = pickle.load(f)
""" Junctions """
# Scale junction pressure
junction_pressure_scaled = results.node['pressure'][node_names[2]]/pressure_factor.to_numpy()
jun_cl_press = junction_pressure_scaled.groupby(cluster_labels.loc[cl_ind_press], axis=1)
jun_cl_press_mean = jun_cl_press.mean()
jun_cl_press_std = jun_cl_press.std()
jun_cl_demand = results.node['demand'][node_names[2]].groupby(cluster_labels.loc[cl_ind_press], axis=1)
jun_cl_demand_sum = jun_cl_demand.sum()
# quality from Results | for all junctions | difference | group by the quality cluster | create the mean / standard deviation
jun_cl_qual = results.node['quality'][node_names[2]].diff(axis=0).groupby(cluster_labels.loc[cl_ind_press], axis=1)
qual_cl_qual_mean = jun_cl_qual.mean()
qual_cl_qual_std = jun_cl_qual.std()
""" Tanks """
tank_press = results.node['pressure'][node_names[0]]
# Subtract tank elevation from tank head to obtain tank_level
tank_level = results.node['head'][node_names[0]]-nw_node_df[node_names[0]].loc['elevation']
tank_qual = results.node['quality'][node_names[0]]
""" Reservoirs """
reservoir_press = results.node['pressure'][node_names[1]]
reservoir_level = results.node['head'][node_names[1]]-nw_node_df[node_names[1]].loc['elevation']
reservoir_qual = results.node['quality'][node_names[1]]
""" Pumps """
# Overview over all pumps in nw: nw_link_df[nw_link_df.keys()[nw_link_df.loc['link_type'] == 'Pump']]
# https://wntr.readthedocs.io/en/latest/apidoc/wntr.network.elements.html#wntr.network.elements.HeadPump
# The setting for head pump is alias for normalized speed (usually in the range of 0-1)
head_pump_speed = results.link['setting'][nw_link_df.keys()[nw_link_df.loc['link_type'] == 'Pump']]
head_pump_status = results.link['status'][nw_link_df.keys()[nw_link_df.loc['link_type'] == 'Pump']]
head_pump_speed_corr = head_pump_speed
pump_energy = economics.pump_energy(results.link['flowrate'], results.node['head'], ctown.wn)[link_names[0]]
pump_energy /= 1000
pump_energy.head(3)
""" Valves """
# Overview over all valves in nw: nw_link_df[nw_link_df.keys()[nw_link_df.loc['link_type'] == 'Valve']]
# https://wntr.readthedocs.io/en/latest/apidoc/wntr.network.elements.html#wntr.network.elements.TCValve
# TODO: Only one valve type in the future?
PRValve_dp = results.link['setting'][nw_link_df.keys()[nw_link_df.loc['valve_type'] == 'PRV']]
TCValve_throttle = results.link['setting'][nw_link_df.keys()[nw_link_df.loc['valve_type'] == 'TCV']]
"""
--------------------------------------------------
Data Pre-Processing: 02 - Create states + inputs
--------------------------------------------------
"""
state_dict = {
'tank_press': tank_press,
}
sys_states = pd.concat(state_dict.values(), axis=1, keys=state_dict.keys())
input_dict = { # 'head_pump_speed': head_pump_speed,
'head_pump_speed': head_pump_speed_corr,
'PRValve_dp': PRValve_dp,
'TCValve_throttle': TCValve_throttle,
'jun_cl_demand_sum': jun_cl_demand_sum}
sys_inputs = pd.concat(input_dict.values(), axis=1, keys=input_dict.keys())
aux_output_dict = {'pump_energy': pump_energy,
'jun_cl_press_mean': jun_cl_press_mean, }
aux_outputs = pd.concat(aux_output_dict.values(), axis=1, keys=aux_output_dict.keys())
"""
--------------------------------------------------
Data Pre-Processing: 03 - Neural Network I/O
--------------------------------------------------
"""
nn_input_dict = {'sys_states': sys_states,
'sys_inputs': sys_inputs}
nn_input = pd.concat(nn_input_dict.values(), axis=1, keys=nn_input_dict.keys(), names=['type', 'name', 'index'])
if narx_input:
arx_input = []
for i in range(narx_horizon):
arx_input.append(nn_input.shift(i, axis=0))
arx_input = pd.concat(arx_input, keys=np.arange(narx_horizon), names=['NARX', 'type', 'name', 'index'], axis=1)
nn_input = arx_input
if narx_output:
nn_output = nn_input.shift(-1, axis=0)
else:
sys_states_next = sys_states.shift(-1, axis=0)
dsys_states = sys_states.diff(axis=0)
dsys_states_next = dsys_states.shift(-1, axis=0)
nn_output_dict = { # 'sys_states': sys_states_next,
'sys_states': dsys_states_next,
'aux_outputs': aux_outputs}
nn_output = pd.concat(nn_output_dict.values(), axis=1, keys=nn_output_dict.keys())
# Filter nan:
output_filter = nn_output.isnull().any(axis=1)
if output_filter.any():
nn_input = nn_input[~output_filter]
nn_output = nn_output[~output_filter]
input_filter = nn_input.isnull().any(axis=1)
if input_filter.any():
nn_input = nn_input[~input_filter]
nn_output = nn_output[~input_filter]
nn_input_list.append(nn_input)
nn_output_list.append(nn_output)
nn_input_concat = pd.concat(nn_input_list, axis=0)
nn_output_concat = pd.concat(nn_output_list, axis=0)
if return_lists:
return nn_input_list, nn_output_list
else:
return nn_input_concat, nn_output_concat
