def main():
fp = r'../data/CS3dv.dss'
fp_out = r'../data/CS3_test_out.dss'
fp_out2 = r'../data/CS3_test_out2.dss'
part_b = 'S_SHSTA S_FOLSM C_KSWCK'.split()
df = cs.read_dss(fp, b=part_b, end_date='2015-09-30')
df = df.cs.condense()
df.cs.to_dss(fp_out2, a='CalSim3', f='JAS_dev')
return 0
def main():
fp = r'../../HEC5Q/NAA/AR_HEC5Q_Q5/Model/AR_WQ_Report.dss'
# fp = r'../../HEC5Q/NAA/AR_HEC5Q_Q5/Model/CALSIMII_HEC5Q.dss'
fp_out = r'../data/HEC5Q_test_out.dss'
part_b = 'AMERICAN'.split()
# part_b = 'GERBER_1920'.split()
# df = cs.read_dss(fp, b=part_b, end_date='2015-09-30')
# df = cs.read_dss(fp, b=part_b, c='SWRAD', end_date='2015-09-30')
df = cs.read_dss(fp, e='1DAY', end_date='2015-09-30')
df = df.cs.wide()
df.cs.to_dss(fp_out, a='CalSim3', f='JAS_dev')
return 0
def main():
fp0 = r'../data/CS3dv_copy.dss'
# fp0 = r'../data/CS3dv.dss'
fp1 = r'../data/2019-09-12_USBR_DV.dss'
DVs = [fp0, fp0] * 4
# part_b = 'S_SHSTA S_FOLSM C_KSWCK C_NTOMA D_SAC196_MTC000'.split()
part_b = 'S_SHSTA S_FOLSM'.split()
studies = 'Base Alt1'.split()
df = cs.read_dss(DVs, b=part_b, end_date='2015-09-30')
# df = cs.read_dss(DVs, b=part_b, studies=studies, end_date='2015-09-30')
# df = cs.read_dss(fp0, b=part_b, end_date='2015-09-30')
df = df.cs.plot('ma')
return 0
def main():
# Identify all model runs.
working_subdir = os.listdir('../__models/CalSim3')
models = list()
for sd in working_subdir:
models.append(sd)
# Generate list of DSS output files.
list_dss = list()
for m in models:
srce_dir = os.path.join('../__models/CalSim3', m)
fpDV = os.path.join(srce_dir, 'CONV/DSS/CS3ROC_COS.dss')
list_dss.append(fpDV)
# Query DSS files.
df = cs.read_dss(list_dss,
b='C_KSWCK',
end_date='2015-09-30',
studies=models).cs.wide()
diff_list = list()
for m in models[1:]:
diff_list.append((df[m] - df[models[0]]).copy())
diff = pd.concat(diff_list, keys=models[1:], names=['Study'], axis=1)
_ = diff.cs.plot()
# Return success indicator.
return 0
def main():
t_0 = dt.datetime.now()
# Identify all HEC5Q runs.
HEC5Q_dir = '../__models/HEC5Q/HEC5Q'
sub_dirs = os.listdir(HEC5Q_dir)
models = list()
for sub_dir in sub_dirs:
if 'CS3' in sub_dir:
models.append(sub_dir)
flow_fps = list()
temp_fps = list()
for model in models:
flow_fp = os.path.join(HEC5Q_dir, model, 'SR_HEC5Q_Q0', 'Model',
'CALSIMII_HEC5Q.DSS')
flow_fps.append(flow_fp)
temp_fp = os.path.join(HEC5Q_dir, model, 'SR_HEC5Q_Q0', 'Model',
'SR_WQ_Report.dss')
temp_fps.append(temp_fp)
# Acquire daily Storage, Inflow, Outflow, and Temperature data from HEC5Q.
df_S = cs.read_dss(flow_fps, b='S_SHSTA', e='1DAY', studies=models)
df_I = cs.read_dss(flow_fps, b='I_SHSTA', e='1DAY',
studies=models).cs.wide()
df_O = cs.read_dss(flow_fps, b='C_KSWCK', e='1DAY',
studies=models).cs.wide()
df_T = cs.read_dss(temp_fps,
b='SACR_BLW_CLR_CR',
c='TEMP_F',
studies=models).cs.wide()
df_S.fillna(0, inplace=True)
df_I.fillna(0, inplace=True)
df_O.fillna(0, inplace=True)
df_T.fillna(0, inplace=True)
# Convert df_S from acre-feet to TAF.
df_S['Value'] = df_S['Value'] / 1000
df_S['Units'] = 'TAF'
df_S = df_S.cs.wide()
# Aggregate data to monthly timestep.
df_S = df_S.resample('M').last()
df_I = df_I.resample('M').mean()
df_O = df_O.resample('M').mean()
df_T = df_T.resample('M').mean()
# Process data in preparation for weight training.
S_min = df_S.min().min()
S_max = df_S.max().max()
df_S_n = (df_S - S_min) / (S_max - S_min) * (0.99 - (-0.99)) + (-0.99)
I_min = df_I.min().min()
I_max = df_I.max().max()
df_I_n = (df_I - I_min) / (I_max - I_min) * (0.99 - (-0.99)) + (-0.99)
O_min = df_O.min().min()
O_max = df_O.max().max()
df_O_n = (df_O - O_min) / (O_max - O_min) * (0.99 - (-0.99)) + (-0.99)
T_min = df_T.min().min()
T_max = df_T.max().max()
df_T_n = (df_T - T_min) / (T_max - T_min) * (0.99 - (-0.99)) + (-0.99)
records = list()
i = 0
idx = df_T_n.index[4:]
for model in models:
for row in idx:
S_col = df_S_n[[model]].columns[0]
I_col = df_I_n[[model]].columns[0]
O_col = df_O_n[[model]].columns[0]
T_col = df_T_n[[model]].columns[0]
record = [i]
i += 1
record += df_S_n.loc[:row, S_col].to_list()[:-5:-1]
record += df_I_n.loc[:row, I_col].to_list()[:-5:-1]
record += df_O_n.loc[:row, O_col].to_list()[:-5:-1]
record += df_T_n.loc[:row, T_col].to_list()[-1:]
records.append(record)
arr_records = np.array(records)
# Print read time.
t_1 = dt.datetime.now()
print('Read Time: {}'.format(t_1 - t_0))
t_0 = dt.datetime.now()
# Randomly split data into Training, Validation, and Testing sets.
np.random.shuffle(arr_records)
len_trn = int(0.6 * arr_records.shape[0])
len_val = int(0.2 * arr_records.shape[0])
x_Trn = arr_records[:len_trn, 1:-1].transpose()
t_Trn = arr_records[:len_trn, -1:].transpose()
i_Trn = arr_records[:len_trn, :1].transpose()
x_Val = arr_records[len_trn:len_trn + len_val, 1:-1].transpose()
t_Val = arr_records[len_trn:len_trn + len_val, -1:].transpose()
i_Val = arr_records[len_trn:len_trn + len_val, :1].transpose()
x_Tst = arr_records[len_trn + len_val:, 1:-1].transpose()
t_Tst = arr_records[len_trn + len_val:, -1:].transpose()
i_Tst = arr_records[len_trn + len_val:, :1].transpose()
t_1 = dt.datetime.now()
# Define weight structure.
nHidWeights = [512, 256, 128, 64]
# nHidWeights = [64]
print('Preprocessing Time for {}: {}'.format(nHidWeights, t_1 - t_0))
# Train weights.
Output_Dict = ann.FFANN_SGD(x_Trn,
t_Trn,
x_Val,
t_Val,
x_Tst,
t_Tst,
nHidWeights,
vSeed=6332,
L_Rate=0.0001,
Max_Iter=20000,
SummaryFreq=1000,
ReportFreq=1000)
# Print returned dictionary.
print('ann.FFANN_SGD Returned Dictionary of Key & Value Types:')
for k, v in Output_Dict.items():
print('{!s:>16}: {!s}'.format(k, type(v)))
# Post process data to DataFrame.
y = np.concatenate(
(Output_Dict['y_Trn'], Output_Dict['y_Val'], Output_Dict['y_Tst']),
axis=1)
arr_data = np.concatenate((arr_records, y.transpose()), axis=1)
sx_col = ['S_SHSTA(0)', 'S_SHSTA(-1)', 'S_SHSTA(-2)', 'S_SHSTA(-3)']
ix_col = ['I_SHSTA(0)', 'I_SHSTA(-1)', 'I_SHSTA(-2)', 'I_SHSTA(-3)']
ox_col = ['C_KSWCK(0)', 'C_KSWCK(-1)', 'C_KSWCK(-2)', 'C_KSWCK(-3)']
ty_col = ['T_Target(0)', 'T_Output(0)']
cols = ['Record'] + sx_col + ix_col + ox_col + ty_col
df = pd.DataFrame(arr_data, columns=cols)
df.sort_values('Record', inplace=True)
df.set_index('Record', inplace=True)
df[sx_col] = (df[sx_col] - (-0.99)) / (0.99 -
(-0.99)) * (S_max - S_min) + S_min
df[ix_col] = (df[ix_col] - (-0.99)) / (0.99 -
(-0.99)) * (I_max - I_min) + I_min
df[ox_col] = (df[ox_col] - (-0.99)) / (0.99 -
(-0.99)) * (O_max - O_min) + O_min
df[ty_col] = (df[ty_col] - (-0.99)) / (0.99 -
(-0.99)) * (T_max - T_min) + T_min
# Output model DataFrames to disk.
len_idx = idx.size
for i in range(len(models)):
df_temp = df.loc[i * len_idx:(i + 1) * len_idx - 1, :].copy()
df_temp['DateTime'] = idx
df_temp.set_index('DateTime', inplace=True)
fp = '../data/training_results/{}.xlsx'.format(models[i])
df_temp.to_excel(fp)
# Output weights to disk.
for k, v in Output_Dict['WeightDict'].items():
W = v['W']
B = v['B']
np.savetxt('../__trained_tnn/W{}.txt'.format(k), W)
np.savetxt('../__trained_tnn/B{}.txt'.format(k), B)
# Plot output.
# if show_plot:
# plot_results(Output_Dict, t_Trn, t_Val, t_Tst, i_Trn, i_Val, i_Tst,
# T_max, T_min)
plot_results(Output_Dict, t_Trn, t_Val, t_Tst, i_Trn, i_Val, i_Tst, T_max,
T_min)
# Return accuracy.
A_trn = Output_Dict['ReportSummary'][('Accuracy', 'Training')].iloc[-1]
A_val = Output_Dict['ReportSummary'][('Accuracy', 'Validation')].iloc[-1]
A_tst = Output_Dict['ReportSummary'][('Accuracy', 'Testing')].iloc[-1]
return (A_trn, A_val, A_tst)
def transfer_data(fpSV, fpDV, OutDir=None):
r"""
No documentation as of 2019-01-03.
"""
# %% Create list of CalSim3 variables needed for HEC-5Q-CS3.
# The list below comes from SR_CS-CS3.dat variables in the 'get' function.
CS3_Var = [ # Trinity River
'S_TRNTY',
'I_TRNTY',
'E_TRNTY',
'C_TRNTY',
'Something Crazy',
'E_LWSTN',
'I_LWSTN',
'D_LWSTN_CCT011',
# Clear Creek
'S_WKYTN',
'I_CLR025',
'E_WKYTN',
'C_WKYTN',
'D_WKYTN_SPT003',
'D_WKYTN_02_PU',
'D_WKYTN_WTPBUK',
'D_WKYTN_WTPCSD',
# Sacramento River
'S_SHSTA',
'I_SHSTA',
'E_SHSTA',
'C_SHSTA',
'D_SHSTA_WTPJMS',
'C_KSWCK',
'C_SAC287',
'C_SAC277',
'C_SAC271',
'C_SAC269',
'C_SAC259',
'C_SAC257',
'C_SAC254',
'C_SAC247',
'C_SAC229',
'C_SAC217',
'C_SAC201',
'C_SAC178',
'C_SAC169',
'C_SAC162',
'C_SAC146',
'C_SAC122',
'SP_SAC193_BTC003',
'SP_SAC188_BTC003',
'SP_SAC178_BTC003',
'SP_SAC159_BTC003',
'SP_SAC148_BTC003',
'SP_SAC122_SBP021',
'C_SAC097',
'C_SAC091',
# Sutter Bypass
'C_SSL001',
# Stony Creek
'S_BLKBT',
'I_BLKBT',
'C_SGRGE',
'E_BLKBT',
'C_BLKBT',
'C_STN004'
]
# %% Set additional process parameters.
# Establish DSS Output parameters.
OutSV = '2020D09ESV_3.dss'
OutDV = '2020D09EDV_3.dss'
Part_A = 'CALSIM'
Part_F = '2020D09E'
# Construct output DSS file paths.
if OutDir:
oSVpth = os.path.realpath(os.path.join(OutDir, OutSV))
oDVpth = os.path.realpath(os.path.join(OutDir, OutDV))
else:
oSVpth = os.path.realpath(os.path.join(os.getcwd(), OutSV))
oDVpth = os.path.realpath(os.path.join(os.getcwd(), OutDV))
# Read data.
SV_DF = cs.read_dss(fpSV, b=CS3_Var)
DV_DF = cs.read_dss(fpDV, b=CS3_Var)
# Replace Part F.
cs.transform.split_pathname(SV_DF, inplace=True)
SV_DF['Part F'] = Part_F
cs.transform.join_pathname(SV_DF, inplace=True)
cs.transform.split_pathname(DV_DF, inplace=True)
DV_DF['Part F'] = Part_F
cs.transform.join_pathname(DV_DF, inplace=True)
# Write DSS files to disk.
SV_DF.cs.to_dss(oSVpth)
DV_DF.cs.to_dss(oDVpth)
# Return success indicator.
return 0
def main():
fp = r'../data/CS3dv.dss'
part_b = 'S_SHSTA S_FOLSM C_KSWCK'.split()
df = cs.read_dss(fp, b=part_b, end_date='2015-09-30')
print(df)