def corrected_day_average(corrected: pd.DataFrame, simulated: pd.DataFrame, observed: pd.DataFrame,
merged_sim_obs: pd.DataFrame = False, merged_cor_obs: pd.DataFrame = False,
titles: dict = None, outformat: str = 'plotly') -> go.Figure or str:
"""
Calculates and plots the daily average streamflow. This function uses
hydrostats.data.merge_data on the 3 inputs. If you have already computed these because you are doing a full
comparison of bias correction, you can provide them to save time
Args:
corrected: the response from the geoglows.bias.correct_historical_simulation function
simulated: the csv response from historic_simulation
merged_sim_obs: (optional) if you have already computed it, hydrostats.data.merge_data(simulated, observed)
merged_cor_obs: (optional) if you have already computed it, hydrostats.data.merge_data(corrected, observed)
observed: the dataframe of observed data. Must have a datetime index and a single column of flow values
outformat: either 'plotly' or 'plotly_html' (default plotly)
titles: (dict) Extra info to show on the title of the plot. For example:
{'Reach ID': 1234567, 'Drainage Area': '1000km^2'}
Returns:
plotly.GraphObject: plotly object, especially for use with python notebooks and the .show() method
"""
if corrected is False and simulated is False and observed is False:
if merged_sim_obs is not False and merged_cor_obs is not False:
pass # if you provided the merged dataframes already, we use those
else:
# merge the datasets together
merged_sim_obs = hd.merge_data(sim_df=simulated, obs_df=observed)
merged_cor_obs = hd.merge_data(sim_df=corrected, obs_df=observed)
daily_avg = hd.daily_average(merged_sim_obs)
daily_avg2 = hd.daily_average(merged_cor_obs)
scatters = [
go.Scatter(x=daily_avg.index, y=daily_avg.iloc[:, 1].values, name='Observed Data'),
go.Scatter(x=daily_avg.index, y=daily_avg.iloc[:, 0].values, name='Simulated Data'),
go.Scatter(x=daily_avg2.index, y=daily_avg2.iloc[:, 0].values, name='Corrected Simulated Data'),
]
layout = go.Layout(
title=_build_title('Daily Average Streamflow Comparison', titles),
xaxis=dict(title='Days'), yaxis=dict(title='Discharge (m<sup>3</sup>/s)', autorange=True),
showlegend=True)
if outformat == 'plotly':
return go.Figure(data=scatters, layout=layout)
elif outformat == 'plotly_html':
return offline_plot(
go.Figure(data=scatters, layout=layout),
config={'autosizable': True, 'responsive': True},
output_type='div',
include_plotlyjs=False
)
raise ValueError('Invalid outformat chosen. Choose plotly or plotly_html')
def get_dailyAverages(request):
"""
Get observed data from csv files in Hydroshare
Get historic simulations from ERA Interim
"""
get_data = request.GET
global nomRiver
global nomEstacion
global simulated_df
global observed_df
global corrected_df
try:
'''Merge Data'''
merged_df = hd.merge_data(sim_df=simulated_df, obs_df=observed_df)
merged_df2 = hd.merge_data(sim_df=corrected_df, obs_df=observed_df)
'''Plotting Data'''
daily_avg = hd.daily_average(merged_df)
daily_avg2 = hd.daily_average(merged_df2)
daily_avg_obs_Q = go.Scatter(x=daily_avg.index, y=daily_avg.iloc[:, 1].values, name='Observed', )
daily_avg_sim_Q = go.Scatter(x=daily_avg.index, y=daily_avg.iloc[:, 0].values, name='Simulated', )
daily_avg_corr_sim_Q = go.Scatter(x=daily_avg2.index, y=daily_avg2.iloc[:, 0].values,
name='Corrected Simulated', )
layout = go.Layout(
title='Daily Average Streamflow for River {0} at {1}'.format(nomRiver, nomEstacion),
xaxis=dict(title='Days', ), yaxis=dict(title='Discharge (m<sup>3</sup>/s)', autorange=True),
showlegend=True)
chart_obj = PlotlyView(go.Figure(data=[daily_avg_obs_Q, daily_avg_sim_Q, daily_avg_corr_sim_Q], layout=layout))
context = {
'gizmo_object': chart_obj,
}
return render(request, 'historical_validation_tool_somalia/gizmo_ajax.html', context)
except Exception as e:
print(str(e))
return JsonResponse({'error': 'No data found for the selected station.'})
def test_plot_seasonal(self):
daily_avg_df = hd.daily_average(df=self.merged_df)
daily_std_error = hd.daily_std_error(merged_data=self.merged_df)
# Creating test image array
hv.plot(merged_data_df=daily_avg_df,
title='Daily Average Streamflow (Standard Error)',
legend=('SFPT', 'GLOFAS'),
x_season=True,
labels=['Datetime', 'Streamflow (csm)'],
linestyles=['r-', 'k-'],
fig_size=(14, 8),
ebars=daily_std_error,
ecolor=('r', 'k'),
tight_xlim=True)
buf = BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
img_test = mpimg.imread(buf)
buf.close()
# Reading Original Image
img_original = mpimg.imread(
r'baseline_images/plot_tests/plot_seasonal.png')
# Comparing
self.assertTrue(np.all(np.isclose(img_test, img_original)))
def test_daily_average(self):
original_df = pd.read_csv(r'Comparison_Files/daily_average.csv', index_col=0)
original_df.index = original_df.index.astype(np.object)
test_df = hd.daily_average(self.merged_df)
self.assertIsNone(pd.testing.assert_frame_equal(original_df, test_df))
def historic_averages_handler(request, average_type):
"""
Controller for retrieving averages
"""
# handle the parameters from the user
try:
reach_id, region, units, return_format = handle_parameters(request)
units_title, units_title_long = get_units_title(units)
forcing = request.args.get('forcing', 'era_5')
hist_df = get_historical_dataframe(reach_id, region, units, forcing)
except Exception as e:
raise e
hist_df.index = pd.to_datetime(hist_df.index)
if average_type == 'daily':
hist_df = hd.daily_average(hist_df, rolling=True)
else:
hist_df = hd.monthly_average(hist_df)
hist_df.index.name = 'datetime'
if return_format == 'csv':
response = make_response(hist_df.to_csv())
response.headers['content-type'] = 'text/csv'
response.headers[
'Content-Disposition'] = f'attachment; filename=seasonal_average_{forcing}_{reach_id}.csv'
return response
if return_format == "json":
return jsonify({
'region':
region,
'simulation_forcing':
forcing,
'forcing_fullname':
forcing.replace('era_', 'ERA ').title(),
'comid':
reach_id,
'gendate':
datetime.datetime.utcnow().isoformat() + 'Z',
'time_series': {
'datetime': hist_df.index.tolist(),
'flow': hist_df[f'streamflow_{units_title}^3/s'].tolist(),
},
'units': {
'name': 'Streamflow',
'short': f'{units_title}3/s',
'long': f'Cubic {units_title_long} per Second'
}
})
else:
raise ValueError(f'Invalid return_format: {return_format}')
plt.savefig(volume_analysis_out_dir + '/Volume Analysis for ' + str(id) +
' - ' + name + '. COMID - ' + str(comid) + '.png')
hv.plot(merged_df,
legend=('Simulated', 'Observed'),
grid=True,
title='Hydrograph for ' + str(id) + ' - ' + name + '\n River: ' +
rio + '. COMID: ' + str(comid),
labels=['Datetime', 'Streamflow (m$^3$/s)'],
linestyles=['b-', 'r-'],
fig_size=(15, 9))
plt.savefig(
path.join(plot_out_dir,
'{0}_{1}_hydrographs.png'.format(str(id), name)))
daily_avg = hd.daily_average(merged_df)
daily_std_error = hd.daily_std_error(merged_data=merged_df)
hv.plot(merged_data_df=daily_avg,
legend=('Simulated', 'Observed'),
grid=True,
x_season=True,
title='Daily Average Streamflow (Standard Error) for ' + str(id) +
' - ' + name + '\n River: ' + rio + '. COMID: ' + str(comid),
labels=['Datetime', 'Streamflow (m$^3$/s)'],
linestyles=['b-', 'r-'],
fig_size=(15, 9),
ebars=daily_std_error,
ecolor=('b', 'r'),
tight_xlim=False)
plt.savefig(
path.join(daily_average_out_dir,
# Now there is a dictionary called 'seas_avg_dict' that has the seasonal averages stored in a pandas DataFrame.
# Each array has the datetime and flowrate.
# Each data frame is named in the format '{state}-{streamID}' (eg: 'az-7' or 'col-9').
# There are a total of 180 streams, or 180 keys in the dictionary: seas_avg_dict['az-7']
# list_streams_condensed = list of all the stream names, or names of the data frames.
# ***************************************************************************************************************
# ***************************************************************************************************************
az_9 = streamflow_dict['mo-7']
az_avg_9 = seas_avg_dict['mo-avg-7']
merged_df = hd.merge_data(sim_df=az_9,
obs_df=streamflow_dict['az-21'],
column_names=['9-calc', '21-calc'])
dailyavg2 = hd.daily_average(merged_df)
avg_calc = dailyavg2.drop(columns='21-calc')
az_avg_9.index = pd.date_range("2001-01-01", "2001-12-31").strftime("%m/%d")
group = [avg_calc, az_avg_9]
calc_vs_ncdf = pd.concat(group, axis=1)
calc_vs_ncdf.drop(index='02/29', inplace=True)
labels = ['Datetime', 'Streamflow (cms)']
hv.plot(merged_data_df=calc_vs_ncdf,
title="MO Daily Avg: Hydrostats vs Rapid ",
linestyles=['r-', 'k-'],
legend=('Hydrostats', 'Rapid NetCDF'),
labels=labels,
x_season=True,
# This list controls the axis labels:
labels=['Datetime', 'Streamflow (cms)']
# End of Dynamic Input. Do NOT Change the following -------------------------------------------
# Create a list of 18 stream modified Time Series
list_riv_part = []
for riv in list_riv_mouth:
riv_part = riv.loc[begin_date:end_date]
list_riv_part.append(riv_part)
for s, o, t, c1, c2, s1, s2 in zip(list_sim, list_obs, range(18), color1, color2, series1, series2):
merged_df = hd.merge_data(sim_df=list_riv_part[s], obs_df=list_riv_part[o])
davg_df = hd.daily_average(merged_df)
filename = list_titles[t]
hv.plot(merged_data_df=davg_df,
title=filename,
linestyles=[c1, c2],
legend=(s1, s2),
labels=labels,
metrics = metrics,
x_season=True,
grid=True)
plt.savefig('/home/chrisedwards/Documents/rapid_output/graphs/{}.png'.format(filename))
or_obs_full = pd.read_csv(
'/home/chrisedwards/Documents/gauge_data/14306500_1-1-1980_12-31-2014.csv',
index_col=0)
or_obs_cms = or_obs_full.drop(columns=["Flow-cfs", "Estimation"])
list_obs_df = [az_obs_cms, id_obs_cms, mo_obs_cms, ny_obs_cms, or_obs_cms]
x_df = or_obs_full.drop(columns=['Flow-cms', 'Estimation'])
# ------------------------------------------------------------------------------------------------------
merged_df = hd.merge_data(sim_df=x_df,
obs_df=or_obs_cms,
column_names=['Delete', 'Observed'])
temp_da = hd.daily_average(merged_df)
avg_obs = temp_da.drop(columns='Delete')
or_lowres.index = pd.date_range("2001-01-01", "2001-12-31").strftime("%m/%d")
group = [or_lowres, avg_obs]
rapid_vs_obs = pd.concat(group, axis=1)
rapid_vs_obs.drop(index='02/29', inplace=True)
# print(rapid_vs_obs)
labels = ['Datetime', 'Streamflow (cms)']
hv.plot(merged_data_df=rapid_vs_obs,
title="OR Rapid Daily Avg: Low vs Obs",
linestyles=['r-', 'k-'],
legend=('Low Res', 'Obs'),
'Low Res', 'Med Res', 'High Res', 'Low Res', 'Med Res', 'High Res',
'Low Res', 'Med Res', 'High Res'
]
s2 = 'Observed'
# This list specifies which metrics to use:
metrics = []
# This list controls the axis labels:
labels = ['Datetime', 'Streamflow (cms)']
# End of Dynamic Input. Do NOT Change the following -------------------------------------------
for s, o, t, c1, s1 in zip(list_sim, list_obs, range(15), color1, series1):
temp_merged = hd.merge_data(sim_df=x_df, obs_df=list_obs_df[o])
temp_davg = hd.daily_average(temp_merged)
obs_loop_df = temp_davg.drop(index='02/29', columns='Simulated')
sim_temp_df = list_riv_mouth[s]
sim_loop_in = sim_temp_df.index = pd.date_range(
'2001-01-01', '2001-12-31').strftime("%m/%d")
# dates=pd.date_range('2001-01-01', '2001-12-31').strftime("%m/%d")
sim_loop_df = pd.DataFrame(
sim_loop_in,
index=pd.date_range('2001-01-01', '2001-12-31').strftime("%m/%d"))
group = [sim_loop_df, obs_loop_df]
merged_df = pd.concat(group, axis=1)
filename = list_titles[t]
hv.plot(merged_data_df=merged_df,
title=filename,
