def corrected_day_average(corrected: pd.DataFrame, simulated: pd.DataFrame, observed: pd.DataFrame, merged_sim_obs: pd.DataFrame = False, merged_cor_obs: pd.DataFrame = False, titles: dict = None, outformat: str = 'plotly') -> go.Figure or str: """ Calculates and plots the daily average streamflow. This function uses hydrostats.data.merge_data on the 3 inputs. If you have already computed these because you are doing a full comparison of bias correction, you can provide them to save time Args: corrected: the response from the geoglows.bias.correct_historical_simulation function simulated: the csv response from historic_simulation merged_sim_obs: (optional) if you have already computed it, hydrostats.data.merge_data(simulated, observed) merged_cor_obs: (optional) if you have already computed it, hydrostats.data.merge_data(corrected, observed) observed: the dataframe of observed data. Must have a datetime index and a single column of flow values outformat: either 'plotly' or 'plotly_html' (default plotly) titles: (dict) Extra info to show on the title of the plot. For example: {'Reach ID': 1234567, 'Drainage Area': '1000km^2'} Returns: plotly.GraphObject: plotly object, especially for use with python notebooks and the .show() method """ if corrected is False and simulated is False and observed is False: if merged_sim_obs is not False and merged_cor_obs is not False: pass # if you provided the merged dataframes already, we use those else: # merge the datasets together merged_sim_obs = hd.merge_data(sim_df=simulated, obs_df=observed) merged_cor_obs = hd.merge_data(sim_df=corrected, obs_df=observed) daily_avg = hd.daily_average(merged_sim_obs) daily_avg2 = hd.daily_average(merged_cor_obs) scatters = [ go.Scatter(x=daily_avg.index, y=daily_avg.iloc[:, 1].values, name='Observed Data'), go.Scatter(x=daily_avg.index, y=daily_avg.iloc[:, 0].values, name='Simulated Data'), go.Scatter(x=daily_avg2.index, y=daily_avg2.iloc[:, 0].values, name='Corrected Simulated Data'), ] layout = go.Layout( title=_build_title('Daily Average Streamflow Comparison', titles), xaxis=dict(title='Days'), yaxis=dict(title='Discharge (m<sup>3</sup>/s)', autorange=True), showlegend=True) if outformat == 'plotly': return go.Figure(data=scatters, layout=layout) elif outformat == 'plotly_html': return offline_plot( go.Figure(data=scatters, layout=layout), config={'autosizable': True, 'responsive': True}, output_type='div', include_plotlyjs=False ) raise ValueError('Invalid outformat chosen. Choose plotly or plotly_html')
def get_dailyAverages(request): """ Get observed data from csv files in Hydroshare Get historic simulations from ERA Interim """ get_data = request.GET global nomRiver global nomEstacion global simulated_df global observed_df global corrected_df try: '''Merge Data''' merged_df = hd.merge_data(sim_df=simulated_df, obs_df=observed_df) merged_df2 = hd.merge_data(sim_df=corrected_df, obs_df=observed_df) '''Plotting Data''' daily_avg = hd.daily_average(merged_df) daily_avg2 = hd.daily_average(merged_df2) daily_avg_obs_Q = go.Scatter(x=daily_avg.index, y=daily_avg.iloc[:, 1].values, name='Observed', ) daily_avg_sim_Q = go.Scatter(x=daily_avg.index, y=daily_avg.iloc[:, 0].values, name='Simulated', ) daily_avg_corr_sim_Q = go.Scatter(x=daily_avg2.index, y=daily_avg2.iloc[:, 0].values, name='Corrected Simulated', ) layout = go.Layout( title='Daily Average Streamflow for River {0} at {1}'.format(nomRiver, nomEstacion), xaxis=dict(title='Days', ), yaxis=dict(title='Discharge (m<sup>3</sup>/s)', autorange=True), showlegend=True) chart_obj = PlotlyView(go.Figure(data=[daily_avg_obs_Q, daily_avg_sim_Q, daily_avg_corr_sim_Q], layout=layout)) context = { 'gizmo_object': chart_obj, } return render(request, 'historical_validation_tool_somalia/gizmo_ajax.html', context) except Exception as e: print(str(e)) return JsonResponse({'error': 'No data found for the selected station.'})
def test_plot_seasonal(self): daily_avg_df = hd.daily_average(df=self.merged_df) daily_std_error = hd.daily_std_error(merged_data=self.merged_df) # Creating test image array hv.plot(merged_data_df=daily_avg_df, title='Daily Average Streamflow (Standard Error)', legend=('SFPT', 'GLOFAS'), x_season=True, labels=['Datetime', 'Streamflow (csm)'], linestyles=['r-', 'k-'], fig_size=(14, 8), ebars=daily_std_error, ecolor=('r', 'k'), tight_xlim=True) buf = BytesIO() plt.savefig(buf, format='png') buf.seek(0) img_test = mpimg.imread(buf) buf.close() # Reading Original Image img_original = mpimg.imread( r'baseline_images/plot_tests/plot_seasonal.png') # Comparing self.assertTrue(np.all(np.isclose(img_test, img_original)))
def test_daily_average(self): original_df = pd.read_csv(r'Comparison_Files/daily_average.csv', index_col=0) original_df.index = original_df.index.astype(np.object) test_df = hd.daily_average(self.merged_df) self.assertIsNone(pd.testing.assert_frame_equal(original_df, test_df))
def historic_averages_handler(request, average_type): """ Controller for retrieving averages """ # handle the parameters from the user try: reach_id, region, units, return_format = handle_parameters(request) units_title, units_title_long = get_units_title(units) forcing = request.args.get('forcing', 'era_5') hist_df = get_historical_dataframe(reach_id, region, units, forcing) except Exception as e: raise e hist_df.index = pd.to_datetime(hist_df.index) if average_type == 'daily': hist_df = hd.daily_average(hist_df, rolling=True) else: hist_df = hd.monthly_average(hist_df) hist_df.index.name = 'datetime' if return_format == 'csv': response = make_response(hist_df.to_csv()) response.headers['content-type'] = 'text/csv' response.headers[ 'Content-Disposition'] = f'attachment; filename=seasonal_average_{forcing}_{reach_id}.csv' return response if return_format == "json": return jsonify({ 'region': region, 'simulation_forcing': forcing, 'forcing_fullname': forcing.replace('era_', 'ERA ').title(), 'comid': reach_id, 'gendate': datetime.datetime.utcnow().isoformat() + 'Z', 'time_series': { 'datetime': hist_df.index.tolist(), 'flow': hist_df[f'streamflow_{units_title}^3/s'].tolist(), }, 'units': { 'name': 'Streamflow', 'short': f'{units_title}3/s', 'long': f'Cubic {units_title_long} per Second' } }) else: raise ValueError(f'Invalid return_format: {return_format}')
plt.savefig(volume_analysis_out_dir + '/Volume Analysis for ' + str(id) + ' - ' + name + '. COMID - ' + str(comid) + '.png') hv.plot(merged_df, legend=('Simulated', 'Observed'), grid=True, title='Hydrograph for ' + str(id) + ' - ' + name + '\n River: ' + rio + '. COMID: ' + str(comid), labels=['Datetime', 'Streamflow (m$^3$/s)'], linestyles=['b-', 'r-'], fig_size=(15, 9)) plt.savefig( path.join(plot_out_dir, '{0}_{1}_hydrographs.png'.format(str(id), name))) daily_avg = hd.daily_average(merged_df) daily_std_error = hd.daily_std_error(merged_data=merged_df) hv.plot(merged_data_df=daily_avg, legend=('Simulated', 'Observed'), grid=True, x_season=True, title='Daily Average Streamflow (Standard Error) for ' + str(id) + ' - ' + name + '\n River: ' + rio + '. COMID: ' + str(comid), labels=['Datetime', 'Streamflow (m$^3$/s)'], linestyles=['b-', 'r-'], fig_size=(15, 9), ebars=daily_std_error, ecolor=('b', 'r'), tight_xlim=False) plt.savefig( path.join(daily_average_out_dir,
# Now there is a dictionary called 'seas_avg_dict' that has the seasonal averages stored in a pandas DataFrame. # Each array has the datetime and flowrate. # Each data frame is named in the format '{state}-{streamID}' (eg: 'az-7' or 'col-9'). # There are a total of 180 streams, or 180 keys in the dictionary: seas_avg_dict['az-7'] # list_streams_condensed = list of all the stream names, or names of the data frames. # *************************************************************************************************************** # *************************************************************************************************************** az_9 = streamflow_dict['mo-7'] az_avg_9 = seas_avg_dict['mo-avg-7'] merged_df = hd.merge_data(sim_df=az_9, obs_df=streamflow_dict['az-21'], column_names=['9-calc', '21-calc']) dailyavg2 = hd.daily_average(merged_df) avg_calc = dailyavg2.drop(columns='21-calc') az_avg_9.index = pd.date_range("2001-01-01", "2001-12-31").strftime("%m/%d") group = [avg_calc, az_avg_9] calc_vs_ncdf = pd.concat(group, axis=1) calc_vs_ncdf.drop(index='02/29', inplace=True) labels = ['Datetime', 'Streamflow (cms)'] hv.plot(merged_data_df=calc_vs_ncdf, title="MO Daily Avg: Hydrostats vs Rapid ", linestyles=['r-', 'k-'], legend=('Hydrostats', 'Rapid NetCDF'), labels=labels, x_season=True,
# This list controls the axis labels: labels=['Datetime', 'Streamflow (cms)'] # End of Dynamic Input. Do NOT Change the following ------------------------------------------- # Create a list of 18 stream modified Time Series list_riv_part = [] for riv in list_riv_mouth: riv_part = riv.loc[begin_date:end_date] list_riv_part.append(riv_part) for s, o, t, c1, c2, s1, s2 in zip(list_sim, list_obs, range(18), color1, color2, series1, series2): merged_df = hd.merge_data(sim_df=list_riv_part[s], obs_df=list_riv_part[o]) davg_df = hd.daily_average(merged_df) filename = list_titles[t] hv.plot(merged_data_df=davg_df, title=filename, linestyles=[c1, c2], legend=(s1, s2), labels=labels, metrics = metrics, x_season=True, grid=True) plt.savefig('/home/chrisedwards/Documents/rapid_output/graphs/{}.png'.format(filename))
or_obs_full = pd.read_csv( '/home/chrisedwards/Documents/gauge_data/14306500_1-1-1980_12-31-2014.csv', index_col=0) or_obs_cms = or_obs_full.drop(columns=["Flow-cfs", "Estimation"]) list_obs_df = [az_obs_cms, id_obs_cms, mo_obs_cms, ny_obs_cms, or_obs_cms] x_df = or_obs_full.drop(columns=['Flow-cms', 'Estimation']) # ------------------------------------------------------------------------------------------------------ merged_df = hd.merge_data(sim_df=x_df, obs_df=or_obs_cms, column_names=['Delete', 'Observed']) temp_da = hd.daily_average(merged_df) avg_obs = temp_da.drop(columns='Delete') or_lowres.index = pd.date_range("2001-01-01", "2001-12-31").strftime("%m/%d") group = [or_lowres, avg_obs] rapid_vs_obs = pd.concat(group, axis=1) rapid_vs_obs.drop(index='02/29', inplace=True) # print(rapid_vs_obs) labels = ['Datetime', 'Streamflow (cms)'] hv.plot(merged_data_df=rapid_vs_obs, title="OR Rapid Daily Avg: Low vs Obs", linestyles=['r-', 'k-'], legend=('Low Res', 'Obs'),
'Low Res', 'Med Res', 'High Res', 'Low Res', 'Med Res', 'High Res', 'Low Res', 'Med Res', 'High Res' ] s2 = 'Observed' # This list specifies which metrics to use: metrics = [] # This list controls the axis labels: labels = ['Datetime', 'Streamflow (cms)'] # End of Dynamic Input. Do NOT Change the following ------------------------------------------- for s, o, t, c1, s1 in zip(list_sim, list_obs, range(15), color1, series1): temp_merged = hd.merge_data(sim_df=x_df, obs_df=list_obs_df[o]) temp_davg = hd.daily_average(temp_merged) obs_loop_df = temp_davg.drop(index='02/29', columns='Simulated') sim_temp_df = list_riv_mouth[s] sim_loop_in = sim_temp_df.index = pd.date_range( '2001-01-01', '2001-12-31').strftime("%m/%d") # dates=pd.date_range('2001-01-01', '2001-12-31').strftime("%m/%d") sim_loop_df = pd.DataFrame( sim_loop_in, index=pd.date_range('2001-01-01', '2001-12-31').strftime("%m/%d")) group = [sim_loop_df, obs_loop_df] merged_df = pd.concat(group, axis=1) filename = list_titles[t] hv.plot(merged_data_df=merged_df, title=filename,