def plot_aggs_stat(self):
# This function plots the all aggregated statistics
# x: aggregated time;
# y: particular metrics
list_func = func_lists()
dis_mat = read_dis_file('Rain_gauges.xlsx', self.gauge_all)
for col in self.gauge_all.columns:
for key in list_func.keys():
df = self.single_stat(key)
Stations = cluster_num(dis_mat[col], 5)
df_new = df.loc[:, Stations.keys()]
trace = []
for col2 in df_new.columns:
trace.append(
go.Scatter(x=df_new.index,
y=df_new[col2],
name=str(col2) + ' ' +
str(round(Stations[col2] / 1000)) + 'km'))
layout = go.Layout(title=f'{col}-{key}')
fig = go.Figure(data=trace, layout=layout)
os.chdir('./metrics-analysis')
plotly.offline.plot(fig,
filename=f'{col}-{key}.html',
auto_open=False)
os.chdir('../')
def cal_all_agg_stat(self):
aggs = np.arange(60, 1500, 60)
aggs = np.r_[10, 30, aggs]
aggs = [str(i) + 'min' for i in aggs]
# aggs = ['30min', '60min', '120min', '180min', '240min', '300min']
gauge_10min, radar_10min = self.read_gauge_radar_10min()
writer = pd.ExcelWriter('agg_all_stat_new.xlsx')
for agg in aggs:
if agg == '10min':
stat = statistics(gauge_10min, radar_10min, func_lists()).T
stat.to_excel(writer, sheet_name=f'{agg}')
else:
gauge, radar = self.data_agg(
gauge_10min, freq=agg), self.data_agg(radar_10min,
freq=agg)
stat = statistics(gauge, radar, func_lists()).T
stat.to_excel(writer, sheet_name=f'{agg}')
print(f'{agg} completed')
def statistics_cal_station(writer):
'''
for station based statistics, concatenate all events together to calculate one value
'''
func_list = func_lists()
gauge, radar, inds = eventselect()
gauge_station = pd.DataFrame()
radar_station = pd.DataFrame()
for i, ind in enumerate(inds):
_gauge_station = gauge[ind]
_radar_station = radar[ind]
gauge_station = gauge_station.append(_gauge_station, ignore_index=True)
radar_station = radar_station.append(_radar_station, ignore_index=True)
stat_station = statistics(gauge_station, radar_station, func_list)
stat_station.to_excel(writer, sheet_name='station')
return stat_station
def statistics_cal_event(writer):
'''
for event based statistics, concatenate all stations in one event
'''
func_list = func_lists()
gauge, radar, inds = eventselect()
col_names = [f'event{i+1}' for i in range(len(inds))]
stats = pd.DataFrame(columns=col_names, index=func_list.keys())
col = gauge.columns
radar = radar.loc[:, col]
for i, ind in enumerate(inds):
_gauge_event = gauge[ind]
_radar_event = radar[ind]
for key, func in func_list.items():
stats[f'event{i+1}'][key] = func(_gauge_event.agg('sum', axis=1),
_radar_event.agg('sum', axis=1))
stats.to_excel(writer, sheet_name='event')
return stats
from AggregateAllScripts import RadarGauge
from RadarGauge_statistics import func_lists, statistics
#try with 1 hour data
os.chdir('D:\\Radar Projects\\lizhi\\for LiZhi')
#RG = RadarGauge('D:\\Radar Projects\\lizhi\\for LiZhi')
#RG.cal_all_agg_stat() #calculate all statistics over each station output agg_all_stat_new.xlsx
#_ = RG.read_gauge_all_data() # this just initialize all needed data no need to store
#RG.plot_aggs_stat() #this plot every statistic at each station
#adjusted_rain = pd.read_excel('10min-bias-adjusted.xlsx')
radar_1h = pd.read_excel('60min-gauge-radar.xlsx', sheet_name='radar')
radar_30min = pd.read_excel('30min-gauge-radar.xlsx', sheet_name='radar')
radar_10min = pd.read_excel('10min-gauge-radar.xlsx', sheet_name='radar')
radar_180min = pd.read_excel('180min-gauge-radar.xlsx', sheet_name='radar')
radars = [radar_1h, radar_30min, radar_10min, radar_180min]
xl = pd.ExcelFile('marked-data-old.xlsx')
sheet_names = xl.sheet_names
writer = pd.ExcelWriter('Statistics-marked-old.xlsx')
for i, sheet_name in enumerate(sheet_names):
gauge = xl.parse(sheet_name)
stat = statistics(gauge, radars[i], func_lists())
stat.to_excel(writer, sheet_name=sheet_name)
writer.close()
gauge_10min = pd.read_excel('10min-bias-adjusted-each.xlsx')
func_lists = func_lists()
stats = statistics(gauge_10min[gauge_10min.index > '2018-11-14'],
radar_10min[radar_10min.index > '2018-11-14'], func_lists)
func_lists['pearson'](gauge_10min['S100'][gauge_10min.index > '2018-11-14'],
radar_10min['S100'][radar_10min.index > '2018-11-14'])
def statistics_day(self, writer):
return daily(func_lists(), writer)
def statistics_10min(self, writer):
return tens(func_lists(), writer)