def trend_mav(fc_dict, yn, t_node, db_list, w_days, k_days):
"""
w_days : [day] +/-w_days のデータを使う
k_days : [day] 指数移動平均の時定数
"""
_tt = []
_vv = []
for db in db_list:
_tt = _tt + list(db.db['T'])
ff = [fc_dict[yn][db.label](t) for t in db.db['T']]
vv = [a/b for a, b in zip(db.db[yn], ff)]
_vv = _vv + list(vv)
_tt = np.array(_tt)
_vv = np.array(_vv)
tt_min = min(_tt)
def _mav(t):
# 時刻 t における窓付き指数移動平均
# w_days : [day] 窓幅
# k_days : [day] 時定数
#
w_post = max(0, tt_min + w_days - t)
if 1:
w_post = w_days
ndx = (_tt >= t - w_days) & (_tt <= t + w_post)
tt = _tt[ndx]
vv = _vv[ndx]
num = len(tt)
if num >= 2:
ww = np.exp(-np.abs(tt - t)/k_days)
avg = np.sum(ww*vv)/np.sum(ww)
else:
avg = np.NaN
return avg, num
# 全社平均(指数移動平均)を求める。
# f_mav(t) は補間関数
# t_node = [a for a in sorted(np.arange(t_max(db_list), t_min(db_list), -2))]
v_node, n_node = zip(*[_mav(a) for a in t_node])
v_node = np.array(v_node)
f_mav = interp(t_node, v_node)
def _err(t):
w_post = max(0, tt_min + w_days - t)
if 1:
w_post = w_days
ndx = (_tt >= t - w_days) & (_tt <= t + w_post)
tt = _tt[ndx]
vv = _vv[ndx]
d = [v - f_mav(t) for t, v in zip(tt, vv)]
return np.std(d, ddof=1)
e_node = [_err(a) for a in t_node]
return TVEN(t_node, v_node, e_node, n_node)
def proc_hilo():
""" HI - LO のトレンド
"""
fig, axes = plt.subplots(2, 1)
fig.subplots_adjust(left=0.16, right=0.86, bottom=0.15)
for (ndx, yn) in enumerate(['APP_RATE', 'NAP_RATE']):
tim2, avg2, _err, _num = tven_buf['H'][yn].by_column()
tim3, avg3, _err, _num = tven_buf['L'][yn].by_column()
f2 = interp(tim2, avg2)
f3 = interp(tim3, avg3)
tmin = max(tim2[0], tim3[0])
tmax = min(tim2[-1], tim3[-1])
t_node = np.arange(tmin, tmax, 1)
d_node = [dt_fm_sn(a) for a in t_node]
y_node = [f2(t) - f3(t) for t in t_node]
axes[ndx].plot(d_node, y_node)
axes[ndx].grid(True)
axes[ndx].set_ylim(-4, 12)
if ndx == 0:
set_date_tick(axes[ndx], (1, 4, 7, 10), '%m', 0)
else:
set_date_tick(axes[ndx], (1, 7), '%Y/%m', 30)
def main():
# オプション設定 (cfg['args'] に記録)
#
opt = options()
cfg['args'] = opt.parse_args()
args = cfg['args']
# DB 読み込み
# DB の読み込み日付の指定.
# ma_days [day] 感度解析のウィンドウ(直近 ma_days のデータから感度求める)
#
args = cfg['args']
d0 = _d(args.db_begin)
df = _d(args.db_end)
pp2, pp3, ppj = db_defs(d0, df, args.db_folder)
ppa = ppj + pp2 + pp3
# ファクター設定
fc_dict = {}
for yn in ['APP_RATE', 'NAP_RATE']:
fc_dict[yn] = {}
tstp = 10
tt = np.arange(t_min(ppa), t_max(ppa), tstp)
ff_step = calc_fact(ppa, yn, tt, d_window=6*30)
for p, f in zip(ppa, ff_step):
fc_dict[yn][p.label] = interp(tt, f)
# 補正後の平均
#
for k in ['APP_RATE', 'NAP_RATE']:
if args.k_days > 0:
t0 = sn_fm_dt(d0)
t_node = np.arange(t0, t_max(ppa) + 1, 1) # 移動平均を求める時刻
tven_buf[k] = trend_mav(fc_dict, k, t_node, ppa, w_days=30, k_days=args.k_days)
else:
t0 = sn_fm_dt(d0)
t0_sunday = t0 + (6 - d0.weekday()) # 0:月曜 6:日曜
t_node = np.arange(t0_sunday, t_max(ppa) + 3, 7) # 移動平均を求める時刻
tven_buf[k] = trend_sunday(fc_dict, k, t_node, ppa)
if 1:
proc_trend()
proc_trend_x(ppa, fc_dict)
if 1:
# 公表値/補正値/残差
fig, axes = plt.subplots(3, 2, figsize=(10, 7))
fig.subplots_adjust(left=0.1, bottom=0.1, right=0.90, top=0.95, wspace=0.44)
fig.text(0.20, 0.97, '支持する')
if args.k_days > 0:
fig.text(0.29, 0.62, '平均は指数移動平均')
fig.text(0.29, 0.60, '(時定数 %d 日)' % args.k_days)
else:
fig.text(0.29, 0.62, '平均は日曜前後数日')
proc_raw_cal_sdv(fc_dict, axes[:,0], 'APP_RATE', ppa)
fig.text(0.70, 0.97, '支持しない')
if args.k_days > 0:
fig.text(0.75, 0.62, '平均は指数移動平均')
fig.text(0.75, 0.60, '(時定数 %d 日)' % args.k_days)
else:
fig.text(0.75, 0.62, '平均は日曜前後数日')
proc_raw_cal_sdv(fc_dict, axes[:,1], 'NAP_RATE', ppa)
if args.gout:
fig.savefig(os.path.join(args.gout_folder, 'Fig%d_%s.png' % (args.gout_ndx + 2, cfg['gout_date'])))
if 1:
proc_factor(ppa, 'APP_RATE', '内閣 支持率 感度係数', fc_dict, 3)
proc_factor(ppa, 'NAP_RATE', '内閣 不支持率 感度係数', fc_dict, 4)
# 表示
# イメージファイルの出力は proc_last() 内で行う
# 複数の図を出力するアプリの plot.show() は一つなので main の最後で実施する
if args.gout == False:
plt.show()
def proc_raw_cal_sdv(fc_dict, axes, k_app_nap, db_list):
""" 発表値、補正値、補正値の残差のグラフ
"""
args = cfg['args']
if args.rainbow:
cm = plt.get_cmap('rainbow')
def _c(j):
return cm(1.0 - j/len(db_list))
else:
cm = plt.get_cmap('tab10')
def _c(j):
return cm(j)
tim, val, err, num = tven_buf[k_app_nap].by_column()
ax =axes[0]
ax.set_ylim(20, 70)
ax.set_ylabel('調査結果(発表値) %')
for (j, db) in enumerate(db_list):
dd = [dt_fm_sn(a) for a in db.db['T']]
if db.label != 'SSRC':
ax.plot(dd, db.db[k_app_nap], db.marker+'-', ms=db.size*0.5, color=_c(j), label=db.label, alpha=0.5)
else:
ddvv = [a for a in zip(dd, db.db[k_app_nap]) if a[0] > datetime(2020, 4, 1)]
dd, vv = [a for a in zip(*ddvv)]
ax.plot(dd, vv, db.marker+'-', ms=db.size*0.5, color=_c(j), label=db.label, alpha=0.5)
set_date_tick(ax, (1,4,7,10), '%m', 0)
ax.grid(True)
ax.legend(loc='upper left', bbox_to_anchor=(1.0, 1.0))
ax =axes[1]
ax.set_ylim(20, 70)
ax.set_ylabel('感度補正後(曲線は平均値) %')
ax.set_ylabel('感度補正後 %')
for (j, db) in enumerate(db_list):
dd = [dt_fm_sn(a) for a in db.db['T']]
vv = [a/fc_dict[k_app_nap][db.label](b) for a, b in zip(db.db[k_app_nap], db.db['T'])]
if db.label != 'SSRC':
ax.plot(dd, vv, db.marker, ms=db.size*0.5, color=_c(j), label=db.label, alpha=0.5)
else:
ddvv = [a for a in zip(dd, vv) if a[0] > datetime(2020, 4, 1)]
dd, vv = [a for a in zip(*ddvv)]
ax.plot(dd, vv, db.marker+'-', ms=db.size*0.5, color=_c(j), label=db.label, alpha=0.5)
dd = [dt_fm_sn(a) for a in tim]
ee = err/np.sqrt(num)
ax.fill_between(dd, val-ee, val+ee, color='blue', alpha=0.1)
ax.plot(dd, val,'-', color='blue', lw=1, alpha=1)
set_date_tick(ax, (1,4,7,10), '%m', 0)
ax.grid(True)
ax =axes[2]
ax.set_ylim(-8, 8)
ax.set_ylabel('感度補正後の残差 %')
for (j, db) in enumerate(db_list):
vv = [a/fc_dict[k_app_nap][db.label](b) for a, b in zip(db.db[k_app_nap], db.db['T'])]
dv, sd = deviation(db.db['T'], vv, interp(tim, val))
dd = [dt_fm_sn(a) for a in db.db['T']]
ax.plot(dd, dv, '-', color=_c(j), label=db.label, alpha=0.5)
set_date_tick(ax, (1, 7), '%Y/%m', 30)
ax.grid(True)
ax.legend(loc='upper left', bbox_to_anchor=(1.0, 1.0))
def main():
# オプション設定 (cfg['args'] に記録)
#
opt = options()
cfg['args'] = opt.parse_args()
args = cfg['args']
# DB 読み込み
# DB の読み込み日付の指定.
# ma_days [day] 感度解析のウィンドウ(直近 ma_days のデータから感度求める)
#
args = cfg['args']
d0 = _d(args.db_begin)
df = _d(args.db_end)
pp2, pp3, ppj = db_defs(d0, df, args.db_folder)
# ファクター設定
fc_dict = {}
for yn in ['APP_RATE', 'NAP_RATE']:
fc_dict[yn] = {}
for db_list in [ppj, pp2, pp3]:
tstp = 10
tt = np.arange(t_min(db_list), t_max(db_list), tstp)
ff_step = calc_fact(db_list, yn, tt, d_window=6 * 30)
for db, f in zip(db_list, ff_step):
fc_dict[yn][db.label] = interp(tt, f)
# 補正後の平均
#
t0 = sn_fm_dt(d0)
t_node2 = np.arange(t0, t_max(pp2) + 1, 1)
t_node3 = np.arange(t0, t_max(pp3) + 1, 1)
t_nodej = np.arange(t0, t_max(ppj) + 1, 1)
for k in ['APP_RATE', 'NAP_RATE']:
if args.k_days > 0:
tven_buf['H'][k] = trend_mav(fc_dict,
k,
t_node2,
pp2,
w_days=30,
k_days=args.k_days)
else:
t0 = sn_fm_dt(d0)
t0_sunday = t0 + (6 - d0.weekday()) # 0:月曜 6:日曜
t_node = np.arange(t0_sunday, t_max(pp2) + 3, 7) # 移動平均を求める時刻
tven_buf['H'][k] = trend_sunday(fc_dict, k, t_node, pp2)
for k in ['APP_RATE', 'NAP_RATE']:
if args.k_days > 0:
tven_buf['L'][k] = trend_mav(fc_dict,
k,
t_node3,
pp3,
w_days=30,
k_days=args.k_days)
else:
t0 = sn_fm_dt(d0)
t0_sunday = t0 + (6 - d0.weekday()) # 0:月曜 6:日曜
t_node = np.arange(t0_sunday, t_max(pp3) + 3, 7) # 移動平均を求める時刻
tven_buf['L'][k] = trend_sunday(fc_dict, k, t_node, pp3)
for k in ['APP_RATE', 'NAP_RATE']:
if args.k_days > 0:
tven_buf['J'][k] = trend_mav(fc_dict,
k,
t_nodej,
ppj,
w_days=30,
k_days=args.k_days)
else:
t0 = sn_fm_dt(d0)
t0_sunday = t0 + (6 - d0.weekday()) # 0:月曜 6:日曜
t_node = np.arange(t0_sunday, t_max(ppj) + 3, 7) # 移動平均を求める時刻
tven_buf['J'][k] = trend_sunday(fc_dict, k, t_node, ppj)
if 1:
proc_trend()
if 1:
# 公表値/補正値/残差
for yn in ['APP_RATE', 'NAP_RATE']:
fig, axes = plt.subplots(3, 2, figsize=(10, 7))
fig.subplots_adjust(left=0.1,
bottom=0.1,
right=0.90,
top=0.95,
wspace=0.44)
fig.text(0.15, 0.97, '支持%s(グループH)' % {
'APP_RATE': 'する',
'NAP_RATE': 'しない'
}[yn])
if args.k_days > 0:
fig.text(0.29, 0.62, '平均は指数移動平均')
fig.text(0.29, 0.60, '(時定数 %d 日)' % args.k_days)
else:
fig.text(0.29, 0.62, '平均は日曜前後数日')
tt2, avg, err, num = tven_buf['H'][yn].by_column()
proc_raw_cal_sdv(fc_dict, axes[:, 0], yn, pp2, tt2, avg)
fig.text(0.65, 0.97, '支持%s(グループL)' % {
'APP_RATE': 'する',
'NAP_RATE': 'しない'
}[yn])
if args.k_days > 0:
fig.text(0.75, 0.62, '平均は指数移動平均')
fig.text(0.75, 0.60, '(時定数 %d 日)' % args.k_days)
else:
fig.text(0.75, 0.62, '平均は日曜前後数日')
tt3, avg, err, num = tven_buf['L'][yn].by_column()
proc_raw_cal_sdv(fc_dict, axes[:, 1], yn, pp3, tt3, avg)
if args.gout:
if yn == 'APP_RATE':
fig.savefig(
os.path.join(
args.gout_folder, 'Fig%d_%s.png' %
(args.gout_ndx + 2, cfg['gout_date'])))
else:
fig.savefig(
os.path.join(
args.gout_folder, 'Fig%d_%s.png' %
(args.gout_ndx + 3, cfg['gout_date'])))
if 1:
proc_yn(pp2, pp3, ppj)
if 1:
proc_fc(fc_dict, pp2, pp3, ppj)
if 1:
proc_hilo()
# 表示
# イメージファイルの出力は proc_last() 内で行う
# 複数の図を出力するアプリの plot.show() は一つなので main の最後で実施する
if args.gout == False:
plt.show()
def proc_raw_cal_sdv(fc_dict, axes, yn, db_list, tim, avg):
ax = axes[0]
ax.set_ylim(20, 70)
ax.set_ylabel('調査結果(発表値) %')
for db in db_list:
dd = [dt_fm_sn(a) for a in db.db['T']]
if db.label != 'SSRC':
ax.plot(dd,
db.db[yn],
db.marker,
ms=db.size * 0.5,
label=db.label,
alpha=0.5)
else:
ddvv = [
a for a in zip(dd, db.db[yn]) if a[0] > datetime(2020, 4, 1)
]
dd, vv = [a for a in zip(*ddvv)]
ax.plot(dd,
vv,
db.marker + '-',
ms=db.size * 0.5,
label=db.label,
alpha=0.5)
set_date_tick(ax, (1, 4, 7, 10), '%m', 0)
ax.grid(True)
ax.legend(loc='upper left', bbox_to_anchor=(1.0, 1.0))
ax = axes[1]
ax.set_ylim(20, 70)
ax.set_ylabel('感度補正後(線は平均値) %')
for db in db_list:
dd = [dt_fm_sn(a) for a in db.db['T']]
ff = [fc_dict[yn][db.label](t) for t in db.db['T']]
vv = [a / b for a, b in zip(db.db[yn], ff)]
if db.label != 'SSRC':
ax.plot(dd,
vv,
db.marker,
ms=db.size * 0.5,
label=db.label,
alpha=0.5)
else:
ddvv = [a for a in zip(dd, vv) if a[0] > datetime(2020, 4, 1)]
dd, vv = [a for a in zip(*ddvv)]
ax.plot(dd,
vv,
db.marker + '-',
ms=db.size * 0.5,
label=db.label,
alpha=0.5)
dd = [dt_fm_sn(a) for a in tim]
ax.plot(dd, avg, '-', color='royalblue', lw=2) #, alpha=0.1)
set_date_tick(ax, (1, 4, 7, 10), '%m', 0)
ax.grid(True)
ax.legend(loc='upper left', bbox_to_anchor=(1.0, 1.0))
ax = axes[2]
ax.set_ylim(-8, 8)
ax.set_ylabel('感度補正後の残差 %')
for db in db_list:
vv = [
a / fc_dict[yn][db.label](b)
for (a, b) in zip(db.db[yn], db.db['T'])
]
ss, sd = deviation(db.db['T'], vv, interp(tim, avg))
dd = [dt_fm_sn(a) for a in db.db['T']]
ax.plot(dd, ss, '-', label=db.label, alpha=0.5)
set_date_tick(ax, (1, 7), '%Y/%m', 30)
ax.grid(True)
ax.legend(loc='upper left', bbox_to_anchor=(1.0, 1.0))