def residuals(df,
forecast,
skip_first=0,
skip_last=0,
title="Patiend ID CGM",
savefig=False):
"""Plot the input residuals.
Parameters
-------------------
df : DataFrame, the output returned by gluco_extract(..., return_df=True)
forecast : array of float, the prediction for the given time-series
skip_first : number, the number of initial samples to exclude
skip_last : number, the number of final samples to exclude
title : string, the title of the plot (optional)
savefig : bool, if True save title.png
"""
# Evaluate the residuals (exclude learning and open loop ph samples)
residuals = df.as_matrix()[skip_first:-skip_last].ravel(
) - forecast[skip_first:-skip_last]
plt.figure(figsize=(12, 4), dpi=300)
plt.plot(df.index[skip_first:-skip_last], residuals)
DW = sm.stats.durbin_watson(residuals)
plt.title('Durbin-Watson: {:.3f}'.format(DW))
if savefig: plt.savefig(title + '_residuals.png')
def plot_kim_curve(tmp):
sns.set_context("notebook", font_scale=1.5, rc={"lines.linewidth": 5})
sns.set_style("darkgrid")
plt.figure(figsize=(20, 10))
plt.hold('on')
plt.plot(np.linspace(0, 0.3, 100), tmp['kc_avg'])
plt.ylim([0, 1])
def plot(dirname, field, vert_range, label):
with catch_warnings():
# SPEEDY output is not CF compliant
simplefilter('ignore', UserWarning)
# Load cubes
print(f'Plotting {field}')
analy = load_cube(f'{dirname}/mean.nc', field)
nature = load_cube('nature.nc', field)
sprd = load_cube(f'{dirname}/sprd.nc', field)
# Get minimum duration of data
time = min(
analy.coord('time').points[-1],
nature.coord('time').points[-1])
analy = analy.extract(Constraint(time=lambda t: t < time))
nature = nature.extract(Constraint(time=lambda t: t < time))
sprd = sprd.extract(Constraint(time=lambda t: t < time))
# Extract vertically over chosen vertical range
lev_constraint_lambda = lambda s: s <= vert_range[
0] and s > vert_range[1]
# RMSE
rmse = ((analy - nature)**2)\
.extract(Constraint(atmosphere_sigma_coordinate=lev_constraint_lambda))\
.collapsed(['atmosphere_sigma_coordinate', 'longitude'], MEAN)**0.5
# Spread
sprd = sprd.extract(Constraint(atmosphere_sigma_coordinate=lev_constraint_lambda))\
.collapsed(['atmosphere_sigma_coordinate', 'longitude'], MEAN)
# Compute time mean after March 1st 00:00
with FUTURE.context(cell_datetime_objects=True):
rmse = rmse.extract(Constraint(time=lambda t: t > PartialDateTime(month=3,day=1)))\
.collapsed('time', MEAN)
sprd = sprd.extract(Constraint(time=lambda t: t > PartialDateTime(month=3,day=1)))\
.collapsed('time', MEAN)
latitude_coord = rmse.coord('latitude')
rmse_h, = plt.plot(latitude_coord.points,
rmse.data,
label=f'{label} error',
linestyle='-')
sprd_h, = plt.plot(latitude_coord.points,
sprd.data,
label=f'{label} spread',
linestyle='--',
color=rmse_h.get_color())
return [rmse_h, sprd_h]
def cgm(df,
gluco_fit=None,
hypo=70,
hyper=126,
title="Patiend ID CGM",
savefig=False):
"""Plot the CGM signal on an input time span.
Parameters
-------------------
df : DataFrame, the output returned by gluco_extract(..., return_df=True)
gluco_fit : array of float, the results of a fitted model (optional)
hypo : number, hypoglicaemia threshold in
mg/dl (default is 70, i.e. 3.9 mmol/l)
hyper : number, hyperglicaemia threshold
in mg/dl (default is 126, i.e. 7 mmol/l)
title : string, the title of the plot (optional)
savefig : bool, if True save title.png
"""
plt.figure(figsize=(10, 6), dpi=300)
plt.hlines(hypo,
df.index[0],
df.index[-1],
linestyles='dashed',
label='hypoglicaemia')
plt.hlines(hyper,
df.index[0],
df.index[-1],
linestyles='dotted',
label='hyperglicaemia')
plt.ylim([10, 410])
plt.plot_date(df.index, df.as_matrix(), '-', label='real CGM')
if gluco_fit is not None:
plt.plot(df.index, gluco_fit, '--', label='predicted CGM')
plt.title(title)
plt.ylabel('mg/dL')
plt.xticks(rotation='vertical')
plt.legend(bbox_to_anchor=(1.1, 1.0))
if savefig: plt.savefig(title + '_fit.png')
for t in np.arange(Ns):
# Xnew, Pnew = kf.filter_update(filtered_state_mean=Xold.ravel(),
# filtered_state_covariance=Pold,
# observation=ys[t])
# Pold = np.array(Pnew)
# Xold = np.array(Xnew)
y[t] = np.dot(Xnew[t, :], H)
# y[t] = np.dot(Xnew,H)
# ---------------------------------------- #
# In[36]:
# ---------- Plot ------------------------ #
plt.subplot(211)
plt.plot(ts, ys, '-', label='noisy') # noisy measures
plt.plot(ts, y, '--', label='filtered') # filtered measures
plt.grid()
plt.legend(bbox_to_anchor=(0., 1.02, 1., .102),
loc=3,
ncol=2,
mode="expand",
borderaxespad=0.)
plt.subplot(212)
residuals = ys.ravel()[0:Ns] - y.ravel()[0:Ns]
plt.plot(ts, residuals)
plt.title('Durbin-Watson: {:.3f}'.format(sm.stats.durbin_watson(residuals)))
plt.tight_layout()
plt.grid()
# ## Translate `pykalman` names into our convention
def plot(dirname, label_in):
with catch_warnings():
label = label_in
# SPEEDY output is not CF compliant
simplefilter('ignore', UserWarning)
print(f'Plotting {fields[0]}')
analy_ps = load_cube(f'{dirname}/mean.nc', fields[0])
nature_ps = load_cube('nature.nc', fields[0])
# Get minimum duration of data
time = min(
analy_ps.coord('time').points[-1],
nature_ps.coord('time').points[-1])
analy_ps = analy_ps.extract(Constraint(time=lambda t: t < time))
nature_ps = nature_ps.extract(Constraint(time=lambda t: t < time))
# Generate x date axis
with FUTURE.context(cell_datetime_objects=True):
time_axis = [x.point for x in nature_ps.coord('time').cells()]
rmse = (((analy_ps - nature_ps) / obs_errors[0])**2).collapsed(
['latitude', 'longitude'], SUM)
for field, obs_error in zip(fields[1:], obs_errors[1:]):
print(f'Plotting {field}')
for lev in levels:
# Build iris constraint object
lev_con = Constraint(atmosphere_sigma_coordinate=lev)
analy_ps = load_cube(f'{dirname}/mean.nc', field)
nature_ps = load_cube(f'nature.nc', field)
analy_ps = analy_ps.extract(
Constraint(time=lambda t: t < time) & lev_con)
nature_ps = nature_ps.extract(
Constraint(time=lambda t: t < time) & lev_con)
rmse += (((analy_ps - nature_ps) / obs_error)**2).collapsed(
['latitude', 'longitude'], SUM)
# Divide by the total number of fields (4 3D fields x 8 levels + 1 2D field) and gridpoints (96*48)
rmse = rmse / (33.0 * 96.0 * 48.0)
# Square root to get RMSE
rmse = rmse**0.5
label = label_in + ' RMSE'
# Try to compute time mean after March 1st 00:00 (if data doesn't go up to March 1st yet,
# an AttributeError will be thrown - this is ignored
try:
with FUTURE.context(cell_datetime_objects=True):
after_march = rmse.extract(
Constraint(
time=lambda t: t > PartialDateTime(month=3, day=1)))
mean = float(after_march.collapsed('time', MEAN).data)
label += f' ({mean:{4}.{3}})'
except AttributeError:
pass
rmse_h, = plt.plot(time_axis, rmse.data, label=label)
for field, obs_error in zip(fields, obs_errors):
print(f'Plotting {field}')
analy_cb = load_cube(f'{dirname}/sprd.nc', field)
analy_cb = analy_cb.extract(Constraint(time=lambda t: t < time))
if field == 'Surface Pressure [Pa]':
try:
sprd += (analy_cb / obs_error).collapsed(
['latitude', 'longitude'], SUM)
except NameError:
sprd = (analy_cb / obs_error).collapsed(
['latitude', 'longitude'], SUM)
else:
for lev in levels:
analy_cb_lev = analy_cb.extract(
Constraint(atmosphere_sigma_coordinate=lev))
try:
sprd += (analy_cb_lev / obs_error).collapsed(
['latitude', 'longitude'], SUM)
except NameError:
sprd = (analy_cb_lev / obs_error).collapsed(
['latitude', 'longitude'], SUM)
# Divide by the total number of fields (4 3D fields x 8 levels + 1 2D field) and gridpoints (96*48)
sprd = sprd / (33.0 * 96.0 * 48.0)
label = label_in + ' spread'
# Try to compute time mean after March 1st 00:00 (if data doesn't go up to March 1st yet,
# an AttributeError will be thrown - this is ignored
try:
with FUTURE.context(cell_datetime_objects=True):
after_march = sprd.extract(
Constraint(
time=lambda t: t > PartialDateTime(month=3, day=1)))
mean = float(after_march.collapsed('time', MEAN).data)
label += f' ({mean:{4}.{3}})'
except AttributeError:
pass
sprd_h, = plt.plot(time_axis,
sprd.data,
linestyle='--',
label=label,
color=rmse_h.get_color())
return [rmse_h, sprd_h]
labels= ['64 bits', '22 bits']
for d, c in zip(dirs, colors):
print(f'Plotting {d}')
plot_fields(d, c)
# Get axis limits and equalise
xlim = plt.gca().get_xlim()
ylim = plt.gca().get_ylim()
xlim = ylim = [min(xlim[0], ylim[0]), max(xlim[1], ylim[1])]
plt.xlim(xlim)
plt.ylim(ylim)
# Plot diagonal line
plt.plot(xlim, ylim, ls="--", c=".3")
plt.title('')
plt.xlabel('RMSE')
plt.ylabel('Spread')
# Legend for symbol
for field in fields:
plt.scatter(-1, -1, s=90, marker=field[2], color='.3', label=field[1])
# Legend for color
plot_lines = [plt.plot([-1,-1.5], [-1,-1.5], color=colors[0])[0]]
plot_lines.append(plt.plot([-1,-1.5], [-1,-1.5], color=colors[1])[0])
col_leg = plt.legend(plot_lines, labels, loc=6, fontsize=12)
# Add both legends
# perform moving-window arma
errs, forecast = moving_window_ARIMA(df,
w_size=w_size,
ph=ph,
p=p,
d=d,
q=q,
start_params=None,
verbose=True)
# In[13]:
# plot results
gluco_plot(df.index, df.as_matrix(), title='Patient ' + str(k))
plt.plot(df.index, forecast['ts'], linestyle='dashed', label='forecast')
plt.legend(bbox_to_anchor=(1.2, 1.0))
MAE_6 = np.mean(errs['err_6'])
MAE_12 = np.mean(errs['err_12'])
MAE_18 = np.mean(errs['err_18'])
RMSE_6 = np.linalg.norm(errs['err_6']) / np.sqrt(len(errs['err_6']))
RMSE_12 = np.linalg.norm(errs['err_12']) / np.sqrt(len(errs['err_12']))
RMSE_18 = np.linalg.norm(errs['err_18']) / np.sqrt(len(errs['err_18']))
print("MAE (30') = {:2.3f}\t|\tMAE (60') = {:2.3f}\t|\tMAE (90') = {:2.3f}".
format(MAE_6, MAE_12, MAE_18))
print("RMSE (30') = {:2.3f}\t|\tRMSE (60') = {:2.3f}\t|\tRMSE (90') = {:2.3f}".
format(RMSE_6, RMSE_12, RMSE_18))
# In[14]:
residuals = df.as_matrix()[w_size:-ph].ravel() - forecast['ts'][w_size:-ph]
def plot(dirname, field, label_in):
with catch_warnings():
label = label_in
# SPEEDY output is not CF compliant
simplefilter('ignore', UserWarning)
print(f'Plotting {field}')
analy_ps = load_cube(f'{dirname}/mean.nc', field)
nature_ps = load_cube('nature.nc', field)
# Get minimum duration of data
time = min(
analy_ps.coord('time').points[-1],
nature_ps.coord('time').points[-1])
analy_ps = analy_ps.extract(Constraint(time=lambda t: t < time))
nature_ps = nature_ps.extract(Constraint(time=lambda t: t < time))
# Generate x date axis
with FUTURE.context(cell_datetime_objects=True):
time_axis = [x.point for x in nature_ps.coord('time').cells()]
coords = ['latitude', 'longitude', 'atmosphere_sigma_coordinate']
rmse = ((analy_ps - nature_ps)**2).collapsed(coords, MEAN)**0.5
label = label_in + ' RMSE'
# Try to compute time mean after March 1st 00:00 (if data doesn't go up to March 1st yet,
# an AttributeError will be thrown - this is ignored
try:
with FUTURE.context(cell_datetime_objects=True):
after_march = rmse.extract(
Constraint(
time=lambda t: t > PartialDateTime(month=3, day=1)))
mean = float(after_march.collapsed('time', MEAN).data)
label += f' ({mean:{4}.{3}})'
except AttributeError:
pass
rmse_h, = plt.plot(time_axis, rmse.data, label=label)
analy_cb = load_cube(f'{dirname}/sprd.nc', field)
analy_cb = analy_cb.extract(Constraint(time=lambda t: t < time))
sprd = analy_cb.collapsed(coords, MEAN)
label = label_in + ' spread'
# Try to compute time mean after March 1st 00:00 (if data doesn't go up to March 1st yet,
# an AttributeError will be thrown - this is ignored
try:
with FUTURE.context(cell_datetime_objects=True):
after_march = sprd.extract(
Constraint(
time=lambda t: t > PartialDateTime(month=3, day=1)))
mean = float(after_march.collapsed('time', MEAN).data)
label += f' ({mean:{4}.{3}})'
except AttributeError:
pass
sprd_h, = plt.plot(time_axis,
sprd.data,
linestyle='--',
label=label,
color=rmse_h.get_color())
return [rmse_h, sprd_h]
resid = y - beta_0 - beta_1 * x
beta_new = beta + np.sum(resid * resid) / 2
return np.random.gamma(alpha_new, 1 / beta_new)
#data_df = pd.read_csv("1113.HK.csv", header=0, delimiter=",")
beta_0_true = 0.00008503
beta_1_true = 0.60954186
tau_true = 1
N = 100
x = np.random.uniform(low = 0, high = 4, size = N)
y = np.random.normal(beta_0_true + beta_1_true * x, 1 / np.sqrt(tau_true))
synth_plot = plt.plot(x, y, "o")
plt.xlabel("x")
plt.ylabel("y")
# specify initial values
init = {"beta_0": 0,
"beta_1": 0,
"tau": 2}
## specify hyper parameters
hypers = {"mu_0": 0,
"tau_0": 1,
"mu_1": 0,
"tau_1": 1,
"alpha": 2,
"beta": 1}