def hp_detrend(times, interpolated_data):
"""
Does a Hodrick-Prescott detrending always using an estimated period
of 24h.
"""
timer = plo.laptimer()
print "Detrending data... time: ",
detrended_data = np.zeros(interpolated_data.shape)
trendlines = np.zeros(interpolated_data.shape)
for idx, idata in enumerate(interpolated_data.T):
# copy data for editing
cell = np.copy(idata)
model = blu.Bioluminescence(times, cell, period_guess=24)
# all we care about is the HP filter, no need to fit everything
model.filter()
model.detrend()
baseline = model.yvals['mean']
# find where recording starts and there are no nans
valid = ~np.isnan(cell)
# subtrect baseline from recording
cell[valid] = cell[valid] - baseline
# put the detrended cell in the detrended_data matrix,
# and same for baseline
detrended_data[:, idx] = cell
trendlines[valid, idx] = baseline
print str(np.round(timer(), 1)) + "s"
return times, detrended_data, trendlines
def analyze_experiment(path, filename):
""" plots the result for a single experiment """
data = load_invivo_data(path + filename, binning=8)
phase_results = cwt_phases(data['times'], data['counts'])
fig = plt.figure()
plo.PlotOptions(ticks='in')
ax = plt.subplot()
#plot stimulations
ax.add_patch(
patches.Rectangle(
(data['L'][0],
-data['L'][2] * 2 * np.pi / 24 + 2 * np.pi - 2), # (x,y)
data['L'][1] - data['L'][0], # width
1.0 * 2 * np.pi / 24, # height
color='h',
label='Lo'))
ax.add_patch(
patches.Rectangle(
(data['H'][0],
-data['H'][2] * 2 * np.pi / 24 + 2 * np.pi - 2), # (x,y)
data['H'][1] - data['H'][0], # width
1.0 * 2 * np.pi / 24, # height
color='j',
label='Hi/Lo'))
ax.plot(phase_results['days'],
phase_results['phis'],
'kx',
mew=1,
label=filename[:-4])
#ax.set_ylim([0,24])
plo.format_2pi_axis(ax, y0=True, x=False)
ax.set_xlabel('Day')
ax.set_ylabel('Running Phase (rad)')
plt.legend()
plt.tight_layout()
fig.savefig(path + filename[:-4] + 'phases.png')
plt.clf()
plt.close(fig)
def butterworth_lowpass(times, data, cutoff_period=4):
"""
Does a Butterworth low pass filtering of the data.
"""
timer = plo.laptimer()
print "Butterworth filter... time: ",
denoised_data = np.zeros(data.shape)
for idx in range(len(data.T)):
# copy data for editing
idata = np.copy(data[:, idx])
valid = ~np.isnan(idata)
_, filtdata = blu.lowpass_filter(times[valid],
idata[valid],
cutoff_period=cutoff_period)
# subtrect baseline from recording
denoised_data[valid, idx] = filtdata
print str(np.round(timer(), 1)) + "s"
return times, denoised_data
def LS_pgram(times, ls_data, circ_low=18, circ_high=30, alpha=0.05):
"""Calculates a LS periodogram for each data sequence,
and returns the p-values for each peak. If the largest significant
peak is in the circadian range as specified by the args, it is
rhythmic."""
timer = plo.laptimer()
print "Lomb-Scargle Periodogram... time: ",
rhythmic_or_not = np.zeros(len(ls_data.T))
pgram_data = np.zeros((300, len(ls_data.T)))
circadian_peaks = np.zeros(len(ls_data.T))
circadian_peak_periods = np.zeros(len(ls_data.T))
np.seterr(divide='ignore', invalid='ignore')
# pgram
for data_idx, d1 in enumerate(ls_data.T):
# remove nans
t1 = np.copy(times[~np.isnan(d1)])
d1 = np.copy(d1[~np.isnan(d1)])
pers, pgram, sig = blu.periodogram(t1,
d1,
period_low=1,
period_high=60,
res=300)
peak = np.argmax(pgram)
if (pers[peak] >= circ_low and pers[peak] <= circ_high
and sig[peak] <= alpha):
rhythmic_or_not[data_idx] = 1
#
circadian_peaks[data_idx] = pgram[peak]
circadian_peak_periods[data_idx] = pers[peak]
else:
minpeak = np.argmax(pers >= circ_low)
maxpeak = np.argmin(pers < circ_high)
circadian_peaks[data_idx] = np.max(pgram[minpeak:maxpeak])
circadian_peak_periods[data_idx] =\
pers[minpeak:maxpeak][np.argmax(pgram[minpeak:maxpeak])]
# return either normed or un-normed data
pgram_data[:, data_idx] = pgram
print str(np.round(timer(), 1)) + "s"
return pers, pgram_data, circadian_peaks, circadian_peak_periods, rhythmic_or_not
def plot_bounds(axs, bounds_dicts):
ax, bx, cx = axs
delta_phi_fs = np.asarray(bounds_dicts[0]['delta_phi_fs'])
numcycles = np.asarray(bounds_dicts[0]['numcycles'])
directions = np.asarray(bounds_dicts[0]['directions'])
dir_switch = delta_phi_fs[np.argmax(np.abs(np.diff(directions)))]
ax.plot(delta_phi_fs, numcycles, color='k')
ax.axvline(dir_switch, color='k', ls=":")
ax.set_ylabel("Number Cycles,\nContinuous Control")
ax.set_ylim([0, 3.4])
colors = ['l', 'h', 'f']
labels = ['4h', '2h', '1h']
for idx, bounds_dict in enumerate(bounds_dicts):
cyc_to_reachs = np.asarray(bounds_dict['cyc_to_reachs'])
regions_missed = np.asarray(bounds_dict['regions_missed'])
del_phis = np.asarray(bounds_dict['del_phis'])
bx.plot(delta_phi_fs, del_phis, color=colors[idx], label=labels[idx])
bx.set_ylabel("Minimal-time\nphase error (rad)")
cx.set_ylabel("Max Additional\nCycles to Reset")
cx.plot(delta_phi_fs,
cyc_to_reachs - numcycles,
color=colors[idx],
label=labels[idx])
cx.set_xlabel(r"$\Delta\phi_f$")
bx.axvline(dir_switch, color='k', ls=":")
cx.axvline(dir_switch, color='k', ls=":")
bx.set_ylim([0, 1.5])
cx.set_ylim([0, 8])
plo.format_2pi_axis(ax)
plo.format_2pi_axis(bx)
plo.format_2pi_axis(cx)
ax.set_xticklabels([])
bx.set_xticklabels([])
# load reference optimal
t_opts_ref = np.load('Data/t_opts.npy')
#get how l-infty norm, how tstar max changes
linfty = [0]
step_taus = np.arange(0.5, 13, 0.5)
max_tstar = [np.max(t_opts_ref)]
for step in step_taus:
t_opts, delta_phi_fs = t_opts_calc(phi0s, step)
linfty.append(np.max(t_opts - t_opts_ref))
max_tstar.append(np.max(t_opts))
step_taus = np.hstack([[0], step_taus])
# plot results
plo.PlotOptions(uselatex=True, ticks='in')
plt.figure(figsize=(3.5, 4.2))
gs = gridspec.GridSpec(4, 2)
ax = plt.subplot(gs[0, :])
img = ax.pcolormesh(phi0s_plt,
delta_phi_fs,
t_opts_ref,
label='Time (h)',
cmap='inferno',
vmax=60,
vmin=0)
cbar = plt.colorbar(img)
img.set_zorder(-20)
cbar.set_label('$t^{opt}$ (h)')
ax.set_ylabel('$\Delta\phi_f$')
def plot_subfig_results(axes, sfr, shift=0, pred_win=2):
cx, dx, ex = axes
control_inputs, errors, time_steps, mpc_phis, ext_phis_output, mpc_ps, mpc_ts, tphases, contphases = sfr
def shift_time(ts, shift_t=1.75 * pmodel.T, shift_val=shift):
if np.size(ts) is 1:
if ts >= shift_t:
return ts + shift_val
else:
return ts
elif np.size(ts) > 1:
ts = np.copy(ts)
ts[np.where(ts >= shift_t)] += shift_val
return np.asarray(ts)
else:
print "unknown type supplied"
def phase_of_time(times):
return (times * 2 * np.pi / pmodel.T) % (2 * np.pi)
cx.plot(
np.asarray(tphases) / 23.7 * 24. - 0.25 * 24., np.sin(contphases), 'l')
cx.plot(
np.asarray(tphases) / 23.7 * 24. - 0.25 * 24.,
np.sin(phase_of_time(shift_time(tphases))), 'k--')
cx.set_xlim([12, 72])
cx.set_xticks([24, 36, 48, 60])
cx.set_xticklabels([])
cx.set_ylim([-1., 1.45])
cx.set_ylabel('sin($\phi$)')
dx.step(control_inputs[1:, 0] / 23.7 * 24 - .25 * 24,
-control_inputs[:-1, 1],
'0.7',
label='Control Input')
dx.set_xlim([12, 72])
dx.set_ylim([0, 0.11])
dx.set_xticks([12, 24, 36, 48, 60, 72])
dx.set_yticks([0, 0.05, 0.10])
dx.set_xticklabels([''])
dx.set_ylabel('u')
for tl in dx.get_yticklabels():
tl.set_color('0.7')
dx2 = dx.twinx()
dx2.plot(
np.asarray(tphases) / 23.7 * 24. - 0.25 * 24.,
-pmodel.pPRC_interp(pmodel._phi_to_t(np.asarray(contphases)))[:, 15],
'f')
dx2.set_xticks([12, 24, 36, 48, 60, 72])
dx2.set_xlim([12, 72])
plo.format_4pi_axis(dx2, x=False, y=True)
for tl in dx2.get_yticklabels():
tl.set_color('f')
ex.step(time_steps[1:-pred_win + 1] / 23.7 * 24 - .25 * 24,
errors[:],
'k',
label='Phase Error')
ex.set_xlim([12, 72])
ex.set_xticks([12, 24, 36, 48, 60, 72])
ex.set_yticks([0, 0.5, 1.0, 1.5])
ex.set_ylabel('Error (rad$^2$)')
ex.set_xlabel('Time (h)')
result = executor.map(mpc_problem_fig2, inputs, chunksize=1)
for idx,res in enumerate(result):
subfig_results.append(res)
np.save('Data/fig2b.npy',subfig_results[0])
np.save('Data/fig2c.npy',subfig_results[1])
'''
#load the results
sfr1 = np.load('Data/fig2b.npy')
sfr2 = np.load('Data/fig2c.npy')
# plotting regions =======================
#first plot: part a and b
plo.PlotOptions(ticks='in')
plt.figure(figsize=(3.5, 3.0))
gs = gridspec.GridSpec(3, 2, height_ratios=(1.5, 1, 1))
axes_set1 = [plt.subplot(gs[i, 0]) for i in range(3)]
axes_set2 = [plt.subplot(gs[i, 1]) for i in range(3)]
def plot_subfig_results(axes, sfr, shift=0, pred_win=2):
cx, dx, ex = axes
control_inputs, errors, time_steps, mpc_phis, ext_phis_output, mpc_ps, mpc_ts, tphases, contphases = sfr
def shift_time(ts, shift_t=1.75 * pmodel.T, shift_val=shift):
if np.size(ts) is 1:
if ts >= shift_t:
return ts + shift_val
for key in chr2_results.keys():
chr2_results[key] = np.hstack([chr2_results[key], result[key]])
control_results = {
'h_angles': [],
'h_changes': [],
'l_angles': [],
'l_changes': [],
'off_diffs': []
}
for filename in control_files:
result = breakdown_experiment(control_paths, filename)
for key in control_results.keys():
control_results[key] = np.hstack([control_results[key], result[key]])
plo.PlotOptions()
# distribution of shifts
boxplot_data0 = chr2_results['h_changes']
boxplot_data1 = chr2_results['l_changes']
boxplot_data2 = chr2_results['off_diffs']
boxplot_data3 = control_results['h_changes']
boxplot_data4 = control_results['l_changes']
boxplot_data5 = control_results['off_diffs']
boxplot_data = [
boxplot_data5, boxplot_data3, boxplot_data4, boxplot_data2, boxplot_data0,
boxplot_data1
]
names = [
axx.set_xlim([-2*np.pi, 2*np.pi])
axx.set_xticklabels([r'-2$\pi$', r'-$\pi$','0',
r'$\pi$', r'2$\pi$'])
ax.set_xticklabels([])
bx.set_xticklabels([])
def plot_errors(ax, error_dicts):
colors = ['l', 'h', 'f']
for idx, error_dict in enumerate(error_dicts):
delta_phi_fs = error_dict['phases']
ax.plot(delta_phi_fs, error_dict['errors'],
marker = 'x', color = colors[idx], ls='')
plo.PlotOptions(ticks='in')
plo.PlotOptions(uselatex=True, ticks='in')
plt.figure(figsize=(3.5,3.85))
gs = gridspec.GridSpec(3,1)
ax = plt.subplot(gs[0,0])
bx = plt.subplot(gs[1,0])
cx = plt.subplot(gs[2,0])
plot_errors(bx, [errors_4h, errors_2h, errors_1h])
plot_bounds([ax,bx,cx], [res_4h, res_2h, res_1h])
bx.legend()
for delta_phi in delta_phi_fs:
direction = int(delta_phi > delta_phi_star)
if direction == 0:
ncyc = 1 + delta_phi // adv_per_cycle
numcycles.append(ncyc)
adv_del.append(0)
del_phi.append(
(ncyc) * (np.abs(slowdown_pos_loss + slowdown_pos_loss) +
loss_neg + loss_pos))
elif direction == 1:
ncyc = 1 + (2 * np.pi - delta_phi) // -del_per_cycle
adv_del.append(1)
numcycles.append(ncyc)
del_phi.append((ncyc) * (loss_neg + loss_pos))
plo.PlotOptions(ticks='in')
plo.PlotOptions(uselatex=True, ticks='in')
plt.figure(figsize=(3.5, 2.85))
gs = gridspec.GridSpec(3, 1)
ax = plt.subplot(gs[0, 0])
bx = plt.subplot(gs[1, 0])
cx = plt.subplot(gs[2, 0])
ax.plot(delta_phi_fs, numcycles)
ax.axvline(delta_phi_star, color='k', ls=":")
ax.set_ylabel("Number Cycles")
bx.set_ylim([0, 5.2])
bx.plot(delta_phi_fs, np.asarray(del_phi))
bx.set_ylim([0, 1.])
def sinusoidal_fitting(times,
data,
rhythmic_or_not,
fit_times=None,
forced_periods=None):
"""
Takes detrended and denoised data and times, and fits a damped sinusoid to
the data. Returns: times, sine_data, phase_data, periods, amplitudes,
decay parameters, and pseudo rsq values.
If forced_periods are supplied, the sine fit will be within 1h of the forced period.
"""
timer = plo.laptimer()
tot_time = 0
print "\nSinusoidal fitting.\n----------"
ncells = len(data.T)
if fit_times is None:
fit_times = np.copy(times)
# these will be the outputs
sine_data = np.nan * np.ones((len(fit_times), len(data[0])))
phase_data = np.nan * np.ones((len(fit_times), len(data[0])))
phases = np.nan * np.ones(len(data[0]))
meaningful_phases = np.nan * np.ones(len(data[0]))
periods = np.nan * np.ones(len(data[0]))
amplitudes = np.nan * np.ones(len(data[0]))
decays = np.nan * np.ones(len(data[0]))
pseudo_r2s = np.nan * np.ones(len(data[0]))
for idx, idata in enumerate(data.T):
if np.std(idata) > 1E-12:
# try a sinusoidal fit but only if data is not perfectly flat
if idx % 100 == 0:
print str(np.round(100 * idx / ncells,
2)) + " pct complete... time: ",
tot_time += timer()
print str(np.round(tot_time, 1)) + "s"
# copy data for editing
cell = np.copy(idata)
model = dsin.DecayingSinusoid(times, cell)
# fit the data with the polynomial+sinusoid
# note we are not using model averaging, we are
# just taking the single best model by AICc
# if no estimate is given just do the normal fitting
# if an estimate is given, bound the period within two
model.run()
params = model.best_model.result.params
if forced_periods is not None:
# only use force if necessary
if np.abs(params['period'].value - forced_periods[idx]) > 1:
# force is necessary
model._estimate_parameters()
model._fit_models(period_force=forced_periods[idx])
model._calculate_averaged_parameters()
params = model.best_model.result.params
phases[idx] = (fit_times[0] * 2 * np.pi / params['period'] +
params['phase']) % (2 * np.pi)
if rhythmic_or_not[idx] == 1:
phase_data[:,
idx] = (fit_times * 2 * np.pi / params['period'] +
params['phase']) % (2 * np.pi)
sine_data[:, idx] = dsin.sinusoid_component(params, fit_times)
# summary stats
periods[idx] = model.best_model.result.params['period']
amplitudes[idx] = model.best_model.result.params['amplitude']
decays[idx] = model.best_model.result.params['decay']
pseudo_r2s[idx] = model.best_model._calc_r2()
meaningful_phases[idx] = (
fit_times[0] * 2 * np.pi / params['period'] +
params['phase']) % (2 * np.pi)
sinefit = {
'ts': fit_times,
'cosines': sine_data,
'phase_timeseries': phase_data,
'phi0all': phases,
'periods': periods,
'amplitudes': amplitudes,
'decays': decays,
'r2s': pseudo_r2s,
'phi0r': meaningful_phases
}
return sinefit
def eigensmooth(times, detrended_data, ev_threshold=0.05, dim=40, min_ev=2):
"""
Uses an eigendecomposition to keep only elements with >threshold of the
data. Then it returns the denoised data.
Notes: This should take the covariance matrix of the data using fwd-backward method. Then it
eigendecomposes it, then it finds the biggest (2+) eigenvalues and returns only the
components associated with them.
For an intuitive explanation of how this works:
http://www.visiondummy.com/2014/04/geometric-interpretation-covariance-matrix/
"""
timer = plo.laptimer()
print "Eigendecomposition... time: ",
#set up results - by default set to NaNs
denoised_data = np.nan * np.ones(detrended_data.shape)
eigenvalues_list = []
# denoise each piece of data
for data_idx, d0 in enumerate(detrended_data.T):
# remove nans from times, d1. keep nans in d0
t1 = times[~np.isnan(d0)]
d1 = np.copy(d0[~np.isnan(d0)])
# using spectrum to get the covariance matrix
X = spectrum.linalg.corrmtx(d1, dim - 1, method='autocorrelation')
# the embedding matrix
X = (1 / np.sqrt(len(X.T))) * np.array(X)
XT = np.transpose(X)
C = XT.dot(X)
# now eigendecompose
evals, Q = np.linalg.eig(C)
# find evals that matter, use a minimum of 2
eval_goodness = np.max(
[2, np.sum(evals / (1E-12 + np.sum(evals)) >= ev_threshold)])
QT = np.transpose(Q)
# and return the reconstruction
P = QT.dot(XT)
denoised = np.sum(P[:eval_goodness], 0)
# find alignment - for some reason the signal can be flipped.
# this catches it
# truncate the first 24h during alignment
align, atype = alignment(d1, denoised, d=dim, dstart=96)
# fix alignment if leading nans
nanshift = 0
if np.isnan(d0[0]):
nanshift = np.argmin(np.isnan(d0))
# get the correctly-shaped denoised data
denoised = denoised[align:align + len(d1)] * atype
#align denoised data in matrix
denoised_data[nanshift:len(denoised) + nanshift, data_idx] = denoised
eigenvalues_list.append(evals / np.sum(evals + 1E-12))
print str(np.round(timer(), 1)) + "s"
return times, denoised_data, eigenvalues_list