data = {}
dat_bin = {}
dat_mean = {}
dat_var = {}
dat = {}
stage_bins = {}
dat_bin_s = {}
for strain in strains:
print 'processing %s...' % strain
data[strain] = exp.load_data(strain)
dat[strain] = getattr(data[strain], feat)
sbinsb = exp.stage_bins(data[strain], nbins=sbins)
dat_bin[strain] = exp.bin_data(dat[strain], sbinsb)
dat_mean[strain] = np.mean(dat_bin[strain], axis=0)
dat_var[strain] = np.var(dat_bin[strain], axis=0)
stage_bins[strain] = exp.stage_bins(data[strain], nbins=1)
dat_bin_s[strain] = exp.bin_data(dat[strain], stage_bins[strain])
#%% Order by activity
order = {}
for strain in strains:
order[strain] = np.argsort(np.sum(dat[strain], axis=1))
#%% Plot Data
if save_fig is not None:
fig.savefig(
os.path.join(save_fig, '%s_%s.png' % (strain, 'stage_durations')))
### Mean feature accross worms and stages
d = getattr(data, feat)
#average+order
dm = np.sum(d, axis=1) * 1.0 / data.total_time
dorder = np.argsort(dm)
exp.save(dorder, '%s_%s_order.npy' % (strain, feat))
#stage resolved averages
sbins = exp.stage_bins(data, nbins=1)
dms = exp.bin_data(d, sbins)
dms_shuffle = dms.copy()
for i in range(dms.shape[1]):
dms_shuffle[:, i] = np.random.permutation(dms_shuffle[:, i])
dms_data = [dms, dms_shuffle]
fig = plt.figure(20 + i)
plt.clf()
plt.subplot(3, 2, 1)
plt.plot(dm[dorder])
for s in range(data.nstages):
plt.plot(dm[np.argsort(dms[:, s])])
plt.title('%s %s - total mean' % (strain, feat))
for s in range(data.nstages):
res = average(signal, kernel_flat, scales)
#res = average(signal, kernel_gauss, scales);
fres = exp.load('%s_%s_scales_mean.npy' % (strain, feat), memmap='r+')
fres[i, :, :] = res[0]
from multiprocessing import Pool #, cpu_count;
#pool = Pool(processes = cpu_count()-6);
pool = Pool(processes=12)
pool.map(run, range(data.nworms))
### Time normalize data
fres = exp.load('%s_%s_scales_mean.npy' % (strain, feat), memmap='r')
sbins = exp.stage_bins(data, nbins=8192)
res_shape = (fres.shape[0], fres.shape[1], sbins[0].shape[1])
exp.memmap('%s_%s_time_normalized_scales_mean.npy' % (strain, feat),
shape=res_shape,
mode='w+')
def norm(i):
print 'scale %d' % i
fres = exp.load('%s_%s_scales_mean.npy' % (strain, feat), memmap='r')
fres_tn = exp.load('%s_%s_time_normalized_scales_mean.npy' %
(strain, feat),
memmap='r+')
fres_tn[:, i, :] = exp.bin_data(fres[:, i, :], sbins)
pool = Pool(processes=12)
else:
return -dd;
#%% Load / Prepare data
data = {}; dat_bin = {}; dat_mean = {}; dat_var = {};
dat = {}; stage_bins = {}; dat_bin_s = {};
for strain in strains:
print 'processing %s...'% strain;
data[strain] = exp.load_data(strain);
dat[strain] = getattr(data[strain], feat);
rids = getattr(data[strain], 'roam') > 0;
bins, bine = exp.stage_bins(data[strain], nbins= sbins);
nworms = rids.shape[0];
dat_bin[strain] = np.zeros((nworms, tbins));
for t in range(tbins):
for w in range(nworms):
ids = np.arange(bins[w,t], bine[w,t]);
dd = dat[strain][w, ids][rids[w][ids]];
dat_bin[strain][w,t] = np.std(dd) / np.mean(dd);
#dat_bin[strain] = exp.bin_data(dat[strain], sbinsb);
dat_mean[strain] = np.mean(dat_bin[strain], axis = 0);
dat_var[strain] = np.var(dat_bin[strain], axis = 0);
#stage_bins[strain] = exp.stage_bins(data[strain], nbins=1);
plt.plot(model.predict(d[wid, tstart:tend]), 'r')
plt.plot(-d[wid, tstart:tend] + 2.1, 'b')
#plt.plot(d[wid, tstart:tend] - idx_from_path(path))
## see how the hmm parameter change over time
def model_to_param(model):
emis = []
for s in model.states[:-2]:
emis.append(s.distribution.values()[-1])
trans = model.dense_transition_matrix()
return np.hstack([trans[0, 0], trans[1, 1], emis, trans[2, :2]])
sbins_start, sbins_end = exp.stage_bins(data, nbins=2**5)
nbins_total = sbins_start.shape[1]
#nparam = nstates * (nstates-1) + nstates; #transition parameter + emission prob for 1
nparam = 6
nworms = d.shape[0]
probs = np.zeros((nparam, nbins_total))
for i in range(nbins_total):
print '%d/%d' % (i, nbins_total)
model = hmm(nstates=2)
dd = [d[:, sbins_start[j][i]:sbins_end[j][i]] for j in range(nworms)]
model.fit(dd, max_iterations=500, stop_threshold=10e-5, n_jobs=12)
probs[:, i] = model_to_param(model)
probs[3, :] = 1 - probs[3, :]
droam = exp.bin_data(d, (sbins_start, sbins_end))
idx_dw = np.logical_and(bin_st[b] < t_dw + d_dw, t_dw < bin_ed[b]);
durs_up.append(d_up[idx_up]);
durs_dw.append(d_dw[idx_dw]);
return (durs_up, durs_dw);
def bin_durations(t_up, t_dw, d_up, d_dw, bin_st, bin_ed):
nworms = len(t_up);
durs_up = [];
durs_dw = [];
for w in range(nworms):
du,dd = bin_durations_for_worm(t_up[w], t_dw[w], d_up[w], d_dw[w], bin_st[w], bin_ed[w]);
durs_up.append(du); durs_dw.append(dd);
return (durs_up, durs_dw);
sbin_st, sbin_ed = exp.stage_bins(data, nbins=10);
durs_up, durs_dw = bin_durations(times_up, times_dw, dur_up, dur_dw, sbin_st, sbin_ed);
## distributions for individual worms
# durs have shape (nworms, nbins, xx)
nworms = len(durs_up);
nbins = len(sbin_st[0]);
# number of periods
ndurs_up = np.zeros((nworms, nbins));
ndurs_dw = np.zeros((nworms, nbins));
for w in range(nworms):
t = np.random.rand(100) > 0.5
t = np.array(t, dtype=int)
p, c = pk.maximum_likelihood_probabilities(t)
os.chdir(dir_base)
import experiment as exp
strain = 'N2'
feat = 'roam'
data = exp.load_data(strain)
d = getattr(data, feat)
sbins = exp.stage_bins(data, nbins=2**4)
c = np.zeros((data.nworms, sbins[0].shape[1], 2))
p = np.zeros((data.nworms, sbins[0].shape[1], 2))
for wid in range(data.nworms):
print 'worm %d' % wid
st = sbins[0][wid]
ed = sbins[1][wid]
for i, se in enumerate(zip(st, ed)):
s, e = se
pp, cc = pk.maximum_likelihood_probabilities(d[wid, s:e])
c[wid, i] = [cc[(0, 1)], cc[(1, 0)]]
p[wid, i] = [pp[0], pp[1]]
### plot the result