def new_filter_reduce(res, filter_keys, reduce_key, regularize='min'):
out = defaultdict(list)
if isinstance(filter_keys, str):
filter_keys = [filter_keys]
unique_combinations, ixs = filter.retrieve_unique_entries(res, filter_keys)
for v in unique_combinations:
filter_dict = {
filter_key: val
for filter_key, val in zip(filter_keys, v)
}
cur_res = filter.filter(res, filter_dict)
if len(cur_res[reduce_key]):
try:
if regularize == 'min':
_regularize_length(cur_res, reduce_key)
elif regularize == 'max':
_regularize_length_cristian_data(cur_res, reduce_key)
else:
raise ValueError('did not recognize regularize keyword')
except:
print('cannot regularize the length of {}'.format(reduce_key))
temp_res = reduce_by_mean(cur_res, reduce_key)
append_defaultdicts(out, temp_res)
bad = []
for key, val in out.items():
try:
out[key] = np.array(val)
except:
bad.append(key)
print('{} could not be reduced'.format(key))
for badkey in bad:
out.pop(badkey)
return out
def _get_overlap_water(res, arg):
def _helper(list_of_name_ix_tuple, desired_tuple):
for tuple in list_of_name_ix_tuple:
if tuple[0] == desired_tuple:
ix = tuple[1]
assert len(ix) == 1, 'more than 1 unique entry'
return ix[0]
res['Overlap'] = np.zeros(res['day'].shape)
names, ixs = filter.retrieve_unique_entries(
res, ['mouse', 'day', 'odor_standard'])
list_of_name_ix_tuples = list(zip(names, ixs))
mice = np.unique(res['mouse'])
for mouse in mice:
mouse_res = filter.filter(res, filter_dict={'mouse': mouse})
days = np.unique(mouse_res['day'])
for day in days:
mouse_day_res = filter.filter(mouse_res, filter_dict={'day': day})
odors = np.unique(mouse_day_res['odor_standard'])
if 'US' in odors:
us_ix = _helper(list_of_name_ix_tuples, (mouse, day, 'US'))
us_cells = np.where(res['sig'][us_ix])[0]
for odor in odors:
odor_ix = _helper(list_of_name_ix_tuples,
(mouse, day, odor))
odor_cells = np.where(res['sig'][odor_ix])[0]
if arg == 'US/CS+':
overlap = _overlap(us_cells, odor_cells, arg='over')
elif arg == 'CS+/US':
overlap = _overlap(odor_cells, us_cells, arg='over')
else:
raise ValueError('overlap arg not recognized')
res['Overlap'][odor_ix] = overlap
def shuffle_data(res, shuffle_keys):
out = copy.deepcopy(res)
data = out['data']
if isinstance(shuffle_keys, str):
shuffle_keys = [shuffle_keys]
unique_combinations, combination_ixs = filter.retrieve_unique_entries(
out, shuffle_keys)
for ixs in combination_ixs:
current_data = data[ixs]
trials_per_condition = [x.shape[0] for x in current_data]
concatenated_data = np.concatenate(current_data, axis=0)
shuffled_ixs = np.random.permutation(concatenated_data.shape[0])
shuffled_ixs_per_condition = np.split(
shuffled_ixs, np.cumsum(trials_per_condition))[:-1]
for i in range(len(shuffled_ixs_per_condition)):
assert (trials_per_condition[i] == len(
shuffled_ixs_per_condition[i])
), 'bad partitioning of shuffled ixs'
new_data = np.zeros(len(ixs), dtype=object)
for i, ix in enumerate(shuffled_ixs_per_condition):
new_data[i] = concatenated_data[ix]
out['data'][ixs] = new_data
return out
def _helper(start_end_day_res):
combs, list_of_ixs = filter.retrieve_unique_entries(
start_end_day_res, ['mouse', 'odor_standard'])
for i, comb in enumerate(combs):
ixs = list_of_ixs[i]
assert len(ixs) == 2
if start_end_day_res['training_day'][0] == 'Naive':
ref = ixs[0]
test = ixs[1]
elif start_end_day_res['training_day'][0] == 'Learned':
ref = ixs[1]
test = ixs[0]
else:
raise ValueError('cannot find ref day')
if normalize:
start_end_day_res[key][test] = start_end_day_res['Fraction Responsive'][test] / \
start_end_day_res['Fraction Responsive'][ref]
start_end_day_res[key][ref] = 1
else:
start_end_day_res[key][test] = start_end_day_res['Fraction Responsive'][test] - \
start_end_day_res['Fraction Responsive'][ref]
start_end_day_res[key][ref] = 0
def _fix_trials(res):
import filter
combinations, ixs = filter.retrieve_unique_entries(res,
loop_keys=['mouse'])
res['old_trials'] = np.copy(res['trials'])
for ix in ixs:
for i in ix:
trials = res['trials'][i]
new_trials = np.arange(1, len(trials) + 1)
res['trials'][i] = new_trials
def _normalize_across_days(res):
combinations, list_of_ixs = filter.retrieve_unique_entries(
res, loop_keys=['mouse', 'odor'])
for ixs in list_of_ixs:
assert res['day'][ixs[0]] < res['day'][
ixs[1]], 'not the first day as reference'
first_dff = res['max_dff'][ixs[0]]
second_dff = res['max_dff'][ixs[1]]
second_dff = second_dff / first_dff
res['max_dff'][ixs[0]] = 1.0
res['max_dff'][ixs[1]] = second_dff
def _compare_dff(res, loop_keys, arg):
new = defaultdict(list)
combinations, list_of_ixs = filter.retrieve_unique_entries(
res, loop_keys=loop_keys)
for ixs in list_of_ixs:
assert len(ixs) == 2, 'more than two entries for days'
assert res['day'][ixs[0]] < res['day'][
ixs[1]], 'not the first day as reference'
mask_a = res['sig'][ixs[0]]
mask_b = res['sig'][ixs[1]]
if arg == 'all':
mask = np.array([a or b
for a, b in zip(mask_a, mask_b)]).astype(bool)
elif arg == 'first':
mask = np.array(mask_a).astype(bool)
elif arg == 'last':
mask = np.array(mask_b).astype(bool)
elif arg == 'none':
mask = np.ones_like(mask_b).astype(bool)
else:
raise ValueError('arg not recognized')
amplitudes = []
for ix in ixs:
s = res['DAQ_O_ON_F'][ix]
e = res['DAQ_W_ON_F'][ix]
dff = res['dff'][ix][mask]
amplitude = np.max(dff[:, s:e], axis=1)
amplitudes.append(amplitude)
new['day_0'].append(np.array(amplitudes[0]))
new['day_1'].append(np.array(amplitudes[1]))
update_keys = loop_keys.copy()
update_keys.append('odor_valence')
for update_key in update_keys:
new[update_key].append(res[update_key][ixs[0]])
# amplitudes = np.transpose(amplitudes)
# update_keys = loop_keys.copy()
# update_keys.append('odor_valence')
# for amplitude_tup in amplitudes:
# for update_key in update_keys:
# new[update_key].append(res[update_key][ixs[0]])
# new['data'].append(amplitude_tup)
# new['day'].append([0, 1])
# new['ix'].append(tup_ix)
# tup_ix += 1
for k, v in new.items():
new[k] = np.array(v)
return new
def get_roc(res):
def _dprime(a, b):
u1, u2 = np.mean(a), np.mean(b)
s1, s2 = np.std(a), np.std(b)
return (u1 - u2) / np.sqrt(.5 * (np.square(s1) + np.square(s2)))
def _roc(a, b):
import sklearn.metrics
data = np.concatenate((a, b))
labels = np.concatenate(
(np.ones_like(a), np.zeros_like(b))).astype('bool')
roc = sklearn.metrics.roc_auc_score(labels, data)
return roc
def _rolling_window(a, window):
shape = a.shape[:-1] + (a.shape[-1] - window + 1, window)
strides = a.strides + (a.strides[-1], )
return np.lib.stride_tricks.as_strided(a, shape=shape, strides=strides)
key = 'lick'
print_key = 'roc'
x_key = 'roc_trial'
window = 10
res[print_key] = np.copy(res[key])
res[x_key] = np.copy(res['trial'])
res = filter.exclude(res, {'odor_valence': 'US'})
res_ = reduce.new_filter_reduce(res,
filter_keys=['mouse', 'odor_valence'],
reduce_key=key)
combinations, list_of_ixs = filter.retrieve_unique_entries(res_, ['mouse'])
for i, ixs in enumerate(list_of_ixs):
assert len(ixs) == 2
assert res_['odor_valence'][ixs[0]] == 'CS+'
assert res_['odor_valence'][ixs[1]] == 'CS-'
a = _rolling_window(res_[key][ixs[0]], window)
b = _rolling_window(res_[key][ixs[1]], window)
dprimes = np.array([_roc(x, y) for x, y in zip(a, b)])
res_[print_key][ixs[0]] = dprimes
res_[print_key][ixs[1]] = dprimes
res_[x_key][ixs[0]] = np.arange(len(dprimes))
res_[x_key][ixs[1]] = np.arange(len(dprimes))
return res_
def shift_discrimination_index(res):
combs, ixs = filter.retrieve_unique_entries(res, ['mouse'])
res['together_trials'] = np.copy(res['trials'])
res['together_days'] = np.copy(res['days'])
for comb, ix in zip(combs, ixs):
phases = res['phase'][ix]
pt_mask = ix[phases == 'Pretraining']
dt_mask = ix[phases == 'Discrimination']
pt_trials = res['trials'][pt_mask]
total_pt_trials = np.max([len(x) for x in pt_trials])
dt_trials = res['trials'][dt_mask]
modified_dt_trials = [x + total_pt_trials for x in dt_trials]
res['together_trials'][dt_mask] = modified_dt_trials
pt_days = res['days'][pt_mask]
total_pt_days = np.sum([x[-1] for x in pt_days])
dt_days = res['days'][dt_mask]
modified_dt_days = [x + total_pt_days for x in dt_days]
res['together_days'][dt_mask] = modified_dt_days
for k, v in res.items():
res[k] = np.array(v)
def get_reversal_sig(res):
key = 'ssig'
def _helper(res):
assert res['odor_valence'][0] == 'CS+', 'wrong odor'
assert res['odor_valence'][1] == 'CS-', 'wrong odor'
on = res['DAQ_O_ON_F'][0]
off = res['DAQ_W_ON_F'][0]
sig_p = res[key][0]
sig_m = res[key][1]
dff_p = res['dff'][0]
dff_m = res['dff'][1]
sig_p_mask = sig_p == 1
sig_m_mask = sig_m == 1
dff_mask = dff_p - dff_m
dff_mask = np.mean(dff_mask[:, on:off], axis=1)
p = [a and b for a, b in zip(sig_p_mask, dff_mask > 0)]
m = [a and b for a, b in zip(sig_m_mask, dff_mask < 0)]
return np.array(p), np.array(m)
mice = np.unique(res['mouse'])
res = filter.filter(res, filter_dict={'odor_valence': ['CS+', 'CS-']})
sig_res = reduce.new_filter_reduce(
res, reduce_key=key, filter_keys=['mouse', 'day', 'odor_valence'])
dff_res = reduce.new_filter_reduce(
res, reduce_key='dff', filter_keys=['mouse', 'day', 'odor_valence'])
sig_res['dff'] = dff_res['dff']
reversal_res = defaultdict(list)
day_strs = ['Lrn', 'Rev']
for mouse in mice:
mouse_res = filter.filter(sig_res, filter_dict={'mouse': mouse})
days = np.unique(mouse_res['day'])
p_list = []
m_list = []
for i, day in enumerate(days):
mouse_day_res = filter.filter(mouse_res, filter_dict={'day': day})
p, m = _helper(mouse_day_res)
reversal_res['mouse'].append(mouse)
reversal_res['mouse'].append(mouse)
reversal_res['day'].append(day_strs[i])
reversal_res['day'].append(day_strs[i])
reversal_res['odor_valence'].append('CS+')
reversal_res['odor_valence'].append('CS-')
reversal_res[key].append(p)
reversal_res[key].append(m)
reversal_res['Fraction'].append(np.mean(p))
reversal_res['Fraction'].append(np.mean(m))
p_list.append(p)
m_list.append(m)
for k, val in reversal_res.items():
reversal_res[k] = np.array(val)
stats_res = defaultdict(list)
for mouse in mice:
mouse_res = filter.filter(reversal_res, filter_dict={'mouse': mouse})
combinations, list_of_ixs = filter.retrieve_unique_entries(
mouse_res, ['day', 'odor_valence'])
assert len(combinations) == 4, 'not equal to 4'
assert combinations[0][-1] == 'CS+'
assert combinations[1][-1] == 'CS-'
assert combinations[2][-1] == 'CS+'
assert combinations[3][-1] == 'CS-'
assert combinations[0][0] == day_strs[0]
assert combinations[1][0] == day_strs[0]
assert combinations[2][0] == day_strs[1]
assert combinations[3][0] == day_strs[1]
p_before = mouse_res[key][0]
m_before = mouse_res[key][1]
n_before = np.invert([a or b for a, b in zip(p_before, m_before)])
p_after = mouse_res[key][2]
m_after = mouse_res[key][3]
n_after = np.invert([a or b for a, b in zip(p_after, m_after)])
list_before = [p_before, m_before, n_before]
list_after = [p_after, m_after, n_after]
str = ['p', 'm', 'none']
for i, before in enumerate(list_before):
for j, after in enumerate(list_after):
ix_intersect = np.intersect1d(
np.where(before)[0],
np.where(after)[0])
fraction = len(ix_intersect) / np.sum(before)
stats_res['mouse'].append(mouse)
stats_res['condition'].append(str[i] + '-' + str[j])
stats_res['Fraction'].append(fraction)
for key, val in stats_res.items():
stats_res[key] = np.array(val)
return reversal_res, stats_res
def behavior_vs_neural_power(neural_res, behavior_res, start, end, figure_path, neural_arg ='power', behavior_arg ='onset'):
neural_res = copy.copy(neural_res)
neural_key = 'neural'
behavior_key = 'behavior'
if behavior_arg == 'onset':
behavior_data_key = 'time_first_lick_raw'
elif behavior_arg == 'magnitude':
behavior_data_key = 'lick_5s'
elif behavior_arg == 'com':
behavior_data_key = 'lick_com_raw'
else:
raise ValueError('bad key')
neural_data_key = 'max_power'
_power(neural_res, excitatory=True)
for i, p in enumerate(neural_res['Power']):
s, e = neural_res['DAQ_O_ON_F'][i], neural_res['DAQ_W_ON_F'][i]
mp = np.max(p[s:e]) - np.mean(p[:s])
neural_res[neural_data_key].append(mp)
neural_res[neural_data_key] = np.array(neural_res[neural_data_key])
list_of_days = [np.arange(s, e+1) for s, e in zip(start,end)]
print(list_of_days)
neural_res_filtered = filter.filter_days_per_mouse(neural_res, days_per_mouse=list_of_days)
neural_res_filtered = filter.filter(neural_res_filtered, filter_dict={'odor_valence':'CS+'})
behavior_res_filtered = filter.filter(behavior_res, filter_dict={'odor_valence':'CS+'})
names_neu, ixs_neu = filter.retrieve_unique_entries(neural_res_filtered, ['mouse','day','odor_standard'])
out = defaultdict(list)
for ix, names in zip(ixs_neu, names_neu):
mouse = names[0]
day = names[1]
odor_standard = names[2]
assert len(ix) == 1
neural = neural_res_filtered[neural_data_key][ix[0]]
temp = filter.filter(behavior_res_filtered, {'mouse': mouse, 'odor_standard': odor_standard})
assert len(temp[behavior_data_key]) == 1
ix_day = [x == day for x in temp['day'][0]]
lick = temp[behavior_data_key][0][ix_day]
lick = lick[lick>-1]
out['mouse'].append(mouse)
out['day'].append(day)
out['odor_standard'].append(odor_standard)
out[neural_key + neural_arg].append(np.mean(neural))
out[behavior_key + behavior_arg].append(np.mean(lick))
out['neural_raw'].append(neural)
out['lick_raw'].append(lick)
for k, v in out.items():
out[k] = np.array(v)
## versus
if behavior_arg in ['onset', 'com']:
xlim = [0, 5]
xticks = [0, 2, 5]
xticklabels = ['ON', 'OFF', 'US']
else:
xlim = [0, 35]
xticks = [0, 10, 20, 30]
xticklabels = [0, 10, 20, 30]
ax_args = {'xlim': xlim, 'ylim': [0, .1], 'xticks': xticks, 'yticks': [0, .05, .1],
'xticklabels': xticklabels}
path, name = plot.plot_results(out, x_key=behavior_key + behavior_arg, y_key=neural_key + neural_arg,
loop_keys='mouse',
plot_function=plt.scatter,
plot_args=scatter_args,
ax_args=ax_args,
colormap='jet',
path=figure_path,
save=False)
from sklearn import linear_model
from sklearn.metrics import mean_squared_error, r2_score
scores = []
for mouse in np.unique(out['mouse']):
res = filter.filter(out, {'mouse':mouse})
regr = linear_model.LinearRegression()
x = res[behavior_key + behavior_arg].reshape(-1,1)
y = res[neural_key + neural_arg].reshape(-1,1)
regr.fit(x, y)
y_pred = regr.predict(x)
score = regr.score(x, y)
scores.append(score)
print('regression: {}'.format(scores))
xlim = plt.xlim()
ylim = plt.ylim()
plt.text((xlim[1] - xlim[0])/2, ylim[1]-.01, 'Average R = {:.2f}'.format(np.mean(scores)))
name += '_' + behavior_arg
plot._easy_save(path, name)
def behavior_vs_neural_onset(neural_res, behavior_res, start, end, figure_path, neural_arg ='onset', behavior_arg ='onset'):
neural_key = 'neural'
behavior_key = 'behavior'
if behavior_arg == 'onset':
behavior_data_key = 'time_first_lick_raw'
elif behavior_arg == 'magnitude':
behavior_data_key = 'lick_5s'
elif behavior_arg == 'com':
behavior_data_key = 'lick_com_raw'
else:
raise ValueError('bad key')
if neural_arg == 'onset':
neural_data_key = 'onset'
elif neural_arg == 'magnitude':
neural_data_key = 'dff'
else:
raise ValueError('bad key')
list_of_days = [np.arange(s, e+1) for s, e in zip(start,end)]
neural_res_filtered = filter.filter_days_per_mouse(neural_res, days_per_mouse=list_of_days)
neural_res_filtered = filter.filter(neural_res_filtered, filter_dict={'odor_valence':'CS+'})
behavior_res_filtered = filter.filter(behavior_res, filter_dict={'odor_valence':'CS+'})
names_neu, ixs_neu = filter.retrieve_unique_entries(neural_res_filtered, ['mouse','day','odor_standard'])
out = defaultdict(list)
for ix, names in zip(ixs_neu, names_neu):
mouse = names[0]
day = names[1]
odor_standard = names[2]
assert len(ix) == 1
neural = neural_res_filtered[neural_data_key][ix[0]]
#test
ixs = np.where(neural > -1)[0]
for a in ixs:
plt.plot(np.transpose(neural_res_filtered['data'][ix[0]][a]))
plt.title(neural[a])
plt.show()
neural = neural[neural > -1] * .229
temp = filter.filter(behavior_res_filtered, {'mouse': mouse, 'odor_standard': odor_standard})
assert len(temp[behavior_data_key]) == 1
ix_day = [x == day for x in temp['day'][0]]
lick = temp[behavior_data_key][0][ix_day]
lick = lick[lick>-1]
out['mouse'].append(mouse)
out['day'].append(day)
out['odor_standard'].append(odor_standard)
out[neural_key + neural_arg].append(np.mean(neural))
out[behavior_key + behavior_arg].append(np.mean(lick))
out['neural_raw'].append(neural)
out['lick_raw'].append(lick)
for k, v in out.items():
out[k] = np.array(v)
## distribution
def _helper(real, label, bin, range, ax):
density, bins = np.histogram(real, bins=bin, density=True, range= range)
unity_density = density / density.sum()
widths = bins[:-1] - bins[1:]
ax.bar(bins[1:], unity_density, width=widths, alpha=.5, label=label)
all_neural = np.concatenate(out['neural_raw'])
all_licks = np.concatenate(out['lick_raw'])
bins = 20
range = [0, 5]
xticks = [0, 2, 5]
xticklabels = ['Odor ON', 'Odor OFF', 'US']
fig = plt.figure(figsize=(2, 1.5))
ax = fig.add_axes([0.2, 0.2, 0.7, 0.7])
_helper(all_neural, neural_key + neural_arg, bins, range, ax)
_helper(all_licks, behavior_key + behavior_arg, bins, range, ax)
plt.xticks(xticks, xticklabels)
plt.xlim([range[0]-0.5, range[1] + .5])
plt.ylabel('Density')
ax.spines["right"].set_visible(False)
ax.spines["top"].set_visible(False)
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')
folder = 'onset_distribution_lick_neural_'
name = behavior_arg + '_' + ','.join([str(x) for x in start]) + '_' + ','.join([str(x) for x in end])
plt.legend(frameon=False)
_easy_save(os.path.join(figure_path, folder), name, dpi=300, pdf=True)
print('lick mean: {}'.format(np.mean(all_licks)))
print('neural mean: {}'.format(np.mean(all_neural)))
## versus
if behavior_arg in ['onset', 'com']:
xlim = [0, 5]
xticks = [0, 2, 5]
xticklabels = ['ON','OFF', 'US']
else:
xlim = [0, 35]
xticks = [0, 10, 20, 30]
xticklabels = [0, 10, 20, 30]
ax_args = {'xlim':xlim, 'ylim':[0, 3.5], 'xticks':xticks, 'yticks':[0, 2, 5],
'xticklabels':xticklabels, 'yticklabels': ['ON','OFF','US']}
path, name = plot.plot_results(out, x_key=behavior_key + behavior_arg, y_key=neural_key + neural_arg, loop_keys='mouse',
plot_function=plt.scatter,
plot_args= scatter_args,
ax_args=ax_args,
colormap='jet',
path = figure_path,
save=False)
from sklearn import linear_model
from sklearn.metrics import mean_squared_error, r2_score
scores = []
for mouse in np.unique(out['mouse']):
res = filter.filter(out, {'mouse':mouse})
regr = linear_model.LinearRegression()
x = res[behavior_key + behavior_arg].reshape(-1,1)
y = res[neural_key + neural_arg].reshape(-1,1)
regr.fit(x, y)
y_pred = regr.predict(x)
score = regr.score(x, y)
scores.append(score)
print('average regression: {}'.format(np.mean(scores)))
xlim = plt.xlim()
ylim = plt.ylim()
plt.text((xlim[1] - xlim[0])/2, ylim[1]-.5, 'Average R = {:.2f}'.format(np.mean(scores)))
name += '_' + behavior_arg
plot._easy_save(path, name)
x = np.concatenate(temp[xname])
y = np.concatenate(temp[yname])
x, y, z = _get_density(x, y)
x_jitter = _add_jitter(x, 5)
y_jitter = _add_jitter(y, 1)
_plot(x_jitter,
y_jitter,
z,
xlim=xlim,
ylim=ylim,
xticks=xticks,
yticks=yticks,
name='late')
#individual
tuples, _ = filter.retrieve_unique_entries(
res, loop_keys=['condition', 'phase', 'mouse'])
for tuple in tuples:
condition = tuple[0]
phase = tuple[1]
mouse = tuple[2]
temp = filter.filter(
res, {
'odor_valence': 'CS+',
'condition': condition,
'phase': phase,
'mouse': mouse
})
sessions = np.unique(temp['session'][0])
for session in sessions:
temp = filter.filter(
def _correlation(res, loop_keys, shuffle, odor_end=True, direction=0):
res = filter.exclude(res, {'odor_valence': 'US'})
for i, dff in enumerate(res['dff']):
s = res['DAQ_O_ON_F'][i]
e = res['DAQ_W_ON_F'][i]
if odor_end:
amplitude = np.max(dff[:, s:e], axis=1)
else:
amplitude = np.max(dff[:, s:], axis=1)
res['corr_amp'].append(amplitude)
res['corr_amp'] = np.array(res['corr_amp'])
combinations, list_of_ixs = filter.retrieve_unique_entries(
res, loop_keys=loop_keys)
loop_keys_ = loop_keys + ['odor_valence', 'odor_standard']
corrcoefs = defaultdict(list)
for ixs in list_of_ixs:
data = res['corr_amp'][ixs]
for i, data_1 in enumerate(data):
for j, data_2 in enumerate(data):
if shuffle:
n_iter = 10
corrcoef = 0
for k in range(n_iter):
corrcoef += np.corrcoef(
(np.random.permutation(data_1),
np.random.permutation(data_2)))[0, 1]
corrcoef /= (n_iter * 1.0)
else:
if i != j:
datas = res['data'][ixs[i]]
s = res['DAQ_O_ON_F'][ixs[i]]
e = res['DAQ_W_ON_F'][ixs[i]]
ds = []
for cell_data in datas:
config = psth.psth_helper.PSTHConfig()
d = psth.psth_helper.subtract_baseline(
cell_data, config.baseline_start,
s - config.baseline_end)
ds.append(d)
datas_i = np.array(ds)
datas = res['data'][ixs[j]]
s = res['DAQ_O_ON_F'][ixs[j]]
e = res['DAQ_W_ON_F'][ixs[j]]
ds = []
for cell_data in datas:
config = psth.psth_helper.PSTHConfig()
d = psth.psth_helper.subtract_baseline(
cell_data, config.baseline_start,
s - config.baseline_end)
ds.append(d)
datas_j = np.array(ds)
corrcoefs_ = []
for rep in np.arange(100):
s_ix_a = np.random.choice(datas_i.shape[1],
datas_i.shape[1] // 2,
replace=False)
s_ix_b = np.random.choice(datas_j.shape[1],
datas_j.shape[1] // 2,
replace=False)
dffa = np.mean(datas_i[:, s_ix_a, :], axis=1)
dffb = np.mean(datas_j[:, s_ix_b, :], axis=1)
if odor_end:
dffa = dffa[:, s:e]
dffb = dffb[:, s:e]
else:
dffa = dffa[:, s:]
dffb = dffb[:, s:]
if direction == 1:
dffa[dffa < 0] = 0
dffb[dffb < 0] = 0
amplitudea = np.max(dffa, axis=1)
amplitudeb = np.max(dffb, axis=1)
elif direction == -1:
dffa[dffa > 0] = 0
dffb[dffb > 0] = 0
amplitudea = np.min(dffa, axis=1)
amplitudeb = np.min(dffb, axis=1)
elif direction == 0:
amplitudea = np.max(dffa, axis=1)
amplitudeb = np.max(dffb, axis=1)
else:
raise ValueError('no direction given')
# if odor_end:
# amplitudea = np.max(dffa[:, s:e], axis=1)
# amplitudeb = np.max(dffb[:, s:e], axis=1)
# else:
# amplitudea = np.max(dffa[:, s:], axis=1)
# amplitudeb = np.max(dffb[:, s:], axis=1)
corrcoefs_.append(
np.corrcoef(amplitudea, amplitudeb)[0, 1])
corrcoef = np.mean(corrcoefs_)
# corrcoef = np.corrcoef((data_1, data_2))[0, 1]
else:
# corrcoef = np.corrcoef((data_1, data_2))[0, 1]
datas = res['data'][ixs[i]]
s = res['DAQ_O_ON_F'][ixs[i]]
e = res['DAQ_W_ON_F'][ixs[i]]
ds = []
for cell_data in datas:
config = psth.psth_helper.PSTHConfig()
d = psth.psth_helper.subtract_baseline(
cell_data, config.baseline_start,
s - config.baseline_end)
ds.append(d)
datas = np.array(ds)
corrcoefs_ = []
for rep in np.arange(100):
s_ix_a = np.random.choice(datas.shape[1],
datas.shape[1] // 2,
replace=False)
s_ix_b = [
x for x in np.arange(datas.shape[1])
if x not in s_ix_a
]
dffa = np.mean(datas[:, s_ix_a, :], axis=1)
dffb = np.mean(datas[:, s_ix_b, :], axis=1)
# if odor_end:
# amplitudea = np.max(dffa[:, s:e], axis=1)
# amplitudeb = np.max(dffb[:, s:e], axis=1)
# else:
# amplitudea = np.max(dffa[:, s:], axis=1)
# amplitudeb = np.max(dffb[:, s:], axis=1)
if direction == 1:
dffa[dffa < 0] = 0
dffb[dffb < 0] = 0
amplitudea = np.max(dffa, axis=1)
amplitudeb = np.max(dffb, axis=1)
elif direction == -1:
dffa[dffa > 0] = 0
dffb[dffb > 0] = 0
amplitudea = np.min(dffa, axis=1)
amplitudeb = np.min(dffb, axis=1)
elif direction == 0:
amplitudea = np.max(dffa, axis=1)
amplitudeb = np.max(dffb, axis=1)
else:
raise ValueError('no direction given')
corrcoefs_.append(
np.corrcoef(amplitudea, amplitudeb)[0, 1])
corrcoef = np.mean(corrcoefs_)
corrcoefs['corrcoef'].append(corrcoef)
corrcoefs['Odor_A'].append(i)
corrcoefs['Odor_B'].append(j)
for loop_key in loop_keys_:
corrcoefs[loop_key].append(res[loop_key][ixs[0]])
for key, value in corrcoefs.items():
corrcoefs[key] = np.array(value)
return corrcoefs
def _windowed_analysis(neural_res,
behavior_res,
window=13,
smooth_length=3,
excitatory=True,
valence='CS+'):
def _moving_window(catdata, window):
n_trials = catdata.shape[1]
x = []
for i in range(n_trials - window):
temp = _power(catdata[:, i:i + window, :], s, e, excitatory)
if i == 0:
for _ in range(1 + window // 2):
x.append(temp)
else:
x.append(temp)
while len(x) != n_trials:
x.append(temp)
return x
neural_res = copy.copy(neural_res)
neural_res = filter.filter(
neural_res, {'odor_valence': [valence]}) # filter only CS+ responses
names, list_of_ixs = filter.retrieve_unique_entries(
neural_res, ['mouse', 'odor_standard'])
out = defaultdict(list)
for i, ixs in enumerate(list_of_ixs):
mouse = names[i][0]
odor_standard = names[i][1]
out['mouse'].append(mouse)
out['odor_standard'].append(odor_standard)
out['odor_valence'].append(odor_standard[:-1])
#neural analysis
if len(np.unique(neural_res['DAQ_O_ON_F'])) > 1:
print('Odor times not the same')
if len(np.unique(neural_res['DAQ_W_ON_F'])) > 1:
print('Water times not the same')
s = np.min(neural_res['DAQ_O_ON_F'][ixs])
e = np.min(neural_res['DAQ_W_ON_F'][ixs])
d = neural_res['data'][ixs]
catdata = np.concatenate(d, axis=1)
n_trials = catdata.shape[1]
x = _moving_window(catdata, window)
x = savgol_filter(x, smooth_length, 0)
if excitatory:
out['power'].append(np.array(x))
else:
out['power'].append(np.array(-1 * x))
out['trials'].append(np.arange(n_trials))
#behavior analysis
ix_lick = np.logical_and(
behavior_res['odor_standard'] == odor_standard,
behavior_res['mouse'] == mouse)
assert np.sum(ix_lick) == 1, '{},{},{}'.format(odor_standard, mouse,
ix_lick)
y = behavior_res['boolean_smoothed'][ix_lick][0]
out[ykey_b].append(y)
# both
temp = (x - np.min(x)) / (np.max(x) - np.min(x))
temp[:10] = 0
half_pow = np.argwhere(temp > 0.5)[0][0]
if np.any(y > 50):
half_lick = np.argwhere(y > 50)[0][0]
else:
half_lick = -1
out['half_power'].append(half_pow)
out['half_lick'].append(half_lick)
for k, v in out.items():
out[k] = np.array(v)
#average by odor
out = reduce.new_filter_reduce(out,
filter_keys=['mouse', 'odor_valence'],
reduce_key='power')
for i, power in enumerate(out['power']):
min = np.min(power)
max = np.max(power)
out['power'][i] = (power - min) / (max - min)
temp = reduce.new_filter_reduce(out,
filter_keys=['mouse', 'odor_valence'],
reduce_key=ykey_b)
for i in range(len(out['power'])):
bhv = temp[ykey_b][i]
neural = out['power'][i]
if len(neural) > len(
bhv
): # when there is naive day but no training / behavioral data
bhv_trials = np.arange(len(neural) - len(bhv), len(neural))
else:
bhv_trials = np.arange(len(neural))
out[xkey_b].append(bhv_trials)
out[xkey_b] = np.array(out[xkey_b])
# resample
# f = lambda a: ((resample_trials - a[0]) / (a[-1] - a[0])) * (a - a[0]) + a[0]
# for i in range(len(out[xkey_b])):
# out[xkey_b][i] = f(out[xkey_b][i])
# out['trials'][i] = f(out['trials'][i])
return out
def plot_results(res,
x_key,
y_key,
loop_keys=None,
select_dict=None,
path=None,
colors=None,
colormap='cool',
plot_function=plt.plot,
ax_args={},
plot_args={},
xjitter=0,
save=True,
reuse=False,
twinax=False,
sort=False,
error_key='_sem',
fig_size=(2, 1.5),
rect=(.25, .25, .6, .6),
legend=True,
name_str=''):
'''
:param res: flattened dict of results
:param x_key:
:param y_key:
:param loop_key:
:param select_dict:
:param path: save path
:param ax_args: additional args to pass to ax, such as ylim, etc. in dictionary format
:return:
'''
if select_dict is not None:
res = filter.filter(res, select_dict)
if reuse:
ax = plt.gca()
if twinax:
ax = ax.twinx()
else:
fig = plt.figure(figsize=fig_size)
ax = fig.add_axes(rect)
if sort:
ind_sort = np.argsort(res[x_key])
for key, val in res.items():
res[key] = val[ind_sort]
if loop_keys != None:
if isinstance(loop_keys, str):
loop_keys = [loop_keys]
loop_combinations, loop_indices = filter.retrieve_unique_entries(
res, loop_keys)
if save:
labels = [
str(','.join(str(e) for e in cur_combination))
for cur_combination in loop_combinations
]
else:
labels = [None] * len(loop_combinations)
if colormap is None:
cmap = plt.get_cmap('cool')
else:
cmap = plt.get_cmap(colormap)
if colors is None:
colors = [
cmap(i) for i in np.linspace(0, 1, len(loop_combinations))
]
loop_lines = len(loop_combinations)
else:
loop_lines = 1
loop_indices = [np.arange(len(res[y_key]))]
if colors is None:
colors = ['black']
else:
colors = [colors]
labels = [None]
for i in range(loop_lines):
color = colors[i]
label = labels[i]
plot_ix = loop_indices[i]
x_plot = res[x_key][plot_ix]
if type(x_plot[0]) != np.ndarray:
x_plot = np.array([nice_names(x) for x in list(x_plot)])
y_plot = res[y_key][plot_ix]
if plot_function == plt.errorbar:
error_plot = res[error_key][plot_ix]
_plot_error(plot_function,
x_plot,
y_plot,
error_plot,
color=color,
label=label,
plot_args=plot_args)
elif plot_function == plt.fill_between:
error_plot = res[error_key][plot_ix]
_plot_fill(plot_function,
x_plot,
y_plot,
error_plot,
color=color,
label=label,
plot_args=plot_args)
elif plot_function == sns.swarmplot:
t = defaultdict(list)
for k, v in res.items():
t[k] = res[k][plot_ix]
sns.swarmplot(x=x_key,
y=y_key,
hue=loop_keys[0],
data=t,
**plot_args)
ax.get_legend().remove()
elif plot_function == sns.barplot:
import pandas as pd
t = defaultdict(list)
for k, v in res.items():
t[k] = res[k][plot_ix]
sns.barplot(x=x_key, y=y_key, data=t, **plot_args)
ax.get_legend().remove()
else:
_plot(plot_function,
x_plot,
y_plot,
color=color,
label=label,
plot_args=plot_args,
xjitter=xjitter)
ax.set(**ax_args)
#format
# plt.xticks(rotation=45)
ax.set_ylabel(nice_names(y_key), fontsize=7)
ax.set_xlabel(nice_names(x_key), fontsize=7)
if x_key == 'time':
xticks = res['xticks'][0]
xticklabels = ['On', 'Off', 'US']
ax.set_xticks(xticks)
ax.set_xticklabels(xticklabels)
if not twinax:
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')
else:
ax.spines['top'].set_visible(False)
ax.yaxis.label.set_color('green')
ax.tick_params(axis='y', colors='green')
if loop_keys and legend:
nice_loop_str = '+'.join([nice_names(x) for x in loop_keys])
handles, labels = ax.get_legend_handles_labels()
by_label = OrderedDict(zip(labels, handles))
l = ax.legend(by_label.values(),
by_label.keys(),
ncol=2,
fontsize=7,
frameon=False)
try:
for handle in l.legendHandles:
handle.set_sizes([5])
except:
pass
if not loop_keys and legend:
plt.legend(frameon=False)
# l.set_title(nice_loop_str)
# plt.setp(l.get_title(), fontsize=4)
if select_dict is None:
name = 'figure'
else:
name = ''
for k, v in select_dict.items():
name += k + '_' + str(v) + '_'
name += name_str
folder_name = y_key + '_vs_' + x_key
if loop_keys:
loop_str = '+'.join(loop_keys)
folder_name += '_vary_' + loop_str
save_path = os.path.join(path, folder_name)
if save:
_easy_save(save_path, name, dpi=300, pdf=True)
else:
return save_path, name
def plot_stability_across_days(res, start_days_per_mouse, learned_days_per_mouse, figure_path):
strengthen_starting_threshold = .1
strengthen_absolute_threshold = .2
strengthen_relative_threshold = 2
weaken_starting_threshold = .3
weaken_absolute_threshold = .15
weaken_relative_threshold = .5
res = copy.copy(res)
res = filter.filter(res, {'odor_valence':['CS+', 'CS-']})
list_odor_on = res['DAQ_O_ON_F']
list_water_on = res['DAQ_W_ON_F']
for i in range(len(list_odor_on)):
dff_list = res['dff'][i]
dff_max = np.max(dff_list[:,list_odor_on[i]: list_water_on[i]],axis=1)
res['dff_max'].append(dff_max)
res['dff_max'] = np.array(res['dff_max'])
out = defaultdict(list)
combinations, ixs = filter.retrieve_unique_entries(res, ['mouse','odor'])
for combination, ix in zip(combinations, ixs):
odor_valence = np.unique(res['odor_valence'][ix])
mouse = np.unique(res['mouse'][ix])
days = res['day'][ix]
assert len(days) == len(np.unique(days))
assert len(mouse) == 1
sort_ix = np.argsort(days)
list_of_dff_max = res['dff_max'][ix][sort_ix]
list_of_ssig = res['sig'][ix][sort_ix]
data = [x * y for x, y in zip(list_of_ssig, list_of_dff_max)]
data = np.array(data)
strengthen_mask = data[0,:] > strengthen_starting_threshold
strengthen_overall_threshold = np.max((data[0,:] * strengthen_relative_threshold, data[0,:] + strengthen_absolute_threshold), axis=0)
strengthen_passed_mask = strengthen_overall_threshold < data
strengthen_any = np.any(strengthen_passed_mask[1:,:], axis=0)
n_strengthen_passed = np.sum(strengthen_any * strengthen_mask)
n_strengthen_denom = np.sum(strengthen_mask)
new_mask = np.invert(strengthen_mask)
n_new_passed = np.sum(strengthen_any * new_mask)
n_new_denom = np.sum(new_mask)
weaken_mask = data[0,:] > weaken_starting_threshold
weaken_overall_threshold = np.min((data[0,:] * weaken_relative_threshold, data[0,:] - weaken_absolute_threshold), axis=0)
weaken_passed_mask = weaken_overall_threshold > data
weaken_any = np.any(weaken_passed_mask[1:,:], axis=0)
n_weaken_passed = np.sum(weaken_any * weaken_mask)
n_weaken_denom = np.sum(weaken_mask)
strs = ['strengthen_denom', 'strengthen_pass', 'new_denom', 'new_pass','weaken_denom','weaken_pass']
vals = [n_strengthen_denom, n_strengthen_passed, n_new_denom, n_new_passed, n_weaken_denom, n_weaken_passed]
for k, v in zip(strs,vals):
out['Type'].append(k)
out['Count'].append(v)
out['mouse'].append(mouse[0])
out['odor_valence'].append(odor_valence[0])
for k, v in out.items():
out[k] = np.array(v)
def _helper(res):
out = defaultdict(list)
strs = [['strengthen_denom', 'strengthen_pass'], ['new_denom', 'new_pass'], ['weaken_denom', 'weaken_pass']]
out_strs = ['up', 'new', 'down']
for i, keys in enumerate(strs):
ix_denom = res['Type'] == keys[0]
ix_numer = res['Type'] == keys[1]
numer = np.sum(res['Count'][ix_numer])
denom = np.sum(res['Count'][ix_denom])
fraction = numer/ denom
out['Type'].append(out_strs[i])
out['Fraction'].append(fraction)
out['numer'].append(numer)
out['denom'].append(denom)
for k, v in out.items():
out[k] = np.array(v)
return out
ax_args_copy = ax_args.copy()
ax_args_copy.update({'ylim':[0, 0.5], 'yticks':[0, .1, .2, .3, .4, .5]})
overall = copy.copy(out)
csm_res = filter.filter(overall, {'odor_valence':'CS-'})
csp_res = filter.filter(overall, {'odor_valence':'CS+'})
csp_stats = _helper(csp_res)
csp_stats['Condition'] = np.array(['CS+'] * len(csp_stats['Type']))
csm_stats = _helper(csm_res)
csm_stats['Condition'] = np.array(['CS-'] * len(csm_stats['Type']))
overall_stats = reduce.chain_defaultdicts(csp_stats, csm_stats, copy_dict=True)
filter.assign_composite(overall_stats, ['Type', 'Condition'])
bar_args = {'alpha': 1, 'edgecolor':'black','linewidth':1}
colors = ['Green','Red']
save_path, name = plot.plot_results(overall_stats, x_key='Type_Condition', y_key='Fraction', sort=True,
colors=colors,
path=figure_path,
plot_function=plt.bar, plot_args=bar_args, ax_args=ax_args_copy,
save=False, reuse=False)
strs = ['down','new','up']
fontsize = 4
for i in range(3):
ix = csp_stats['Type'] == strs[i]
numer = csp_stats['numer'][ix][0]
denom = csp_stats['denom'][ix][0]
x = i * 2
y = numer/denom
y_offset = .025
plt.text(x, y + y_offset, str(numer) + '/' + str(denom), horizontalalignment = 'center', fontsize= fontsize)
strs = ['down','new','up']
for i in range(3):
ix = csm_stats['Type'] == strs[i]
numer = csm_stats['numer'][ix][0]
denom = csm_stats['denom'][ix][0]
x = i * 2 + 1
y = numer/denom
y_offset = .025
plt.text(x, y + y_offset, str(numer) + '/' + str(denom), horizontalalignment = 'center', fontsize= fontsize)
plot._easy_save(save_path, name, pdf=True)
overall_stats_no_distinction = _helper(overall)
save_path, name = plot.plot_results(overall_stats_no_distinction, x_key='Type', y_key='Fraction', sort=True,
colors=['Grey']*10,
path=figure_path,
plot_function=plt.bar, plot_args=bar_args, ax_args=ax_args_copy,
save=False, reuse=False)
strs = ['down','new','up']
for i in range(3):
ix = overall_stats_no_distinction['Type'] == strs[i]
numer = overall_stats_no_distinction['numer'][ix][0]
denom = overall_stats_no_distinction['denom'][ix][0]
x = i
y = numer/denom
y_offset = .025
plt.text(x, y + y_offset, str(numer) + '/' + str(denom), horizontalalignment = 'center', fontsize= fontsize)
plot._easy_save(save_path, name, pdf=True)
