def __init__(self,basePath,mouse,s,dataSet='OnACID',SNR_thr=2,rval_thr=0):
pxtomu = 530.68/512
pathMouse = pathcat([basePath,mouse])
pathSession = pathcat([pathMouse,'Session%02d'%s])
pathData = pathcat([pathSession,'results_OnACID.mat'])
ld = loadmat(pathData,variable_names=['Cn'])
pathData = pathcat([pathSession,'results_%s.mat'%dataSet])
self.data = loadmat(pathData,variable_names=['A','C','SNR','r_values'])
self.data['Cn'] = ld['Cn'].T
self.idx_good = (self.data['SNR']>2).T & (self.data['r_values']>0).T
self.cm = com(self.data['A'],512,512) * pxtomu
self.cm[np.squeeze(~self.idx_good),:] = np.NaN
self.D_ROIs = sp.spatial.distance.pdist(self.cm)
self.D_ROIs_mat = sp.spatial.distance.squareform(self.D_ROIs)
nCells = self.data['C'].shape[0]
self.corrcoef = np.zeros((nCells,nCells))
for n in tqdm(np.where(self.idx_good)[0]):
for nn in range(n+1,nCells):
if self.idx_good[nn]:
self.corrcoef[n,nn] = np.corrcoef(self.data['C'][n,:],self.data['C'][nn,:])[0,1]
self.Acorr = np.zeros((nCells,nCells))
for n in tqdm(np.where(self.idx_good)[0]):#range(nCells)):
idx_close=np.where(self.D_ROIs_mat[n,:]<20)[0]
for nn in idx_close:
self.Acorr[n,nn],_ = calculate_img_correlation(self.data['A'][:,n],self.data['A'][:,nn],shift=False)
self.Asz = (self.data['A']>0).sum(0)
def plot_session_shifts(basePath,
mouse,
sessions=None,
dataSet='OnACID',
pltSave=False):
pathMouse = pathcat([basePath, mouse])
pathSession1 = pathcat(
[pathMouse,
'Session%02d/results_%s.mat' % (sessions[0], dataSet)])
ROIs1_ld = loadmat(pathSession1, variable_names=['A', 'Cn'])
pathSession2 = pathcat(
[pathMouse,
'Session%02d/results_%s.mat' % (sessions[1], dataSet)])
ROIs2_ld = loadmat(pathSession2, variable_names=['A', 'Cn'])
A1 = ROIs1_ld['A']
A2 = ROIs2_ld['A']
#dims = Cn.shape
get_shift_and_flow(A1, A2, plot_bool=True)
#Cn2 = cv2.remap(ROIs2_ld['Cn'].astype(np.float32), x_remap, y_remap, cv2.INTER_NEAREST)
plt.figure()
plt.subplot(1, 2, 1)
plt.imshow(ROIs1_ld['Cn'], origin='lower')
plt.subplot(1, 2, 2)
plt.imshow(ROIs2_ld['Cn'], origin='lower')
plt.show(block=False)
if pltSave:
plt.savefig('%ssession_shift.png' % pathPlots)
def __init__(self, basePath=None, mouse=None, s=None):
self.f = 15 ## recording rate in Hz
if not (basePath is None):
pathMouse = pathcat([basePath, mouse])
self.pathSession = pathcat([pathMouse, 'Session%02d' % s])
self.data = {}
def run_detection(basePath,
mouse,
nP,
s_start=1,
s_end=None,
nSession=None,
session_order=None,
rerun=False):
pathMouse = pathcat([basePath, mouse])
if nSession is None:
nSession, tmp = get_nPaths(pathMouse, 'Session')
cMouse = cluster(basePath,
mouse,
nSession,
dataSet='redetect',
session_order=session_order,
suffix='')
cMouse.process_sessions(n_processes=nP, reprocess=True)
cMouse.get_IDs()
cMouse.get_stats(n_processes=nP)
# cMouse.load([True,True,True,False,False])
cMouse.update_status(complete=False,
SNR_thr=3,
rval_thr=0.5,
reliability_thr=0.1,
Bayes_thr=10,
nCluster_thr=2)
# return cMouse
s_end = nSession + 1 if s_end is None else s_end + 1
print(s_start, s_end)
print('run detection on %d sessions for mouse %s' % (nSession, mouse))
for s in range(s_start, s_end):
# try:
IDs = cMouse.IDs['neuronID'][cMouse.status[:, s - 1, 1], s - 1,
1].astype('int')
s_range = np.unique(cMouse.IDs['neuronID'][:, s - 1, 2])
s0 = int(np.unique(cMouse.IDs['neuronID'][:, s - 1, 2])[0])
print('Now processing session %d' % s0)
dPC = detect_PC(basePath,
mouse,
s0,
nP,
nbin=100,
plt_bool=False,
sv_bool=False,
suffix='')
dPC.run_detection(f_max=1,
rerun=rerun,
dataSet='redetect',
assignment=IDs)
def find_overlaps(self, s, reprocess=None):
if not (reprocess is None):
self.reprocess = reprocess
if self.reprocess:
pathLoad = pathcat([
self.cluster.meta['pathMouse'],
'Session%02d/results_redetect.mat' % (s + 1)
])
data = loadmat(pathLoad, variable_names=['A', 'C'])
self.D_ROIs = sp.spatial.distance.squareform(
sp.spatial.distance.pdist(self.cluster.stats['com'][:, s, :]))
idx_close = self.D_ROIs < 10
self.corrcoef = np.zeros((self.nC, self.nC)) * np.NaN
self.Acorr = np.zeros((self.nC, self.nC)) * np.NaN
for c in tqdm(np.where(self.cluster.status[:, s, 1])[0]):
n = int(self.cluster.IDs['neuronID'][c, s, 1])
for cc in np.where(idx_close[c, :])[0]:
nn = int(self.cluster.IDs['neuronID'][cc, s, 1])
self.corrcoef[c, cc] = np.corrcoef(data['C'][n, :],
data['C'][nn, :])[0, 1]
self.Acorr[c, cc], _ = calculate_img_correlation(
data['A'][:, n], data['A'][:, nn], shift=False)
self.reprocess = False
np.fill_diagonal(self.corrcoef, np.NaN)
np.fill_diagonal(self.Acorr, np.NaN)
idx_overlap = np.where((self.Acorr > 0.2) & (self.corrcoef > 0.5))
for (i, j) in zip(idx_overlap[0], idx_overlap[1]):
print(
'neurons: %d,%d, \t Acorr: %.3f, Ccorr: %.3f, \t SNR: %.2f,%.2f'
% (i, j, self.Acorr[i, j], self.corrcoef[i, j],
self.cluster.stats['SNR'][i, s],
self.cluster.stats['SNR'][j, s]))
print(len(idx_overlap[0]))
def load_data(self, C=None, S=None, SNR=None, r_values=None):
## do I need noise level for either method? (rerun redetection??)
if C is None:
pathResults = pathcat([self.pathSession, 'results_redetect.mat'])
ld = loadmat(pathResults,
variable_names=[
'C', 'S', 'A', 'b', 'f', 'SNR', 'r_values', 'CNN'
],
squeeze_me=True)
for key in ld.keys():
self.data[key] = ld[key]
self.data['C'] -= self.data['C'].min(1)[:, np.newaxis]
self.data['S'][self.data['S'] < 0] = 0
else:
self.data['C'] = C
self.data['S'] = S
self.data['C'] -= self.data['C'].min(1)[:, np.newaxis]
self.data['S'][self.data['S'] < 0] = 0
self.data['SNR'] = SNR
self.data['r_values'] = r_values
#self.data['A'] = normalize_sparse_array(self.data['A'])
self.nCells, self.T = self.data['C'].shape
def analyze_matchStats(pathMouse,std,thr,w,OnACID=True,old=False):
match = {}
if OnACID:
suffix = '_OnACID'
else:
suffix = ''
if not old:
pathAss = pathcat([pathMouse,sprintf('matching/results_matching_multi_std=%d_thr=%d_w=%d%s.mat'%(std,thr*100,round(w*100),suffix]))
ld = loadmat(pathAss)
scores = ld['scores']
assignments = ld['assignments']
for s in range(1,15):
N_union = max(np.where(~np.isnan(assignments[:,s-1]))[0]);
N_s = sum(~isnan(assignments(:,s)));
match[s] = {}
match[s]['bool'] = np.zeros((N_union,N_s)).astype('bool')
match[s]['d_s'] = np.zeros((N_union,N_s))*np.NaN
ld_manual = loadmat(pathcat([pathMouse,'backup/save_final/matching_results.mat']))
matchStats_arr{1} = get_matchState(data,clusters_sv);
if old:
ld_old = loadmat(pathcat([pathMouse,'matching_results.mat']))
[matchStats_arr{2}, data2, idx_ref_match, idx_match_ref] = OnACID_get_matchState(pathMouse,std,thr,w,OnACID,old);
else:
[matchStats_arr{2}, data2, idx_ref_match, idx_match_ref] = OnACID_get_matchState(pathMouse,std,thr,w,OnACID,old);
col = ['b','r']
nSes = len(matchStats_arr{1}.clusters(1).list)
plt.figure('position',[100 100 800 400])
nROI_base = zeros(nSes,1);
for s = 1:nSes
nROI_base[s] = len(idx_ref_match{s});#data.session(s).nROI;
# disp(sprintf('nROIs: %d , %d',sum(~isnan(idx_ref_match{s})),sum(~isnan(idx_match_ref{s}))))
data.session(s).nROI = sum(~isnan(idx_match_ref{s}));
end
ROI_recurr_frac = zeros(nSes-1,2);
## clean up from non-matched neurons
for s=1:nSes
for n = 1:length(idx_ref_match{s})
if isnan(idx_ref_match{s}(n)) && n <= length(matchStats_arr{1}.session(s).neuron)
c = matchStats_arr{1}.session(s).neuron(n).cluster_ID;
if ~isempty(c)
matchStats_arr{1}.session(s).neuron(n).cluster_ID = [];
matchStats_arr{1}.clusters(c).list(s) = 0;
end
end
end
for n = max(idx_match_ref{s})+1:length(matchStats_arr{1}.session(s).neuron)
c = matchStats_arr{1}.session(s).neuron(n).cluster_ID;
matchStats_arr{1}.session(s).neuron(n).cluster_ID = [];
if ~isempty(c)
matchStats_arr{1}.clusters(c).list(s) = 0;
end
end
# for n = 1:length(idx_match_ref{s})
# n
# if isnan(idx_match_ref{s}(n))# && n <= length(matchStats_arr{2}.session(s).neuron)
# c = matchStats_arr{2}.session(s).neuron(n).cluster_ID;
# if ~isempty(c)
# matchStats_arr{2}.session(s).neuron(n).cluster_ID = [];
# matchStats_arr{2}.clusters(c).list(s) = 0;
# end
# end
# end
#
# for n = max(idx_ref_match{s})+1:length(matchStats_arr{2}.session(s).neuron)
# c = matchStats_arr{2}.session(s).neuron(n).cluster_ID;
# matchStats_arr{2}.session(s).neuron(n).cluster_ID = [];
# if ~isempty(c)
# matchStats_arr{2}.clusters(c).list(s) = 0;
# end
# end
end
for i = 1:length(matchStats_arr)
nC = length(matchStats_arr{i}.clusters);
cluster_nROI = zeros(nSes,1);
nROI = zeros(nSes,1);
ROI_recurr = zeros(nSes,nSes-1);
for c = 1:nC
matchStats_arr{i}.clusters(c).nROI = nnz(matchStats_arr{i}.clusters(c).list);
if matchStats_arr{i}.clusters(c).nROI
cluster_nROI(matchStats_arr{i}.clusters(c).nROI) = cluster_nROI(matchStats_arr{i}.clusters(c).nROI) + 1;
end
nROI = nROI + (matchStats_arr{i}.clusters(c).list>0);
end
nROI_total = zeros(nSes-1,1);
for s = 1:nSes
for sm = s+1:nSes
ds = sm-s;
# nROI_total(ds) = nROI_total(ds) + nROI(s);
nROI_total(ds) = nROI_total(ds) + nROI_base(s);
end
end
# subplot(4,2,1)
# hold on
# bar(nROI,'FaceAlpha',0.6,'FaceColor',col{i})
# xlabel('Session #')
# ylabel('# ROIs')
# subplot(2,2,2)
# hold on
# bar(nROI_total,'FaceAlpha',0.6)
subplot(2,2,1)
if i == 1
dispName = 'pre';
else
dispName = 'post';
end
hold on
bar(cluster_nROI,'FaceAlpha',0.6,'FaceColor',col{i},'DisplayName',dispName)
xlabel('# ROIs / cluster')
ylabel('# clusters')
ylim([0,400])
legend('location','NorthEast')
pause(0.0001)
for s = 1:nSes
for sm = s+1:nSes
ds = sm-s;
for c = 1:nC
if matchStats_arr{i}.clusters(c).list(s) && matchStats_arr{i}.clusters(c).list(sm)
ROI_recurr(s,ds) = ROI_recurr(s,ds) + 1;
end
end
end
end
ROI_recurr_frac(:,i) = sum(ROI_recurr,1)./nROI_total.T;
subplot(2,2,2)
hold on
plot([0,15],[0.5,0.5],'k:','LineWidth',0.5,'HandleVisibility','off')
plot(ROI_recurr_frac(:,i),'o--','DisplayName','p_{recurr}')
xticks([])
# ylim([0,1])
ylabel('p_{rec}')
end
subplot(2,2,2)
hold on
plot([0,15],[0,0],'k:','LineWidth',0.5,'HandleVisibility','off')
plot(ROI_recurr_frac(:,2)-ROI_recurr_frac(:,1),'ko--','DisplayName','\Delta p')
ylim([-0.1,1])
xlabel('\Delta s')
ylabel('\Delta p_{rec}')
disp_txt = false;
matches = zeros(nSes-1,4);
xlim([0.5,15.5])
legend('Location','NorthEast')
# figure
subplot(2,2,4)
hold on
p_matches_correct = plot(matches(:,1)./sum(matches,2),'g','DisplayName','correct match');
p_matches_wrong = plot(matches(:,2)./sum(matches,2),'r','DisplayName','wrong match');
p_matches_missing = plot(matches(:,3)./sum(matches,2),'r--','DisplayName','missing');
p_matches_unnec = plot(matches(:,4)./sum(matches,2),'r:','DisplayName','unnec. match');
legend('Location','northeastoutside')
xlabel('\Delta s')
yticks(linspace(0,1,6))
yticklabels(linspace(0,1,6))
ylabel('fraction')
c_matches = zeros(length(matchStats_arr{1}.clusters),4); ## col-# = cluster-ID, cols: 1: new cluster-ID; 2: old #ROIs, 3: new #ROIs, 4: # correct ROIs
xlim([0.5,15.5])
ylim([0,1])
pause(0.0001)
nROI_match = zeros(nSes+1,4);
## now, compare both matchStats
for s = 1:nSes
disp(sprintf('Now processing s=%d',s))
for n = 1:length(idx_ref_match{s})
if isnan(idx_ref_match{s}(n)) || n > length(matchStats_arr{2}.session(s).neuron)
# disp('why?')
continue
end
c2 = matchStats_arr{2}.session(s).neuron(n).cluster_ID;
c_ref = matchStats_arr{1}.session(s).neuron(n).cluster_ID;
# matchStats_arr{1}.clusters(c_ref).list
# matchStats_arr{2}.clusters(c2).list
if ~isempty(c_ref)
if isempty(c2)
if ~c_matches(c_ref,4)
c_matches(c_ref,1) = NaN;
c_matches(c_ref,2) = nnz(matchStats_arr{1}.clusters(c_ref).list);
c_matches(c_ref,3) = 0;
c_matches(c_ref,4) = 0;
end
else
init_correct = sum(matchStats_arr{1}.clusters(c_ref).list & matchStats_arr{1}.clusters(c_ref).list==matchStats_arr{2}.clusters(c2).list);
if init_correct > c_matches(c_ref,4);
c_matches(c_ref,1) = c2;
c_matches(c_ref,2) = nnz(matchStats_arr{1}.clusters(c_ref).list);
c_matches(c_ref,3) = nnz(matchStats_arr{2}.clusters(c2).list);
c_matches(c_ref,4) = init_correct;
end
end
end
# if ~isempty(c_ref)# && nnz(matchStats_arr{1}.clusters(c_ref).list) < 2
# continue
# end
# if ~isempty(c2)# && nnz(matchStats_arr{2}.clusters(c2).list) < 2
# continue
# end
# init_correct
# waitforbuttonpress
# if ~isempty(c2) && matchStats_arr{i}.clusters(c2).nROI == 15
for sm = s+1:nSes
ds = sm-s;
if disp_txt
[s sm ds]
n
[c_ref,c2]
matchStats_arr{1}.clusters(c_ref).list
matchStats_arr{2}.clusters(c2).list
# [matchStats_arr{1}.clusters(c_ref).list(s) matchStats_arr{2}.clusters(c2).list(s)]
# [matchStats_arr{1}.clusters(c_ref).list(sm) matchStats_arr{2}.clusters(c2).list(sm)]
end
# m_true = [];
# if ~isempty(matchStats_arr{2}.clusters(c2).list(sm))
# m_true = matchStats_arr{2}.clusters(c2).list(sm);
# end
if isempty([c_ref,c2])
if disp_txt
disp('none assigned')
end
elseif isempty(c_ref)
m2 = matchStats_arr{2}.clusters(c2).list(sm);
if m2
matches(ds,4) = matches(ds,4) + 1;
if ~old
match(sm).bool(c2,idx_ref_match{sm}(m2)) = 4;
match(sm).d_s(c2,idx_ref_match{sm}(m2)) = sm-s;
nROI = 16;
nROI_match(nROI,4) = nROI_match(nROI,4) + 1;
end
end
if disp_txt
disp('ref n not assigned')
end
elseif isempty(c2)
m = matchStats_arr{1}.clusters(c_ref).list(sm);
if m && ~isnan(idx_ref_match{sm}(m))
matches(ds,3) = matches(ds,3) + 1;
if ~old
match(sm).bool(c2,idx_ref_match{sm}(m)) = 3;
nROI = matchStats_arr{1}.clusters(c_ref).nROI;
nROI_match(nROI,3) = nROI_match(nROI,3) + 1;
end
end
if disp_txt
disp('processed n not assigned')
end
else
m2 = matchStats_arr{2}.clusters(c2).list(sm);
m = matchStats_arr{1}.clusters(c_ref).list(sm);
# if m<=length(idx_ref_match{sm}) && (~m || ~isnan(idx_ref_match{sm}(m))) && m2<=length(idx_ref_match{sm}) && (~m2 || ~isnan(idx_ref_match{sm}(m2)))
if ~m && ~m2
if disp_txt
disp('nothing, both are empty!')
end
elseif ~m
matches(ds,4) = matches(ds,4) + 1;
if ~old
match(sm).bool(c2,idx_ref_match{sm}(m2)) = 4;
match(sm).d_s(c2,idx_ref_match{sm}(m2)) = sm-s;
nROI = matchStats_arr{1}.clusters(c_ref).nROI;
nROI_match(nROI,4) = nROI_match(nROI,4) + 1;
end
if disp_txt
disp('ref sm not assigned')
end
elseif ~m2
matches(ds,3) = matches(ds,3) + 1;
if ~old
match(sm).bool(c2,idx_ref_match{sm}(m)) = 3;
nROI = matchStats_arr{1}.clusters(c_ref).nROI;
nROI_match(nROI,3) = nROI_match(nROI,3) + 1;
end
if disp_txt
disp('processed sm not assigned')
end
elseif m2 ~= m
matches(ds,2) = matches(ds,2) + 1;
if ~old
match(sm).bool(c2,idx_ref_match{sm}(m2)) = 2;
match(sm).bool(c2,idx_ref_match{sm}(m)) = 3;
match(sm).d_s(c2,idx_ref_match{sm}(m2)) = sm-s;
nROI = matchStats_arr{1}.clusters(c_ref).nROI;
nROI_match(nROI,2) = nROI_match(nROI,2) + 1;
end
if disp_txt
disp('wrongly assigned')
end
else
matches(ds,1) = matches(ds,1) + 1;
if ~old
match(sm).bool(c2,idx_ref_match{sm}(m2)) = 1;
match(sm).d_s(c2,idx_ref_match{sm}(m2)) = sm-s;
nROI = matchStats_arr{1}.clusters(c_ref).nROI;
nROI_match(nROI,1) = nROI_match(nROI,1) + 1;
end
if disp_txt
disp('properly assigned')
end
end
# end
end
# if disp_txt
# waitforbuttonpress
# end
end
if ~mod(n,200)
norm_matches = sum(matches(:,1:3),2);
set(p_matches_correct,'YData',matches(:,1)./norm_matches)
set(p_matches_wrong,'YData',matches(:,2)./norm_matches)
set(p_matches_missing,'YData',matches(:,3)./norm_matches)
set(p_matches_unnec,'YData',matches(:,4)./norm_matches)
drawnow
# pause(0.0001)
# waitforbuttonpress
end
end
end
subplot(2,2,3)
scatter(c_matches(:,3)+0.5*rand(size(c_matches,1),1)-0.25,c_matches(:,2)+0.5*rand(size(c_matches,1),1)-0.25,5,c_matches(:,4)./c_matches(:,3));
colormap('jet');
xlim([1.5,15.5])
ylim([1.5,15.5])
xlabel('#ROIs pre')
ylabel('#ROIs post')
cbar = colorbar;
ylabel(cbar, 'ratio correct matches')
pathSv = pathcat(pathMouse,'Figures',sprintf('results_matching_multi_std=%d_thr=%d_w=%d%s.png',std,thr*100,round(w*100),suffix))
# path = pathcat(pathFigures,'results_manual_matching.png');
print(pathSv,'-dpng','-r300')
figure
c_idx = c_matches(:,2)>1;
# d_matches = c_matches(c_idx,2)-c_matches(c_idx,4);
d_matches = c_matches(c_idx,2)-c_matches(c_idx,3);
histogram2(c_matches(c_idx,2),d_matches,linspace(-0.5,15.5,16),linspace(-15.5,15.5,31))
if ~old
disp('now, find avg and STD of corr. and distance of wrongly assigned matches / missing matches, as well as same distr from proper matches (able to characterize those based on those values?). also, find the shifted fp_corr of those groups (better match with this?). can i find a better value, which describes the matching yet better?')
disp('get something, saying how many clusters are actually estimated right, vs wrong, + how many errors are made in the clusters (per nROI?)')
### gather scores for each entry in cluster, compute average or something and plot vs # of correct matches
#
col = {'g','r','b','m'}
# col = {[0,1,0],[1,0,0],[0,0,1],[1,0,1]}
dispName = {'correct','wrong','missing','unnecessary'}
figure
matches_ecdf = cell(4,1);
nROI_match
subplot(1,3,2)
h = bar(nROI_match,'stacked');
for i=1:4
set(h(i),'FaceColor',col{i})
end
# set(h,{'FaceColor'},col)
hold on
subplot(1,3,3)
h = bar(transpose(nROI_match),'stacked');
# for i=1:4
# set(h(i),'FaceColor',col{i})
# end
hold on
for i = 1:4
for s = 2:nSes
# for sm = s+1:nSes
# disp(sprintf('session %d',s))
mask = logical(match(s).bool==i); #(1:size(scores{s,1},1),1:size(scores{s,1},2))
matches_ecdf{i} = horzcat(matches_ecdf{i},transpose(scores{s-1,1}(mask)));
# subplot(2,2,1)
# hold on
# scatter(scores{s-1,1}(mask),scores{s-1,3}(mask),'Color',col{i})
# end
end
subplot(1,3,1)
[f,x,flo,fup] = ecdf(matches_ecdf{i});
stairs(x,f,'Color',col{i},'DisplayName',dispName{i});
hold on
stairs(x,flo,':','Color',col{i},'HandleVisibility','off');
stairs(x,fup,':','Color',col{i},'HandleVisibility','off');
# subplot(1,2,2)
# hold on
# hist(matches_ecdf{i})#,'FaceAlpha',0.5,'FaceColor',col{i})
end
legend('Location','NorthWest')
xlim([0,1])
pathSv = pathcat(pathMouse,'Figures',sprintf('results_matching2_multi_std=%d_thr=%d_w=%d%s.png',std,thr*100,round(w*100),suffix))
# path = pathcat(pathFigures,'results_manual_matching.png');
print(pathSv,'-dpng','-r300')
end
# for i = -1:1
# for s = 1:nSes
# for sm = s+1:nSes
# mask = match(s,sm).bool(1:size(xdata(1,2).corr,1),1:size(xdata(1,2).corr,2))==i;
# scatter(xdata(s,sm).dist(mask),xdata(s,sm).corr(mask),col{i})
# end
# end
# end
return c_matches,match
def get_session_perspective(basePath, mouse, s_in):
pathMouse = pathcat([basePath, mouse])
dims = (512, 512)
Cn = np.zeros((2, 512, 512))
for i, s in enumerate(s_in):
pathSession = pathcat([pathMouse, 'Session%02d' % s])
pathResults = pathcat([pathSession, 'results_redetect.mat'])
ld = loadmat(pathResults, variable_names=['A'])
A = ld['A']
#A=normalize_sparse_array(A)
Cn[i, ...] = A.sum(1).reshape(512, 512).astype('float32')
shift, flow, (x_grid, y_grid) = get_shift_and_flow(Cn[0, ...],
Cn[1, ...],
dims,
projection=None,
plot_bool=False)
#out = cv2.getPerspectiveTransform(Cn[0,...],Cn[1,...])
#grid_ref = np.stack([x_grid,y_grid,np.zeros(dims)],2).reshape(-1,3)
#grid2 = np.stack([x_grid+flow[...,0],y_grid+flow[...,1],np.zeros(dims)],2).reshape(-1,3)
#print(shift)
#grid_ref = np.stack([x_grid,y_grid],2).reshape(-1,2)
#grid2 = np.stack([x_grid-shift[0]+flow[...,0],y_grid-shift[1]+flow[...,1]],2).reshape(-1,2)
#return grid_ref
#print(grid_ref.shape)
#out = cv2.estimateAffine2D(grid_ref,grid2)
#out = cv2.estimateAffine3D(grid_ref,grid2)
#out = cv2.calibrateCamera(grid_ref,grid2,dims,None,None)
print(Cn[0, ...].max())
print(Cn[1, ...].max())
plt.figure(figsize=(10, 6))
plt.subplot(231)
plt.imshow(Cn[0], origin='lower')
plt.title('Session 1')
plt.subplot(234)
plt.imshow(Cn[1], origin='lower')
plt.title('Session 2')
ax = plt.subplot(232)
C = signal.convolve(
Cn[0, ...] - Cn[0, ...].mean(),
Cn[1, ::-1, ::-1] - Cn[1, ...].mean(),
mode='same') / (np.prod(dims) * Cn[0, ...].std() * Cn[1, ...].std())
ax.imshow(C, origin='lower')
ax.set_xticks(range(240, 270, 5))
ax.set_xticklabels(range(-15, 15, 5))
ax.set_yticks(range(240, 270, 5))
ax.set_yticklabels(range(-15, 15, 5))
#plt.set('yticks',range(240,270,7),'yticklabels',range(-15,15,7))
ax.set_xlim([240, 271])
ax.set_ylim([240, 271])
#plt.colorbar()
plt.title('image correlation (= %5.3g)' % C.max())
plt.subplot(235)
idxes = 15
plt.quiver(x_grid[::idxes, ::idxes],
y_grid[::idxes, ::idxes],
flow[::idxes, ::idxes, 0],
flow[::idxes, ::idxes, 1],
angles='xy',
scale_units='xy',
scale=0.5,
headwidth=4,
headlength=4,
width=0.002,
units='width')
plt.xlim([0, 512])
plt.ylim([0, 512])
plt.title('optical flow')
plt.subplot(233)
Cn_plot = np.zeros(dims + (3, ))
Cn_plot[..., 0] = Cn[0]
Cn_plot[..., 1] = Cn[1]
Cn_plot /= Cn_plot.max()
plt.imshow(Cn_plot, origin='lower')
plt.title('1+2 (unaligned)')
plt.subplot(236)
x_remap = (x_grid - shift[0] + flow[:, :, 0])
y_remap = (y_grid - shift[1] + flow[:, :, 1])
Cn_new = cv2.remap(Cn[1], x_remap, y_remap, cv2.INTER_CUBIC)
Cn_plot = np.zeros(dims + (3, ))
Cn_plot[..., 0] = Cn[0]
Cn_plot[..., 1] = Cn_new
Cn_plot /= Cn_plot.max()
plt.imshow(Cn_plot, origin='lower')
plt.title('1+2 (aligned)')
plt.tight_layout()
plt.show(block=False)
svPath = pathcat([pathMouse, 'alignExample.png'])
plt.savefig(svPath, format='png', dpi=300)
#plt.figure()
#plt.imshow(Cn[0])
#plt.show(block=False)
#plt.figure()
#plt.imshow(Cn_new)
#plt.show(block=False)
#grid_new = np.zeros(grid2.shape)
#im_out = cv2.warpAffine(Cn[1],out[0],(Cn.shape[1:]))
#print(Cn.shape[1:])
#plt.figure()
#plt.subplot(121)
#plt.imshow(im_out)
#plt.subplot(122)
#plt.imshow(im_out)
#plt.show(block=False)
#shift, flow, (x_grid,y_grid) = get_shift_and_flow(im_out,Cn[1],dims,projection=None,plot_bool=False)
return out #,im_out
def set_para(basePath,mouse,s,nP=0,nbin=100,plt_bool=False,sv_bool=False,suffix='2'):
## set paths:
pathMouse = pathcat([basePath,mouse])
pathSession = pathcat([pathMouse,'Session%02d'%s])
coarse_factor = int(nbin/20)
#nbin = 100
#coarse_factor = 5
qtl_steps = 4
fact = 1 ## factor from path length to bin number
gate_mice = ["34","35","65","66","72","839","840","841","842","879","882","884","886","67","68","91","549","551","756","757","758","918shKO","931wt","943shKO"]
nogate_mice = ["231","232","236","243","245","246","762","",""]
zone_idx = {}
if any(mouse==m for m in gate_mice): ## gate
zone_idx['gate'] = [18,33]
zone_idx['reward'] = [75,95]
have_gt = True;
elif any(mouse==m for m in nogate_mice): ## no gate
zone_idx['reward'] = [50,70]#[50,66]#
zone_idx['gate'] = [np.NaN,np.NaN]
have_gt = False;
zone_mask = {}
zone_mask['reward'] = np.zeros(nbin).astype('bool')#range(zone_idx['reward'][0],zone_idx['reward'][-1])
zone_mask['gate'] = np.zeros(nbin).astype('bool')
zone_mask['others'] = np.ones(nbin).astype('bool')
zone_mask['reward'][zone_idx['reward'][0]:zone_idx['reward'][-1]] = True
zone_mask['others'][zone_mask['reward']] = False
if have_gt:
zone_mask['gate'][zone_idx['gate'][0]:zone_idx['gate'][-1]] = True
zone_mask['others'][zone_mask['gate']] = False
# zone_mask['others'][40:50] = False ## remove central wall pattern change?!
zone_mask['active'] = nbin+1
zone_mask['silent'] = nbin+2
print('now')
para = {'nbin':nbin,'f':15,
'bin_array':np.linspace(0,nbin-1,nbin),
'bin_array_centers':np.linspace(0,nbin,nbin+1)-0.5,
'coarse_factor':coarse_factor,
'nbin_coarse':int(nbin/coarse_factor),
'pxtomu':536/512,
'L_track':100,
'rate_thr':4,
'width_thr':5,
'trials_min_count':3,
'trials_min_fraction':0.2,
'Ca_thr':0,
't_measures': get_t_measures(mouse),
'nP':nP,
'N_bs':10000,'repnum':1000,
'qtl_steps':qtl_steps,'sigma':5,
'qtl_weight':np.ones(qtl_steps)/qtl_steps,
'names':['A_0','A','SD','theta'],
#'CI_arr':[0.001,0.025,0.05,0.159,0.5,0.841,0.95,0.975,0.999],
'CI_arr':[0.025,0.05,0.95,0.975],
'plt_bool':plt_bool&(nP==0),
'plt_theory_bool':True&(nP==0),
'plt_sv':sv_bool&(nP==0),
'mouse':mouse,
'session':s,
'pathSession':pathSession,
'pathMouse':pathMouse,
'pathFigs':'/home/wollex/Data/Science/PhD/Thesis/pics/Methods',#'/home/wollex/Data/Documents/Uni/2016-XXXX_PhD/Japan/Work/Results/pics/Methods',
### provide names for figures
'svname_status': 'PC_fields%s_status.mat'%suffix,
'svname_fields': 'PC_fields%s_para.mat'%suffix,
'svname_firingstats': 'PC_fields%s_firingstats.mat'%suffix,
### modes, how to perform PC detection
'modes':{'activity':'calcium',#'spikes',# ## data provided: 'calcium' or 'spikes'
'info':'MI', ## information calculated: 'MI', 'Isec' (/second), 'Ispike' (/spike)
'shuffle':'shuffle_trials' ## how to shuffle: 'shuffle_trials', 'shuffle_global', 'randomize'
},
'zone_idx':zone_idx,
'zone_mask':zone_mask
}
## -----------------------------------------------------------------------------------------------------------------------
#if nargin == 3:
#para.t_s = get_t_measures(mouse);
#para.nSes = length(para.t_s);
#time_real = false;
#if time_real
#t_measures = get_t_measures(mouse);
#t_mask_m = false(1,t_measures(nSes));
#for s = 1:nSes-1
#for sm = s+1:nSes
#dt = t_measures(sm)-t_measures(s);
#t_mask_m(dt) = true;
#end
#end
#t_data_m = find(t_mask_m);
#t_ses = t_measures;
#t_mask = t_mask_m;
#t_data = t_data_m;
#nT = length(t_data);
#else
#t_measures = get_t_measures(mouse);
#t_measures = t_measures(s_offset:s_offset+nSes-1);
## t_measures = 1:nSes; ## remove!
## t_measures
#t_ses = linspace(1,nSes,nSes);
#t_data = linspace(1,nSes,nSes);
#t_mask = true(nSes,1);
#nT = nSes;
#end
return para
def plot_thesis(self, n, sig=[2, 3, 4], SNR_thr=2, r_thr=0):
print('plotting for neurons %d and %d' % (n[0], n[1]))
# n=[0,2452] ## in session 10 of mouse 762
self.nCells, self.T = self.data['C'].shape
T = self.T
t_arr = np.linspace(0, self.T / self.f, self.T)
plt.rcParams['font.size'] = 12
idxes = (self.data['SNR'] > SNR_thr) & (self.data['r_values'] > r_thr)
zlen = 500
zstart = 1500
zrange = range(zstart, (zstart + zlen))
fig = plt.figure(figsize=(7, 5), dpi=150)
col = np.array([1, 0, 0])
ax_C_zoom = plt.axes([0.075, 0.7, 0.2, 0.25])
add_number(fig, ax_C_zoom, order=1, offset=[-50, 0])
ax_C_zoom = plt.axes([0.075 + 0.29, 0.7, 0.2, 0.25])
add_number(fig, ax_C_zoom, order=2, offset=[-50, 0])
for i in range(2):
ax_C_zoom = plt.axes([0.075 + (0.29 * i), 0.7, 0.2, 0.25])
ax_C_zoom.plot(t_arr[zrange],
self.data['C'][n[i], zrange],
'k',
linewidth=0.5)
ax_C_zoom.set_ylim([0, 10 * self.baseline_C[n[i]]])
ax_C_zoom.set_xlim([t_arr[zstart], t_arr[zstart + zlen]])
ax_C_zoom.set_xticks([])
ax_C_zoom.set_yticks([])
ax_C_zoom.spines['top'].set_visible(False)
ax_C_zoom.spines['right'].set_visible(False)
if i == 0:
ax_C_zoom.set_ylabel('C (a.u.)')
ax_C = plt.axes([0.1 + (0.29 * i), 0.85, 0.125, 0.1])
ax_C.plot(t_arr, self.data['C'][n[i], :], 'k', linewidth=0.5)
ax_C.set_yticks([])
ax_C.set_xlim([0, self.T / self.f])
ax_C.spines['top'].set_visible(False)
ax_C.spines['right'].set_visible(False)
#print(self.T)
ax_S_zoom = plt.axes([0.075 + (0.29 * i), 0.425, 0.2, 0.275])
y_arr = np.linspace(0,
self.baseline_S[n[i]] + 4 * self.noise_S[n[i]],
100)
x1 = norm.pdf(y_arr,
loc=self.baseline_S[n[i]],
scale=self.noise_S[n[i]])
x1 = x1 / x1.max() * 5 + t_arr[zrange[-1]] - 5
x2 = t_arr[zrange[-1]] * np.ones(100) - 5
#plt.plot(x_offset,A_0,'ko')
ax_S_zoom.fill_betweenx(y_arr,
x1,
x2,
facecolor='r',
alpha=1,
edgecolor=None)
ax_S_zoom.vlines(t_arr[zrange],
np.zeros(zlen),
self.data['S'][n[i], zrange],
colors='k',
linewidth=0.5)
#ax_S_zoom.bar(t_arr,self.data['S'][n[i],:],width=1/self.f,facecolor='r')
ax_S_zoom.plot(t_arr[zrange],
np.ones(zlen) * self.baseline_S[n[i]],
'r-',
linewidth=0.8)
for j, s in enumerate(np.flipud(sig)):
ax_S_zoom.plot(t_arr[zrange],
np.ones(zlen) * self.baseline_S[n[i]] +
s * self.noise_S[n[i]],
color=np.array(
[1, 0.3 * (2 - j), 0.3 * (2 - j)]),
linestyle='--',
label='$\\theta_S = %d$' % s,
linewidth=0.8)
ax_S_zoom.set_ylim([0, 10 * self.baseline_S[n[i]]])
ax_S_zoom.set_xlabel('time in s')
ax_S_zoom.set_yticks([])
ax_S_zoom.set_xticks(np.linspace(0, 600, 61))
ax_S_zoom.set_xlim([t_arr[zstart], t_arr[zstart + zlen]])
ax_S_zoom.spines['top'].set_visible(False)
ax_S_zoom.spines['right'].set_visible(False)
if i == 0:
ax_S_zoom.set_ylabel('S (a.u.)')
ax_S = plt.axes([0.1 + (0.29 * i), 0.575, 0.125, 0.1])
ax_S.vlines(t_arr,
np.zeros(T),
self.data['S'][n[i], :],
colors='k',
linewidth=0.5)
ax_S.set_yticks([])
ax_S.set_xlim([0, self.T / self.f])
ax_S.spines['top'].set_visible(False)
ax_S.spines['right'].set_visible(False)
else:
ax_S_zoom.legend(loc='upper right',
bbox_to_anchor=[1, 1],
fontsize=8)
xlim_arr = [2, 1, 0.5]
ax_fr = plt.axes([0.075, 0.125, 0.175, 0.15])
add_number(fig, ax_fr, order=3, offset=[-50, 10])
for j, s in enumerate(sig):
self.N_spikes_S = np.floor(
self.data['S'] / (self.baseline_S[:, np.newaxis] +
s * self.noise_S[:, np.newaxis])).sum(1)
ax_fr = plt.axes([0.075 + 0.22 * j, 0.125, 0.175, 0.15])
ax_fr.hist(self.N_spikes_S[idxes] / (T / self.f),
np.linspace(0, xlim_arr[j], 51),
facecolor='tab:blue',
alpha=0.5,
label='SNR$\geq$%d' % SNR_thr)
ax_fr.hist(self.N_spikes_S[~idxes] / (T / self.f),
np.linspace(0, xlim_arr[j], 51),
facecolor='tab:red',
alpha=0.5,
label='SNR<%d' % SNR_thr)
#ax_fr.bar(0,0,color=np.array([1,0.3*j,0.3*j]),label='$\\theta_S = %d$'%s)
ax_fr.set_xlim([0, xlim_arr[j]])
ax_fr.set_yticks([])
_, ymax = ax_fr.get_ylim()
if j > 0:
ax_fr.text(xlim_arr[j] * 0.5,
ymax * 0.8,
'$\\theta_S = %d$' % s,
fontsize=10)
if j == 0:
ax_fr.text(0.2,
ymax * 0.8,
'$\\theta_S = %d$' % s,
fontsize=10)
ax_fr.set_ylabel('count')
elif j == 1:
ax_fr.set_xlabel('$\\bar{\\nu}$ [Hz]')
ax_fr.spines['top'].set_visible(False)
ax_fr.spines['right'].set_visible(False)
Ns = (self.data['S'] > 0).sum(1)
s_adapt = norm.ppf((1 - 0.01)**(1 / Ns))
self.N_spikes_S = np.floor(
self.data['S'] /
(self.baseline_S[:, np.newaxis] +
s_adapt[:, np.newaxis] * self.noise_S[:, np.newaxis])).sum(1)
ax_fr = plt.axes([0.75, 0.125, 0.175, 0.15])
ax_fr.hist(self.N_spikes_S[idxes] / (T / self.f),
np.linspace(0, (0.5)**j, 51),
facecolor='tab:blue',
alpha=0.5,
label='SNR$\geq$%d' % SNR_thr)
ax_fr.hist(self.N_spikes_S[~idxes] / (T / self.f),
np.linspace(0, (0.5)**j, 51),
facecolor='tab:red',
alpha=0.5,
label='SNR<%d' % SNR_thr)
#ax_fr.bar(0,0,color=np.array([1,0.3*j,0.3*j]),label='$\\theta_S = %d$'%s)
ax_fr.set_xlim([0, 0.5**j])
ax_fr.set_xlabel('$\\bar{\\nu}$ [Hz]')
ax_fr.set_yticks([])
if j == 0:
ax_fr.set_ylabel('count')
ax_fr.legend(loc='lower right', bbox_to_anchor=[1.2, 0.4], fontsize=10)
ax_fr.spines['top'].set_visible(False)
ax_fr.spines['right'].set_visible(False)
_, ymax = ax_fr.get_ylim()
#ax_fr.text(0.5**j*0.5,ymax*0.8,'$\\theta_S$ adapti'%s,fontsize=10)
#ax_fr.text(0.2**j,ymax*0.9,'$\\theta_S = %d$'%s,fontsize=10)
ax_theory = plt.axes([0.725, 0.75, 0.25, 0.2])
add_number(fig, ax_theory, order=4, offset=[-50, 10])
x_arr = np.linspace(1, 10, 101)
for j, t in enumerate([500, 2000, 10000]):
ax_theory.plot(x_arr,
1 - np.exp(t * np.log(norm.cdf(x_arr))),
color=np.array([0.3 * j, 0.3 * j, 0.3 * j]),
label='T=%d' % t)
for j, s in enumerate(sig):
ax_theory.plot(s, 1.1, 'v', color=np.array([1, 0.3 * j, 0.3 * j]))
ax_theory.legend(fontsize=10,
loc='upper right',
bbox_to_anchor=[1.1, 1.1])
ax_theory.set_xlabel('x')
ax_theory.set_ylabel('$p(M_T > x$)')
ax_theory.spines['top'].set_visible(False)
ax_theory.spines['right'].set_visible(False)
ax_theory2 = plt.axes([0.725, 0.425, 0.25, 0.2])
add_number(fig, ax_theory2, order=5, offset=[-50, 10])
T_arr = np.linspace(0, 10000, 1000)
plt.plot(T_arr, norm.ppf((1 - 0.1)**(1 / T_arr)), 'k')
plt.xlabel('$T$')
plt.ylabel('$\\theta_S$')
plt.tight_layout()
plt.show(block=False)
svPath = pathcat([self.pathSession, 'get_firingrates.png'])
plt.savefig(svPath, format='png', dpi=150)
def dewarp_sessions(basePath,
mouse,
sessions,
use_opt_flow=True,
plot_bool=False):
max_thr = 0.01
idxes = 15
t_start = time.time()
pathMouse = pathcat([basePath, mouse])
nSes = sessions[-1] - sessions[0] + 1
pathSession = pathcat([pathMouse, 'Session%02d' % sessions[0]])
pathData = pathcat([pathSession, 'results_OnACID.mat'])
ld = loadmat(pathData)
template1 = ld['Cn']
template1 -= template1.min()
template1 /= template1.max()
dims = template1.shape
A1 = ld['A']
A1 = scipy.sparse.vstack(
[a.multiply(a > max_thr * a.max()) / a.sum() for a in A1.T]).T
if 'csc_matrix' not in str(type(A1)):
A1 = scipy.sparse.csc_matrix(A1)
pathSave = pathcat([pathSession, 'footprints_dewarped'])
#pickleData(A1,'%s.pkl'%pathSave,'save')
savemat('%s.mat' % pathSave, {'A': A1, 'Cn': template1})
for s in range(sessions[0], nSes):
pathSession = pathcat([pathMouse, 'Session%02d' % (s + 1)])
pathData = pathcat([pathSession, 'results_OnACID.mat'])
print('Now processing %s' % pathData)
ld = loadmat(pathData)
template2 = ld['Cn']
template2 -= template2.min()
template2 /= template2.max()
A2 = ld['A']
if 'ndarray' not in str(type(A2)):
A2 = A2.toarray()
x_grid, y_grid = np.meshgrid(
np.arange(0., dims[1]).astype(np.float32),
np.arange(0., dims[0]).astype(np.float32))
## align ROIs from session 2 to the template from session 1
print('adjust session position')
C = np.fft.fftshift(
np.real(
np.fft.ifft2(
np.fft.fft2(template1) *
np.fft.fft2(np.rot90(template2, 2)))))
max_pos = np.where(C == np.max(C))
x_shift = (max_pos[1] - (dims[1] / 2 - 1)).astype(int)
y_shift = (max_pos[0] - (dims[0] / 2 - 1)).astype(int)
print('shift by x,y: %5.3f,%5.3f' % (x_shift, y_shift))
x_remap = (x_grid - x_shift).astype(np.float32)
y_remap = (y_grid - y_shift).astype(np.float32)
A_2t = np.reshape(A2, dims + (-1, ), order='F').transpose(
2, 0, 1) # cast A2 to (x,y,n) -> (n,x,y) representation
A2_shift = np.stack([
cv2.remap(img.astype(np.float32), x_remap, y_remap,
cv2.INTER_NEAREST) for img in A_2t
],
axis=0)
template2 = cv2.remap(template2, x_remap, y_remap, cv2.INTER_NEAREST)
if use_opt_flow: ## for each pixel, find according position in other map
template1_norm = np.uint8(template1 * (template1 > 0) * 255)
template2_norm = np.uint8(template2 * (template2 > 0) * 255)
flow = cv2.calcOpticalFlowFarneback(np.uint8(template1_norm * 255),
np.uint8(template2_norm * 255),
None, 0.5, 3, 128, 3, 7, 1.5,
0)
x_remap = (flow[:, :, 0] + x_grid).astype(np.float32)
y_remap = (flow[:, :, 1] + y_grid).astype(np.float32)
A2 = np.stack([
cv2.remap(img.astype(np.float32), x_remap, y_remap,
cv2.INTER_NEAREST) for img in A2_shift
],
axis=0)
template2 = cv2.remap(template2, x_remap, y_remap,
cv2.INTER_NEAREST)
else:
A2 = A2_shift
A2 = np.reshape(
A2.transpose(1, 2, 0), # cast A2 to (x,y,n) again
(A1.shape[0], A_2t.shape[0]),
order='F') # and
print("Aligning done, t = %5.3fs" % (time.time() - t_start))
#if max_thr > 0:
A2 = scipy.sparse.csc_matrix(A2)
A2 = scipy.sparse.vstack([
a.multiply(a > max_thr * a.max()) / a.sum() for a in A2.T
]).T ## normalizing
pathSave = pathcat([pathSession, 'footprints_dewarped'])
#pickleData(A2,'%s.pkl'%pathSave,'save')
savemat('%s.mat' % pathSave, {'A': A2, 'Cn': template2})
if plot_bool:
plt.figure()
plt.quiver(x_grid[::idxes, ::idxes],
y_grid[::idxes, ::idxes],
flow[::idxes, ::idxes, 0],
flow[::idxes, ::idxes, 1],
angles='xy',
scale_units='xy',
scale=0.25,
headwidth=4,
headlength=4,
width=0.002,
units='width')
plt.show(block=False)