def plot_with_labels_interact(self,
ta,
x=None,
cmap='tab20',
figsize=(20, 10),
marker_size=10,
line_alpha=0.2,
ylim=(-150, 150),
title='Tail angles with GMM labels'):
import plotly.graph_objs as go
if isinstance(cmap, str):
cmap = eval(f'plt.cm.{cmap}')
labels, features = self.predict(ta)
if x is None:
x = np.arange(ta.shape[1])
y = ta[-1]
if self.pk_thr_ is not None:
pks = spt.findPeaks(y, thr=self.pk_thr_, pol=0, thrType='rel')[0]
else:
pks = np.arange(len(y))
line = go.Scatter(x=x,
y=y,
mode='lines',
opacity=line_alpha,
marker=dict(color='black'),
name='ta')
scatters = []
scatters.append(line)
for iLbl, lbl in enumerate(np.unique(labels)):
clr = f'rgba{cmap(lbl/self.n_gmm_)}'
inds = np.where(labels == lbl)[0]
inds = np.intersect1d(inds, pks)
scatter = go.Scatter(x=x[inds],
y=y[inds],
mode='markers',
marker=dict(color=clr,
symbol=lbl,
size=marker_size),
name=f'Lbl-{lbl}')
scatters.append(scatter)
fig = go.Figure(scatters)
if ylim is not None:
ylim = np.array(ylim)
ylim[0] = np.minimum(ylim[0], y.min())
ylim[1] = np.maximum(ylim[1], y.max())
fig.layout.yaxis.range = ylim
fig.layout.xaxis.range = [x[0], x[-1]]
fig.update_layout(title=title)
# fig.show()
# figName = f'Fig-{util.timestamp()}_trl-{iTrl}.html'
# fig.write_html(os.path.join(figDir,figName))
return fig
def fit(self, ta):
"""Fit model to tail angles. This includes preprocessing wherein
SVD-based feature extraction is performed, followed by PCA for
dimensionality reduction, if specfied.
Parameters
----------
self: object
Instance of initiated SvdGmm class
ta: array, (nPointsAlongTail, nTimePoints)
Tail angles array
Returns
-------
self: object
Trained SvdGmm model.
"""
if self.svd is None:
svd = TruncatedSVD(n_components=self.n_svd_,
random_state=self.random_state_).fit(ta.T)
else:
svd = self.svd
V = svd.transform(ta.T)
dv = np.gradient(V)[0]
ddv = np.gradient(dv)[0]
X = np.c_[V, dv, ddv]
if self.use_envelopes_:
features = max_min_envelopes(X.T).T
scaler = StandardScaler(with_mean=self.scaler_withMean_).fit(features)
features = scaler.transform(features)
if self.pk_thr_ is not None:
y = ta[-1]
pks = spt.findPeaks(y, thr=self.pk_thr_, thrType='rel', pol=0)[0]
print(f'Peaks are {round(100*len(pks)/len(y), 1)}% of all samples')
features = features[pks, :]
if self.pca_percVar_ is not None:
pca = PCA(n_components=self.pca_percVar_,
random_state=self.random_state_).fit(features)
features = pca.transform(features)
pca.n_components_ = features.shape[1]
else:
pca = None
print('Fitting GMM..')
gmm = GMM(n_components=self.n_gmm_,
random_state=self.random_state_,
covariance_type=self.covariance_type_,
**self.gmm_kwargs_)
gmm = gmm.fit(features)
self.svd = svd
self.scaler = scaler
self.pca = pca
self.gmm = gmm
return self
def getStimInfo(bas,ch_stim:str = 'patch3',ch_switch:str = 'patch2',\
ch_camTrig = 'camTrigger', minPkDist =5*6000):
import numpy as np
import apCode.SignalProcessingTools as spt
pks, amps = spt.findPeaks(np.array(bas[ch_stim]).flatten(),
thr=0.5,
pol=1,
minPkDist=minPkDist)
inds_keep = np.where(amps > 0)[0]
pks = pks[inds_keep]
amps = amps[inds_keep]
foo = np.array(bas[ch_switch]).flatten()[pks]
ht = np.array(['T'] * len(pks))
ht[np.where(foo > 1)] = 'H'
return pks, amps, ht
def stimInds(bas,thr = 1, stimCh:str = 'stim0', minStimDist:int =5*6000,
normalize:bool = True):
"""
bas: dic
BehaveAndScan file read by "importCh".
thr: scalar or str
Threshold for detecting stimuli. If 'auto', then estimates a threshold (assuming)
that stimulus polarity is positive (i.e. large positive values are stimuli)
stimChannels: list of strings
List of the names of stimulus channels.
minStimDist: int
Minimum distance (# of samples) between successive stimuli. Set 0
normalize:bool
Whether to normalize the stimulus channel before detecting stimuli.
If True, converts stim channel to z-score units. Useful option when
absolute threshold value is unknown.
"""
import numpy as np
import apCode.SignalProcessingTools as spt
from apCode.volTools import getGlobalThr
x = bas[stimCh].copy()
if normalize:
x = spt.zscore(x)
if isinstance(thr, str):
if thr.lower() == 'auto':
x_pos = x[np.where(x>=0)]
thr = getGlobalThr(x_pos)
pks = spt.findPeaks(x,thr = thr, pol =1, minPkDist = minStimDist)
if len(pks)>0:
return pks[0]
else:
print('No stimuli found!')
return None
saveDir = os.path.join(inputDir, 'proc')
imgName = 'K-means clustering of Alx cells_clstrs unord' + '.tif'
tff.imsave(os.path.join(saveDir, imgName),
np.transpose(imgStack, [0, 3, 1, 2]))
#%% REGRESSION
import apCode.AnalyzeEphysData as aed
# Chose # 2 after visual inspection (Note: Run Kmeans with k-means++
# init and unordered clusters before runing this)
centroid_M = centroids[0, :]
dt = data['time'][2] - data['time'][1]
centroid_M = spt.zscore(spt.chebFilt(centroid_M, dt, 0.01, btype='high'))
thr_Ca = volt.getGlobalThr(centroid_M)
pks = spt.findPeaks(centroid_M, thr=thr_Ca, minPkDist=30)
plt.figure(figsize=(16, 6))
plt.style.use('dark_background')
plt.subplot(131)
plt.plot(data['time'], centroid_M)
plt.plot(data['time'][pks[0]], centroid_M[pks[0]], 'ro')
plt.xlim(100, 500)
plt.xlabel('Time (s)')
plt.ylabel('dF/F')
plt.title('Some stim-elicited responses in chosen centroid'
'\n Shown peaks used to average responses')
centroid_M_seg = aed.SegmentDataByEvents(centroid_M, pks[0], 20, 50, axis=0)
trlLens = np.array([len(trl) for trl in centroid_M_seg])
shortTrls = np.where(trlLens < scipy.stats.mode(trlLens)[0])[0]
centroid_M_seg = np.delete(centroid_M_seg, shortTrls, axis=0)
# In[112]:
importlib.reload(aed)
pre = aed.import10ch(os.path.join(epDir,preFile))
post = aed.import10ch(os.path.join(epDir,postFile))
print(pre.keys())
print(time.ctime())
# In[155]:
#%% Detect and check stimulus indices
pre['stimInds'] = spt.findPeaks(spt.zscore(pre[stimCh]),thr=3)[0]-2
post['stimInds'] = spt.findPeaks(spt.zscore(post[stimCh]),thr=3)[0]-2
xmin = np.max((pre['t'][0],post['t'][0]))
xmax = np.min((pre['t'][-1],post['t'][-1]))
plt.style.use(('seaborn-dark','seaborn-colorblind','seaborn-poster'))
plt.subplot(2,1,1)
plt.plot(pre['t'],pre[stimCh])
plt.plot(pre['t'][pre['stimInds']],pre[stimCh][pre['stimInds']],'o',markersize =15)
plt.xlim(xmin,xmax)
plt.title('Pre-Ablation')
plt.subplot(2,1,2)
plt.plot(post['t'],post[stimCh])
plt.plot(post['t'][post['stimInds']],post[stimCh][post['stimInds']],'o',markersize =15)
plt.xlim(xmin,xmax)
def expand_on_bends(df_trl,
Fs=500,
tPre_ms=100,
bendThr=10,
minLat_ms=5,
maxGap_ms=100):
"""Takes dataframe where each row contains single trial information and
expands such that each row contains single bend information
Parameters
----------
df_trl: pandas dataframe, (nTrlsInTotal, nVariables)
Fs: int
Sampling frequency when collecting data(images)
nPre_ms: scalar
"""
import apCode.SignalProcessingTools as spt
minPkDist = int((10e-3) * Fs)
nPre = tPre_ms * 1e-3 * Fs
minLat = minLat_ms * 1e-3 * Fs
maxGap = maxGap_ms * 1e-3 * Fs
df_bend = []
for iTrl in np.unique(df_trl.trlIdx_glob):
df_now = df_trl.loc[df_trl.trlIdx_glob == iTrl]
y = df_now.iloc[0]['tailAngles'][-1]
y = spt.chebFilt(y, 1 / Fs, (5, 60), btype='bandpass')
pks = spt.findPeaks(y,
thr=bendThr,
thrType='rel',
pol=0,
minPkDist=minPkDist)[0]
if len(pks) > 3:
dpks = np.diff(pks)
tooSoon = np.where(pks < (nPre + minLat))[0]
tooSparse = np.where(dpks > maxGap)[0] + 1
inds_del = np.union1d(tooSoon, tooSparse)
pks = np.delete(pks, inds_del, axis=0)
if len(pks) > 3:
nBends = len(pks)
bendIdx = np.arange(nBends)
bendSampleIdxInTrl = pks
bendAmp = y[pks]
bendAmp_abs = np.abs(bendAmp)
bendAmp_rel = np.insert(np.abs(np.diff(bendAmp)), 0,
np.abs(bendAmp[0]))
bendInt_ms = np.gradient(pks) * (1 / Fs) * 1000
onset_ms = (pks[0] - nPre + 1) * (1 / Fs) * 1000
else:
nBends = 0
bendIdx, bendAmp, bendAmp_abs, bendAmp_rel, bendInt_ms =\
[np.nan for _ in range(5)]
bendsampleIdxInTrl, onset_ms = [np.nan for _ in range(2)]
dic = dict(trlIdx_glob=iTrl,
nBends=nBends,
bendIdx=bendIdx,
bendSampleIdxInTrl=bendSampleIdxInTrl,
bendAmp=bendAmp,
bendAmp_abs=bendAmp_abs,
bendAmp_rel=bendAmp_rel,
bendInt_ms=bendInt_ms,
onset_ms=onset_ms)
df_now = pd.DataFrame(dic)
df_bend.append(df_now)
df_bend = pd.concat(df_bend, ignore_index=True)
return pd.merge(df_trl, df_bend, on='trlIdx_glob')
saveDir = os.path.join(inputDir, 'proc')
imgName = 'K-means clustering of Alx cells_clstrs unord' + '.tif'
tff.imsave(os.path.join(saveDir,imgName),np.transpose(imgStack,[0,3,1,2]))
#%% REGRESSION
import apCode.AnalyzeEphysData as aed
# Chose # 2 after visual inspection (Note: Run Kmeans with k-means++
# init and unordered clusters before runing this)
centroid_M = centroids[0,:]
dt = data['time'][2] - data['time'][1]
centroid_M = spt.zscore(spt.chebFilt(centroid_M,dt,0.01,btype='high'))
thr_Ca = volt.getGlobalThr(centroid_M)
pks = spt.findPeaks(centroid_M,thr=thr_Ca, minPkDist=30)
plt.figure(figsize = (16,6))
plt.style.use('dark_background')
plt.subplot(131)
plt.plot(data['time'],centroid_M)
plt.plot(data['time'][pks[0]],centroid_M[pks[0]],'ro')
plt.xlim(100,500)
plt.xlabel('Time (s)')
plt.ylabel('dF/F')
plt.title('Some stim-elicited responses in chosen centroid'
'\n Shown peaks used to average responses')
centroid_M_seg = aed.SegmentDataByEvents(centroid_M,pks[0],20,50,axis =0)
trlLens = np.array([len(trl) for trl in centroid_M_seg])
shortTrls = np.where(trlLens < scipy.stats.mode(trlLens)[0])[0]
centroid_M_seg = np.delete(centroid_M_seg,shortTrls,axis = 0)
dt = 1 / Fs
preStimPts = preStimPer * Fs
postStimPts = postStimPer * Fs
# In[112]:
importlib.reload(aed)
pre = aed.import10ch(os.path.join(epDir, preFile))
post = aed.import10ch(os.path.join(epDir, postFile))
print(pre.keys())
print(time.ctime())
# In[155]:
#%% Detect and check stimulus indices
pre['stimInds'] = spt.findPeaks(spt.zscore(pre[stimCh]), thr=3)[0] - 2
post['stimInds'] = spt.findPeaks(spt.zscore(post[stimCh]), thr=3)[0] - 2
xmin = np.max((pre['t'][0], post['t'][0]))
xmax = np.min((pre['t'][-1], post['t'][-1]))
plt.style.use(('seaborn-dark', 'seaborn-colorblind', 'seaborn-poster'))
plt.subplot(2, 1, 1)
plt.plot(pre['t'], pre[stimCh])
plt.plot(pre['t'][pre['stimInds']],
pre[stimCh][pre['stimInds']],
'o',
markersize=15)
plt.xlim(xmin, xmax)
plt.title('Pre-Ablation')
plt.subplot(2, 1, 2)
def estimateCaDecayKinetics(time,
signals,
p0=None,
thr=2,
preTime=10,
postTime=40):
"""
Given a time vector and Ca signal matrix of shape = (C,T), where
C = # of cells, and T = # of time points (must match length of time
vector), returns output of shape = (nSamples, 2), where the 1st and
2nd columns contain the fast and slow decay tau estimates after
fitting Ca2+ signals with double exponential
Parameters:
time - Time vector of length T
signals - Ca signals array of shape (nSamples,T)
p0 - Array-like, (tau_fast, tau_slow, wt_fast), where tau_fast is the
fast decay time constant (in sec), tau_slow is the slow decay
constant, and wt_fast is the weight of the fast exponential (<1)
for fitting the signal as a weighted sum of the fast and slow
exponential. Default is None, in which case fitting optimization
begins without initial estimate
thr - Threshold for peak detection in Ca signals, in units of zscore
preTime - Pre-peak time length of the Ca signals to include for segmentation
postTime - Post-peak " " " "
Avinash Pujala, JRC, 2017
"""
import numpy as np
from scipy.optimize import curve_fit as cf
import apCode.SignalProcessingTools as spt
import apCode.AnalyzeEphysData as aed
def doubleExp(time, tau1, tau2, wt1):
wt2 = 1 - wt1
time = time - time[0]
e = wt1 * np.exp(-time / tau1) + wt2 * np.exp(-time / tau2)
return e
def listToArray(x):
lens = [len(item) for item in x]
lenOfLens = len(lens)
lens = lens[np.min((lenOfLens - 1, 2))]
a = np.zeros((len(x), lens))
delInds = []
for itemNum, item in enumerate(x):
if len(item) == lens:
a[itemNum, :] = item
else:
delInds.append(itemNum)
a = np.delete(a, delInds, axis=0)
return a, delInds
if np.ndim(signals) == 1:
signals = np.reshape(signals, (1, len(signals)))
dt = time[2] - time[1]
pts_post = np.round(postTime / dt).astype(int)
pts_pre = np.round(preTime / dt).astype(int)
x_norm = spt.zscore(signals, axis=1)
x_seg, params, x_seg_fit = [], [], []
nSamples = np.shape(signals)[0]
excludedSamples = np.zeros((nSamples, 1))
for nSample in np.arange(nSamples):
inds_pk = spt.findPeaks(x_norm[nSample, :], thr=thr, ampType='rel')[0]
if len(inds_pk) == 0:
print('Peak detection failed for sample #', nSample,
'. Try lowering threshold')
excludedSamples[nSample] = 1
else:
blah = aed.SegmentDataByEvents(signals[nSample, :],
inds_pk,
pts_pre,
pts_post,
axis=0)
blah = listToArray(blah)[0]
blah = np.mean(blah, axis=0)
x_seg.append(blah)
ind_max = np.where(blah == np.max(blah))[0][0]
y = spt.standardize(blah[ind_max:])
t = np.arange(len(y)) * dt
popt, pcov = cf(doubleExp, t, y, p0=[10, 20, 0.5], bounds=(0, 20))
if popt[0] > popt[1]:
popt[0:2] = popt[2:0:-1]
popt[-1] = 1 - popt[-1]
params.append(popt)
foo = doubleExp(t, popt[0], popt[1], popt[2])
x_seg_fit.append(foo)
excludedSamples = np.where(excludedSamples)[0]
includedSamples = np.setdiff1d(np.arange(nSamples), excludedSamples)
x_seg, delInds = listToArray(x_seg)
params = np.delete(np.array(params), delInds, axis=0)
delInds = includedSamples[delInds]
if len(delInds) > 0:
print(
'Sample #', delInds,
'excluded for short segment length. Consider decreasing pre-peak time length'
)
excludedSamples = np.union1d(delInds, excludedSamples)
x_seg = spt.standardize(np.array(x_seg), axis=1)
x_seg_fit = np.array(listToArray(x_seg_fit)[0])
out = {
'raw': x_seg,
'fit': x_seg_fit,
'params': np.array(params),
'excludedSamples': excludedSamples
}
return out
def estimateCaDecayKinetics(time, signals, p0 = None, thr = 2, preTime = 10,
postTime = 40):
"""
Given a time vector and Ca signal matrix of shape = (C,T), where
C = # of cells, and T = # of time points (must match length of time
vector), returns output of shape = (nSamples, 2), where the 1st and
2nd columns contain the fast and slow decay tau estimates after
fitting Ca2+ signals with double exponential
Parameters:
time - Time vector of length T
signals - Ca signals array of shape (nSamples,T)
p0 - Array-like, (tau_fast, tau_slow, wt_fast), where tau_fast is the
fast decay time constant (in sec), tau_slow is the slow decay
constant, and wt_fast is the weight of the fast exponential (<1)
for fitting the signal as a weighted sum of the fast and slow
exponential. Default is None, in which case fitting optimization
begins without initial estimate
thr - Threshold for peak detection in Ca signals, in units of zscore
preTime - Pre-peak time length of the Ca signals to include for segmentation
postTime - Post-peak " " " "
Avinash Pujala, JRC, 2017
"""
import numpy as np
from scipy.optimize import curve_fit as cf
import apCode.SignalProcessingTools as spt
import apCode.AnalyzeEphysData as aed
def doubleExp(time, tau1, tau2, wt1):
wt2 = 1-wt1
time = time - time[0]
e = wt1*np.exp(-time/tau1) + wt2*np.exp(-time/tau2)
return e
def listToArray(x):
lens = [len(item) for item in x]
lenOfLens = len(lens)
lens = lens[np.min((lenOfLens-1,2))]
a = np.zeros((len(x),lens))
delInds = []
for itemNum,item in enumerate(x):
if len(item) == lens:
a[itemNum,:] = item
else:
delInds.append(itemNum)
a = np.delete(a,delInds,axis = 0)
return a, delInds
if np.ndim(signals)==1:
signals = np.reshape(signals,(1,len(signals)))
dt = time[2]-time[1]
pts_post = np.round(postTime/dt).astype(int)
pts_pre = np.round(preTime/dt).astype(int)
x_norm = spt.zscore(signals,axis = 1)
x_seg, params, x_seg_fit = [],[],[]
nSamples = np.shape(signals)[0]
excludedSamples = np.zeros((nSamples,1))
for nSample in np.arange(nSamples):
inds_pk = spt.findPeaks(x_norm[nSample,:],thr = thr,ampType = 'rel')[0]
if len(inds_pk)==0:
print('Peak detection failed for sample #', nSample, '. Try lowering threshold')
excludedSamples[nSample] = 1
else:
blah = aed.SegmentDataByEvents(signals[nSample,:],inds_pk,pts_pre,pts_post,axis = 0)
blah = listToArray(blah)[0]
blah = np.mean(blah,axis=0)
x_seg.append(blah)
ind_max = np.where(blah == np.max(blah))[0][0]
y = spt.standardize(blah[ind_max:])
t = np.arange(len(y))*dt
popt,pcov = cf(doubleExp,t,y,p0 = [10,20, 0.5], bounds = (0,20))
if popt[0]> popt[1]:
popt[0:2] = popt[2:0:-1]
popt[-1] = 1-popt[-1]
params.append(popt)
foo = doubleExp(t,popt[0],popt[1],popt[2])
x_seg_fit.append(foo)
excludedSamples = np.where(excludedSamples)[0]
includedSamples = np.setdiff1d(np.arange(nSamples),excludedSamples)
x_seg,delInds = listToArray(x_seg)
params = np.delete(np.array(params),delInds,axis = 0)
delInds = includedSamples[delInds]
if len(delInds)>0:
print('Sample #', delInds, 'excluded for short segment length. Consider decreasing pre-peak time length')
excludedSamples = np.union1d(delInds,excludedSamples)
x_seg = spt.standardize(np.array(x_seg),axis = 1)
x_seg_fit = np.array(listToArray(x_seg_fit)[0])
out = {'raw': x_seg,'fit': x_seg_fit,'params': np.array(params),'excludedSamples': excludedSamples}
return out