def clipWave(W,freq,time,freqRange,timeRange,coi = []):
'''
W_clip, freq_clip,time_clip,coi_clip = clipWave(W,freq,time,freqRange, \
timeRange,coi = [])
Clips the matrix of wavelet coefficients to the specified freq and time
range
Inputs:
W - 2D matrix of wavelet coefficients
freq - Freq vector of length = number of rows of W
time - Time vector of length = number of cols of W
freqRange - 2 element array specifying the min and max freq to clip W to
timeRange - 2 element array specifying the min and max time to clip W to
coi - Cone of influence; can be empty
Outputs:
W_clip - Clipped W
freq_clip - Clipped freq
...
'''
import apCode.SignalProcessingTools as spt
fInds = spt.nearestMatchingInds(freqRange,freq)
print(fInds)
tInds = spt.valsToNearestInds(timeRange,time)
W_clip = W[fInds[1]:fInds[0],tInds[0]:tInds[1]]
freq_clip = freq[fInds[1]:fInds[0]]
time_clip = time[tInds[0]:tInds[1]]
if len(coi) >0:
coi_clip = coi[tInds[0]:tInds[1]]
else:
coi_clip = coi
return W_clip, freq_clip,time_clip,coi_clip
def basToDf(bas, n_pre_bas=200e-3, n_post_bas=1.5):
import apCode.SignalProcessingTools as spt
import pandas as pd
import numpy as np
import apCode.util as util
df = {}
dt = bas['t'][1] - bas['t'][0]
Fs = int(1 / dt)
inds_stim, amps_stim, ch_stim = getStimInfo(bas)
keys = list(bas.keys())
ind_mtr = util.findStrInList('den', keys)[-1]
x = np.squeeze(np.array(bas[keys[ind_mtr]]))
if x.shape[0] == 2:
x = x.T
motor_trl = spt.segmentByEvents(spt.zscore(x, axis=0), inds_stim,
int(n_pre_bas * Fs), int(n_post_bas * Fs))
motorTime_trl = spt.segmentByEvents(bas['t'], inds_stim,
int(n_pre_bas * Fs),
int(n_post_bas * Fs))
trlNum = np.arange(len(inds_stim)) + 1
df['trlNum'] = trlNum
df['stimInd'] = inds_stim
df['stimAmp'] = amps_stim
df['stimHT'] = ch_stim
df['motorActivity'] = motor_trl
df['motorTime'] = motorTime_trl
return pd.DataFrame(df)
def clipWave(W, freq, time, freqRange, timeRange, coi=[]):
'''
W_clip, freq_clip,time_clip,coi_clip = clipWave(W,freq,time,freqRange, \
timeRange,coi = [])
Clips the matrix of wavelet coefficients to the specified freq and time
range
Inputs:
W - 2D matrix of wavelet coefficients
freq - Freq vector of length = number of rows of W
time - Time vector of length = number of cols of W
freqRange - 2 element array specifying the min and max freq to clip W to
timeRange - 2 element array specifying the min and max time to clip W to
coi - Cone of influence; can be empty
Outputs:
W_clip - Clipped W
freq_clip - Clipped freq
...
'''
import apCode.SignalProcessingTools as spt
fInds = spt.nearestMatchingInds(freqRange, freq)
print(fInds)
tInds = spt.valsToNearestInds(timeRange, time)
W_clip = W[fInds[1]:fInds[0], tInds[0]:tInds[1]]
freq_clip = freq[fInds[1]:fInds[0]]
time_clip = time[tInds[0]:tInds[1]]
if len(coi) > 0:
coi_clip = coi[tInds[0]:tInds[1]]
else:
coi_clip = coi
return W_clip, freq_clip, time_clip, coi_clip
def plotCentroids(time, centroids, stimTimes, time_ephys, ephys, scaled = False,
colors = None, xlabel = '',ylabel = '', title = ''):
"""
Plots centroids resulting from some clustering method
Parameters:
time - Time vectors for centroids (optical samplign interval)
centroids - Array of shape (M, N), where M is the number of centroids, and N is the #
number of features (or time points)
stimTimes - Times of stimulus onsets for overlaying vertical dashed lines
time_ephys - Time axis for ephys data (usually sampled at higher rate)
ephys - Ephys time series
scaled - Boolean; If true, scales centroids individually, else scales jointly.
colors - Array of shape (M,3) or (M,4). Colormap to use for plotting centroids
"""
import apCode.SignalProcessingTools as spt
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
if scaled:
centroids = spt.standardize(centroids,axis = 1)
else:
centroids = spt.standardize(centroids)
ephys = spt.standardize(ephys)
n_clusters = np.shape(centroids)[0]
if np.any(colors == None):
colors = np.array(sns.color_palette('colorblind',np.shape(centroids)[0]))
elif np.shape(colors)[0] < np.shape(centroids)[0]:
colors = np.tile(colors,(np.shape(centroids)[0],1))
colors = colors[:np.shape(centroids)[0],:]
if np.any(time == None):
time = np.arange(np.shape(centroids)[1])
plt.style.use(['dark_background','seaborn-poster'])
for cc in np.arange(np.shape(centroids)[0]):
plt.plot(time,centroids[cc,:]-np.mean(centroids[cc,:])-cc,color = colors[cc,:])
plt.plot(time_ephys,ephys-np.mean(ephys)-cc-1,color = colors[0,:])
yt = np.arange(n_clusters + 1)
ytl = list(yt)
ytl[-1] = 'ephys'
plt.yticks(-yt, ytl)
plt.xlabel(xlabel,fontsize = 16)
plt.ylabel(ylabel, fontsize = 16)
plt.box('off')
plt.title(title, fontsize = 20)
plt.grid('off')
plt.xlim(time[0],time[-1])
for st in stimTimes:
plt.axvline(x = st, ymin = 0, ymax =1, alpha = 0.3,
color = 'w',linestyle = '--')
def getClrMap(x,cMap,maskInds):
cm = np.zeros((len(x),4))*np.nan
negInds = np.where(x<0)[0]
posInds = np.where(x>=0)[0]
x[negInds] = spt.mapToRange(np.hstack((x[negInds],[0,-1])),[0,127])[0:-2]
x[posInds] = spt.mapToRange(np.hstack((x[posInds],[0,1])),[128,255])[0:-2]
cm[negInds] = cMap(x[negInds].astype(int))
cm[posInds] = cMap(x[posInds].astype(int))
if len(maskInds) == 0:
pass
else:
cm[maskInds,-1] = 0
return cm
def plotWave(W,
freq,
time,
coi=[],
powScale='log',
cmap='coolwarm',
xlabel='Time (sec)',
ylabel='Freq (Hz)'):
'''
fh = plotWave(W,freq,time,...)
Plots the matrix of wavelet coefficients W, using the specified freq and time axes
'''
import numpy as np
import apCode.SignalProcessingTools as spt
import matplotlib.pyplot as plt
if powScale.lower() == 'log':
W = np.log2(np.abs(W))
else:
W = np.abs(W)
period = 1 / freq
dt = time[1] - time[0]
tt = np.hstack((time[[0, 0]] - dt * 0.5, time, time[[-1, -1]] + dt * 0.5))
if len(coi) == 0:
coi = time
coi_ext = np.log2(np.hstack((freq[[-1, 1]], 1 / coi, freq[[1, -1]])))
tt = spt.nearestMatchingInds(tt, time)
coi_ext = spt.nearestMatchingInds(coi_ext, freq)
freq_log = np.unique(spt.nextPow2(freq))
fTick = 2**freq_log
yTick = 1 / fTick
inds = spt.nearestMatchingInds(yTick, period)
ytl = (1 / period[inds]).astype(int).ravel()
fig = plt.imshow(W, aspect='auto', cmap=cmap)
fig.axes.set_yticks(inds)
fig.axes.set_yticklabels(np.array(ytl))
plt.colorbar()
xTick = np.linspace(0, len(time) - 1, 5).astype(int)
xtl = np.round(time[xTick] / (time[-1] / 4)) * 0.5
fig.axes.set_xticks(xTick)
fig.axes.set_xticklabels(xtl.ravel())
plt.xlabel(xlabel)
plt.ylabel(ylabel)
#plt.show()
plt.plot(tt, coi_ext, 'k--')
return fig
def plot_with_labels_interact(self,
ta,
x=None,
cmap='tab20',
figsize=(20, 10),
marker_size=10,
line_alpha=0.2,
ylim=(-150, 150),
title='Tail angles with GMM labels'):
import plotly.graph_objs as go
if isinstance(cmap, str):
cmap = eval(f'plt.cm.{cmap}')
labels, features = self.predict(ta)
if x is None:
x = np.arange(ta.shape[1])
y = ta[-1]
if self.pk_thr_ is not None:
pks = spt.findPeaks(y, thr=self.pk_thr_, pol=0, thrType='rel')[0]
else:
pks = np.arange(len(y))
line = go.Scatter(x=x,
y=y,
mode='lines',
opacity=line_alpha,
marker=dict(color='black'),
name='ta')
scatters = []
scatters.append(line)
for iLbl, lbl in enumerate(np.unique(labels)):
clr = f'rgba{cmap(lbl/self.n_gmm_)}'
inds = np.where(labels == lbl)[0]
inds = np.intersect1d(inds, pks)
scatter = go.Scatter(x=x[inds],
y=y[inds],
mode='markers',
marker=dict(color=clr,
symbol=lbl,
size=marker_size),
name=f'Lbl-{lbl}')
scatters.append(scatter)
fig = go.Figure(scatters)
if ylim is not None:
ylim = np.array(ylim)
ylim[0] = np.minimum(ylim[0], y.min())
ylim[1] = np.maximum(ylim[1], y.max())
fig.layout.yaxis.range = ylim
fig.layout.xaxis.range = [x[0], x[-1]]
fig.update_layout(title=title)
# fig.show()
# figName = f'Fig-{util.timestamp()}_trl-{iTrl}.html'
# fig.write_html(os.path.join(figDir,figName))
return fig
def getClrMapsForEachRegressor(betas,normed = True, cMap = 'PiYG',
scaling =1, betaThr= None):
"""
Given the coeffiecients(betas) from regression, returns a list of color
maps, with each color map corresponding to the betas for a single regressor.
These can be used by colorCellsInImgStack to create image stacks with
cells colored by betas.
Parameters:
betas - Array-like with shape (nSamples, nFeatures).
normed - Boolean; If True, normalizes betas such that for each feature
the values range from -1 to 1.
scaling - Not yet implemented
betaThr - None(default),scalar,'auto'; Determines if any thresholding
should be applied based on beta values. If None, then no thresholding,
if scalar, then for beta values whose magnitude is less than this
scalar, the alpha value in the color maps is set to zero. If 'auto'
then automatically determines threshold
"""
import apCode.SignalProcessingTools as spt
import matplotlib.pyplot as plt
import apCode.volTools as volt
def getClrMap(x,cMap,maskInds):
cm = np.zeros((len(x),4))*np.nan
negInds = np.where(x<0)[0]
posInds = np.where(x>=0)[0]
x[negInds] = spt.mapToRange(np.hstack((x[negInds],[0,-1])),[0,127])[0:-2]
x[posInds] = spt.mapToRange(np.hstack((x[posInds],[0,1])),[128,255])[0:-2]
cm[negInds] = cMap(x[negInds].astype(int))
cm[posInds] = cMap(x[posInds].astype(int))
if len(maskInds) == 0:
pass
else:
cm[maskInds,-1] = 0
return cm
if isinstance(cMap,str):
cMap = plt.cm.get_cmap(cMap)
if normed:
betas = spt.standardize(betas, preserveSign = True, axis = 0)*scaling
clrMaps = []
for beta in betas.T:
if betaThr == None:
maskInds = []
elif betaThr == 'auto':
betaThr = volt.getGlobalThr(np.abs(beta))
maskInds = np.where(np.abs(beta)<betaThr)
else:
maskInds = np.where(np.abs(beta)<betaThr)
clrMaps.append(getClrMap(beta,cMap,maskInds))
#clrMaps = [getClrMap(beta,cMap,betaThr) for beta in betas.transpose()]
return clrMaps
def plotWave(W,freq,time,coi = [],powScale = 'log',cmap = 'coolwarm', xlabel = 'Time (sec)', ylabel = 'Freq (Hz)'):
'''
fh = plotWave(W,freq,time,...)
Plots the matrix of wavelet coefficients W, using the specified freq and time axes
'''
import numpy as np
import apCode.SignalProcessingTools as spt
import matplotlib.pyplot as plt
if powScale.lower() == 'log':
W = np.log2(np.abs(W))
else:
W = np.abs(W)
period = 1/freq
dt = time[1]-time[0]
tt = np.hstack((time[[0,0]]-dt*0.5,time,time[[-1,-1]]+dt*0.5))
if len(coi) == 0:
coi = time
coi_ext = np.log2(np.hstack((freq[[-1,1]],1/coi,freq[[1,-1]])))
tt = spt.nearestMatchingInds(tt,time)
coi_ext = spt.nearestMatchingInds(coi_ext,freq)
freq_log = np.unique(spt.nextPow2(freq))
fTick = 2**freq_log
yTick = 1/fTick
inds = spt.nearestMatchingInds(yTick,period)
ytl = (1/period[inds]).astype(int).ravel()
fig =plt.imshow(W, aspect = 'auto',cmap = cmap)
fig.axes.set_yticks(inds)
fig.axes.set_yticklabels(np.array(ytl))
plt.colorbar()
xTick = np.linspace(0,len(time)-1,5).astype(int)
xtl = np.round(time[xTick]/(time[-1]/4))*0.5
fig.axes.set_xticks(xTick)
fig.axes.set_xticklabels(xtl.ravel())
plt.xlabel(xlabel)
plt.ylabel(ylabel)
#plt.show()
plt.plot(tt,coi_ext,'k--')
return fig
def fit(self, ta):
"""Fit model to tail angles. This includes preprocessing wherein
SVD-based feature extraction is performed, followed by PCA for
dimensionality reduction, if specfied.
Parameters
----------
self: object
Instance of initiated SvdGmm class
ta: array, (nPointsAlongTail, nTimePoints)
Tail angles array
Returns
-------
self: object
Trained SvdGmm model.
"""
if self.svd is None:
svd = TruncatedSVD(n_components=self.n_svd_,
random_state=self.random_state_).fit(ta.T)
else:
svd = self.svd
V = svd.transform(ta.T)
dv = np.gradient(V)[0]
ddv = np.gradient(dv)[0]
X = np.c_[V, dv, ddv]
if self.use_envelopes_:
features = max_min_envelopes(X.T).T
scaler = StandardScaler(with_mean=self.scaler_withMean_).fit(features)
features = scaler.transform(features)
if self.pk_thr_ is not None:
y = ta[-1]
pks = spt.findPeaks(y, thr=self.pk_thr_, thrType='rel', pol=0)[0]
print(f'Peaks are {round(100*len(pks)/len(y), 1)}% of all samples')
features = features[pks, :]
if self.pca_percVar_ is not None:
pca = PCA(n_components=self.pca_percVar_,
random_state=self.random_state_).fit(features)
features = pca.transform(features)
pca.n_components_ = features.shape[1]
else:
pca = None
print('Fitting GMM..')
gmm = GMM(n_components=self.n_gmm_,
random_state=self.random_state_,
covariance_type=self.covariance_type_,
**self.gmm_kwargs_)
gmm = gmm.fit(features)
self.svd = svd
self.scaler = scaler
self.pca = pca
self.gmm = gmm
return self
def getStimInfo(bas,ch_stim:str = 'patch3',ch_switch:str = 'patch2',\
ch_camTrig = 'camTrigger', minPkDist =5*6000):
import numpy as np
import apCode.SignalProcessingTools as spt
pks, amps = spt.findPeaks(np.array(bas[ch_stim]).flatten(),
thr=0.5,
pol=1,
minPkDist=minPkDist)
inds_keep = np.where(amps > 0)[0]
pks = pks[inds_keep]
amps = amps[inds_keep]
foo = np.array(bas[ch_switch]).flatten()[pks]
ht = np.array(['T'] * len(pks))
ht[np.where(foo > 1)] = 'H'
return pks, amps, ht
def stimInds(bas,thr = 1, stimCh:str = 'stim0', minStimDist:int =5*6000,
normalize:bool = True):
"""
bas: dic
BehaveAndScan file read by "importCh".
thr: scalar or str
Threshold for detecting stimuli. If 'auto', then estimates a threshold (assuming)
that stimulus polarity is positive (i.e. large positive values are stimuli)
stimChannels: list of strings
List of the names of stimulus channels.
minStimDist: int
Minimum distance (# of samples) between successive stimuli. Set 0
normalize:bool
Whether to normalize the stimulus channel before detecting stimuli.
If True, converts stim channel to z-score units. Useful option when
absolute threshold value is unknown.
"""
import numpy as np
import apCode.SignalProcessingTools as spt
from apCode.volTools import getGlobalThr
x = bas[stimCh].copy()
if normalize:
x = spt.zscore(x)
if isinstance(thr, str):
if thr.lower() == 'auto':
x_pos = x[np.where(x>=0)]
thr = getGlobalThr(x_pos)
pks = spt.findPeaks(x,thr = thr, pol =1, minPkDist = minStimDist)
if len(pks)>0:
return pks[0]
else:
print('No stimuli found!')
return None
for cNum, cInds in enumerate(data['info']['inds']):
data['info']['inds'][cNum] = cInds.astype(int)-1
data['info']['center'] = data['info']['center'].astype(int)-1
data['info']['slice'] = data['info']['slice'].astype(int)-1
data['info']['x_minmax'] = data['info']['x_minmax'].astype(int)-1
data['info']['y_minmax'] = data['info']['y_minmax'].astype(int)-1
data['inds']['inMask'] = data['inds']['inMask'].astype(int)[0]-1
if alreadyRegressed:
data['regr']= pyData['regr']
data['pathToData'] = os.path.join(inputDir,pyFileName)
print('\n', int(time.time()-tic),'sec')
print(time.ctime())
print('Geting indices for stacks in ephys data and stims in optical data...')
data['inds']['stim'] = spt.nearestMatchingInds(pyData['time'].ravel()[pyData['stim']['inds']],data['time'])
data['inds']['stackInit'] = spt.nearestMatchingInds(data['time'],pyData['time'],processing = 'parallel')
# Detrending dF/F signals
print('Detrending dF/F signals...')
for cc in np.arange(np.shape(data['dFF'])[1]):
data['dFF'][:,cc] = ia.detrendCa(data['dFF'][:,cc],data['inds']['stim'])
# An adjustment for some datasets
data['avg'] = np.transpose(data['avg'],[0,2,1])
np.shape(data['avg'])
#%% Check to see if stimulus onset indices have been detected correctly
stimInd = 4
periTime_ephys = 0.1
periTime_opt = 8
def dataFrameOfMatchedMtrAndCaTrls_singleFish(bas, ca = None, ch_camTrig = 'camTrigger', ch_stim = 'patch3',
ch_switch = 'patch2', thr_camTrig= 4, Fs_bas = 6000, t_pre_bas = 0.2,\
t_post_bas = 1.5, t_pre_ca = 1, t_post_ca = 10, n_jobs = 20):
"""
Parameters
----------
bas: dict
Dictionary resulting from reading of BAS file
ca: array, (nRois, nSamples)
nSamples is expected to match the number of Camera triggers
Returns
-------
df: Pandas dataframe
"""
import numpy as np
from scipy.stats import mode
import apCode.SignalProcessingTools as spt
from apCode import util
import pandas as pd
def motorFromBas(bas):
keys = list(bas.keys())
ind_mtr = util.findStrInList('den', keys)
if len(ind_mtr) > 0:
ind_mtr = ind_mtr[-1]
x = np.squeeze(np.array(bas[keys[ind_mtr]]))
else:
if 'ch3' in bas:
x = np.array([bas['ch3'], bas['ch4']])
else:
x = np.array([bas['ch1'], bas['ch2']])
if x.shape[0] < x.shape[1]:
x = x.T
return x
mtr = motorFromBas(bas)
stimInds, stimAmps, stimHt = getStimInfo(bas, ch_stim = ch_stim,\
ch_camTrig= ch_camTrig, ch_switch= ch_switch)
camTrigInds = getCamTrigInds(bas[ch_camTrig], thr=thr_camTrig)
# dt_bas = 1/Fs_bas
dt_ca = np.mean(np.diff(bas['t'][camTrigInds]))
Fs_ca = int(np.round(1 / dt_ca))
if np.any(ca == None):
nFrames = len(camTrigInds)
# t_ca = np.arange(nFrames)*dt_ca
else:
nFrames = ca.shape[1]
# t_ca = np.arange(nFrames)*dt_ca
d = len(camTrigInds) - ca.shape[1]
if d > 0:
print(f'Check threshold, {d} more camera triggers than expected')
elif d < 0:
print('Check threshold, {d} fewer camera triggers than expected')
n_pre_bas = int(np.round(t_pre_bas * Fs_bas))
n_post_bas = int(np.round(t_post_bas * Fs_bas))
n_pre_ca = int(np.round(t_pre_ca * Fs_ca))
n_post_ca = int(np.round(t_post_ca * Fs_ca))
stimFrames = spt.nearestMatchingInds(stimInds, camTrigInds) - 2
mtr_trl = spt.segmentByEvents(mtr,
stimInds,
n_pre_bas,
n_post_bas,
n_jobs=n_jobs)
indsVec_ca = np.arange(nFrames)
indsVec_ca_trl = spt.segmentByEvents(indsVec_ca,
stimFrames,
n_pre_ca,
n_post_ca,
n_jobs=n_jobs)
trlNum_actual = np.arange(len(mtr_trl))
lens_mtr = np.array([len(mtr_) for mtr_ in mtr_trl])
lens_ca = np.array([len(inds_) for inds_ in indsVec_ca_trl])
inds_del_mtr = np.where(lens_mtr != mode(lens_mtr)[0])[0]
inds_del_ca = np.where(lens_ca != mode(lens_ca)[0])[0]
inds_del_trl = np.union1d(inds_del_mtr, inds_del_ca)
trlNum_actual = np.delete(trlNum_actual, inds_del_trl)
mtr_trl = list(np.delete(
mtr_trl, inds_del_trl,
axis=0)) # Converting to list so that can later put in dataframe
indsVec_ca_trl = list(np.delete(indsVec_ca_trl, inds_del_trl, axis=0))
dic = dict(mtr_trl=mtr_trl, caInds_trl=indsVec_ca_trl)
if not np.any(ca == None):
if np.ndim(ca) == 1:
ca = ca[np.newaxis, ...]
ca_trl = [ca[:, inds_] for inds_ in indsVec_ca_trl]
dic['ca_trl'] = ca_trl
trlNum = np.arange(len(mtr_trl))
dic['trlNum'] = trlNum
dic['trlNum_actual'] = trlNum_actual
dic['stimAmp'] = np.delete(stimAmps, inds_del_trl)
dic['stimLoc'] = np.delete(stimHt, inds_del_trl)
return pd.DataFrame(dic, columns=dic.keys())
def wavelet(y,
t,
dj=1 / 32,
mother='morlet',
pad=1,
param=-1,
freqScale='log',
freqRange=None,
**kwargs):
'''
Computes the wavelet transform of a timeseries
[W,period,scale,coi] = wavelet(...)
Parameters
----------
y: 1D array
Timeseries to obtain wavelet transform for
t: Scalar or 1D array
If scalar,then sampling interval (dt), else time sequence
pad: Boolean
Zero padding; pad = 1 results in padding of the signal with zeros till
the next largest length that is a power of 2.
dj: Scalar
Wavelet scale resolution; defaults to 1/24
mother: String
Mother wavelet function (Morlet (default), Paul, or DOG)
param: Scalar
Wave number; defaults to 6 for Morlet wavelet
freqSCale: String, ['log'] | 'lin'
Base-2 logarithmic or linear frequency scale.
freqRange - 2 tuple or list, or None:
Determines the frequency range over which to compute the wavelets.
If None, then computes for the full possible range, determined by
Nyquist limit the time length of the timeseries signal.
**kwargs
s0 - Smallest scale; defaults to 2*dt, where dt is sampling interval
J1 - Starting wavelet scale
Outputs:
W - Matrix of wavelet coefficients
period - Vector of Fourier periods over which WT is computed
scale - Vector of wavelet scales used for computation
coi - Cone of influence; a vector of values that show where edge effects become significant
'''
import numpy as np
import apCode.SignalProcessingTools as spt
n1 = len(y)
if np.size(t) == 1:
dt = t
t = np.arange(n1) * dt
else:
dt = t[1] - t[0]
# Smallest scale
s0 = kwargs.get('s0', 2 * dt)
J1 = kwargs.get('J1', int(np.fix(np.log2(n1 * dt / s0) / dj)))
#...demean and and zero pad timeseries if specified
y = y - np.mean(y)
if pad == 1:
y = spt.zeroPadToNextPowOf2(y)
#...construct wavenumber array used in transform (eqn 5)
N = len(y)
k = (np.arange(N / 2) + 1) * ((2 * np.pi) / (N * dt))
k_neg = -k[np.int(np.fix((N - 1) / 2)):0:-1]
k = np.hstack((0, k, k_neg))
#... compute fft of the padded timeseries (eqn 3)
f = np.fft.fft(y)
#...construct SCALE array & empty PERIOD and WAVE arrays
fourier_factor = get_fourier_factor(mother=mother, k0=param)
if freqScale.lower() == 'log':
scale = s0 * 2**(np.arange(0, J1 + 1) * dj)
if freqRange != None:
freq = 1 / (scale * fourier_factor)
minF = np.min(freqRange)
maxF = np.max(freqRange)
keepInds = np.where((freq >= minF) & (freq <= maxF))[0]
freq = freq[keepInds]
scale = 1. / (freq * fourier_factor)
else:
maxF = 1 / s0
minF = 1 / (n1 * dt)
freq = np.arange(minF, maxF, dj)
if freqRange != None:
minF = np.min(freqRange)
maxF = np.max(freqRange)
keepInds = np.where((freq >= minF) & (freq <= maxF))[0]
freq = freq[keepInds]
scale = 1 / (freq * fourier_factor)
period = scale
wave = np.zeros((len(scale), N)) # instantiate the wavelet array
wave = wave + 1j * wave # Make it complex
for sNum, s in enumerate(scale):
daughter, fourier_factor, coi, dofmin = wave_bases(k,
s,
mother=mother,
param=param)
wave[sNum, :] = np.fft.ifft(f * daughter)
period = fourier_factor * scale
vec1 = np.arange(1, (n1 + 1) / 2 - 1)
vec2 = np.arange((n1 / 2 - 1), 0, -1)
vec = np.hstack((1e-5, vec1, vec2, 1e-5))
coi = coi * dt * vec
wave = wave[:, 0:n1]
return wave, period, scale, coi
def expand_on_bends(df_trl,
Fs=500,
tPre_ms=100,
bendThr=10,
minLat_ms=5,
maxGap_ms=100):
"""Takes dataframe where each row contains single trial information and
expands such that each row contains single bend information
Parameters
----------
df_trl: pandas dataframe, (nTrlsInTotal, nVariables)
Fs: int
Sampling frequency when collecting data(images)
nPre_ms: scalar
"""
import apCode.SignalProcessingTools as spt
minPkDist = int((10e-3) * Fs)
nPre = tPre_ms * 1e-3 * Fs
minLat = minLat_ms * 1e-3 * Fs
maxGap = maxGap_ms * 1e-3 * Fs
df_bend = []
for iTrl in np.unique(df_trl.trlIdx_glob):
df_now = df_trl.loc[df_trl.trlIdx_glob == iTrl]
y = df_now.iloc[0]['tailAngles'][-1]
y = spt.chebFilt(y, 1 / Fs, (5, 60), btype='bandpass')
pks = spt.findPeaks(y,
thr=bendThr,
thrType='rel',
pol=0,
minPkDist=minPkDist)[0]
if len(pks) > 3:
dpks = np.diff(pks)
tooSoon = np.where(pks < (nPre + minLat))[0]
tooSparse = np.where(dpks > maxGap)[0] + 1
inds_del = np.union1d(tooSoon, tooSparse)
pks = np.delete(pks, inds_del, axis=0)
if len(pks) > 3:
nBends = len(pks)
bendIdx = np.arange(nBends)
bendSampleIdxInTrl = pks
bendAmp = y[pks]
bendAmp_abs = np.abs(bendAmp)
bendAmp_rel = np.insert(np.abs(np.diff(bendAmp)), 0,
np.abs(bendAmp[0]))
bendInt_ms = np.gradient(pks) * (1 / Fs) * 1000
onset_ms = (pks[0] - nPre + 1) * (1 / Fs) * 1000
else:
nBends = 0
bendIdx, bendAmp, bendAmp_abs, bendAmp_rel, bendInt_ms =\
[np.nan for _ in range(5)]
bendsampleIdxInTrl, onset_ms = [np.nan for _ in range(2)]
dic = dict(trlIdx_glob=iTrl,
nBends=nBends,
bendIdx=bendIdx,
bendSampleIdxInTrl=bendSampleIdxInTrl,
bendAmp=bendAmp,
bendAmp_abs=bendAmp_abs,
bendAmp_rel=bendAmp_rel,
bendInt_ms=bendInt_ms,
onset_ms=onset_ms)
df_now = pd.DataFrame(dic)
df_bend.append(df_now)
df_bend = pd.concat(df_bend, ignore_index=True)
return pd.merge(df_trl, df_bend, on='trlIdx_glob')
#xtl = (np.floor(np.linspace(data['time'][0],data['time'][-1],10)/100)*100).astype(int)
xt = xtl/(data['time'][1]-data['time'][0])
ax.xaxis.set_ticks(xt)
ax.xaxis.set_ticklabels(xtl)
ax.tick_params(labelsize = 14)
plt.ylabel('Clstr #', fontsize = 16)
plt.xlabel('Time (sec)', fontsize = 16)
plt.title('Cell data, sorted by cluster', fontsize = 18);
#%% Create rgb img stack with cells colored by cluster ID
import nvCode.tifffile as tff
import apCode.volTools as volt
imgStack_norm = spt.standardize(data['avg'].copy())
imgStack = np.transpose(np.tile(np.zeros(np.shape(imgStack_norm)),[3,1,1,1]),[1,2,3,0])
sliceList = np.arange(np.shape(data['avg'])[0])
#sliceList = [30]
for z in sliceList:
inds_slice = np.where(data['info']['slice']==z)[0]
imgStack[z,:,:,:] = volt.gray2rgb(imgStack_norm[z])
for lbl in np.unique(labels):
inds_lbl = data['inds']['inMask'].ravel()[np.where(labels == lbl)[0]]
inds_cell = np.intersect1d(inds_slice,inds_lbl)
if len(inds_cell)>0:
pxls_cell = [data['info']['inds'][ind] for ind in inds_cell]
#pxls = np.squeeze(np.array(pxls_cell)).ravel().astype(int)
rgb= plt.cm.RdYlGn(color_idx[lbl])[0:3]
for pxl in pxls_cell:
pxl = pxl.ravel().astype(int)
dt = 1 / Fs
preStimPts = preStimPer * Fs
postStimPts = postStimPer * Fs
# In[112]:
importlib.reload(aed)
pre = aed.import10ch(os.path.join(epDir, preFile))
post = aed.import10ch(os.path.join(epDir, postFile))
print(pre.keys())
print(time.ctime())
# In[155]:
#%% Detect and check stimulus indices
pre['stimInds'] = spt.findPeaks(spt.zscore(pre[stimCh]), thr=3)[0] - 2
post['stimInds'] = spt.findPeaks(spt.zscore(post[stimCh]), thr=3)[0] - 2
xmin = np.max((pre['t'][0], post['t'][0]))
xmax = np.min((pre['t'][-1], post['t'][-1]))
plt.style.use(('seaborn-dark', 'seaborn-colorblind', 'seaborn-poster'))
plt.subplot(2, 1, 1)
plt.plot(pre['t'], pre[stimCh])
plt.plot(pre['t'][pre['stimInds']],
pre[stimCh][pre['stimInds']],
'o',
markersize=15)
plt.xlim(xmin, xmax)
plt.title('Pre-Ablation')
plt.subplot(2, 1, 2)
def estimateCaDecayKinetics(time, signals, p0 = None, thr = 2, preTime = 10,
postTime = 40):
"""
Given a time vector and Ca signal matrix of shape = (C,T), where
C = # of cells, and T = # of time points (must match length of time
vector), returns output of shape = (nSamples, 2), where the 1st and
2nd columns contain the fast and slow decay tau estimates after
fitting Ca2+ signals with double exponential
Parameters:
time - Time vector of length T
signals - Ca signals array of shape (nSamples,T)
p0 - Array-like, (tau_fast, tau_slow, wt_fast), where tau_fast is the
fast decay time constant (in sec), tau_slow is the slow decay
constant, and wt_fast is the weight of the fast exponential (<1)
for fitting the signal as a weighted sum of the fast and slow
exponential. Default is None, in which case fitting optimization
begins without initial estimate
thr - Threshold for peak detection in Ca signals, in units of zscore
preTime - Pre-peak time length of the Ca signals to include for segmentation
postTime - Post-peak " " " "
Avinash Pujala, JRC, 2017
"""
import numpy as np
from scipy.optimize import curve_fit as cf
import apCode.SignalProcessingTools as spt
import apCode.AnalyzeEphysData as aed
def doubleExp(time, tau1, tau2, wt1):
wt2 = 1-wt1
time = time - time[0]
e = wt1*np.exp(-time/tau1) + wt2*np.exp(-time/tau2)
return e
def listToArray(x):
lens = [len(item) for item in x]
lenOfLens = len(lens)
lens = lens[np.min((lenOfLens-1,2))]
a = np.zeros((len(x),lens))
delInds = []
for itemNum,item in enumerate(x):
if len(item) == lens:
a[itemNum,:] = item
else:
delInds.append(itemNum)
a = np.delete(a,delInds,axis = 0)
return a, delInds
if np.ndim(signals)==1:
signals = np.reshape(signals,(1,len(signals)))
dt = time[2]-time[1]
pts_post = np.round(postTime/dt).astype(int)
pts_pre = np.round(preTime/dt).astype(int)
x_norm = spt.zscore(signals,axis = 1)
x_seg, params, x_seg_fit = [],[],[]
nSamples = np.shape(signals)[0]
excludedSamples = np.zeros((nSamples,1))
for nSample in np.arange(nSamples):
inds_pk = spt.findPeaks(x_norm[nSample,:],thr = thr,ampType = 'rel')[0]
if len(inds_pk)==0:
print('Peak detection failed for sample #', nSample, '. Try lowering threshold')
excludedSamples[nSample] = 1
else:
blah = aed.SegmentDataByEvents(signals[nSample,:],inds_pk,pts_pre,pts_post,axis = 0)
blah = listToArray(blah)[0]
blah = np.mean(blah,axis=0)
x_seg.append(blah)
ind_max = np.where(blah == np.max(blah))[0][0]
y = spt.standardize(blah[ind_max:])
t = np.arange(len(y))*dt
popt,pcov = cf(doubleExp,t,y,p0 = [10,20, 0.5], bounds = (0,20))
if popt[0]> popt[1]:
popt[0:2] = popt[2:0:-1]
popt[-1] = 1-popt[-1]
params.append(popt)
foo = doubleExp(t,popt[0],popt[1],popt[2])
x_seg_fit.append(foo)
excludedSamples = np.where(excludedSamples)[0]
includedSamples = np.setdiff1d(np.arange(nSamples),excludedSamples)
x_seg,delInds = listToArray(x_seg)
params = np.delete(np.array(params),delInds,axis = 0)
delInds = includedSamples[delInds]
if len(delInds)>0:
print('Sample #', delInds, 'excluded for short segment length. Consider decreasing pre-peak time length')
excludedSamples = np.union1d(delInds,excludedSamples)
x_seg = spt.standardize(np.array(x_seg),axis = 1)
x_seg_fit = np.array(listToArray(x_seg_fit)[0])
out = {'raw': x_seg,'fit': x_seg_fit,'params': np.array(params),'excludedSamples': excludedSamples}
return out
def wavelet(y,t,dj = 1/32, mother = 'morlet',pad = 1, param = -1, freqScale = 'log',
freqRange = None, **kwargs):
'''
Computes the wavelet transform of a timeseries
[W,period,scale,coi] = wavelet(...)
Parameters
----------
y: 1D array
Timeseries to obtain wavelet transform for
t: Scalar or 1D array
If scalar,then sampling interval (dt), else time sequence
pad: Boolean
Zero padding; pad = 1 results in padding of the signal with zeros till
the next largest length that is a power of 2.
dj: Scalar
Wavelet scale resolution; defaults to 1/24
mother: String
Mother wavelet function (Morlet (default), Paul, or DOG)
param: Scalar
Wave number; defaults to 6 for Morlet wavelet
freqSCale: String, ['log'] | 'lin'
Base-2 logarithmic or linear frequency scale.
freqRange - 2 tuple or list, or None:
Determines the frequency range over which to compute the wavelets.
If None, then computes for the full possible range, determined by
Nyquist limit the time length of the timeseries signal.
**kwargs
s0 - Smallest scale; defaults to 2*dt, where dt is sampling interval
J1 - Starting wavelet scale
Outputs:
W - Matrix of wavelet coefficients
period - Vector of Fourier periods over which WT is computed
scale - Vector of wavelet scales used for computation
coi - Cone of influence; a vector of values that show where edge effects become significant
'''
import numpy as np
import apCode.SignalProcessingTools as spt
n1 = len(y)
if np.ndim(t)==0:
dt = t
t = np.arange(n1)*dt
else:
dt = t[1]-t[0]
# Smallest scale
s0 = kwargs.get('s0')
if s0 is None:
s0 = 2*dt
J1 = kwargs.get('J1')
if J1 is None:
J1 = int(np.fix(np.log2(n1*dt/s0)/dj))
#...demean and and zero pad timeseries if specified
y = y - np.mean(y)
if pad ==1:
y = spt.zeroPadToNextPowOf2(y)
#...construct wavenumber array used in transform (eqn 5)
N = len(y)
k = (np.arange(N/2) + 1) *((2*np.pi)/(N*dt))
k_neg = -k[np.int(np.fix((N-1)/2)):0:-1]
k = np.hstack((0,k,k_neg))
#... compute fft of the padded timeseries (eqn 3)
f = np.fft.fft(y)
#...construct SCALE array & empty PERIOD and WAVE arrays
fourier_factor = get_fourier_factor(mother = mother, k0 = param)
if freqScale.lower()== 'log':
scale = s0*2**(np.arange(0,J1+1)*dj)
if freqRange != None:
freq = 1/(scale*fourier_factor)
minF = np.min(freqRange)
maxF = np.max(freqRange)
keepInds = np.where((freq>=minF) & (freq <= maxF))[0]
freq = freq[keepInds]
scale = 1./(freq*fourier_factor)
else:
maxF = 1/s0
minF = 1/(n1*dt)
freq = np.arange(minF,maxF,dj)
if freqRange != None:
minF = np.min(freqRange)
maxF = np.max(freqRange)
keepInds = np.where((freq>=minF) & (freq <= maxF))[0]
freq = freq[keepInds]
scale = 1/(freq*fourier_factor)
period = scale
wave = np.zeros((len(scale),N)) # instantiate the wavelet array
wave = wave + 1j*wave # Make it complex
for sNum, s in enumerate(scale):
daughter, fourier_factor,coi, dofmin = wave_bases(k,s, mother = mother, param = param)
wave[sNum,:] = np.fft.ifft(f * daughter)
period = fourier_factor*scale
vec1 = np.arange(1,(n1+1)/2-1)
vec2 = np.arange((n1/2-1),0,-1)
vec = np.hstack((1e-5,vec1,vec2,1e-5))
coi = coi*dt*vec
wave = wave[:,0:n1]
return wave,period,scale,coi
replace=True)
var['sat_score'] = np.random.choice(np.arange(1200, 1601),
size=n_samples,
replace=True)
var['hours_of_research_experience'] = np.random.choice(np.arange(20, 110),
size=n_samples,
replace=True)
var['age_of_applicant'] = np.random.choice(np.arange(18, 22),
size=n_samples,
replace=True)
var['harvard_entrance_test_score'] = np.random.choice(np.arange(70, 101), size = n_samples,\
replace = True)
X = np.array([var['gpa'], var['sat_score'], var['hours_of_research_experience'], var['age_of_applicant'],\
var['harvard_entrance_test_score']])
X = spt.standardize(X.T, axis=0)
y = np.dot(X, wts)
y = y + np.random.rand(len(y)) * np.std(y)
y = spt.standardize(y) * 0.8 + 0.2
var['chances_of_acceptance_into_harvard'] = y
data = var
#%%
from sklearn.linear_model import LinearRegression
reg = LinearRegression().fit(X, y)
print('Regression coefficients are {}, and intercept is {}'.format(
reg.coef_, reg.intercept_))
def estimateCaDecayKinetics(time,
signals,
p0=None,
thr=2,
preTime=10,
postTime=40):
"""
Given a time vector and Ca signal matrix of shape = (C,T), where
C = # of cells, and T = # of time points (must match length of time
vector), returns output of shape = (nSamples, 2), where the 1st and
2nd columns contain the fast and slow decay tau estimates after
fitting Ca2+ signals with double exponential
Parameters:
time - Time vector of length T
signals - Ca signals array of shape (nSamples,T)
p0 - Array-like, (tau_fast, tau_slow, wt_fast), where tau_fast is the
fast decay time constant (in sec), tau_slow is the slow decay
constant, and wt_fast is the weight of the fast exponential (<1)
for fitting the signal as a weighted sum of the fast and slow
exponential. Default is None, in which case fitting optimization
begins without initial estimate
thr - Threshold for peak detection in Ca signals, in units of zscore
preTime - Pre-peak time length of the Ca signals to include for segmentation
postTime - Post-peak " " " "
Avinash Pujala, JRC, 2017
"""
import numpy as np
from scipy.optimize import curve_fit as cf
import apCode.SignalProcessingTools as spt
import apCode.AnalyzeEphysData as aed
def doubleExp(time, tau1, tau2, wt1):
wt2 = 1 - wt1
time = time - time[0]
e = wt1 * np.exp(-time / tau1) + wt2 * np.exp(-time / tau2)
return e
def listToArray(x):
lens = [len(item) for item in x]
lenOfLens = len(lens)
lens = lens[np.min((lenOfLens - 1, 2))]
a = np.zeros((len(x), lens))
delInds = []
for itemNum, item in enumerate(x):
if len(item) == lens:
a[itemNum, :] = item
else:
delInds.append(itemNum)
a = np.delete(a, delInds, axis=0)
return a, delInds
if np.ndim(signals) == 1:
signals = np.reshape(signals, (1, len(signals)))
dt = time[2] - time[1]
pts_post = np.round(postTime / dt).astype(int)
pts_pre = np.round(preTime / dt).astype(int)
x_norm = spt.zscore(signals, axis=1)
x_seg, params, x_seg_fit = [], [], []
nSamples = np.shape(signals)[0]
excludedSamples = np.zeros((nSamples, 1))
for nSample in np.arange(nSamples):
inds_pk = spt.findPeaks(x_norm[nSample, :], thr=thr, ampType='rel')[0]
if len(inds_pk) == 0:
print('Peak detection failed for sample #', nSample,
'. Try lowering threshold')
excludedSamples[nSample] = 1
else:
blah = aed.SegmentDataByEvents(signals[nSample, :],
inds_pk,
pts_pre,
pts_post,
axis=0)
blah = listToArray(blah)[0]
blah = np.mean(blah, axis=0)
x_seg.append(blah)
ind_max = np.where(blah == np.max(blah))[0][0]
y = spt.standardize(blah[ind_max:])
t = np.arange(len(y)) * dt
popt, pcov = cf(doubleExp, t, y, p0=[10, 20, 0.5], bounds=(0, 20))
if popt[0] > popt[1]:
popt[0:2] = popt[2:0:-1]
popt[-1] = 1 - popt[-1]
params.append(popt)
foo = doubleExp(t, popt[0], popt[1], popt[2])
x_seg_fit.append(foo)
excludedSamples = np.where(excludedSamples)[0]
includedSamples = np.setdiff1d(np.arange(nSamples), excludedSamples)
x_seg, delInds = listToArray(x_seg)
params = np.delete(np.array(params), delInds, axis=0)
delInds = includedSamples[delInds]
if len(delInds) > 0:
print(
'Sample #', delInds,
'excluded for short segment length. Consider decreasing pre-peak time length'
)
excludedSamples = np.union1d(delInds, excludedSamples)
x_seg = spt.standardize(np.array(x_seg), axis=1)
x_seg_fit = np.array(listToArray(x_seg_fit)[0])
out = {
'raw': x_seg,
'fit': x_seg_fit,
'params': np.array(params),
'excludedSamples': excludedSamples
}
return out
# In[112]:
importlib.reload(aed)
pre = aed.import10ch(os.path.join(epDir,preFile))
post = aed.import10ch(os.path.join(epDir,postFile))
print(pre.keys())
print(time.ctime())
# In[155]:
#%% Detect and check stimulus indices
pre['stimInds'] = spt.findPeaks(spt.zscore(pre[stimCh]),thr=3)[0]-2
post['stimInds'] = spt.findPeaks(spt.zscore(post[stimCh]),thr=3)[0]-2
xmin = np.max((pre['t'][0],post['t'][0]))
xmax = np.min((pre['t'][-1],post['t'][-1]))
plt.style.use(('seaborn-dark','seaborn-colorblind','seaborn-poster'))
plt.subplot(2,1,1)
plt.plot(pre['t'],pre[stimCh])
plt.plot(pre['t'][pre['stimInds']],pre[stimCh][pre['stimInds']],'o',markersize =15)
plt.xlim(xmin,xmax)
plt.title('Pre-Ablation')
plt.subplot(2,1,2)
plt.plot(post['t'],post[stimCh])
plt.plot(post['t'][post['stimInds']],post[stimCh][post['stimInds']],'o',markersize =15)
plt.xlim(xmin,xmax)
def readPeriStimulusTifImages(tifPaths, basPaths, nBasCh = 16, ch_camTrig = 'patch4', ch_stim = 'patch3',\
tifNameStr = '', time_preStim = 1, time_postStim = 10, thr_stim = 1.5,\
thr_camTrig = 1, maxAllowedTimeBetweenStimAndCamTrig = 0.5, n_jobs = 1):
"""
Given the directory to .tif files stored by ScanImage (Bessel beam image settings) and the full
path to the accompanying bas files returns a dictionary with values holding peri-stimulus
image data (Ca activity) in trialized format along with some other pertinent info.
Parameters
----------
tifDir: string
Path to directory holding .tif files written by ScanImage. In the current setting, each .tif
file holds nCh*3000 images, where nCh = number of channels.
basPath: string
Full path to the bas (BehavAndScan) file accompanying the imaging session.
nBasCh: scalar
Number of signal channels in the bas file.
ch_camTrig: string
Name of the channel in bas corresponding to the camera trigger signal
ch_stim: string
Name of the stimulus signal channel in bas.
tifNameStr: string
Only .tif files containing this will be read.
time_preStim: scalar
The length of the pre-stimulus time to include when reading images
time_postStim: scalar
The length of the post-stimulus time.
thr_stim: scalar
Threshold to use for detection of stimuli in the stimulus channel of bas.
thr_camTrig: scalar
Threshold to use for detection of camera trigger onsets in the camera trigger channel of bas.
maxAllowedTimeBetweenStimAndCamTrig: scalar
If a camera trigger is separated in time by the nearest stimulus by longer than this time
interval, then ignore this stimulus trial.
Returns
-------
D: dict
Dictionary contaning the following keys:
'I': array, (nTrials, nTime, nImageChannels, imageWidth, imageHeight)
Image hyperstack arranged in conveniently-accessible trialized format.
'tifInfo': dict
Dictionary holding useful image metadata. Has following keys:
'filePaths': list of strings
Paths to .tif files
'nImagesInfile': scalar int
Number of images in each .tif file after accounting of number of
image channels
'nChannelsInFile': scalar int
Number of image channels
'inds_stim': array of integers, (nStim,)
Indices in bas coordinates where stimuli occurred.
'inds_stim_img': array of integers, (nStim,)
Indices in image coordinates where stimuli occurred
'inds_camTrig': array of integers, (nCameraTriggers,)
Indices in bas coordinates corresponding to the onsets of camera triggers.
'bas': dict
BehavAndScan data
"""
import tifffile as tff
import numpy as np
import apCode.FileTools as ft
import apCode.ephys as ephys
import apCode.SignalProcessingTools as spt
import apCode.util as util
# import os
def getImgIndsInTifs(tifInfo):
nImgsInFile_cum = np.cumsum(tifInfo['nImagesInFile'] *
tifInfo['nChannelsInFile'])
imgIndsInTifs = []
for i in range(len(nImgsInFile_cum)):
if i == 0:
inds_ = np.arange(0, nImgsInFile_cum[i])
else:
inds_ = np.arange(nImgsInFile_cum[i - 1], nImgsInFile_cum[i])
imgIndsInTifs.append(inds_)
return imgIndsInTifs
### Read relevant metadata from tif files in directory
print('Reading ScanImage metadata from tif files...')
tifInfo = ft.scanImageTifInfo(tifPaths)
nCaImgs = np.sum(tifInfo['nImagesInFile'])
### Check for consistency in the number of image channels in all files.
if len(np.unique(tifInfo['nChannelsInFile'])) > 1:
print('Different number of image channels across files, check files!')
return None
nImgCh = tifInfo['nChannelsInFile'][0]
print(f'{nCaImgs} {nImgCh}-channel images from all tif files')
### Get a list of indices corresponding to images in each of the tif files
inds_imgsInTifs = getImgIndsInTifs(tifInfo)
### Read bas file to get stimulus and camera trigger indices required to align images and behavior
print(
'Reading and joining bas files, detecting stimuli and camera triggers...'
)
basList = [ephys.importCh(bp, nCh=nBasCh) for bp in basPaths]
bas = concatenateBas(basList)
inds_stim = spt.levelCrossings(bas[ch_stim], thr=thr_stim)[0]
if len(inds_stim) == 0:
print(
f'Only {len(inds_stim)} stims detected, check channel specification or threshold'
)
return dict(bas=bas)
inds_camTrig = spt.levelCrossings(bas[ch_camTrig], thr=thr_camTrig)[0]
if len(inds_camTrig) == 0:
print(
f'Only {len(inds_camTrig)} cam trigs detected, check channel specification or threshold'
)
return dict(bas=bas)
dt_vec = np.diff(bas['t'][inds_camTrig])
dt_ca = np.round(np.mean(dt_vec) * 100) / 100
print('Ca sampling rate = {}'.format(1 / dt_ca))
inds_del = np.where(dt_vec <= (0.5 * dt_ca))[0] + 1
inds_camTrig = np.delete(inds_camTrig, inds_del)
### Deal with possible mismatch in number of camera trigger indices and number of images in tif files
if nCaImgs < len(inds_camTrig):
inds_camTrig = inds_camTrig[:nCaImgs]
nCaImgs_extra = 0
elif nCaImgs > len(inds_camTrig):
nCaImgs_extra = nCaImgs - len(inds_camTrig)
else:
nCaImgs_extra = 0
print('{} extra Ca2+ images'.format(nCaImgs_extra))
print('{} stimuli and {} camera triggers'.format(len(inds_stim),
len(inds_camTrig)))
### Indices of ca images closest to stimulus
inds_stim_img = spt.nearestMatchingInds(inds_stim, inds_camTrig)
### Find trials where the nearest cam trigger is farther than the stimulus by a certain amount
inds_camTrigNearStim = inds_camTrig[inds_stim_img]
t_stim = bas['t'][inds_stim]
t_camTrigNearStim = bas['t'][inds_camTrigNearStim]
inds_tooFar = np.where(
np.abs(t_stim -
t_camTrigNearStim) > maxAllowedTimeBetweenStimAndCamTrig)[0]
inds_ca_all = np.arange(nCaImgs)
nPreStim = int(time_preStim / dt_ca)
nPostStim = int(time_postStim / dt_ca)
print("{} pre-stim points, and {} post-stim points".format(
nPreStim, nPostStim))
inds_ca_trl = np.array(
spt.segmentByEvents(inds_ca_all, inds_stim_img + nCaImgs_extra,
nPreStim, nPostStim))
### Find trials that are too short to include the pre- or post-stimulus period
trlLens = np.array([len(trl_) for trl_ in inds_ca_trl])
inds_tooShort = np.where(trlLens < np.max(trlLens))[0]
inds_trl_del = np.union1d(inds_tooFar, inds_tooShort)
inds_trl_keep = np.setdiff1d(np.arange(len(inds_ca_trl)), inds_trl_del)
### Exclude the above 2 types of trials from consideration
if len(inds_trl_del) > 0:
print('Excluding the trials {}'.format(inds_trl_del))
inds_ca_trl = inds_ca_trl[inds_trl_keep]
I = []
print('Reading trial-related images from tif files...')
nTrls = len(inds_ca_trl)
def trlImages(inds_ca_trl, inds_imgsInTifs, nImgCh, tifInfo, trl):
trl_ = np.arange(trl.min() * nImgCh, (trl.max() + 1) * nImgCh)
loc = util.locateItemsInSetsOfItems(trl_, inds_imgsInTifs)
I_ = []
for subInds, supInd in zip(loc['subInds'], loc['supInds']):
with tff.TiffFile(tifInfo['filePaths'][supInd]) as tif:
img = tif.asarray(key=subInds)
I_.extend(img.reshape(-1, nImgCh, *img.shape[1:]))
I_ = np.array(I_)
return I_
if n_jobs < 2:
chunkSize = int(nTrls / 5)
for iTrl, trl in enumerate(inds_ca_trl):
if np.mod(iTrl, chunkSize) == 0:
print('Trl # {}/{}'.format(iTrl + 1, nTrls))
I_ = trlImages(inds_ca_trl, inds_imgsInTifs, nImgCh, tifInfo, trl)
I.append(I_)
else:
print('Processing with dask')
import dask
from dask.diagnostics import ProgressBar
for trl in inds_ca_trl:
I_ = dask.delayed(trlImages)(inds_ca_trl, inds_imgsInTifs, nImgCh,
tifInfo, trl)
I.append(I_)
with ProgressBar():
I = dask.compute(*I)
D = dict(I = np.squeeze(np.array(I)), tifInfo = tifInfo, inds_stim = inds_stim, inds_stim_img = inds_stim_img,\
inds_camTrig = inds_camTrig,bas = bas, inds_trl_excluded = inds_trl_del)
return D
def plotCentroids(time,
centroids,
stimTimes,
time_ephys,
ephys,
scaled=False,
colors=None,
xlabel='',
ylabel='',
title=''):
"""
Plots centroids resulting from some clustering method
Parameters:
time - Time vectors for centroids (optical samplign interval)
centroids - Array of shape (M, N), where M is the number of centroids, and N is the #
number of features (or time points)
stimTimes - Times of stimulus onsets for overlaying vertical dashed lines
time_ephys - Time axis for ephys data (usually sampled at higher rate)
ephys - Ephys time series
scaled - Boolean; If true, scales centroids individually, else scales jointly.
colors - Array of shape (M,3) or (M,4). Colormap to use for plotting centroids
"""
import apCode.SignalProcessingTools as spt
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
if scaled:
centroids = spt.standardize(centroids, axis=1)
else:
centroids = spt.standardize(centroids)
ephys = spt.standardize(ephys)
n_clusters = np.shape(centroids)[0]
if np.any(colors == None):
colors = np.array(
sns.color_palette('colorblind',
np.shape(centroids)[0]))
elif np.shape(colors)[0] < np.shape(centroids)[0]:
colors = np.tile(colors, (np.shape(centroids)[0], 1))
colors = colors[:np.shape(centroids)[0], :]
if np.any(time == None):
time = np.arange(np.shape(centroids)[1])
plt.style.use(['dark_background', 'seaborn-poster'])
for cc in np.arange(np.shape(centroids)[0]):
plt.plot(time,
centroids[cc, :] - np.mean(centroids[cc, :]) - cc,
color=colors[cc, :])
plt.plot(time_ephys,
ephys - np.mean(ephys) - cc - 1,
color=colors[0, :])
yt = np.arange(n_clusters + 1)
ytl = list(yt)
ytl[-1] = 'ephys'
plt.yticks(-yt, ytl)
plt.xlabel(xlabel, fontsize=16)
plt.ylabel(ylabel, fontsize=16)
plt.box('off')
plt.title(title, fontsize=20)
plt.grid('off')
plt.xlim(time[0], time[-1])
for st in stimTimes:
plt.axvline(x=st,
ymin=0,
ymax=1,
alpha=0.3,
color='w',
linestyle='--')
tic = time.time()
with sitr.ScanImageTiffReader(os.path.join(dir_imgs, files_ca[0])) as reader:
I_si = reader.data()
print("SI time = {}".format(time.time() - tic))
#%%
ch_camTrig = 'patch1'
ch_stim = 'patch3'
frameRate = 50
path_bas = os.path.join(os.path.split(dir_imgs)[0], 't01_bas.16ch')
import apCode.ephys as ephys
bas = ephys.importCh(path_bas, nCh=16)
print(bas.keys())
inds_stim = spt.levelCrossings(bas[ch_stim], thr=2)[0]
dInds = np.diff(bas['t'][inds_stim])
inds_del = np.where(dInds < 15)[0] + 1
inds_stim = np.delete(inds_stim, inds_del)
inds_camTrig = spt.levelCrossings(bas[ch_camTrig], thr=2)[0]
dInds = np.diff(bas['t'][inds_camTrig])
inds_del = np.where(dInds <= (0.5 / frameRate))[0] + 1
inds_camTrig = np.delete(inds_camTrig, inds_del)
# plt.plot(bas[ch_camTrig])
# plt.plot(inds_camTrig,bas[ch_camTrig][inds_camTrig],'o')
print('# stims = {}, # of camera triggers = {}'.format(len(inds_stim),
len(inds_camTrig)))
#%%
#%%
maxAllowedTimeBetweenStimAndCamTrig = 0.5 # In sec
# NMFD core method
nmfdW, nmfdH, nmfdV, divKL, _ = NMFD(A, paramNMFD)
# alpha-Wiener filtering
nmfdA, _ = alphaWienerFilter(A, nmfdV, 1.0)
#%% 4. Visualize
paramVis = dict()
paramVis['deltaT'] = deltaT
paramVis['deltaF'] = deltaF
paramVis['endeSec'] = 3.8
paramVis['fontSize'] = 14
fh1, _ = visualizeComponentsNMF(A, nmfdW, nmfdH, nmfdA, paramVis)
import matplotlib.pyplot as plt
plt.show()
#%% 5. Write audio
audios = []
# resynthesize results of NMF with soft constraints and score information
for k in range(numComp):
Y = nmfdA[k] * np.exp(1j * P);
y, _ = inverseSTFT(Y, paramSTFT)
audios.append(y)
# save result
out_filepath = os.path.join(outDir, f'drums_{k}.wav')
y_out = audios[0] + audios[2]
wav.write(filename=out_filepath, rate=fs, data=spt.standardize(y_out)*2-1)
xtl = np.arange(0, data['time'][-1], 500).astype(int)
#xtl = (np.floor(np.linspace(data['time'][0],data['time'][-1],10)/100)*100).astype(int)
xt = xtl / (data['time'][1] - data['time'][0])
ax.xaxis.set_ticks(xt)
ax.xaxis.set_ticklabels(xtl)
ax.tick_params(labelsize=14)
plt.ylabel('Clstr #', fontsize=16)
plt.xlabel('Time (sec)', fontsize=16)
plt.title('Cell data, sorted by cluster', fontsize=18)
#%% Create rgb img stack with cells colored by cluster ID
import nvCode.tifffile as tff
import apCode.volTools as volt
imgStack_norm = spt.standardize(data['avg'].copy())
imgStack = np.transpose(
np.tile(np.zeros(np.shape(imgStack_norm)), [3, 1, 1, 1]), [1, 2, 3, 0])
sliceList = np.arange(np.shape(data['avg'])[0])
#sliceList = [30]
for z in sliceList:
inds_slice = np.where(data['info']['slice'] == z)[0]
imgStack[z, :, :, :] = volt.gray2rgb(imgStack_norm[z])
for lbl in np.unique(labels):
inds_lbl = data['inds']['inMask'].ravel()[np.where(labels == lbl)[0]]
inds_cell = np.intersect1d(inds_slice, inds_lbl)
if len(inds_cell) > 0:
pxls_cell = [data['info']['inds'][ind] for ind in inds_cell]
#pxls = np.squeeze(np.array(pxls_cell)).ravel().astype(int)
rgb = plt.cm.RdYlGn(color_idx[lbl])[0:3]
for pxl in pxls_cell:
def filtImgs(I,filtSize = 5, kernel = 'median', process = 'parallel'):
'''
Processes images so as to make moving particle tracking easier
I_proc = processImagesForTracking(I,filtSize = 5)
Parameters
----------
I: 3D array of shape (T,M,N), where T = # of images, M, N = # of rows
and columns respectively
Image stack to filter
kernel: String or 2D array
['median'] | 'rect' | 'gauss' or array specifying the kernel
filtSize: Scalar or 2-tuple
Size of the kernel to generate if kernel is string
'''
from scipy import signal
import numpy as np
import apCode.SignalProcessingTools as spt
import time
if process.lower() == 'parallel':
from joblib import Parallel, delayed
import multiprocessing
parFlag = True
num_cores = np.min((multiprocessing.cpu_count(),32))
else:
parFlag = False
tic = time.time()
if np.ndim(I)<3:
I = I[np.newaxis,:,:]
N = np.shape(I)[0]
print('Filter dimensions: {0}'.format(np.shape(filtSize)))
I_flt = np.zeros(np.shape(I))
if isinstance(kernel, str):
if kernel.lower()=='median':
print('Median filtering...')
if np.size(filtSize)>1:
filtSize = filtSize[0]
if np.mod(filtSize,2)==0:
filtSize = filtSize+1 # For median, the filter size should be odd
print('Median filter size must be odd, changed to {}'.format(filtSize))
if parFlag:
print('# of cores = {}'.format(num_cores))
I_flt = Parallel(n_jobs = num_cores,verbose = 5)(delayed(signal.medfilt2d)(img,filtSize) for img in I)
I_flt = np.array(I_flt)
else:
for imgNum, img in enumerate(I):
if np.mod(imgNum,300)==0:
print('Img # {0}/{1}'.format(imgNum,N))
I_flt[imgNum,:,:] = signal.medfilt2d(img,filtSize)
elif kernel.lower() == 'rect':
print('Rectangular filtering...')
if np.size(filtSize)==1:
ker = np.ones((filtSize,filtSize))
else:
ker = np.ones(filtSize)
ker = ker/ker.sum()
ker = ker[np.newaxis,:,:]
if parFlag:
I_flt = Parallel(n_jobs = num_cores,verbose = 5)(delayed(signal.convolve)(img,ker) for img in I)
else:
I_flt = signal.convolve(I,ker,mode = 'same')
elif kernel.lower()=='gauss':
print('Gaussian filtering...')
if np.size(filtSize)==1:
ker = spt.gausswin(filtSize)
ker = ker.reshape((-1,1))
ker = ker*ker.T
else:
ker1 = spt.gausswin(filtSize[0]).reshape((-1,1))
ker2 = spt.gausswin(filtSize[0]).reshape((-1,1))
ker = ker1*ker2.T
ker = ker/ker.sum()
if parFlag:
I_flt = Parallel(n_jobs = num_cores,verbose = 5)(delayed(signal.convolve)(img,ker) for img in I)
else:
ker= ker[np.newaxis,:,:]
I_flt = signal.convolve(I,ker,mode = 'same')
else:
ker = ker/ker.sum()
if parFlag:
I_flt = Parallel(n_jobs = num_cores,verbose = 5)(delayed(signal.convolve)(img,ker) for img in I)
else:
ker= ker[np.newaxis,:,:]
I_flt = signal.convolve(I,ker,mode = 'same')
print(int(time.time()-tic), 'sec')
if np.shape(I_flt)[0]==1:
I_flt = np.squeeze(I_flt)
return I_flt
