def plotCentroids(time, centroids, stimTimes, time_ephys, ephys, scaled = False,
colors = None, xlabel = '',ylabel = '', title = ''):
"""
Plots centroids resulting from some clustering method
Parameters:
time - Time vectors for centroids (optical samplign interval)
centroids - Array of shape (M, N), where M is the number of centroids, and N is the #
number of features (or time points)
stimTimes - Times of stimulus onsets for overlaying vertical dashed lines
time_ephys - Time axis for ephys data (usually sampled at higher rate)
ephys - Ephys time series
scaled - Boolean; If true, scales centroids individually, else scales jointly.
colors - Array of shape (M,3) or (M,4). Colormap to use for plotting centroids
"""
import apCode.SignalProcessingTools as spt
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
if scaled:
centroids = spt.standardize(centroids,axis = 1)
else:
centroids = spt.standardize(centroids)
ephys = spt.standardize(ephys)
n_clusters = np.shape(centroids)[0]
if np.any(colors == None):
colors = np.array(sns.color_palette('colorblind',np.shape(centroids)[0]))
elif np.shape(colors)[0] < np.shape(centroids)[0]:
colors = np.tile(colors,(np.shape(centroids)[0],1))
colors = colors[:np.shape(centroids)[0],:]
if np.any(time == None):
time = np.arange(np.shape(centroids)[1])
plt.style.use(['dark_background','seaborn-poster'])
for cc in np.arange(np.shape(centroids)[0]):
plt.plot(time,centroids[cc,:]-np.mean(centroids[cc,:])-cc,color = colors[cc,:])
plt.plot(time_ephys,ephys-np.mean(ephys)-cc-1,color = colors[0,:])
yt = np.arange(n_clusters + 1)
ytl = list(yt)
ytl[-1] = 'ephys'
plt.yticks(-yt, ytl)
plt.xlabel(xlabel,fontsize = 16)
plt.ylabel(ylabel, fontsize = 16)
plt.box('off')
plt.title(title, fontsize = 20)
plt.grid('off')
plt.xlim(time[0],time[-1])
for st in stimTimes:
plt.axvline(x = st, ymin = 0, ymax =1, alpha = 0.3,
color = 'w',linestyle = '--')
def getClrMapsForEachRegressor(betas,normed = True, cMap = 'PiYG',
scaling =1, betaThr= None):
"""
Given the coeffiecients(betas) from regression, returns a list of color
maps, with each color map corresponding to the betas for a single regressor.
These can be used by colorCellsInImgStack to create image stacks with
cells colored by betas.
Parameters:
betas - Array-like with shape (nSamples, nFeatures).
normed - Boolean; If True, normalizes betas such that for each feature
the values range from -1 to 1.
scaling - Not yet implemented
betaThr - None(default),scalar,'auto'; Determines if any thresholding
should be applied based on beta values. If None, then no thresholding,
if scalar, then for beta values whose magnitude is less than this
scalar, the alpha value in the color maps is set to zero. If 'auto'
then automatically determines threshold
"""
import apCode.SignalProcessingTools as spt
import matplotlib.pyplot as plt
import apCode.volTools as volt
def getClrMap(x,cMap,maskInds):
cm = np.zeros((len(x),4))*np.nan
negInds = np.where(x<0)[0]
posInds = np.where(x>=0)[0]
x[negInds] = spt.mapToRange(np.hstack((x[negInds],[0,-1])),[0,127])[0:-2]
x[posInds] = spt.mapToRange(np.hstack((x[posInds],[0,1])),[128,255])[0:-2]
cm[negInds] = cMap(x[negInds].astype(int))
cm[posInds] = cMap(x[posInds].astype(int))
if len(maskInds) == 0:
pass
else:
cm[maskInds,-1] = 0
return cm
if isinstance(cMap,str):
cMap = plt.cm.get_cmap(cMap)
if normed:
betas = spt.standardize(betas, preserveSign = True, axis = 0)*scaling
clrMaps = []
for beta in betas.T:
if betaThr == None:
maskInds = []
elif betaThr == 'auto':
betaThr = volt.getGlobalThr(np.abs(beta))
maskInds = np.where(np.abs(beta)<betaThr)
else:
maskInds = np.where(np.abs(beta)<betaThr)
clrMaps.append(getClrMap(beta,cMap,maskInds))
#clrMaps = [getClrMap(beta,cMap,betaThr) for beta in betas.transpose()]
return clrMaps
xtl = np.arange(0, data['time'][-1], 500).astype(int)
#xtl = (np.floor(np.linspace(data['time'][0],data['time'][-1],10)/100)*100).astype(int)
xt = xtl / (data['time'][1] - data['time'][0])
ax.xaxis.set_ticks(xt)
ax.xaxis.set_ticklabels(xtl)
ax.tick_params(labelsize=14)
plt.ylabel('Clstr #', fontsize=16)
plt.xlabel('Time (sec)', fontsize=16)
plt.title('Cell data, sorted by cluster', fontsize=18)
#%% Create rgb img stack with cells colored by cluster ID
import nvCode.tifffile as tff
import apCode.volTools as volt
imgStack_norm = spt.standardize(data['avg'].copy())
imgStack = np.transpose(
np.tile(np.zeros(np.shape(imgStack_norm)), [3, 1, 1, 1]), [1, 2, 3, 0])
sliceList = np.arange(np.shape(data['avg'])[0])
#sliceList = [30]
for z in sliceList:
inds_slice = np.where(data['info']['slice'] == z)[0]
imgStack[z, :, :, :] = volt.gray2rgb(imgStack_norm[z])
for lbl in np.unique(labels):
inds_lbl = data['inds']['inMask'].ravel()[np.where(labels == lbl)[0]]
inds_cell = np.intersect1d(inds_slice, inds_lbl)
if len(inds_cell) > 0:
pxls_cell = [data['info']['inds'][ind] for ind in inds_cell]
#pxls = np.squeeze(np.array(pxls_cell)).ravel().astype(int)
rgb = plt.cm.RdYlGn(color_idx[lbl])[0:3]
for pxl in pxls_cell:
#xtl = (np.floor(np.linspace(data['time'][0],data['time'][-1],10)/100)*100).astype(int)
xt = xtl/(data['time'][1]-data['time'][0])
ax.xaxis.set_ticks(xt)
ax.xaxis.set_ticklabels(xtl)
ax.tick_params(labelsize = 14)
plt.ylabel('Clstr #', fontsize = 16)
plt.xlabel('Time (sec)', fontsize = 16)
plt.title('Cell data, sorted by cluster', fontsize = 18);
#%% Create rgb img stack with cells colored by cluster ID
import nvCode.tifffile as tff
import apCode.volTools as volt
imgStack_norm = spt.standardize(data['avg'].copy())
imgStack = np.transpose(np.tile(np.zeros(np.shape(imgStack_norm)),[3,1,1,1]),[1,2,3,0])
sliceList = np.arange(np.shape(data['avg'])[0])
#sliceList = [30]
for z in sliceList:
inds_slice = np.where(data['info']['slice']==z)[0]
imgStack[z,:,:,:] = volt.gray2rgb(imgStack_norm[z])
for lbl in np.unique(labels):
inds_lbl = data['inds']['inMask'].ravel()[np.where(labels == lbl)[0]]
inds_cell = np.intersect1d(inds_slice,inds_lbl)
if len(inds_cell)>0:
pxls_cell = [data['info']['inds'][ind] for ind in inds_cell]
#pxls = np.squeeze(np.array(pxls_cell)).ravel().astype(int)
rgb= plt.cm.RdYlGn(color_idx[lbl])[0:3]
for pxl in pxls_cell:
pxl = pxl.ravel().astype(int)
# NMFD core method
nmfdW, nmfdH, nmfdV, divKL, _ = NMFD(A, paramNMFD)
# alpha-Wiener filtering
nmfdA, _ = alphaWienerFilter(A, nmfdV, 1.0)
#%% 4. Visualize
paramVis = dict()
paramVis['deltaT'] = deltaT
paramVis['deltaF'] = deltaF
paramVis['endeSec'] = 3.8
paramVis['fontSize'] = 14
fh1, _ = visualizeComponentsNMF(A, nmfdW, nmfdH, nmfdA, paramVis)
import matplotlib.pyplot as plt
plt.show()
#%% 5. Write audio
audios = []
# resynthesize results of NMF with soft constraints and score information
for k in range(numComp):
Y = nmfdA[k] * np.exp(1j * P);
y, _ = inverseSTFT(Y, paramSTFT)
audios.append(y)
# save result
out_filepath = os.path.join(outDir, f'drums_{k}.wav')
y_out = audios[0] + audios[2]
wav.write(filename=out_filepath, rate=fs, data=spt.standardize(y_out)*2-1)
# Detrending dF/F signals
print('Detrending dF/F signals...')
for cc in np.arange(np.shape(data['dFF'])[1]):
data['dFF'][:,cc] = ia.detrendCa(data['dFF'][:,cc],data['inds']['stim'])
# An adjustment for some datasets
data['avg'] = np.transpose(data['avg'],[0,2,1])
np.shape(data['avg'])
#%% Check to see if stimulus onset indices have been detected correctly
stimInd = 4
periTime_ephys = 0.1
periTime_opt = 8
t_ephys = pyData['time'].ravel()
y_ephys = spt.standardize(pyData['smooth'][:,eCh])
stimInds_ephys = pyData['stim']['inds']
periInd_ephys = np.ceil(periTime_ephys*pyData['samplingRate']).astype(int)
tInds_ephys = np.arange(stimInds_ephys[stimInd]-periInd_ephys,stimInds_ephys[stimInd]+ periInd_ephys)
t_opt = data['time']
y_opt = spt.standardize(np.mean(data['dFF'],axis = 1))
stimInds_opt = data['inds']['stim']
periInd_opt = np.ceil(periTime_opt/(t_opt[1]-t_opt[0]))
tInds_opt = np.arange(stimInds_opt[stimInd]-periInd_opt+1, stimInds_opt[stimInd] + periInd_opt).astype(int)
plt.style.use(['dark_background','seaborn-colorblind','seaborn-poster'])
plt.figure(figsize = (16,10))
plt.subplot(211)
def plotCentroids(time,
centroids,
stimTimes,
time_ephys,
ephys,
scaled=False,
colors=None,
xlabel='',
ylabel='',
title=''):
"""
Plots centroids resulting from some clustering method
Parameters:
time - Time vectors for centroids (optical samplign interval)
centroids - Array of shape (M, N), where M is the number of centroids, and N is the #
number of features (or time points)
stimTimes - Times of stimulus onsets for overlaying vertical dashed lines
time_ephys - Time axis for ephys data (usually sampled at higher rate)
ephys - Ephys time series
scaled - Boolean; If true, scales centroids individually, else scales jointly.
colors - Array of shape (M,3) or (M,4). Colormap to use for plotting centroids
"""
import apCode.SignalProcessingTools as spt
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
if scaled:
centroids = spt.standardize(centroids, axis=1)
else:
centroids = spt.standardize(centroids)
ephys = spt.standardize(ephys)
n_clusters = np.shape(centroids)[0]
if np.any(colors == None):
colors = np.array(
sns.color_palette('colorblind',
np.shape(centroids)[0]))
elif np.shape(colors)[0] < np.shape(centroids)[0]:
colors = np.tile(colors, (np.shape(centroids)[0], 1))
colors = colors[:np.shape(centroids)[0], :]
if np.any(time == None):
time = np.arange(np.shape(centroids)[1])
plt.style.use(['dark_background', 'seaborn-poster'])
for cc in np.arange(np.shape(centroids)[0]):
plt.plot(time,
centroids[cc, :] - np.mean(centroids[cc, :]) - cc,
color=colors[cc, :])
plt.plot(time_ephys,
ephys - np.mean(ephys) - cc - 1,
color=colors[0, :])
yt = np.arange(n_clusters + 1)
ytl = list(yt)
ytl[-1] = 'ephys'
plt.yticks(-yt, ytl)
plt.xlabel(xlabel, fontsize=16)
plt.ylabel(ylabel, fontsize=16)
plt.box('off')
plt.title(title, fontsize=20)
plt.grid('off')
plt.xlim(time[0], time[-1])
for st in stimTimes:
plt.axvline(x=st,
ymin=0,
ymax=1,
alpha=0.3,
color='w',
linestyle='--')
def estimateCaDecayKinetics(time,
signals,
p0=None,
thr=2,
preTime=10,
postTime=40):
"""
Given a time vector and Ca signal matrix of shape = (C,T), where
C = # of cells, and T = # of time points (must match length of time
vector), returns output of shape = (nSamples, 2), where the 1st and
2nd columns contain the fast and slow decay tau estimates after
fitting Ca2+ signals with double exponential
Parameters:
time - Time vector of length T
signals - Ca signals array of shape (nSamples,T)
p0 - Array-like, (tau_fast, tau_slow, wt_fast), where tau_fast is the
fast decay time constant (in sec), tau_slow is the slow decay
constant, and wt_fast is the weight of the fast exponential (<1)
for fitting the signal as a weighted sum of the fast and slow
exponential. Default is None, in which case fitting optimization
begins without initial estimate
thr - Threshold for peak detection in Ca signals, in units of zscore
preTime - Pre-peak time length of the Ca signals to include for segmentation
postTime - Post-peak " " " "
Avinash Pujala, JRC, 2017
"""
import numpy as np
from scipy.optimize import curve_fit as cf
import apCode.SignalProcessingTools as spt
import apCode.AnalyzeEphysData as aed
def doubleExp(time, tau1, tau2, wt1):
wt2 = 1 - wt1
time = time - time[0]
e = wt1 * np.exp(-time / tau1) + wt2 * np.exp(-time / tau2)
return e
def listToArray(x):
lens = [len(item) for item in x]
lenOfLens = len(lens)
lens = lens[np.min((lenOfLens - 1, 2))]
a = np.zeros((len(x), lens))
delInds = []
for itemNum, item in enumerate(x):
if len(item) == lens:
a[itemNum, :] = item
else:
delInds.append(itemNum)
a = np.delete(a, delInds, axis=0)
return a, delInds
if np.ndim(signals) == 1:
signals = np.reshape(signals, (1, len(signals)))
dt = time[2] - time[1]
pts_post = np.round(postTime / dt).astype(int)
pts_pre = np.round(preTime / dt).astype(int)
x_norm = spt.zscore(signals, axis=1)
x_seg, params, x_seg_fit = [], [], []
nSamples = np.shape(signals)[0]
excludedSamples = np.zeros((nSamples, 1))
for nSample in np.arange(nSamples):
inds_pk = spt.findPeaks(x_norm[nSample, :], thr=thr, ampType='rel')[0]
if len(inds_pk) == 0:
print('Peak detection failed for sample #', nSample,
'. Try lowering threshold')
excludedSamples[nSample] = 1
else:
blah = aed.SegmentDataByEvents(signals[nSample, :],
inds_pk,
pts_pre,
pts_post,
axis=0)
blah = listToArray(blah)[0]
blah = np.mean(blah, axis=0)
x_seg.append(blah)
ind_max = np.where(blah == np.max(blah))[0][0]
y = spt.standardize(blah[ind_max:])
t = np.arange(len(y)) * dt
popt, pcov = cf(doubleExp, t, y, p0=[10, 20, 0.5], bounds=(0, 20))
if popt[0] > popt[1]:
popt[0:2] = popt[2:0:-1]
popt[-1] = 1 - popt[-1]
params.append(popt)
foo = doubleExp(t, popt[0], popt[1], popt[2])
x_seg_fit.append(foo)
excludedSamples = np.where(excludedSamples)[0]
includedSamples = np.setdiff1d(np.arange(nSamples), excludedSamples)
x_seg, delInds = listToArray(x_seg)
params = np.delete(np.array(params), delInds, axis=0)
delInds = includedSamples[delInds]
if len(delInds) > 0:
print(
'Sample #', delInds,
'excluded for short segment length. Consider decreasing pre-peak time length'
)
excludedSamples = np.union1d(delInds, excludedSamples)
x_seg = spt.standardize(np.array(x_seg), axis=1)
x_seg_fit = np.array(listToArray(x_seg_fit)[0])
out = {
'raw': x_seg,
'fit': x_seg_fit,
'params': np.array(params),
'excludedSamples': excludedSamples
}
return out
replace=True)
var['sat_score'] = np.random.choice(np.arange(1200, 1601),
size=n_samples,
replace=True)
var['hours_of_research_experience'] = np.random.choice(np.arange(20, 110),
size=n_samples,
replace=True)
var['age_of_applicant'] = np.random.choice(np.arange(18, 22),
size=n_samples,
replace=True)
var['harvard_entrance_test_score'] = np.random.choice(np.arange(70, 101), size = n_samples,\
replace = True)
X = np.array([var['gpa'], var['sat_score'], var['hours_of_research_experience'], var['age_of_applicant'],\
var['harvard_entrance_test_score']])
X = spt.standardize(X.T, axis=0)
y = np.dot(X, wts)
y = y + np.random.rand(len(y)) * np.std(y)
y = spt.standardize(y) * 0.8 + 0.2
var['chances_of_acceptance_into_harvard'] = y
data = var
#%%
from sklearn.linear_model import LinearRegression
reg = LinearRegression().fit(X, y)
print('Regression coefficients are {}, and intercept is {}'.format(
reg.coef_, reg.intercept_))
def estimateCaDecayKinetics(time, signals, p0 = None, thr = 2, preTime = 10,
postTime = 40):
"""
Given a time vector and Ca signal matrix of shape = (C,T), where
C = # of cells, and T = # of time points (must match length of time
vector), returns output of shape = (nSamples, 2), where the 1st and
2nd columns contain the fast and slow decay tau estimates after
fitting Ca2+ signals with double exponential
Parameters:
time - Time vector of length T
signals - Ca signals array of shape (nSamples,T)
p0 - Array-like, (tau_fast, tau_slow, wt_fast), where tau_fast is the
fast decay time constant (in sec), tau_slow is the slow decay
constant, and wt_fast is the weight of the fast exponential (<1)
for fitting the signal as a weighted sum of the fast and slow
exponential. Default is None, in which case fitting optimization
begins without initial estimate
thr - Threshold for peak detection in Ca signals, in units of zscore
preTime - Pre-peak time length of the Ca signals to include for segmentation
postTime - Post-peak " " " "
Avinash Pujala, JRC, 2017
"""
import numpy as np
from scipy.optimize import curve_fit as cf
import apCode.SignalProcessingTools as spt
import apCode.AnalyzeEphysData as aed
def doubleExp(time, tau1, tau2, wt1):
wt2 = 1-wt1
time = time - time[0]
e = wt1*np.exp(-time/tau1) + wt2*np.exp(-time/tau2)
return e
def listToArray(x):
lens = [len(item) for item in x]
lenOfLens = len(lens)
lens = lens[np.min((lenOfLens-1,2))]
a = np.zeros((len(x),lens))
delInds = []
for itemNum,item in enumerate(x):
if len(item) == lens:
a[itemNum,:] = item
else:
delInds.append(itemNum)
a = np.delete(a,delInds,axis = 0)
return a, delInds
if np.ndim(signals)==1:
signals = np.reshape(signals,(1,len(signals)))
dt = time[2]-time[1]
pts_post = np.round(postTime/dt).astype(int)
pts_pre = np.round(preTime/dt).astype(int)
x_norm = spt.zscore(signals,axis = 1)
x_seg, params, x_seg_fit = [],[],[]
nSamples = np.shape(signals)[0]
excludedSamples = np.zeros((nSamples,1))
for nSample in np.arange(nSamples):
inds_pk = spt.findPeaks(x_norm[nSample,:],thr = thr,ampType = 'rel')[0]
if len(inds_pk)==0:
print('Peak detection failed for sample #', nSample, '. Try lowering threshold')
excludedSamples[nSample] = 1
else:
blah = aed.SegmentDataByEvents(signals[nSample,:],inds_pk,pts_pre,pts_post,axis = 0)
blah = listToArray(blah)[0]
blah = np.mean(blah,axis=0)
x_seg.append(blah)
ind_max = np.where(blah == np.max(blah))[0][0]
y = spt.standardize(blah[ind_max:])
t = np.arange(len(y))*dt
popt,pcov = cf(doubleExp,t,y,p0 = [10,20, 0.5], bounds = (0,20))
if popt[0]> popt[1]:
popt[0:2] = popt[2:0:-1]
popt[-1] = 1-popt[-1]
params.append(popt)
foo = doubleExp(t,popt[0],popt[1],popt[2])
x_seg_fit.append(foo)
excludedSamples = np.where(excludedSamples)[0]
includedSamples = np.setdiff1d(np.arange(nSamples),excludedSamples)
x_seg,delInds = listToArray(x_seg)
params = np.delete(np.array(params),delInds,axis = 0)
delInds = includedSamples[delInds]
if len(delInds)>0:
print('Sample #', delInds, 'excluded for short segment length. Consider decreasing pre-peak time length')
excludedSamples = np.union1d(delInds,excludedSamples)
x_seg = spt.standardize(np.array(x_seg),axis = 1)
x_seg_fit = np.array(listToArray(x_seg_fit)[0])
out = {'raw': x_seg,'fit': x_seg_fit,'params': np.array(params),'excludedSamples': excludedSamples}
return out