def clipWave(W,freq,time,freqRange,timeRange,coi = []):
'''
W_clip, freq_clip,time_clip,coi_clip = clipWave(W,freq,time,freqRange, \
timeRange,coi = [])
Clips the matrix of wavelet coefficients to the specified freq and time
range
Inputs:
W - 2D matrix of wavelet coefficients
freq - Freq vector of length = number of rows of W
time - Time vector of length = number of cols of W
freqRange - 2 element array specifying the min and max freq to clip W to
timeRange - 2 element array specifying the min and max time to clip W to
coi - Cone of influence; can be empty
Outputs:
W_clip - Clipped W
freq_clip - Clipped freq
...
'''
import apCode.SignalProcessingTools as spt
fInds = spt.nearestMatchingInds(freqRange,freq)
print(fInds)
tInds = spt.valsToNearestInds(timeRange,time)
W_clip = W[fInds[1]:fInds[0],tInds[0]:tInds[1]]
freq_clip = freq[fInds[1]:fInds[0]]
time_clip = time[tInds[0]:tInds[1]]
if len(coi) >0:
coi_clip = coi[tInds[0]:tInds[1]]
else:
coi_clip = coi
return W_clip, freq_clip,time_clip,coi_clip
def clipWave(W, freq, time, freqRange, timeRange, coi=[]):
'''
W_clip, freq_clip,time_clip,coi_clip = clipWave(W,freq,time,freqRange, \
timeRange,coi = [])
Clips the matrix of wavelet coefficients to the specified freq and time
range
Inputs:
W - 2D matrix of wavelet coefficients
freq - Freq vector of length = number of rows of W
time - Time vector of length = number of cols of W
freqRange - 2 element array specifying the min and max freq to clip W to
timeRange - 2 element array specifying the min and max time to clip W to
coi - Cone of influence; can be empty
Outputs:
W_clip - Clipped W
freq_clip - Clipped freq
...
'''
import apCode.SignalProcessingTools as spt
fInds = spt.nearestMatchingInds(freqRange, freq)
print(fInds)
tInds = spt.valsToNearestInds(timeRange, time)
W_clip = W[fInds[1]:fInds[0], tInds[0]:tInds[1]]
freq_clip = freq[fInds[1]:fInds[0]]
time_clip = time[tInds[0]:tInds[1]]
if len(coi) > 0:
coi_clip = coi[tInds[0]:tInds[1]]
else:
coi_clip = coi
return W_clip, freq_clip, time_clip, coi_clip
def plotWave(W,
freq,
time,
coi=[],
powScale='log',
cmap='coolwarm',
xlabel='Time (sec)',
ylabel='Freq (Hz)'):
'''
fh = plotWave(W,freq,time,...)
Plots the matrix of wavelet coefficients W, using the specified freq and time axes
'''
import numpy as np
import apCode.SignalProcessingTools as spt
import matplotlib.pyplot as plt
if powScale.lower() == 'log':
W = np.log2(np.abs(W))
else:
W = np.abs(W)
period = 1 / freq
dt = time[1] - time[0]
tt = np.hstack((time[[0, 0]] - dt * 0.5, time, time[[-1, -1]] + dt * 0.5))
if len(coi) == 0:
coi = time
coi_ext = np.log2(np.hstack((freq[[-1, 1]], 1 / coi, freq[[1, -1]])))
tt = spt.nearestMatchingInds(tt, time)
coi_ext = spt.nearestMatchingInds(coi_ext, freq)
freq_log = np.unique(spt.nextPow2(freq))
fTick = 2**freq_log
yTick = 1 / fTick
inds = spt.nearestMatchingInds(yTick, period)
ytl = (1 / period[inds]).astype(int).ravel()
fig = plt.imshow(W, aspect='auto', cmap=cmap)
fig.axes.set_yticks(inds)
fig.axes.set_yticklabels(np.array(ytl))
plt.colorbar()
xTick = np.linspace(0, len(time) - 1, 5).astype(int)
xtl = np.round(time[xTick] / (time[-1] / 4)) * 0.5
fig.axes.set_xticks(xTick)
fig.axes.set_xticklabels(xtl.ravel())
plt.xlabel(xlabel)
plt.ylabel(ylabel)
#plt.show()
plt.plot(tt, coi_ext, 'k--')
return fig
def plotWave(W,freq,time,coi = [],powScale = 'log',cmap = 'coolwarm', xlabel = 'Time (sec)', ylabel = 'Freq (Hz)'):
'''
fh = plotWave(W,freq,time,...)
Plots the matrix of wavelet coefficients W, using the specified freq and time axes
'''
import numpy as np
import apCode.SignalProcessingTools as spt
import matplotlib.pyplot as plt
if powScale.lower() == 'log':
W = np.log2(np.abs(W))
else:
W = np.abs(W)
period = 1/freq
dt = time[1]-time[0]
tt = np.hstack((time[[0,0]]-dt*0.5,time,time[[-1,-1]]+dt*0.5))
if len(coi) == 0:
coi = time
coi_ext = np.log2(np.hstack((freq[[-1,1]],1/coi,freq[[1,-1]])))
tt = spt.nearestMatchingInds(tt,time)
coi_ext = spt.nearestMatchingInds(coi_ext,freq)
freq_log = np.unique(spt.nextPow2(freq))
fTick = 2**freq_log
yTick = 1/fTick
inds = spt.nearestMatchingInds(yTick,period)
ytl = (1/period[inds]).astype(int).ravel()
fig =plt.imshow(W, aspect = 'auto',cmap = cmap)
fig.axes.set_yticks(inds)
fig.axes.set_yticklabels(np.array(ytl))
plt.colorbar()
xTick = np.linspace(0,len(time)-1,5).astype(int)
xtl = np.round(time[xTick]/(time[-1]/4))*0.5
fig.axes.set_xticks(xTick)
fig.axes.set_xticklabels(xtl.ravel())
plt.xlabel(xlabel)
plt.ylabel(ylabel)
#plt.show()
plt.plot(tt,coi_ext,'k--')
return fig
pyData['swim']['startInds'] = pyData_disk['swim']['startInds'][:].astype(
int) - 1
pyData['swim']['distFromLastStim'] = pyData_disk['swim']['distFromLastStim'][:]
pyData['time'] = pyData_disk['t'][:]
pyData['smooth'] = np.transpose(pyData_disk['smooth']['burst'][:])
pyData['samplingRate'] = np.int(1 / (pyData['time'][1] - pyData['time'][0]))
data['pathToEphysData'] = os.path.join(pyDir, pyFileName)
#%% Geting indices for stacks in ephys data and stims in optical data (can take upto 20 mins)
tic = time.time()
maxInt = int(20e-3 * pyData['samplingRate'])
print('Geting indices for stacks in ephys data and stims in optical data...')
data['inds']['stim'] = spt.nearestMatchingInds(
pyData['time'].ravel()[pyData['stim']['inds']], data['time'])
data['inds']['stackInit'] = spt.nearestMatchingInds(data['time'],
pyData['time'],
processing='parallel')
print(int(time.time() - tic), 'sec')
#%% Subsampling ePhys signals to speed up subsequent regressor convolution and also obtaining some useful indices
tic = time.time()
lpf = 100 # Low pass filter value
stimChan = 1
# Subsample(to speed up with convolution with Ca kernel) smoothed ePhys traces and time vector at about 4 times the Nyquist value
data['ephys'] = pyData.copy()
dt_sub = int(pyData['samplingRate'] / (lpf * 4))
data['ephys']['smooth'] = spt.chebFilt(pyData['smooth'],
inds_stim = np.delete(inds_stim, inds_del)
inds_camTrig = spt.levelCrossings(bas[ch_camTrig], thr=2)[0]
dInds = np.diff(bas['t'][inds_camTrig])
inds_del = np.where(dInds <= (0.5 / frameRate))[0] + 1
inds_camTrig = np.delete(inds_camTrig, inds_del)
# plt.plot(bas[ch_camTrig])
# plt.plot(inds_camTrig,bas[ch_camTrig][inds_camTrig],'o')
print('# stims = {}, # of camera triggers = {}'.format(len(inds_stim),
len(inds_camTrig)))
#%%
#%%
maxAllowedTimeBetweenStimAndCamTrig = 0.5 # In sec
t_preStim = 1
t_postStim = 20
Fs_bas = 6000
Fs_ca = 50
n_preStim = t_preStim * Fs_bas
n_postStim = t_postStim * Fs_bas
inds_camTrigNearStim = inds_camTrig[spt.nearestMatchingInds(
inds_stim, inds_camTrig)]
t_stim = bas['t'][inds_stim]
t_camTrigNearStim = bas['t'][inds_camTrigNearStim]
inds_tooFar = np.where(
np.abs(t_stim -
t_camTrigNearStim) > maxAllowedTimeBetweenStimAndCamTrig)[0]
# plt.plot(bas['t'],bas[ch_stim])
# plt.plot(bas['t'],- bas[ch_camTrig])
# plt.plot(bas['t'][inds_camTrigNearStim], bas[ch_camTrig][inds_camTrigNearStim],'o')
def readPeriStimulusTifImages(tifPaths, basPaths, nBasCh = 16, ch_camTrig = 'patch4', ch_stim = 'patch3',\
tifNameStr = '', time_preStim = 1, time_postStim = 10, thr_stim = 1.5,\
thr_camTrig = 1, maxAllowedTimeBetweenStimAndCamTrig = 0.5, n_jobs = 1):
"""
Given the directory to .tif files stored by ScanImage (Bessel beam image settings) and the full
path to the accompanying bas files returns a dictionary with values holding peri-stimulus
image data (Ca activity) in trialized format along with some other pertinent info.
Parameters
----------
tifDir: string
Path to directory holding .tif files written by ScanImage. In the current setting, each .tif
file holds nCh*3000 images, where nCh = number of channels.
basPath: string
Full path to the bas (BehavAndScan) file accompanying the imaging session.
nBasCh: scalar
Number of signal channels in the bas file.
ch_camTrig: string
Name of the channel in bas corresponding to the camera trigger signal
ch_stim: string
Name of the stimulus signal channel in bas.
tifNameStr: string
Only .tif files containing this will be read.
time_preStim: scalar
The length of the pre-stimulus time to include when reading images
time_postStim: scalar
The length of the post-stimulus time.
thr_stim: scalar
Threshold to use for detection of stimuli in the stimulus channel of bas.
thr_camTrig: scalar
Threshold to use for detection of camera trigger onsets in the camera trigger channel of bas.
maxAllowedTimeBetweenStimAndCamTrig: scalar
If a camera trigger is separated in time by the nearest stimulus by longer than this time
interval, then ignore this stimulus trial.
Returns
-------
D: dict
Dictionary contaning the following keys:
'I': array, (nTrials, nTime, nImageChannels, imageWidth, imageHeight)
Image hyperstack arranged in conveniently-accessible trialized format.
'tifInfo': dict
Dictionary holding useful image metadata. Has following keys:
'filePaths': list of strings
Paths to .tif files
'nImagesInfile': scalar int
Number of images in each .tif file after accounting of number of
image channels
'nChannelsInFile': scalar int
Number of image channels
'inds_stim': array of integers, (nStim,)
Indices in bas coordinates where stimuli occurred.
'inds_stim_img': array of integers, (nStim,)
Indices in image coordinates where stimuli occurred
'inds_camTrig': array of integers, (nCameraTriggers,)
Indices in bas coordinates corresponding to the onsets of camera triggers.
'bas': dict
BehavAndScan data
"""
import tifffile as tff
import numpy as np
import apCode.FileTools as ft
import apCode.ephys as ephys
import apCode.SignalProcessingTools as spt
import apCode.util as util
# import os
def getImgIndsInTifs(tifInfo):
nImgsInFile_cum = np.cumsum(tifInfo['nImagesInFile'] *
tifInfo['nChannelsInFile'])
imgIndsInTifs = []
for i in range(len(nImgsInFile_cum)):
if i == 0:
inds_ = np.arange(0, nImgsInFile_cum[i])
else:
inds_ = np.arange(nImgsInFile_cum[i - 1], nImgsInFile_cum[i])
imgIndsInTifs.append(inds_)
return imgIndsInTifs
### Read relevant metadata from tif files in directory
print('Reading ScanImage metadata from tif files...')
tifInfo = ft.scanImageTifInfo(tifPaths)
nCaImgs = np.sum(tifInfo['nImagesInFile'])
### Check for consistency in the number of image channels in all files.
if len(np.unique(tifInfo['nChannelsInFile'])) > 1:
print('Different number of image channels across files, check files!')
return None
nImgCh = tifInfo['nChannelsInFile'][0]
print(f'{nCaImgs} {nImgCh}-channel images from all tif files')
### Get a list of indices corresponding to images in each of the tif files
inds_imgsInTifs = getImgIndsInTifs(tifInfo)
### Read bas file to get stimulus and camera trigger indices required to align images and behavior
print(
'Reading and joining bas files, detecting stimuli and camera triggers...'
)
basList = [ephys.importCh(bp, nCh=nBasCh) for bp in basPaths]
bas = concatenateBas(basList)
inds_stim = spt.levelCrossings(bas[ch_stim], thr=thr_stim)[0]
if len(inds_stim) == 0:
print(
f'Only {len(inds_stim)} stims detected, check channel specification or threshold'
)
return dict(bas=bas)
inds_camTrig = spt.levelCrossings(bas[ch_camTrig], thr=thr_camTrig)[0]
if len(inds_camTrig) == 0:
print(
f'Only {len(inds_camTrig)} cam trigs detected, check channel specification or threshold'
)
return dict(bas=bas)
dt_vec = np.diff(bas['t'][inds_camTrig])
dt_ca = np.round(np.mean(dt_vec) * 100) / 100
print('Ca sampling rate = {}'.format(1 / dt_ca))
inds_del = np.where(dt_vec <= (0.5 * dt_ca))[0] + 1
inds_camTrig = np.delete(inds_camTrig, inds_del)
### Deal with possible mismatch in number of camera trigger indices and number of images in tif files
if nCaImgs < len(inds_camTrig):
inds_camTrig = inds_camTrig[:nCaImgs]
nCaImgs_extra = 0
elif nCaImgs > len(inds_camTrig):
nCaImgs_extra = nCaImgs - len(inds_camTrig)
else:
nCaImgs_extra = 0
print('{} extra Ca2+ images'.format(nCaImgs_extra))
print('{} stimuli and {} camera triggers'.format(len(inds_stim),
len(inds_camTrig)))
### Indices of ca images closest to stimulus
inds_stim_img = spt.nearestMatchingInds(inds_stim, inds_camTrig)
### Find trials where the nearest cam trigger is farther than the stimulus by a certain amount
inds_camTrigNearStim = inds_camTrig[inds_stim_img]
t_stim = bas['t'][inds_stim]
t_camTrigNearStim = bas['t'][inds_camTrigNearStim]
inds_tooFar = np.where(
np.abs(t_stim -
t_camTrigNearStim) > maxAllowedTimeBetweenStimAndCamTrig)[0]
inds_ca_all = np.arange(nCaImgs)
nPreStim = int(time_preStim / dt_ca)
nPostStim = int(time_postStim / dt_ca)
print("{} pre-stim points, and {} post-stim points".format(
nPreStim, nPostStim))
inds_ca_trl = np.array(
spt.segmentByEvents(inds_ca_all, inds_stim_img + nCaImgs_extra,
nPreStim, nPostStim))
### Find trials that are too short to include the pre- or post-stimulus period
trlLens = np.array([len(trl_) for trl_ in inds_ca_trl])
inds_tooShort = np.where(trlLens < np.max(trlLens))[0]
inds_trl_del = np.union1d(inds_tooFar, inds_tooShort)
inds_trl_keep = np.setdiff1d(np.arange(len(inds_ca_trl)), inds_trl_del)
### Exclude the above 2 types of trials from consideration
if len(inds_trl_del) > 0:
print('Excluding the trials {}'.format(inds_trl_del))
inds_ca_trl = inds_ca_trl[inds_trl_keep]
I = []
print('Reading trial-related images from tif files...')
nTrls = len(inds_ca_trl)
def trlImages(inds_ca_trl, inds_imgsInTifs, nImgCh, tifInfo, trl):
trl_ = np.arange(trl.min() * nImgCh, (trl.max() + 1) * nImgCh)
loc = util.locateItemsInSetsOfItems(trl_, inds_imgsInTifs)
I_ = []
for subInds, supInd in zip(loc['subInds'], loc['supInds']):
with tff.TiffFile(tifInfo['filePaths'][supInd]) as tif:
img = tif.asarray(key=subInds)
I_.extend(img.reshape(-1, nImgCh, *img.shape[1:]))
I_ = np.array(I_)
return I_
if n_jobs < 2:
chunkSize = int(nTrls / 5)
for iTrl, trl in enumerate(inds_ca_trl):
if np.mod(iTrl, chunkSize) == 0:
print('Trl # {}/{}'.format(iTrl + 1, nTrls))
I_ = trlImages(inds_ca_trl, inds_imgsInTifs, nImgCh, tifInfo, trl)
I.append(I_)
else:
print('Processing with dask')
import dask
from dask.diagnostics import ProgressBar
for trl in inds_ca_trl:
I_ = dask.delayed(trlImages)(inds_ca_trl, inds_imgsInTifs, nImgCh,
tifInfo, trl)
I.append(I_)
with ProgressBar():
I = dask.compute(*I)
D = dict(I = np.squeeze(np.array(I)), tifInfo = tifInfo, inds_stim = inds_stim, inds_stim_img = inds_stim_img,\
inds_camTrig = inds_camTrig,bas = bas, inds_trl_excluded = inds_trl_del)
return D
def dataFrameOfMatchedMtrAndCaTrls_singleFish(bas, ca = None, ch_camTrig = 'camTrigger', ch_stim = 'patch3',
ch_switch = 'patch2', thr_camTrig= 4, Fs_bas = 6000, t_pre_bas = 0.2,\
t_post_bas = 1.5, t_pre_ca = 1, t_post_ca = 10, n_jobs = 20):
"""
Parameters
----------
bas: dict
Dictionary resulting from reading of BAS file
ca: array, (nRois, nSamples)
nSamples is expected to match the number of Camera triggers
Returns
-------
df: Pandas dataframe
"""
import numpy as np
from scipy.stats import mode
import apCode.SignalProcessingTools as spt
from apCode import util
import pandas as pd
def motorFromBas(bas):
keys = list(bas.keys())
ind_mtr = util.findStrInList('den', keys)
if len(ind_mtr) > 0:
ind_mtr = ind_mtr[-1]
x = np.squeeze(np.array(bas[keys[ind_mtr]]))
else:
if 'ch3' in bas:
x = np.array([bas['ch3'], bas['ch4']])
else:
x = np.array([bas['ch1'], bas['ch2']])
if x.shape[0] < x.shape[1]:
x = x.T
return x
mtr = motorFromBas(bas)
stimInds, stimAmps, stimHt = getStimInfo(bas, ch_stim = ch_stim,\
ch_camTrig= ch_camTrig, ch_switch= ch_switch)
camTrigInds = getCamTrigInds(bas[ch_camTrig], thr=thr_camTrig)
# dt_bas = 1/Fs_bas
dt_ca = np.mean(np.diff(bas['t'][camTrigInds]))
Fs_ca = int(np.round(1 / dt_ca))
if np.any(ca == None):
nFrames = len(camTrigInds)
# t_ca = np.arange(nFrames)*dt_ca
else:
nFrames = ca.shape[1]
# t_ca = np.arange(nFrames)*dt_ca
d = len(camTrigInds) - ca.shape[1]
if d > 0:
print(f'Check threshold, {d} more camera triggers than expected')
elif d < 0:
print('Check threshold, {d} fewer camera triggers than expected')
n_pre_bas = int(np.round(t_pre_bas * Fs_bas))
n_post_bas = int(np.round(t_post_bas * Fs_bas))
n_pre_ca = int(np.round(t_pre_ca * Fs_ca))
n_post_ca = int(np.round(t_post_ca * Fs_ca))
stimFrames = spt.nearestMatchingInds(stimInds, camTrigInds) - 2
mtr_trl = spt.segmentByEvents(mtr,
stimInds,
n_pre_bas,
n_post_bas,
n_jobs=n_jobs)
indsVec_ca = np.arange(nFrames)
indsVec_ca_trl = spt.segmentByEvents(indsVec_ca,
stimFrames,
n_pre_ca,
n_post_ca,
n_jobs=n_jobs)
trlNum_actual = np.arange(len(mtr_trl))
lens_mtr = np.array([len(mtr_) for mtr_ in mtr_trl])
lens_ca = np.array([len(inds_) for inds_ in indsVec_ca_trl])
inds_del_mtr = np.where(lens_mtr != mode(lens_mtr)[0])[0]
inds_del_ca = np.where(lens_ca != mode(lens_ca)[0])[0]
inds_del_trl = np.union1d(inds_del_mtr, inds_del_ca)
trlNum_actual = np.delete(trlNum_actual, inds_del_trl)
mtr_trl = list(np.delete(
mtr_trl, inds_del_trl,
axis=0)) # Converting to list so that can later put in dataframe
indsVec_ca_trl = list(np.delete(indsVec_ca_trl, inds_del_trl, axis=0))
dic = dict(mtr_trl=mtr_trl, caInds_trl=indsVec_ca_trl)
if not np.any(ca == None):
if np.ndim(ca) == 1:
ca = ca[np.newaxis, ...]
ca_trl = [ca[:, inds_] for inds_ in indsVec_ca_trl]
dic['ca_trl'] = ca_trl
trlNum = np.arange(len(mtr_trl))
dic['trlNum'] = trlNum
dic['trlNum_actual'] = trlNum_actual
dic['stimAmp'] = np.delete(stimAmps, inds_del_trl)
dic['stimLoc'] = np.delete(stimHt, inds_del_trl)
return pd.DataFrame(dic, columns=dic.keys())
pyData['stim']['inds'] = pyData_disk['stim']['inds'][:].ravel().astype(int)-1
pyData['swim']['startInds'] = pyData_disk['swim']['startInds'][:].astype(int)-1
pyData['swim']['distFromLastStim'] = pyData_disk['swim']['distFromLastStim'][:]
pyData['time'] = pyData_disk['t'][:]
pyData['smooth'] = np.transpose(pyData_disk['smooth']['burst'][:])
pyData['samplingRate'] = np.int(1/(pyData['time'][1]-pyData['time'][0]))
data['pathToEphysData'] = os.path.join(pyDir,pyFileName)
#%% Geting indices for stacks in ephys data and stims in optical data (can take upto 20 mins)
tic = time.time()
maxInt = int(20e-3*pyData['samplingRate'])
print('Geting indices for stacks in ephys data and stims in optical data...')
data['inds']['stim'] = spt.nearestMatchingInds(pyData['time'].ravel()[pyData['stim']['inds']],data['time'])
data['inds']['stackInit'] = spt.nearestMatchingInds(data['time'],pyData['time'],processing = 'parallel')
print(int(time.time()-tic),'sec')
#%% Subsampling ePhys signals to speed up subsequent regressor convolution and also obtaining some useful indices
tic = time.time()
lpf = 100 # Low pass filter value
stimChan = 1
# Subsample(to speed up with convolution with Ca kernel) smoothed ePhys traces and time vector at about 4 times the Nyquist value
data['ephys'] = pyData.copy()
dt_sub = int(pyData['samplingRate']/(lpf*4))
data['ephys']['smooth'] = spt.chebFilt(pyData['smooth'],1/pyData['samplingRate'],lpf,
btype = 'low')[::dt_sub]