def make(self, key):
time_back = .0002
time_capacitance = .0001
time_forward = .0002
df_squarepulse = pd.DataFrame((SquarePulse()&key)*ephys_patch.Sweep()*ephys_patch.SweepResponse()*ephys_patch.SweepMetadata())
stimamplitude = df_squarepulse['square_pulse_amplitude'].values[0]
if np.abs(stimamplitude)>=40*10**-12:
trace = df_squarepulse['response_trace'].values[0]
start_idx = df_squarepulse['square_pulse_start_idx'][0]
end_idx = df_squarepulse['square_pulse_end_idx'][0]
sr = df_squarepulse['sample_rate'][0]
step_back = int(np.round(time_back*sr))
step_capacitance = int(np.round(time_capacitance*sr))
step_forward = int(np.round(time_forward*sr))
v0_start = np.mean(trace[start_idx-step_back:start_idx])
vrs_start = np.mean(trace[start_idx+step_capacitance:start_idx+step_capacitance+step_forward])
v0_end = np.mean(trace[end_idx-step_back:end_idx])
vrs_end = np.mean(trace[end_idx+step_capacitance:end_idx+step_capacitance+step_forward])
dv_start = vrs_start-v0_start
RS_start = dv_start/stimamplitude
dv_end = vrs_end-v0_end
RS_end = dv_end/stimamplitude*-1
RS = np.round(np.mean([RS_start,RS_end])/1000000,2)
key['series_resistance_squarepulse'] = RS
self.insert1(key,skip_duplicates=True)
def make(self, key):
try:
#%
# key={'subject_id': 463291, 'session': 1, 'cell_number': 0, 'sweep_number': 64}#{'subject_id': 466769, 'session': 1, 'cell_number': 2, 'sweep_number': 45}
junction_potential = 13.5 / 1000 #mV
e_sr = (ephys_patch.SweepMetadata() & key).fetch1('sample_rate')
uncompensatedRS = float(
(SweepSeriesResistance()
& key).fetch1('series_resistance_residual'))
v = (ephys_patch.SweepResponse() & key).fetch1('response_trace')
i = (ephys_patch.SweepStimulus() & key).fetch1('stimulus_trace')
tau_1_on = .1 / 1000
t = np.arange(0, .001, 1 / e_sr)
f_on = np.exp(t / tau_1_on)
f_on = f_on / np.max(f_on)
kernel = np.concatenate([f_on, np.zeros(len(t))])[::-1]
kernel = kernel / sum(kernel)
i_conv = np.convolve(i, kernel, 'same')
v_comp = (v -
i_conv * uncompensatedRS * 10**6) - junction_potential
key['response_trace_corrected'] = v_comp
#%
self.insert1(key, skip_duplicates=True)
except:
print(key)
def plot_IV(subject_id=454597, cellnum=1, ivnum=0, IVsweepstoplot=None):
#%
key = {'subject_id': subject_id, 'cell_number': cellnum}
sweeps = pd.DataFrame(ephys_patch.Sweep() & key)
protocolnames = sweeps['protocol_name'].unique()
ivprotocolnames = [i for i in protocolnames if 'iv' in i.lower()]
ivprotocolname = ivprotocolnames[ivnum]
key['protocol_name'] = ivprotocolname
#
df_iv = pd.DataFrame()
#%
if IVsweepstoplot == None:
#%
ivsweepnums = (ephys_patch.Sweep()
& key).fetch('protocol_sweep_number')
for iwsweepnum in ivsweepnums:
key['protocol_sweep_number'] = iwsweepnum
apnum = len(ephysanal.ActionPotential() * ephys_patch.Sweep()
& key)
if apnum > 2:
break
IVsweepstoplot = [0, iwsweepnum]
for sweepnum in IVsweepstoplot:
key['protocol_sweep_number'] = sweepnum
df_iv = pd.concat([
df_iv,
pd.DataFrame((ephys_patch.Sweep() & key) *
(ephys_patch.SweepResponse() & key) *
(ephys_patch.SweepStimulus() & key) *
(ephys_patch.SweepMetadata() & key))
])
df_IV = pd.DataFrame()
for line in df_iv.iterrows():
linenow = line[1]
time = np.arange(0, len(
linenow['response_trace'])) / linenow['sample_rate']
linenow['time'] = time
df_IV = pd.concat([df_IV, linenow.to_frame().transpose()])
fig = plt.figure()
ax_IV = fig.add_subplot(211) #add_axes([0,0,1,1])
ax_stim = fig.add_subplot(212) #.add_axes([0,-.4,1,.2])
for line in df_IV.iterrows():
ax_IV.plot(line[1]['time'], line[1]['response_trace'] * 1000, 'k-')
ax_stim.plot(line[1]['time'], line[1]['stimulus_trace'] * 10**12, 'k-')
ax_IV.set_xlabel('Time (s)')
ax_IV.set_xlim([0, 1])
ax_IV.set_ylabel('mV')
ax_IV.set_title('subject: {} cell: {}'.format(subject_id, cellnum))
ax_stim.set_xlabel('Time (s)')
ax_stim.set_xlim([0, 1])
ax_stim.set_ylabel('pA')
#ax_stim.set_title('Stimulus')
return fig
def make(self, key):
#%%
#key = {'subject_id': 454263, 'session': 1, 'cell_number': 1, 'sweep_number': 62}
#print(key)
keynow = key.copy()
if len(ActionPotential()&keynow) == 0:
pd_sweep = pd.DataFrame((ephys_patch.Sweep()&key)*(ephys_patch.SweepResponse()&key)*(ephys_patch.SweepStimulus()&key)*(ephys_patch.SweepMetadata()&key))
if len(pd_sweep)>0:
trace = pd_sweep['response_trace'].values[0]
sr = pd_sweep['sample_rate'][0]
si = 1/sr
sigma = .00005
trace_f = ndimage.gaussian_filter(trace,sigma/si)
d_trace_f = np.diff(trace_f)/si
peaks = d_trace_f > 40
peaks = ndimage.morphology.binary_dilation(peaks,np.ones(int(round(.002/si))))
spikemaxidxes = list()
while np.any(peaks):
#%%
spikestart = np.argmax(peaks)
spikeend = np.argmin(peaks[spikestart:])+spikestart
if spikestart == spikeend:
if sum(peaks[spikestart:]) == len(peaks[spikestart:]):
spikeend = len(trace)
try:
sipeidx = np.argmax(trace[spikestart:spikeend])+spikestart
except:
print(key)
sipeidx = np.argmax(trace[spikestart:spikeend])+spikestart
spikemaxidxes.append(sipeidx)
peaks[spikestart:spikeend] = False
#%%
if len(spikemaxidxes)>0:
spikemaxtimes = spikemaxidxes/sr + float(pd_sweep['sweep_start_time'].values[0])
spikenumbers = np.arange(len(spikemaxidxes))+1
keylist = list()
for spikenumber,spikemaxidx,spikemaxtime in zip(spikenumbers,spikemaxidxes,spikemaxtimes):
keynow =key.copy()
keynow['ap_num'] = spikenumber
keynow['ap_max_index'] = spikemaxidx
keynow['ap_max_time'] = spikemaxtime
keylist.append(keynow)
#%%
self.insert(keylist,skip_duplicates=True)
def plot_IV(wr_name='FOR04', cellnum=1, ivnum=0, IVsweepstoplot=[0, 14]):
subject_id = (lab.WaterRestriction() & 'water_restriction_number = "' +
wr_name + '"').fetch('subject_id')[0]
key = {'subject_id': subject_id, 'cell_number': cellnum}
sweeps = pd.DataFrame(ephys_patch.Sweep() & key)
protocolnames = sweeps['protocol_name'].unique()
ivprotocolnames = [i for i in protocolnames if 'iv' in i.lower()]
ivprotocolname = ivprotocolnames[ivnum]
key['protocol_name'] = ivprotocolname
#%
df_iv = pd.DataFrame()
for sweepnum in IVsweepstoplot:
key['protocol_sweep_number'] = sweepnum
df_iv = pd.concat([
df_iv,
pd.DataFrame((ephys_patch.Sweep() & key) *
(ephys_patch.SweepResponse() & key) *
(ephys_patch.SweepStimulus() & key) *
(ephys_patch.SweepMetadata() & key))
])
df_IV = pd.DataFrame()
for line in df_iv.iterrows():
linenow = line[1]
time = np.arange(0, len(
linenow['response_trace'])) / linenow['sample_rate']
linenow['time'] = time
df_IV = pd.concat([df_IV, linenow.to_frame().transpose()])
fig = plt.figure()
ax_IV = fig.add_axes([0, 0, 2, .8])
ax_stim = fig.add_axes([0, -.6, 2, .4])
for line in df_IV.iterrows():
ax_IV.plot(line[1]['time'], line[1]['response_trace'] * 1000, '-')
ax_stim.plot(line[1]['time'], line[1]['stimulus_trace'] * 10**12, '-')
ax_IV.set_xlabel('Time (s)')
ax_IV.set_xlim([0, 1])
ax_IV.set_ylabel('mV')
ax_IV.set_title('Firing pattern')
ax_stim.set_xlabel('Time (s)')
ax_stim.set_xlim([0, 1])
ax_stim.set_ylabel('pA')
ax_stim.set_title('Stimulus')
def get_sweep(key_sweep,junction_potential = 13.5, downsampled_rate = 10000):
key_sweep['cell_number'] = (imaging_gt.CellMovieCorrespondance()&key_sweep).fetch1('cell_number')
#%
# =============================================================================
# key_sweep = {'subject_id': 456462,'session' : 1, 'movie_number' : 0, 'cell_number' : 3,'sweep_number':28}
# junction_potential = 13.5 #mV
# downsampled_rate = 10000 #Hz
# =============================================================================
neutralizationenable,e_sr= (ephys_patch.SweepMetadata()&key_sweep).fetch1('neutralizationenable','sample_rate')
try:
uncompensatedRS = float((ephysanal.SweepSeriesResistance()&key_sweep).fetch1('series_resistance_residual'))
except:
uncompensatedRS = 0
v = (ephys_patch.SweepResponse()&key_sweep).fetch1('response_trace')
i = (ephys_patch.SweepStimulus()&key_sweep).fetch1('stimulus_trace')
tau_1_on =.1/1000
t = np.arange(0,.001,1/e_sr)
f_on = np.exp(t/tau_1_on)
f_on = f_on/np.max(f_on)
kernel = np.concatenate([f_on,np.zeros(len(t))])[::-1]
kernel = kernel /sum(kernel )
i_conv = np.convolve(i,kernel,'same')
v_comp = (v - i_conv*uncompensatedRS*10**6)*1000 - junction_potential
i = i * 10**12
sweep_start_time = float((ephys_patch.Sweep()&key_sweep).fetch('sweep_start_time'))
trace_t = np.arange(len(v))/e_sr + sweep_start_time
downsample_factor = int(np.round(e_sr/downsampled_rate))
#%downsampling
v_out = moving_average(v_comp, n=downsample_factor)
v_out = v_out[int(downsample_factor/2)::downsample_factor]
i_out = moving_average(i, n=downsample_factor)
i_out = i_out[int(downsample_factor/2)::downsample_factor]
t_out = moving_average(trace_t, n=downsample_factor)
t_out = t_out[int(downsample_factor/2)::downsample_factor]
return v_out, i_out, t_out
def plot_cell_SN_ratio_APwise(roi_type = 'VolPy',v0_max = -35,holding_min = -600,frame_rate_min =300, frame_rate_max = 800 ,bin_num = 10 ):
#%% Show S/N ratios for each AP
cmap = cm.get_cmap('jet')
# =============================================================================
# bin_num = 10
# holding_min = -600 #pA
# v0_max = -35 #mV
# roi_type = 'Spikepursuit'#'Spikepursuit'#'VolPy_denoised'#'SpikePursuit'#'VolPy_dexpF0'#'VolPy'#'SpikePursuit_dexpF0'#'VolPy_dexpF0'#''Spikepursuit'#'VolPy'#
# =============================================================================
key = {'roi_type':roi_type}
gtdata = pd.DataFrame((imaging_gt.GroundTruthROI()&key))
cells = gtdata.groupby(['session', 'subject_id','cell_number','motion_correction_method','roi_type']).size().reset_index(name='Freq')
snratio = list()
v0s = list()
holdings = list()
rss = list()
threshs =list()
mintreshs = list()
f0s_all = list()
snratios_all = list()
peakamplitudes_all = list()
noise_all = list()
for cell in cells.iterrows():
cell = cell[1]
key_cell = dict(cell)
del key_cell['Freq']
snratios,f0,peakamplitudes,noises = (imaging.Movie()*imaging_gt.GroundTruthROI()*imaging_gt.ROIAPWave()*ephysanal.ActionPotentialDetails()&key_cell&'ap_real = 1'&'movie_frame_rate > {}'.format(frame_rate_min)&'movie_frame_rate < {}'.format(frame_rate_max)).fetch('apwave_snratio','apwave_f0','apwave_peak_amplitude','apwave_noise')
#f0 = (imaging_gt.GroundTruthROI()*imaging.ROI()&key_cell).fetch('roi_f0')
sweep = (imaging_gt.GroundTruthROI()*imaging_gt.ROIAPWave()&key_cell).fetch('sweep_number')[0]
thresh = (imaging_gt.GroundTruthROI()*imaging_gt.ROIAPWave()*ephysanal.ActionPotentialDetails()&key_cell&'ap_real = 1').fetch('ap_threshold')
trace = (ephys_patch.SweepResponse()*imaging_gt.GroundTruthROI()&key_cell&'sweep_number = {}'.format(sweep)).fetch('response_trace')
trace =trace[0]
stimulus = (ephys_patch.SweepStimulus()*imaging_gt.GroundTruthROI()&key_cell&'sweep_number = {}'.format(sweep)).fetch('stimulus_trace')
stimulus =stimulus[0]
RS = (ephysanal.SweepSeriesResistance()*imaging_gt.GroundTruthROI()&key_cell&'sweep_number = {}'.format(sweep)).fetch('series_resistance')
RS =RS[0]
medianvoltage = np.median(trace)*1000
holding = np.median(stimulus)*10**12
#print(np.mean(snratios[:100]))
snratio.append(np.mean(snratios[:50]))
v0s.append(medianvoltage)
holdings.append(holding)
rss.append(RS)
threshs.append(thresh)
mintreshs.append(np.min(thresh))
f0s_all.append(f0)
snratios_all.append(snratios)
peakamplitudes_all.append(peakamplitudes)
noise_all.append(noises)
#plot_AP_waveforms(key_cell,AP_tlimits)
#%% for each AP
fig=plt.figure()
ax_sn_f0 = fig.add_axes([0,0,1,1])
ax_sn_f0_binned = fig.add_axes([0,-1.2,1,1])
ax_noise_f0 = fig.add_axes([2.6,0,1,1])
ax_noise_f0_binned = fig.add_axes([2.6,-1.2,1,1])
ax_peakampl_f0 = fig.add_axes([1.3,0,1,1])
ax_peakampl_f0_binned = fig.add_axes([1.3,-1.2,1,1])
for loopidx, (f0,snratio_now,noise_now,peakampl_now,cell_now) in enumerate(zip(f0s_all,snratios_all,noise_all,peakamplitudes_all,cells.iterrows())):
if len(f0)>0:
coloridx = loopidx/len(cells)
cell_now = cell_now[1]
label_now = 'Subject:{}'.format(cell_now['subject_id'])+' Cell:{}'.format(cell_now['cell_number'])
ax_sn_f0.plot(f0,snratio_now,'o',ms=1, color = cmap(coloridx), label= label_now)
ax_noise_f0.plot(f0,noise_now,'o',ms=1, color = cmap(coloridx), label= label_now)
ax_peakampl_f0.plot(f0,peakampl_now,'o',ms=1, color = cmap(coloridx), label= label_now)
lows = np.arange(np.min(f0),np.max(f0),(np.max(f0)-np.min(f0))/(bin_num+1))
highs = lows + (np.max(f0)-np.min(f0))/(bin_num+1)
mean_f0 = list()
sd_f0 = list()
mean_sn = list()
sd_sn =list()
mean_noise = list()
sd_noise =list()
mean_ampl = list()
sd_ampl =list()
for low,high in zip(lows,highs):
idx = (f0 >= low) & (f0 < high)
if len(idx)>10:
mean_f0.append(np.mean(f0[idx]))
sd_f0.append(np.std(f0[idx]))
mean_sn.append(np.mean(snratio_now[idx]))
sd_sn.append(np.std(snratio_now[idx]))
mean_noise.append(np.mean(noise_now[idx]))
sd_noise.append(np.std(noise_now[idx]))
mean_ampl.append(np.mean(peakampl_now[idx]))
sd_ampl.append(np.std(peakampl_now[idx]))
ax_sn_f0_binned.errorbar(mean_f0,mean_sn,sd_sn,sd_f0,'o-', color = cmap(coloridx), label= label_now)
ax_noise_f0_binned.errorbar(mean_f0,mean_noise,sd_noise,sd_f0,'o-', color = cmap(coloridx), label= label_now)
ax_peakampl_f0_binned.errorbar(mean_f0,mean_ampl,sd_ampl,sd_f0,'o-', color = cmap(coloridx), label= label_now)
ax_sn_f0.set_xlabel('F0')
ax_sn_f0.set_ylabel('S/N ratio')
ax_sn_f0_binned.set_xlabel('F0')
ax_sn_f0_binned.set_ylabel('S/N ratio')
#ax_sn_f0_binned.legend()
ax_sn_f0_binned.legend(loc='upper center', bbox_to_anchor=(-.45, 1.5), shadow=True, ncol=1)
ax_noise_f0.set_xlabel('F0')
ax_noise_f0.set_ylabel('Noise (std(dF/F))')
ax_noise_f0_binned.set_xlabel('F0')
ax_noise_f0_binned.set_ylabel('Noise (std(dF/F))')
ax_peakampl_f0.set_xlabel('F0')
ax_peakampl_f0.set_ylabel('Peak amplitude (dF/F)')
ax_peakampl_f0_binned.set_xlabel('F0')
ax_peakampl_f0_binned.set_ylabel('Peak amplitude (dF/F)')
#%%
cells['SN']=snratio
cells['V0']=v0s
cells['holding']=holdings
cells['RS']=np.asarray(rss,float)
print(cells)
cells = cells[cells['V0']<v0_max]
cells = cells[cells['holding']>holding_min]
print(cells)
#% S/N ratio histogram
fig=plt.figure()
ax_hist = fig.add_axes([0,0,1,1])
ax_hist.hist(cells['SN'].values)
ax_hist.set_xlabel('S/N ratio of first 50 spikes')
ax_hist.set_ylabel('# of cells')
ax_hist.set_title(roi_type.replace('_',' '))
ax_hist.set_xlim([0,15])
def plot_AP_waveforms(key,
AP_tlimits = [-.005,.01],
bin_step = .00001,
bin_size = .00025,
save_image = False):
#%
select_high_sn_APs = False
# =============================================================================
# bin_step = .00001
# bin_size = .00025
# =============================================================================
#%
tau_1_on = .64/1000
tau_2_on = 4.1/1000
tau_1_ratio_on = .61
tau_1_off = .78/1000
tau_2_off = 3.9/1000
tau_1_ratio_off = 55
#%
movie_numbers,sweep_numbers,apwavetimes,apwaves,famerates,snratio,apnums,ap_threshold = ((imaging_gt.GroundTruthROI()*imaging.Movie()*imaging_gt.ROIAPWave()*ephysanal.ActionPotentialDetails())&key&'ap_real = 1').fetch('movie_number','sweep_number','apwave_time','apwave_dff','movie_frame_rate','apwave_snratio','ap_num','ap_threshold')
uniquemovienumbers = np.unique(movie_numbers)
for movie_number in uniquemovienumbers:
fig=plt.figure()
ax_ephys=fig.add_axes([0,0,1,1])
ax_raw=fig.add_axes([0,1.1,1,1])
aps_now = movie_numbers == movie_number
ax_bin=fig.add_axes([1.3,1.1,1,1])
ax_e_convolved = fig.add_axes([1.3,0,1,1])
ax_bin.set_title('{} ms binning'.format(bin_size*1000))
if select_high_sn_APs :
medsn = np.median(snratio[aps_now])
aps_now = (movie_numbers == movie_number) & (snratio>medsn)
framerate = famerates[aps_now][0]
apwavetimes_conc = np.concatenate(apwavetimes[aps_now])
apwaves_conc = np.concatenate(apwaves[aps_now])
prev_sweep = None
ephys_vs = list()
for apwavetime,apwave,sweep_number,ap_num in zip(apwavetimes[aps_now],apwaves[aps_now],sweep_numbers[aps_now],apnums[aps_now]):
wave_needed_idx = (apwavetime>=AP_tlimits[0]-1/framerate) & (apwavetime<=AP_tlimits[1]+1/framerate)
ax_raw.plot(apwavetime[wave_needed_idx ]*1000,apwave[wave_needed_idx ])
if prev_sweep != sweep_number:
#%
trace = (ephys_patch.SweepResponse()&key&'sweep_number = {}'.format(sweep_number)).fetch1('response_trace')*1000
e_sr = (ephys_patch.SweepMetadata()&key&'sweep_number = {}'.format(sweep_number)).fetch1('sample_rate')
stepback = int(np.abs(np.round(AP_tlimits[0]*e_sr)))
stepforward = int(np.abs(np.round(AP_tlimits[1]*e_sr)))
ephys_t = np.arange(-stepback,stepforward)/e_sr * 1000
prev_sweep = sweep_number
#%
apmaxindex = (ephysanal.ActionPotential()&key & 'sweep_number = {}'.format(sweep_number) & 'ap_num = {}'.format(ap_num)).fetch1('ap_max_index')
ephys_v = trace[apmaxindex-stepback:apmaxindex+stepforward]
ephys_vs.append(ephys_v)
ax_ephys.plot(ephys_t,ephys_v)
#break
#%
mean_ephys_v = np.mean(np.asarray(ephys_vs),0)
#%
t = np.arange(0,.01,1/e_sr)
f_on = tau_1_ratio_on*np.exp(t/tau_1_on) + (1-tau_1_ratio_on)*np.exp(-t/tau_2_on)
f_off = tau_1_ratio_off*np.exp(t[::-1]/tau_1_off) + (1-tau_1_ratio_off)*np.exp(-t[::-1]/tau_2_off)
f_on = f_on/np.max(f_on)
f_off = f_off/np.max(f_off)
kernel = np.concatenate([f_on,np.zeros(len(f_off))])[::-1]
kernel = kernel /sum(kernel )
trace_conv0 = np.convolve(np.concatenate([mean_ephys_v[::-1],mean_ephys_v,mean_ephys_v[::-1]]),kernel,mode = 'same')
trace_conv0 = trace_conv0[len(mean_ephys_v):2*len(mean_ephys_v)]
kernel = np.ones(int(np.round(e_sr/framerate)))
kernel = kernel /sum(kernel )
trace_conv = np.convolve(np.concatenate([trace_conv0[::-1],trace_conv0,trace_conv0[::-1]]),kernel,mode = 'same')
trace_conv = trace_conv[len(mean_ephys_v):2*len(mean_ephys_v)]
bin_centers = np.arange(np.min(apwavetime),np.max(apwavetime),bin_step)
bin_mean = list()
for bin_center in bin_centers:
bin_mean.append(np.mean(apwaves_conc[(apwavetimes_conc>bin_center-bin_size/2) & (apwavetimes_conc<bin_center+bin_size/2)]))
ax_bin.plot(bin_centers*1000,np.asarray(bin_mean),'g-')
ax_bin.invert_yaxis()
ax_bin.set_xlim(np.asarray(AP_tlimits)*1000)
ax_raw.invert_yaxis()
ax_raw.autoscale(tight = True)
ax_raw.set_xlim(np.asarray(AP_tlimits)*1000)
ax_raw.set_ylabel('dF/F')
ax_raw.set_title('subject: {} cell: {} movie: {} apnum: {}'.format(key['subject_id'],key['cell_number'],movie_number,sum(aps_now)))
ax_ephys.set_xlim(np.asarray(AP_tlimits)*1000)
ax_ephys.set_xlabel('ms')
ax_ephys.set_ylabel('mV')
ax_e_convolved.plot(ephys_t,mean_ephys_v,'k-',label = 'mean')
ax_e_convolved.plot(ephys_t,trace_conv0,'g--',label = 'convolved mean')
ax_e_convolved.plot(ephys_t,trace_conv,'g-',label = 'convolved & binned mean')
ax_e_convolved.legend()
ax_e_convolved.set_xlim(np.asarray(AP_tlimits)*1000)
ax_e_convolved.set_xlabel('ms')
plt.show()
imaging_gt.ROIEphysCorrelation()
if save_image:
fig.savefig('./figures/APwaveforms_subject_{}_cell_{}_movie_{}.png'.format(key['subject_id'],key['cell_number'],movie_number), bbox_inches = 'tight')
def plot_ephys_ophys_trace(key_cell,time_to_plot=None,trace_window = 1,show_stimulus = False,show_e_ap_peaks = False, show_o_ap_peaks = False):
#%%
# =============================================================================
# key_cell = {'session': 1,
# 'subject_id': 456462,
# 'cell_number': 3,
# 'motion_correction_method': 'VolPy',
# 'roi_type': 'VolPy'}
# time_to_plot=None
# trace_window = 100
# show_stimulus = False
# show_e_ap_peaks = True
# show_o_ap_peaks = True
# =============================================================================
fig=plt.figure()
ax_ophys = fig.add_axes([0,0,2,.8])
ax_ephys = fig.add_axes([0,-1,2,.8])
if show_stimulus:
ax_ephys_stim = fig.add_axes([0,-1.5,2,.4])
#%
session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key_cell).fetch1('session_time','cell_recording_start')
first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_start_time'),float))
first_movie_start_time_real = first_movie_start_time + session_time.total_seconds()
if not time_to_plot:
time_to_plot = trace_window/2#ephys_matched_ap_times[0]
session_time_to_plot = time_to_plot+first_movie_start_time # time relative to session start
cell_time_to_plot= session_time_to_plot + session_time.total_seconds() -cell_recording_start.total_seconds() # time relative to recording start
#%
sweep_start_times,sweep_end_times,sweep_nums = (ephys_patch.Sweep()&key_cell).fetch('sweep_start_time','sweep_end_time','sweep_number')
needed_start_time = cell_time_to_plot - trace_window/2
needed_end_time = cell_time_to_plot + trace_window/2
#%
sweep_nums = sweep_nums[((sweep_start_times > needed_start_time) & (sweep_start_times < needed_end_time)) |
((sweep_end_times > needed_start_time) & (sweep_end_times < needed_end_time)) |
((sweep_end_times > needed_end_time) & (sweep_start_times < needed_start_time)) ]
ephys_traces = list()
ephys_trace_times = list()
ephys_sweep_start_times = list()
ephys_traces_stim = list()
for sweep_num in sweep_nums:
sweep = ephys_patch.Sweep()&key_cell&'sweep_number = %d' % sweep_num
trace,sr= (ephys_patch.SweepMetadata()*ephys_patch.SweepResponse()&sweep).fetch1('response_trace','sample_rate')
trace = trace*1000
sweep_start_time = float(sweep.fetch('sweep_start_time'))
trace_time = np.arange(len(trace))/sr + sweep_start_time + cell_recording_start.total_seconds() - first_movie_start_time_real
trace_idx = (time_to_plot-trace_window/2 < trace_time) & (time_to_plot+trace_window/2 > trace_time)
ax_ephys.plot(trace_time[trace_idx],trace[trace_idx],'k-')
ephys_traces.append(trace)
ephys_trace_times.append(trace_time)
ephys_sweep_start_times.append(sweep_start_time)
if show_e_ap_peaks:
ap_max_index = (ephysanal.ActionPotential()&sweep).fetch('ap_max_index')
aptimes = trace_time[np.asarray(ap_max_index,int)]
apVs = trace[np.asarray(ap_max_index,int)]
ap_needed = (time_to_plot-trace_window/2 < aptimes) & (time_to_plot+trace_window/2 > aptimes)
aptimes = aptimes[ap_needed]
apVs = apVs[ap_needed]
ax_ephys.plot(aptimes,apVs,'ro')
if show_stimulus:
trace_stim= (ephys_patch.SweepMetadata()*ephys_patch.SweepStimulus()&sweep).fetch1('stimulus_trace')
trace_stim = trace_stim*10**12
ax_ephys_stim.plot(trace_time[trace_idx],trace_stim[trace_idx],'k-')
ephys_traces_stim.append(trace_stim)
ephysdata = {'ephys_traces':ephys_traces,'ephys_trace_times':ephys_trace_times}
if show_stimulus:
ephysdata['ephys_traces_stimulus'] = ephys_traces_stim
movie_nums, movie_start_times,movie_frame_rates,movie_frame_nums = ((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_number','movie_start_time','movie_frame_rate','movie_frame_num')
movie_start_times = np.asarray(movie_start_times, float)
movie_end_times = np.asarray(movie_start_times, float)+np.asarray(movie_frame_nums, float)/np.asarray(movie_frame_rates, float)
needed_start_time = session_time_to_plot - trace_window/2
needed_end_time = session_time_to_plot + trace_window/2
movie_nums = movie_nums[((movie_start_times >= needed_start_time) & (movie_start_times <= needed_end_time)) |
((movie_end_times >= needed_start_time) & (movie_end_times <= needed_end_time)) |
((movie_end_times >= needed_end_time) & (movie_start_times <= needed_start_time)) ]
dffs=list()
frametimes = list()
for movie_num in movie_nums:
#movie_num = movie_nums[(session_time_to_plot>movie_start_times)&(session_time_to_plot<movie_end_times)][0]
key_movie = key_cell.copy()
key_movie['movie_number'] = movie_num
dff,gt_roi_number = ((imaging.ROI()*imaging_gt.GroundTruthROI())&key_movie).fetch1('roi_dff','roi_number')
dff_all,roi_number_all = (imaging.ROI()&key_movie).fetch('roi_dff','roi_number')
dff_all =dff_all[roi_number_all!=gt_roi_number]
frame_times = ((imaging.MovieFrameTimes()*imaging_gt.GroundTruthROI())&key_movie).fetch1('frame_times') + (session_time).total_seconds() - first_movie_start_time_real #modified to cell time
frame_idx = (time_to_plot-trace_window/2 < frame_times) & (time_to_plot+trace_window/2 > frame_times)
dff_list = [dff]
dff_list.extend(dff_all)
prevminval = 0
for dff_now,alpha_now in zip(dff_list,np.arange(1,1/(len(dff_list)+1),-1/(len(dff_list)+1))):
dfftoplotnow = dff_now[frame_idx] + prevminval
ax_ophys.plot(frame_times[frame_idx],dfftoplotnow,'g-',alpha=alpha_now)
prevminval = np.min(dfftoplotnow) -.005
#ax_ophys.plot(frame_times[frame_idx],dff[frame_idx],'g-')
dffs.append(dff)
frametimes.append(frame_times)
if show_o_ap_peaks:
if 'raw' in key_cell['roi_type']:
apidxes = ((imaging.ROI()*imaging_gt.GroundTruthROI())&key_movie).fetch1('roi_spike_indices')
else:
apidxes = ((imaging.ROI()*imaging_gt.GroundTruthROI())&key_movie).fetch1('roi_spike_indices')-1
oap_times = frame_times[apidxes]
oap_vals = dff[apidxes]
oap_needed = (time_to_plot-trace_window/2 < oap_times) & (time_to_plot+trace_window/2 > oap_times)
oap_times = oap_times[oap_needed]
oap_vals = oap_vals[oap_needed]
ax_ophys.plot(oap_times,oap_vals,'ro')
ophysdata = {'ophys_traces':dffs,'ophys_trace_times':frametimes}
ax_ophys.autoscale(tight = True)
# =============================================================================
# dfff = np.concatenate(dffs)
# dfff[np.argmin(dfff)]=0
# ax_ophys.set_ylim([min(dfff),max(dfff)])
# =============================================================================
ax_ophys.set_xlim([time_to_plot-trace_window/2,time_to_plot+trace_window/2])
ax_ophys.set_ylabel('dF/F')
ax_ophys.spines["top"].set_visible(False)
ax_ophys.spines["right"].set_visible(False)
ax_ophys.invert_yaxis()
ax_ephys.autoscale(tight = True)
ax_ephys.set_xlim([time_to_plot-trace_window/2,time_to_plot+trace_window/2])
ax_ephys.set_ylabel('Vm (mV))')
ax_ephys.spines["top"].set_visible(False)
ax_ephys.spines["right"].set_visible(False)
if show_stimulus:
# ax_ephys_stim.autoscale(tight = True)
ax_ephys_stim.set_xlim([time_to_plot-trace_window/2,time_to_plot+trace_window/2])
ax_ephys_stim.set_ylabel('Injected current (pA))')
ax_ephys_stim.set_xlabel('time from first movie start (s)')
ax_ephys_stim.spines["top"].set_visible(False)
ax_ephys_stim.spines["right"].set_visible(False)
else:
ax_ephys.set_xlabel('time from first movie start (s)')
outdict = {'ephys':ephysdata,'ophys':ophysdata,'figure_handle':fig}
#%%
return outdict
def plot_time_interval(wr_name='FOR04',
cellnum=1,
timeedges=[0, 10],
ylimits_response=None,
plotRS=False):
subject_id = (lab.WaterRestriction() & 'water_restriction_number = "' +
wr_name + '"').fetch('subject_id')[0]
key = {'subject_id': subject_id, 'cell_number': cellnum}
allsweeps = pd.DataFrame((ephys_patch.Sweep() & key))
sweepstoplot = np.where(
np.logical_and(
allsweeps['sweep_end_time'] > float(np.min(timeedges)),
allsweeps['sweep_start_time'] < float(np.max(timeedges))))[0]
df_iv = pd.DataFrame()
for sweepnum in sweepstoplot:
key['sweep_number'] = sweepnum
df_iv = pd.concat([
df_iv,
pd.DataFrame((ephys_patch.Sweep() & key) *
(ephys_patch.SweepResponse() & key) *
(ephys_patch.SweepStimulus() & key) *
(ephys_patch.SweepMetadata() & key))
])
df_IV = pd.DataFrame()
for line in df_iv.iterrows():
linenow = line[1]
time = np.arange(0, len(
linenow['response_trace'])) / linenow['sample_rate']
linenow['time'] = time + float(linenow['sweep_start_time'])
df_IV = pd.concat([df_IV, linenow.to_frame().transpose()])
fig = plt.figure()
ax_IV = fig.add_axes([0, 0, 2, .8])
ax_stim = fig.add_axes([0, -.6, 2, .4])
for line in df_IV.iterrows():
ax_IV.plot(line[1]['time'], line[1]['response_trace'] * 1000, 'k-')
ax_stim.plot(line[1]['time'], line[1]['stimulus_trace'] * 10**12, 'k-')
ax_IV.set_xlabel('Time (s)')
ax_IV.set_xlim([np.min(timeedges), np.max(timeedges)])
if ylimits_response:
ax_IV.set_ylim([np.min(ylimits_response), np.max(ylimits_response)])
ax_IV.set_ylabel('mV')
ax_IV.set_title('Response')
ax_stim.set_xlabel('Time (s)')
ax_stim.set_xlim([np.min(timeedges), np.max(timeedges)])
ax_stim.set_ylabel('pA')
ax_stim.set_title('Stimulus')
if plotRS:
del key['sweep_number']
df_RS = pd.DataFrame((ephysanal.SeriesResistance() *
ephysanal.SquarePulse()) & key)
needed = (df_RS['square_pulse_start_time'].values >
np.min(timeedges)) & (df_RS['square_pulse_start_time'].values
< np.max(timeedges))
ax_RS = fig.add_axes([0, -1.2, 2, .4])
ax_RS.plot(df_RS[needed]['square_pulse_start_time'].values,
df_RS[needed]['series_resistance'].values, 'ko')
ax_RS.set_xlabel('Time (s)')
ax_RS.set_ylabel('RS (MOhm)')
ax_RS.set_xlim([np.min(timeedges), np.max(timeedges)])
def plot_AP(wr_name='FOR04',
cellnum=1,
timeedges=[380, 400],
timeback=.0015,
timeforward=.003,
moving_n_diff=0):
subject_id = (lab.WaterRestriction() & 'water_restriction_number = "' +
wr_name + '"').fetch('subject_id')[0]
key = {'subject_id': subject_id, 'cell_number': cellnum}
APstoplot = pd.DataFrame(ephysanal.ActionPotential() & key
& 'ap_max_time >' + str(np.min(timeedges))
& 'ap_max_time <' + str(np.max(timeedges)))
prevsweepnum = np.nan
Y = list()
dY = list()
T = list()
for ap in APstoplot.iterrows():
sweepnum = ap[1]['sweep_number']
if sweepnum != prevsweepnum:
key['sweep_number'] = sweepnum
sweepdata = pd.DataFrame((ephys_patch.Sweep() & key) *
(ephys_patch.SweepResponse() & key) *
(ephys_patch.SweepMetadata() & key))
sr = sweepdata['sample_rate'].values[0]
trace = sweepdata['response_trace'].values[0]
prevsweepnum = sweepnum
stepback = int(np.round(timeback * sr))
stepforward = int(np.round(timeforward * sr))
apmaxidx = ap[1]['ap_max_index']
if apmaxidx > stepback and apmaxidx < len(trace) - stepforward:
y = trace[apmaxidx - stepback:apmaxidx + stepforward]
if moving_n_diff > 1:
dy = np.diff(movingaverage(y, moving_n_diff)) * sr
else:
dy = np.diff(y) * sr
dy = np.squeeze(
np.asarray(
np.nanmean(
np.asmatrix([
np.concatenate([[np.nan], dy]),
np.concatenate([dy, [np.nan]])
]), 0).transpose()))
t_y = np.arange(-stepback, stepforward) / sr
Y.append(y)
dY.append(dy)
T.append(t_y)
#%b
Y = np.asmatrix(Y).transpose() * 1000
T = np.asmatrix(T).transpose() * 1000
dY = np.asmatrix(dY).transpose()
#%
fig = plt.figure()
ax_v = fig.add_axes([0, 0, .8, .8])
ax_v.plot(T, Y)
ax_v.set_xlabel('ms')
ax_v.set_ylabel('mV')
ax_v.set_xlim([-1 * timeback * 1000, timeforward * 1000])
ax_dv = fig.add_axes([1, 0, .8, .8])
ax_dv.plot(Y, dY)
ax_dv.set_xlabel('mV')
ax_dv.set_ylabel('mV/ms')
continue # if there is only one surgery, we don't care
expression_time = np.diff((lab.Surgery()&'subject_id = {}'.format(cell['subject_id'])).fetch('start_time'))[0].days
virus_id = (lab.Surgery.VirusInjection()&'subject_id = {}'.format(cell['subject_id'])).fetch('virus_id')[0]
if virus_id == 238:
virus = 'Voltron 1'
elif virus_id == 240:
virus = 'Voltron 2'
else:
virus = '??'
squarepulses = ephysanal.SquarePulse()&cell&'square_pulse_amplitude < {}'.format(min_current)&'square_pulse_length > {}'.format(min_pulse_time)
Rins = list()
RSs = list()
v0s = list()
for squarepulse in squarepulses:
trace = (ephys_patch.SweepResponse()&squarepulse).fetch1('response_trace')
stim = (ephys_patch.SweepStimulus()&squarepulse).fetch1('stimulus_trace')
sr = (ephys_patch.SweepMetadata()&squarepulse).fetch1('sample_rate')
step_back = int(integration_window*sr)
if step_back<squarepulse['square_pulse_start_idx'] and np.abs(np.median(stim[:step_back]))<=max_baseline_current and np.median(trace[:step_back])<max_v0:
RS = float((ephysanal.SweepSeriesResistance()&squarepulse).fetch1('series_resistance'))
RS_residual = float((ephysanal.SweepSeriesResistance()&squarepulse).fetch1('series_resistance_residual'))
baseline_v = np.median(trace[squarepulse['square_pulse_start_idx']-step_back:squarepulse['square_pulse_start_idx']])
Rin_v = np.median(trace[squarepulse['square_pulse_end_idx']-step_back:squarepulse['square_pulse_end_idx']])
dv = Rin_v-baseline_v
di = squarepulse['square_pulse_amplitude']
Rin = dv/di/1000000 - RS_residual
Rins.append(Rin)
RSs.append(RS)
v0s.append(baseline_v*1000)
#break
def populateelphys():
#%%
df_subject_wr_sessions = pd.DataFrame(lab.WaterRestriction() *
experiment.Session() *
experiment.SessionDetails)
df_subject_ids = pd.DataFrame(lab.Subject())
if len(df_subject_wr_sessions) > 0:
subject_names = df_subject_wr_sessions[
'water_restriction_number'].unique()
subject_names.sort()
else:
subject_names = list()
subject_ids = df_subject_ids['subject_id'].unique()
#%
sumdata = list()
basedir = Path(dj.config['locations.elphysdata_acq4'])
for setup_dir in basedir.iterdir():
setup_name = setup_dir.name
sessions = np.sort(
os.listdir(setup_dir)
) #configfile.readConfigFile(setup_dir.joinpath('.index'))
for session_acq in sessions[::-1]: #.keys():
if session_acq != '.' and session_acq != 'log.txt':
session_dir = setup_dir.joinpath(session_acq)
try:
cells = configfile.readConfigFile(
session_dir.joinpath('.index'))
except: # if there is no file
cells = None
if cells and 'WR_name/ID' in cells['.'].keys(
): # it needs to have WRname
wrname_ephys = cells['.']['WR_name/ID']
wrname = None
for wrname_potential in subject_names: # look for water restriction number
if wrname_potential.lower() in wrname_ephys.lower():
wrname = wrname_potential
subject_id = (df_subject_wr_sessions.loc[
df_subject_wr_sessions[
'water_restriction_number'] == wrname,
'subject_id']).unique()[0]
if wrname == None: # look for animal identifier:
for wrname_potential in subject_ids: # look for water restriction number
if str(wrname_potential) in wrname_ephys.lower():
subject_id = wrname_potential
if len(df_subject_wr_sessions) > 0 and len(
(df_subject_wr_sessions.loc[
df_subject_wr_sessions['subject_id'] ==
subject_id, 'water_restriction_number']
).unique()) > 0:
wrname = (df_subject_wr_sessions.loc[
df_subject_wr_sessions['subject_id'] ==
subject_id, 'water_restriction_number']
).unique()[0]
else:
wrname = 'no water restriction number for this mouse'
if wrname:
session_date = (
session_acq[0:session_acq.find('_')]).replace(
'.', '-')
print('animal: ' + str(subject_id) + ' - ' +
wrname) ##
if setup_name == 'Voltage_rig_1P':
setupname = 'Voltage-Imaging-1p'
else:
print('unkwnown setup, please add')
timer.wait(1000)
if 'experimenter' in cells['.'].keys():
username = cells['.']['experimenter']
else:
username = '******'
print(
'username not specified in acq4 file, assuming rozsam'
)
### check if session already exists
sessiondata = {
'subject_id':
subject_id, #(lab.WaterRestriction() & 'water_restriction_number = "'+df_behavior_session['subject'][0]+'"').fetch()[0]['subject_id'],
'session': np.nan,
'session_date': session_date,
'session_time':
np.nan, #session_time.strftime('%H:%M:%S'),
'username': username,
'rig': setupname
}
for cell in cells.keys():
if cell != '.' and cell != 'log.txt':
ephisdata_cell = list()
sweepstarttimes = list()
cell_dir = session_dir.joinpath(cell)
serieses = configfile.readConfigFile(
cell_dir.joinpath('.index'))
cellstarttime = datetime.datetime.fromtimestamp(
serieses['.']['__timestamp__'])
for series in serieses.keys():
if series != '.' and series != 'log.txt':
series_dir = cell_dir.joinpath(series)
sweeps = configfile.readConfigFile(
series_dir.joinpath('.index'))
if 'Clamp1.ma' in sweeps.keys():
protocoltype = 'single sweep'
sweepkeys = ['']
else:
protocoltype = 'multiple sweeps'
sweepkeys = sweeps.keys()
for sweep in sweepkeys:
if sweep != '.' and '.txt' not in sweep and '.ma' not in sweep:
sweep_dir = series_dir.joinpath(
sweep)
sweepinfo = configfile.readConfigFile(
sweep_dir.joinpath(
'.index'))
if sweep == '':
sweep = '0'
for file in sweepinfo.keys():
if '.ma' in file:
try: # old file version
#print('new file version')
#%
ephysfile = h5.File(
sweep_dir.
joinpath(file),
"r")
data = ephysfile[
'data'][()]
metadata_h5 = ephysfile[
'info']
metadata = read_h5f_metadata(
metadata_h5)
daqchannels = list(
metadata[2]
['DAQ'].keys())
sweepstarttime = datetime.datetime.fromtimestamp(
metadata[2]
['DAQ']
[daqchannels[
0]]
['startTime'])
relativetime = (
sweepstarttime
- cellstarttime
).total_seconds()
if len(
ephisdata_cell
) > 0 and ephisdata_cell[
-1]['sweepstarttime'] == sweepstarttime:
ephisdata = ephisdata_cell.pop(
)
else:
ephisdata = dict(
)
if 'primary' in daqchannels: # ephys data
ephisdata[
'V'] = data[
1]
ephisdata[
'stim'] = data[
0]
ephisdata[
'data'] = data
ephisdata[
'metadata'] = metadata
ephisdata[
'time'] = metadata[
1]['values']
ephisdata[
'relativetime'] = relativetime
ephisdata[
'sweepstarttime'] = sweepstarttime
ephisdata[
'series'] = series
ephisdata[
'sweep'] = sweep
sweepstarttimes.append(
sweepstarttime
)
else: # other daq stuff
#%
for idx, channel in enumerate(
metadata[
0]
['cols']):
channelname = channel[
'name'].decode(
)
if channelname[
0] == 'u':
channelname = channelname[
2:
-1]
if channelname in [
'OrcaFlashExposure',
'Temperature',
'LED525',
'FrameCommand',
'NextFileTrigger'
]:
ephisdata[
channelname] = data[
idx]
#print('{} added'.format(channelname))
else:
print(
'waiting in the other daq'
)
timer.sleep(
1000
)
ephisdata_cell.append(
ephisdata)
#%
except: # new file version
print(
'old version')
ephysfile = MetaArray(
)
ephysfile.readFile(
sweep_dir.
joinpath(file))
data = ephysfile.asarray(
)
metadata = ephysfile.infoCopy(
)
sweepstarttime = datetime.datetime.fromtimestamp(
metadata[2]
['startTime'])
relativetime = (
sweepstarttime
- cellstarttime
).total_seconds()
ephisdata = dict()
ephisdata[
'V'] = data[1]
ephisdata[
'stim'] = data[
0]
ephisdata[
'data'] = data
ephisdata[
'metadata'] = metadata
ephisdata[
'time'] = metadata[
1]['values']
ephisdata[
'relativetime'] = relativetime
ephisdata[
'sweepstarttime'] = sweepstarttime
ephisdata[
'series'] = series
ephisdata[
'sweep'] = sweep
sweepstarttimes.append(
sweepstarttime)
ephisdata_cell.append(
ephisdata)
# ============================================================================
# if wrname == 'FOR04':
# =============================================================================
# add session to DJ if not present
if len(ephisdata_cell) > 0:
# =============================================================================
# print('waiting')
# timer.sleep(1000)
# =============================================================================
#%
if len(experiment.Session()
& 'subject_id = "' +
str(sessiondata['subject_id']) + '"'
& 'session_date = "' +
str(sessiondata['session_date']) +
'"') == 0:
if len(experiment.Session()
& 'subject_id = "' +
str(sessiondata['subject_id']) +
'"') == 0:
sessiondata['session'] = 1
else:
sessiondata['session'] = len(
(experiment.Session()
& 'subject_id = "' +
str(sessiondata['subject_id'])
+ '"').fetch()['session']) + 1
sessiondata['session_time'] = (
sweepstarttimes[0]
).strftime(
'%H:%M:%S'
) # the time of the first sweep will be the session time
experiment.Session().insert1(
sessiondata)
#%
session = (
experiment.Session()
& 'subject_id = "' +
str(sessiondata['subject_id']) + '"'
& 'session_date = "' +
str(sessiondata['session_date']) +
'"').fetch('session')[0]
cell_number = int(cell[cell.find('_') +
1:])
#add cell if not added already
celldata = {
'subject_id': subject_id,
'session': session,
'cell_number': cell_number,
}
#%
if len(ephys_patch.Cell() & celldata
) == 0 or len(ephys_patch.Cell() *
ephys_patch.Sweep()
& celldata) < len(
ephisdata_cell):
if len(ephys_patch.Cell() *
ephys_patch.Sweep() & celldata
) < len(ephisdata_cell):
print('finishing a recording:')
else:
print('adding new recording:')
print(celldata)
if 'type' in serieses['.'].keys():
if serieses['.'][
'type'] == 'interneuron':
celldata['cell_type'] = 'int'
elif serieses['.'][
'type'] == 'unknown' or serieses[
'.']['type'] == 'fail':
celldata[
'cell_type'] = 'unidentified'
else:
print('unhandled cell type!!')
timer.sleep(1000)
else:
celldata[
'cell_type'] = 'unidentified'
celldata['cell_recording_start'] = (
sweepstarttimes[0]
).strftime('%H:%M:%S')
if 'depth' in serieses['.'].keys(
) and len(serieses['.']['depth']) > 0:
celldata['depth'] = int(
serieses['.']['depth'])
else:
celldata['depth'] = -1
try:
ephys_patch.Cell().insert1(
celldata,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
if 'notes' in serieses['.'].keys():
cellnotes = serieses['.']['notes']
else:
cellnotes = ''
cellnotesdata = {
'subject_id': subject_id,
'session': session,
'cell_number': cell_number,
'notes': cellnotes
}
try:
ephys_patch.CellNotes().insert1(
cellnotesdata,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
#%
for i, ephisdata in enumerate(
ephisdata_cell):
#%
sweep_number = i
print('sweep {}'.format(
sweep_number))
sweep_data = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'sweep_start_time':
(ephisdata['sweepstarttime'] -
sweepstarttimes[0]
).total_seconds(),
'sweep_end_time':
(ephisdata['sweepstarttime'] -
sweepstarttimes[0]
).total_seconds() +
ephisdata['time'][-1],
'protocol_name':
ephisdata[
'series'], #[:ephisdata['series'].find('_')],
'protocol_sweep_number':
int(ephisdata['sweep'])
}
if 'mode' in ephisdata['metadata'][
2]['ClampState']: # old file version
recmode = ephisdata[
'metadata'][2][
'ClampState']['mode']
else:
recmode = ephisdata[
'metadata'][2]['Protocol'][
'mode']
if 'IC' in str(recmode):
recording_mode = 'current clamp'
else:
print(
'unhandled recording mode, please act..'
)
timer.sleep(10000)
channelnames = list()
channelunits = list()
for line_now in ephisdata[
'metadata'][0]['cols']:
if type(line_now['name']
) == bytes:
channelnames.append(
line_now['name'].
decode().strip("'"))
channelunits.append(
line_now['units'].
decode().strip("'"))
else:
channelnames.append(
line_now['name'])
channelunits.append(
line_now['units'])
commandidx = np.where(
np.array(channelnames) ==
'command')[0][0]
dataidx = np.where(
np.array(channelnames) ==
'primary')[0][0]
#%
clampparams_data = ephisdata[
'metadata'][2]['ClampState'][
'ClampParams'].copy()
clampparams_data_new = dict()
for clampparamkey in clampparams_data.keys(
): #6004 is true for some reason.. changing it back to 1
if type(clampparams_data[
clampparamkey]
) == np.int32:
if clampparams_data[
clampparamkey] > 0:
clampparams_data[
clampparamkey] = int(
1)
else:
clampparams_data[
clampparamkey] = int(
0)
else:
clampparams_data[
clampparamkey] = float(
clampparams_data[
clampparamkey])
clampparams_data_new[
clampparamkey.lower(
)] = clampparams_data[
clampparamkey]
#%
sweepmetadata_data = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'recording_mode':
recording_mode,
'sample_rate':
np.round(1 / np.median(
np.diff(
ephisdata['metadata']
[1]['values'])))
}
sweepmetadata_data.update(
clampparams_data_new)
sweepdata_data = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'response_trace':
ephisdata['data'][dataidx, :],
'response_units':
ephisdata['metadata'][0]
['cols'][dataidx]['units']
}
sweepstimulus_data = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'stimulus_trace':
ephisdata['data'][
commandidx, :],
'stimulus_units':
ephisdata['metadata'][0]
['cols'][commandidx]['units']
}
#print('waiting')
#timer.sleep(10000)
try:
ephys_patch.Sweep().insert1(
sweep_data,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
try: # maybe it's a duplicate..
ephys_patch.ClampParams(
).insert1(
clampparams_data_new,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
try:
ephys_patch.SweepMetadata(
).insert1(
sweepmetadata_data,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
try:
ephys_patch.SweepResponse(
).insert1(
sweepdata_data,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
try:
ephys_patch.SweepStimulus(
).insert1(
sweepstimulus_data,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
#%
if 'OrcaFlashExposure' in ephisdata.keys(
):
sweepimagingexposuredata = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'imaging_exposure_trace':
ephisdata[
'OrcaFlashExposure']
}
try:
ephys_patch.SweepImagingExposure(
).insert1(
sweepimagingexposuredata,
allow_direct_insert=True
)
except dj.errors.DuplicateError:
pass #already uploaded
if 'Temperature' in ephisdata.keys(
):
sweeptemperaturedata = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'temperature_trace':
ephisdata['Temperature'] *
10,
'temperature_units':
'degC'
}
try:
ephys_patch.SweepTemperature(
).insert1(
sweeptemperaturedata,
allow_direct_insert=True
)
except dj.errors.DuplicateError:
pass #already uploaded
if 'LED525' in ephisdata.keys():
sweepLEDdata = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'imaging_led_trace':
ephisdata['LED525']
}
try:
ephys_patch.SweepLED(
).insert1(
sweepLEDdata,
allow_direct_insert=True
)
except dj.errors.DuplicateError:
pass #already uploaded
repositories , directories , fnames = (imaging.MovieFile() & movie).fetch('movie_file_repository','movie_file_directory','movie_file_name')
for repository,directory,fname in zip(repositories,directories,fnames):
movie_files.append(os.path.join(dj.config['locations.{}'.format(repository)],directory,fname))
sweepdata_out = list()
sweepmetadata_out = list()
for sweep_number in sweep_numbers:
#%
key_sweep = key.copy()
key_sweep['sweep_number'] = sweep_number
neutralizationenable,e_sr= (ephys_patch.SweepMetadata()&key_sweep).fetch1('neutralizationenable','sample_rate')
try:
uncompensatedRS = float((ephysanal.SweepSeriesResistance()&key_sweep).fetch1('series_resistance_residual'))
except:
uncompensatedRS = 0
v = (ephys_patch.SweepResponse()&key_sweep).fetch1('response_trace')
i = (ephys_patch.SweepStimulus()&key_sweep).fetch1('stimulus_trace')
tau_1_on =.1/1000
t = np.arange(0,.001,1/e_sr)
f_on = np.exp(t/tau_1_on)
f_on = f_on/np.max(f_on)
kernel = np.concatenate([f_on,np.zeros(len(t))])[::-1]
kernel = kernel /sum(kernel )
i_conv = np.convolve(i,kernel,'same')
v_comp = (v - i*uncompensatedRS*10**6)*1000 - junction_potential
i = i * 10**12
sweep_start_time = float((ephys_patch.Sweep()&key_sweep).fetch('sweep_start_time'))
trace_t = np.arange(len(v))/e_sr + sweep_start_time
downsample_factor = int(np.round(e_sr/downsampled_rate))
'session', 'subject_id', 'cell_number', 'motion_correction_method',
'roi_type'
]).size().reset_index(name='Freq')
snratio = list()
v0s = list()
holdings = list()
rss = list()
for cell in cells.iterrows():
cell = cell[1]
key_cell = dict(cell)
del key_cell['Freq']
snratios = (imaging_gt.GroundTruthROI() * imaging_gt.ROIAPWave()
& key_cell).fetch('apwave_snratio')
sweep = (imaging_gt.GroundTruthROI() * imaging_gt.ROIAPWave()
& key_cell).fetch('sweep_number')[0]
trace = (ephys_patch.SweepResponse() * imaging_gt.GroundTruthROI()
& key_cell
& 'sweep_number = {}'.format(sweep)).fetch('response_trace')
trace = trace[0]
stimulus = (ephys_patch.SweepStimulus() * imaging_gt.GroundTruthROI()
& key_cell
& 'sweep_number = {}'.format(sweep)).fetch('stimulus_trace')
stimulus = stimulus[0]
RS = (ephysanal.SweepSeriesResistance() * imaging_gt.GroundTruthROI()
& key_cell
& 'sweep_number = {}'.format(sweep)).fetch('series_resistance')
RS = RS[0]
medianvoltage = np.median(trace) * 1000
holding = np.median(stimulus) * 10**12
def plot_cell_SN_ratio_APwise(roi_type='VolPy',
v0_max=-35,
holding_min=-600,
frame_rate_min=300,
frame_rate_max=1800,
F0_min=50,
bin_num=10):
#%% Show S/N ratios for each AP
bin_num = 10
holding_min = -600 #pA
v0_max = -35 #mV
roi_type = 'VolPy_raw' #'Spikepursuit'#'VolPy_denoised'#'SpikePursuit'#'VolPy_dexpF0'#'VolPy'#'SpikePursuit_dexpF0'#'VolPy_dexpF0'#''Spikepursuit'#'VolPy'#
F0_min = 50
frame_rate_min = 200
frame_rate_max = 800
cmap = cm.get_cmap('jet')
key = {'roi_type': roi_type}
gtdata = pd.DataFrame((imaging_gt.GroundTruthROI() & key))
cells = gtdata.groupby([
'session', 'subject_id', 'cell_number', 'motion_correction_method',
'roi_type'
]).size().reset_index(name='Freq')
snratio = list()
v0s = list()
holdings = list()
rss = list()
threshs = list()
mintreshs = list()
f0s_all = list()
snratios_all = list()
peakamplitudes_all = list()
noise_all = list()
for cell in cells.iterrows():
cell = cell[1]
key_cell = dict(cell)
del key_cell['Freq']
snratios, f0, peakamplitudes, noises = (
imaging.Movie() * imaging_gt.GroundTruthROI() *
imaging_gt.ROIAPWave() * ephysanal.ActionPotentialDetails()
& key_cell & 'ap_real = 1'
& 'movie_frame_rate > {}'.format(frame_rate_min)
& 'movie_frame_rate < {}'.format(frame_rate_max)
& 'apwave_f0 > {}'.format(F0_min)).fetch('apwave_snratio',
'apwave_f0',
'apwave_peak_amplitude',
'apwave_noise')
#f0 = (imaging_gt.GroundTruthROI()*imaging.ROI()&key_cell).fetch('roi_f0')
sweep = (imaging_gt.GroundTruthROI() * imaging_gt.ROIAPWave()
& key_cell).fetch('sweep_number')[0]
thresh = (imaging_gt.GroundTruthROI() * imaging_gt.ROIAPWave() *
ephysanal.ActionPotentialDetails() & key_cell
& 'ap_real = 1').fetch('ap_threshold')
trace = (ephys_patch.SweepResponse() * imaging_gt.GroundTruthROI()
& key_cell
& 'sweep_number = {}'.format(sweep)).fetch('response_trace')
trace = trace[0]
stimulus = (ephys_patch.SweepStimulus() * imaging_gt.GroundTruthROI()
& key_cell &
'sweep_number = {}'.format(sweep)).fetch('stimulus_trace')
stimulus = stimulus[0]
RS = (ephysanal.SweepSeriesResistance() * imaging_gt.GroundTruthROI()
& key_cell
& 'sweep_number = {}'.format(sweep)).fetch('series_resistance')
RS = RS[0]
medianvoltage = np.median(trace) * 1000
holding = np.median(stimulus) * 10**12
#print(np.mean(snratios[:100]))
snratio.append(np.mean(snratios[:50]))
v0s.append(medianvoltage)
holdings.append(holding)
rss.append(RS)
threshs.append(thresh)
mintreshs.append(np.min(thresh))
f0s_all.append(f0)
snratios_all.append(snratios)
peakamplitudes_all.append(peakamplitudes)
noise_all.append(noises)
#plot_AP_waveforms(key_cell,AP_tlimits)
#%%
# =============================================================================
# virus_list=list()
# subject_ids = list()
# for cell in cells.iterrows():
# cell = cell[1]
# key_cell = dict(cell)
# del key_cell['Freq']
# virus_id = (lab.Surgery.VirusInjection()&'subject_id = {}'.format(key_cell['subject_id'])).fetch('virus_id')[0]
# if virus_id == 238:
# virus = 'Voltron 1'
# elif virus_id == 240:
# virus = 'Voltron 2'
# virus_list.append(virus)
# subject_ids.append(key_cell['subject_id'])
# # order = np.argsort(virus_list)
# order = np.lexsort((virus_list, subject_ids))
# snratio = np.asarray(snratio)[order]
# v0s = np.asarray(v0s)[order]
# holdings = np.asarray(holdings)[order]
# rss = np.asarray(rss)[order]
# threshs =np.asarray(threshs)[order]
# mintreshs = np.asarray(mintreshs)[order]
# f0s_all = np.asarray(f0s_all)[order]
# snratios_all = np.asarray(snratios_all)[order]
# peakamplitudes_all = np.asarray(peakamplitudes_all)[order]
# noise_all = np.asarray(noise_all)[order]
# virus_list = np.asarray(virus_list)[order]
# cells = cells.set_index(order, append=True).sort_index(level=1).reset_index(1, drop=True)
# =============================================================================
#%%
apnum = 50
fig = plt.figure(figsize=[10, 10])
ax_exptime_f0 = fig.add_subplot(221)
ax_exptime_dff = fig.add_subplot(222)
ax_exptime_noise = fig.add_subplot(223)
ax_exptime_snration = fig.add_subplot(224)
for loopidx, (f0, snratio_now, noise_now, peakampl_now,
cell_now) in enumerate(
zip(f0s_all, snratios_all, noise_all, peakamplitudes_all,
cells.iterrows())):
if len(
f0
) > 0: # and cell_now[1]['subject_id']==466774:# and cell_now[1]['cell_number']==1:
coloridx = loopidx / len(cells)
cell_now = cell_now[1]
virus_id = (lab.Surgery.VirusInjection()
& 'subject_id = {}'.format(
cell_now['subject_id'])).fetch('virus_id')[0]
if virus_id == 238:
virus = 'Voltron 1'
elif virus_id == 240:
virus = 'Voltron 2'
else:
virus = '??'
label_now = 'Subject:{}'.format(
cell_now['subject_id']) + ' Cell:{} - {}'.format(
cell_now['cell_number'], virus)
expression_time = np.diff(
(lab.Surgery() & 'subject_id = {}'.format(
cell_now['subject_id'])).fetch('start_time'))[0].days
ax_exptime_f0.plot(expression_time,
np.mean(f0[:apnum]),
'o',
ms=10,
color=cmap(coloridx),
label=label_now)
ax_exptime_f0.errorbar(expression_time,
np.mean(f0[:apnum]),
np.std(f0[:apnum]),
ecolor=cmap(coloridx))
ax_exptime_f0.set_xlabel('expression time (days)')
ax_exptime_f0.set_ylabel('F0 (pixel intensity)')
ax_exptime_dff.plot(expression_time,
np.mean(peakampl_now[:apnum]),
'o',
ms=10,
color=cmap(coloridx),
label=label_now)
ax_exptime_dff.errorbar(expression_time,
np.mean(peakampl_now[:apnum]),
np.std(peakampl_now[:apnum]),
ecolor=cmap(coloridx))
ax_exptime_dff.set_xlabel('expression time (days)')
ax_exptime_dff.set_ylabel('AP peak amplitude (dF/F)')
ax_exptime_noise.plot(expression_time,
np.mean(noise_now[:apnum]),
'o',
ms=10,
color=cmap(coloridx),
label=label_now)
ax_exptime_noise.errorbar(expression_time,
np.mean(noise_now[:apnum]),
np.std(noise_now[:apnum]),
ecolor=cmap(coloridx))
ax_exptime_noise.set_xlabel('expression time (days)')
ax_exptime_noise.set_ylabel('noise (dF/F)')
ax_exptime_snration.plot(expression_time,
np.mean(snratio_now[:apnum]),
'o',
ms=10,
color=cmap(coloridx),
label=label_now)
ax_exptime_snration.errorbar(expression_time,
np.mean(snratio_now[:apnum]),
np.std(snratio_now[:apnum]),
ecolor=cmap(coloridx))
ax_exptime_snration.set_xlabel('expression time (days)')
ax_exptime_snration.set_ylabel('S/N ratio')
#%% for each AP
fig = plt.figure()
ax_sn_f0 = fig.add_axes([0, 0, 1, 1])
ax_sn_f0_binned = fig.add_axes([0, -1.2, 1, 1])
ax_noise_f0 = fig.add_axes([2.6, 0, 1, 1])
ax_noise_f0_binned = fig.add_axes([2.6, -1.2, 1, 1])
ax_peakampl_f0 = fig.add_axes([1.3, 0, 1, 1])
ax_peakampl_f0_binned = fig.add_axes([1.3, -1.2, 1, 1])
for loopidx, (f0, snratio_now, noise_now, peakampl_now,
cell_now) in enumerate(
zip(f0s_all, snratios_all, noise_all, peakamplitudes_all,
cells.iterrows())):
if len(
f0
) > 0: # and cell_now[1]['subject_id']==466774:# and cell_now[1]['cell_number']==1:
coloridx = loopidx / len(cells)
cell_now = cell_now[1]
virus_id = (lab.Surgery.VirusInjection()
& 'subject_id = {}'.format(
cell_now['subject_id'])).fetch('virus_id')[0]
if virus_id == 238:
virus = 'Voltron 1'
elif virus_id == 240:
virus = 'Voltron 2'
else:
virus = '??'
label_now = 'Subject:{}'.format(
cell_now['subject_id']) + ' Cell:{} - {}'.format(
cell_now['cell_number'], virus)
ax_sn_f0.plot(f0,
snratio_now,
'o',
ms=1,
color=cmap(coloridx),
label=label_now)
ax_noise_f0.plot(f0,
noise_now,
'o',
ms=1,
color=cmap(coloridx),
label=label_now)
ax_peakampl_f0.plot(f0,
peakampl_now,
'o',
ms=1,
color=cmap(coloridx),
label=label_now)
lows = np.arange(np.min(f0), np.max(f0),
(np.max(f0) - np.min(f0)) / (bin_num + 1))
highs = lows + (np.max(f0) - np.min(f0)) / (bin_num + 1)
mean_f0 = list()
sd_f0 = list()
mean_sn = list()
sd_sn = list()
mean_noise = list()
sd_noise = list()
mean_ampl = list()
sd_ampl = list()
for low, high in zip(lows, highs):
idx = (f0 >= low) & (f0 < high)
if len(idx) > 10:
mean_f0.append(np.mean(f0[idx]))
sd_f0.append(np.std(f0[idx]))
mean_sn.append(np.mean(snratio_now[idx]))
sd_sn.append(np.std(snratio_now[idx]))
mean_noise.append(np.mean(noise_now[idx]))
sd_noise.append(np.std(noise_now[idx]))
mean_ampl.append(np.mean(peakampl_now[idx]))
sd_ampl.append(np.std(peakampl_now[idx]))
ax_sn_f0_binned.errorbar(mean_f0,
mean_sn,
sd_sn,
sd_f0,
'o-',
color=cmap(coloridx),
label=label_now)
ax_noise_f0_binned.errorbar(mean_f0,
mean_noise,
sd_noise,
sd_f0,
'o-',
color=cmap(coloridx),
label=label_now)
ax_peakampl_f0_binned.errorbar(mean_f0,
mean_ampl,
sd_ampl,
sd_f0,
'o-',
color=cmap(coloridx),
label=label_now)
ax_sn_f0.set_xlabel('F0')
ax_sn_f0.set_ylabel('S/N ratio')
ax_sn_f0_binned.set_xlabel('F0')
ax_sn_f0_binned.set_ylabel('S/N ratio')
#ax_sn_f0_binned.legend()
ax_sn_f0_binned.legend(loc='upper center',
bbox_to_anchor=(-.45, 1.5),
shadow=True,
ncol=1)
ax_noise_f0.set_xlabel('F0')
ax_noise_f0.set_ylabel('Noise (std(dF/F))')
ax_noise_f0_binned.set_xlabel('F0')
ax_noise_f0_binned.set_ylabel('Noise (std(dF/F))')
ax_peakampl_f0.set_xlabel('F0')
ax_peakampl_f0.set_ylabel('Peak amplitude (dF/F)')
ax_peakampl_f0_binned.set_xlabel('F0')
ax_peakampl_f0_binned.set_ylabel('Peak amplitude (dF/F)')
#%%
cells['SN'] = snratio
cells['V0'] = v0s
cells['holding'] = holdings
cells['RS'] = np.asarray(rss, float)
print(cells)
cells = cells[cells['V0'] < v0_max]
cells = cells[cells['holding'] > holding_min]
print(cells)
#% S/N ratio histogram
fig = plt.figure()
ax_hist = fig.add_axes([0, 0, 1, 1])
ax_hist.hist(cells['SN'].values)
ax_hist.set_xlabel('S/N ratio of first 50 spikes')
ax_hist.set_ylabel('# of cells')
ax_hist.set_title(roi_type.replace('_', ' '))
ax_hist.set_xlim([0, 15])
