def make(self, key):
#%
# key = { 'subject_id': 462149, 'session':1,'cell_number':1,'movie_number':11}
session_time = (experiment.Session() & key).fetch('session_time')[0]
cell_time = (ephys_patch.Cell() & key).fetch('cell_recording_start')[0]
cell_sweep_start_times = (ephys_patch.Sweep()
& key).fetch('sweep_start_time')
cell_sweep_end_times = (ephys_patch.Sweep()
& key).fetch('sweep_end_time')
time_start = float(
np.min(cell_sweep_start_times)) + cell_time.total_seconds(
) - session_time.total_seconds()
time_end = float(
np.max(cell_sweep_end_times)) + cell_time.total_seconds(
) - session_time.total_seconds()
try:
movie = (imaging.Movie()) & key & 'movie_start_time > ' + str(
time_start) & 'movie_start_time < ' + str(time_end)
sweep_start_times, sweep_end_times, sweep_nums = (
ephys_patch.Sweep() & key).fetch('sweep_start_time',
'sweep_end_time',
'sweep_number')
sweep_start_times = np.asarray(sweep_start_times,
float) + cell_time.total_seconds(
) - session_time.total_seconds()
sweep_end_times = np.asarray(sweep_end_times,
float) + cell_time.total_seconds(
) - session_time.total_seconds()
#for movie in movies_now:
frametimes = (imaging.MovieFrameTimes
& movie).fetch1('frame_times')
needed_start_time = frametimes[0]
needed_end_time = frametimes[-1]
sweep_nums_needed = sweep_nums[
((sweep_start_times > needed_start_time) &
(sweep_start_times < needed_end_time)) |
((sweep_end_times > needed_start_time) &
(sweep_end_times < needed_end_time)) |
((sweep_end_times > needed_end_time) &
(sweep_start_times < needed_start_time))]
if len(sweep_nums_needed) > 0:
key['sweep_numbers'] = sweep_nums_needed
self.insert1(key, skip_duplicates=True)
except:
pass
def plot_precision_recall(key_cell,binwidth = 30,frbinwidth = 0.01,firing_rate_window = 3,save_figures = True,xlimits =None):
#%%
session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key_cell).fetch1('session_time','cell_recording_start')
first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_start_time'),float))
first_movie_start_time_real = first_movie_start_time + session_time.total_seconds()
#%
fr_kernel = np.ones(int(firing_rate_window/frbinwidth))/(firing_rate_window/frbinwidth)
first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_start_time'),float))
ephys_matched_ap_times,ephys_unmatched_ap_times,ophys_matched_ap_times,ophys_unmatched_ap_times = (imaging_gt.GroundTruthROI()&key_cell).fetch('ephys_matched_ap_times','ephys_unmatched_ap_times','ophys_matched_ap_times','ophys_unmatched_ap_times')
#%
ephys_matched_ap_times = np.concatenate(ephys_matched_ap_times) - first_movie_start_time
ephys_unmatched_ap_times = np.concatenate(ephys_unmatched_ap_times) - first_movie_start_time
all_ephys_ap_times = np.concatenate([ephys_matched_ap_times,ephys_unmatched_ap_times])
ophys_matched_ap_times = np.concatenate(ophys_matched_ap_times) - first_movie_start_time
ophys_unmatched_ap_times = np.concatenate(ophys_unmatched_ap_times) - first_movie_start_time
all_ophys_ap_times = np.concatenate([ophys_matched_ap_times,ophys_unmatched_ap_times])
all_times = np.concatenate([ephys_matched_ap_times,ephys_unmatched_ap_times,ophys_matched_ap_times,ophys_unmatched_ap_times])
maxtime = np.max(all_times)
#%
fr_bincenters = np.arange(frbinwidth/2,maxtime+frbinwidth,frbinwidth)
fr_binedges = np.concatenate([fr_bincenters-frbinwidth/2,[fr_bincenters[-1]+frbinwidth/2]])
fr_e = np.histogram(all_ephys_ap_times,fr_binedges)[0]/frbinwidth
fr_e = np.convolve(fr_e, fr_kernel,'same')
fr_o = np.histogram(all_ophys_ap_times,fr_binedges)[0]/frbinwidth
fr_o = np.convolve(fr_o, fr_kernel,'same')
#%
bincenters = np.arange(binwidth/2,maxtime+binwidth,binwidth)
binedges = np.concatenate([bincenters-binwidth/2,[bincenters[-1]+binwidth/2]])
ephys_matched_ap_num_binned,tmp = np.histogram(ephys_matched_ap_times,binedges)
ephys_unmatched_ap_num_binned,tmp = np.histogram(ephys_unmatched_ap_times,binedges)
ophys_matched_ap_num_binned,tmp = np.histogram(ophys_matched_ap_times,binedges)
ophys_unmatched_ap_num_binned,tmp = np.histogram(ophys_unmatched_ap_times,binedges)
precision_binned = ophys_matched_ap_num_binned/(ophys_matched_ap_num_binned+ophys_unmatched_ap_num_binned)
recall_binned = ephys_matched_ap_num_binned/(ephys_matched_ap_num_binned+ephys_unmatched_ap_num_binned)
f1_binned = 2*precision_binned*recall_binned/(precision_binned+recall_binned)
#%
if xlimits == None:
xlimits = [binedges[0],binedges[-1]]
fig=plt.figure()#figsize=(50,0)
ax_rates = fig.add_axes([0,1.4,2,.3])
ax_spikes = fig.add_axes([0,1,2,.3])
ax = fig.add_axes([0,0,2,.8])
ax_snratio = fig.add_axes([0,-1,2,.8])
ax_latency = fig.add_axes([0,-2,2,.8])
ax_latency_hist = fig.add_axes([.5,-3,1,.8])
ax.plot(bincenters,precision_binned,'go-',label = 'precision')
ax.plot(bincenters,recall_binned,'ro-',label = 'recall')
ax.plot(bincenters,f1_binned,'bo-',label = 'f1 score')
ax.set_xlim([binedges[0],binedges[-1]])
ax.set_ylim([0,1])
ax.legend()
ax_spikes.plot(ephys_unmatched_ap_times,np.zeros(len(ephys_unmatched_ap_times)),'r|', ms = 10)
ax_spikes.plot(all_ephys_ap_times,np.zeros(len(all_ephys_ap_times))+.33,'k|', ms = 10)
ax_spikes.plot(ophys_unmatched_ap_times,np.ones(len(ophys_unmatched_ap_times)),'r|',ms = 10)
ax_spikes.plot(all_ophys_ap_times,np.ones(len(all_ophys_ap_times))-.33,'g|', ms = 10)
ax_spikes.set_yticks([0,.33,.67,1])
ax_spikes.set_yticklabels(['false negative','ephys','ophys','false positive'])
ax_spikes.set_ylim([-.2,1.2])
ax_spikes.set_xlim(xlimits)
t,sn = (ephysanal.ActionPotential()*imaging_gt.GroundTruthROI()*imaging_gt.ROIAPWave()&key_cell).fetch('ap_max_time','apwave_snratio')
t= np.asarray(t,float) + cell_recording_start.total_seconds() - first_movie_start_time_real
ax_snratio.plot(t,sn,'ko')
ax_snratio.set_ylabel('signal to noise ratio')
ax_snratio.set_ylim([0,20])
ax_snratio.set_xlim(xlimits)
ax_rates.plot(fr_bincenters,fr_e,'k-',label = 'ephys')
ax_rates.plot(fr_bincenters,fr_o,'g-',label = 'ophys')
ax_rates.legend()
ax_rates.set_xlim(xlimits)
ax_rates.set_ylabel('Firing rate (Hz)')
ax_rates.set_title('subject_id: %d, cell number: %s' %(key_cell['subject_id'],key_cell['cell_number']))
ax_latency.plot(ephys_matched_ap_times,1000*(ophys_matched_ap_times-ephys_matched_ap_times),'ko')
ax_latency.set_ylabel('ephys-ophys spike latency (ms)')
ax_latency.set_ylim([0,10])
ax_latency.set_xlabel('time from first movie start (s)')
ax_latency.set_xlim(xlimits)
ax_latency_hist.hist(1000*(ophys_matched_ap_times-ephys_matched_ap_times),np.arange(-5,15,.1))
ax_latency_hist.set_xlabel('ephys-ophys spike latency (ms)')
ax_latency_hist.set_ylabel('matched ap count')
data = list()
data.append(plot_ephys_ophys_trace(key_cell,ephys_matched_ap_times[0],trace_window = 1,show_e_ap_peaks = True,show_o_ap_peaks = True))
data.append(plot_ephys_ophys_trace(key_cell,ephys_unmatched_ap_times[0],trace_window = 1,show_e_ap_peaks = True,show_o_ap_peaks = True))
data.append(plot_ephys_ophys_trace(key_cell,ophys_unmatched_ap_times[0],trace_window = 1,show_e_ap_peaks = True,show_o_ap_peaks = True))
if save_figures:
fig.savefig('./figures/{}_cell_{}_roi_type_{}.png'.format(key_cell['subject_id'],key_cell['cell_number'],key_cell['roi_type']), bbox_inches = 'tight')
for data_now,fname in zip(data,['matched','false_negative','false_positive']):
data_now['figure_handle'].savefig('./figures/{}_cell_{}_roi_type_{}_{}.png'.format(key_cell['subject_id'],key_cell['cell_number'],key_cell['roi_type'],fname), bbox_inches = 'tight')
def plot_ephys_ophys_trace(key_cell,time_to_plot=None,trace_window = 1,show_stimulus = False,show_e_ap_peaks = False, show_o_ap_peaks = False):
#%%
# =============================================================================
# key_cell = {'session': 1,
# 'subject_id': 456462,
# 'cell_number': 3,
# 'motion_correction_method': 'VolPy',
# 'roi_type': 'VolPy'}
# time_to_plot=None
# trace_window = 100
# show_stimulus = False
# show_e_ap_peaks = True
# show_o_ap_peaks = True
# =============================================================================
fig=plt.figure()
ax_ophys = fig.add_axes([0,0,2,.8])
ax_ephys = fig.add_axes([0,-1,2,.8])
if show_stimulus:
ax_ephys_stim = fig.add_axes([0,-1.5,2,.4])
#%
session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key_cell).fetch1('session_time','cell_recording_start')
first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_start_time'),float))
first_movie_start_time_real = first_movie_start_time + session_time.total_seconds()
if not time_to_plot:
time_to_plot = trace_window/2#ephys_matched_ap_times[0]
session_time_to_plot = time_to_plot+first_movie_start_time # time relative to session start
cell_time_to_plot= session_time_to_plot + session_time.total_seconds() -cell_recording_start.total_seconds() # time relative to recording start
#%
sweep_start_times,sweep_end_times,sweep_nums = (ephys_patch.Sweep()&key_cell).fetch('sweep_start_time','sweep_end_time','sweep_number')
needed_start_time = cell_time_to_plot - trace_window/2
needed_end_time = cell_time_to_plot + trace_window/2
#%
sweep_nums = sweep_nums[((sweep_start_times > needed_start_time) & (sweep_start_times < needed_end_time)) |
((sweep_end_times > needed_start_time) & (sweep_end_times < needed_end_time)) |
((sweep_end_times > needed_end_time) & (sweep_start_times < needed_start_time)) ]
ephys_traces = list()
ephys_trace_times = list()
ephys_sweep_start_times = list()
ephys_traces_stim = list()
for sweep_num in sweep_nums:
sweep = ephys_patch.Sweep()&key_cell&'sweep_number = %d' % sweep_num
trace,sr= (ephys_patch.SweepMetadata()*ephys_patch.SweepResponse()&sweep).fetch1('response_trace','sample_rate')
trace = trace*1000
sweep_start_time = float(sweep.fetch('sweep_start_time'))
trace_time = np.arange(len(trace))/sr + sweep_start_time + cell_recording_start.total_seconds() - first_movie_start_time_real
trace_idx = (time_to_plot-trace_window/2 < trace_time) & (time_to_plot+trace_window/2 > trace_time)
ax_ephys.plot(trace_time[trace_idx],trace[trace_idx],'k-')
ephys_traces.append(trace)
ephys_trace_times.append(trace_time)
ephys_sweep_start_times.append(sweep_start_time)
if show_e_ap_peaks:
ap_max_index = (ephysanal.ActionPotential()&sweep).fetch('ap_max_index')
aptimes = trace_time[np.asarray(ap_max_index,int)]
apVs = trace[np.asarray(ap_max_index,int)]
ap_needed = (time_to_plot-trace_window/2 < aptimes) & (time_to_plot+trace_window/2 > aptimes)
aptimes = aptimes[ap_needed]
apVs = apVs[ap_needed]
ax_ephys.plot(aptimes,apVs,'ro')
if show_stimulus:
trace_stim= (ephys_patch.SweepMetadata()*ephys_patch.SweepStimulus()&sweep).fetch1('stimulus_trace')
trace_stim = trace_stim*10**12
ax_ephys_stim.plot(trace_time[trace_idx],trace_stim[trace_idx],'k-')
ephys_traces_stim.append(trace_stim)
ephysdata = {'ephys_traces':ephys_traces,'ephys_trace_times':ephys_trace_times}
if show_stimulus:
ephysdata['ephys_traces_stimulus'] = ephys_traces_stim
movie_nums, movie_start_times,movie_frame_rates,movie_frame_nums = ((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_number','movie_start_time','movie_frame_rate','movie_frame_num')
movie_start_times = np.asarray(movie_start_times, float)
movie_end_times = np.asarray(movie_start_times, float)+np.asarray(movie_frame_nums, float)/np.asarray(movie_frame_rates, float)
needed_start_time = session_time_to_plot - trace_window/2
needed_end_time = session_time_to_plot + trace_window/2
movie_nums = movie_nums[((movie_start_times >= needed_start_time) & (movie_start_times <= needed_end_time)) |
((movie_end_times >= needed_start_time) & (movie_end_times <= needed_end_time)) |
((movie_end_times >= needed_end_time) & (movie_start_times <= needed_start_time)) ]
dffs=list()
frametimes = list()
for movie_num in movie_nums:
#movie_num = movie_nums[(session_time_to_plot>movie_start_times)&(session_time_to_plot<movie_end_times)][0]
key_movie = key_cell.copy()
key_movie['movie_number'] = movie_num
dff,gt_roi_number = ((imaging.ROI()*imaging_gt.GroundTruthROI())&key_movie).fetch1('roi_dff','roi_number')
dff_all,roi_number_all = (imaging.ROI()&key_movie).fetch('roi_dff','roi_number')
dff_all =dff_all[roi_number_all!=gt_roi_number]
frame_times = ((imaging.MovieFrameTimes()*imaging_gt.GroundTruthROI())&key_movie).fetch1('frame_times') + (session_time).total_seconds() - first_movie_start_time_real #modified to cell time
frame_idx = (time_to_plot-trace_window/2 < frame_times) & (time_to_plot+trace_window/2 > frame_times)
dff_list = [dff]
dff_list.extend(dff_all)
prevminval = 0
for dff_now,alpha_now in zip(dff_list,np.arange(1,1/(len(dff_list)+1),-1/(len(dff_list)+1))):
dfftoplotnow = dff_now[frame_idx] + prevminval
ax_ophys.plot(frame_times[frame_idx],dfftoplotnow,'g-',alpha=alpha_now)
prevminval = np.min(dfftoplotnow) -.005
#ax_ophys.plot(frame_times[frame_idx],dff[frame_idx],'g-')
dffs.append(dff)
frametimes.append(frame_times)
if show_o_ap_peaks:
if 'raw' in key_cell['roi_type']:
apidxes = ((imaging.ROI()*imaging_gt.GroundTruthROI())&key_movie).fetch1('roi_spike_indices')
else:
apidxes = ((imaging.ROI()*imaging_gt.GroundTruthROI())&key_movie).fetch1('roi_spike_indices')-1
oap_times = frame_times[apidxes]
oap_vals = dff[apidxes]
oap_needed = (time_to_plot-trace_window/2 < oap_times) & (time_to_plot+trace_window/2 > oap_times)
oap_times = oap_times[oap_needed]
oap_vals = oap_vals[oap_needed]
ax_ophys.plot(oap_times,oap_vals,'ro')
ophysdata = {'ophys_traces':dffs,'ophys_trace_times':frametimes}
ax_ophys.autoscale(tight = True)
# =============================================================================
# dfff = np.concatenate(dffs)
# dfff[np.argmin(dfff)]=0
# ax_ophys.set_ylim([min(dfff),max(dfff)])
# =============================================================================
ax_ophys.set_xlim([time_to_plot-trace_window/2,time_to_plot+trace_window/2])
ax_ophys.set_ylabel('dF/F')
ax_ophys.spines["top"].set_visible(False)
ax_ophys.spines["right"].set_visible(False)
ax_ophys.invert_yaxis()
ax_ephys.autoscale(tight = True)
ax_ephys.set_xlim([time_to_plot-trace_window/2,time_to_plot+trace_window/2])
ax_ephys.set_ylabel('Vm (mV))')
ax_ephys.spines["top"].set_visible(False)
ax_ephys.spines["right"].set_visible(False)
if show_stimulus:
# ax_ephys_stim.autoscale(tight = True)
ax_ephys_stim.set_xlim([time_to_plot-trace_window/2,time_to_plot+trace_window/2])
ax_ephys_stim.set_ylabel('Injected current (pA))')
ax_ephys_stim.set_xlabel('time from first movie start (s)')
ax_ephys_stim.spines["top"].set_visible(False)
ax_ephys_stim.spines["right"].set_visible(False)
else:
ax_ephys.set_xlabel('time from first movie start (s)')
outdict = {'ephys':ephysdata,'ophys':ophysdata,'figure_handle':fig}
#%%
return outdict
matplotlib.rc('font', **font)
from pathlib import Path
homefolder = dj.config['locations.mr_share']
#%% ephys parameters at different expression times and different viruses
ephys_data=dict()
ephys_data = {'cell_dict':list(),
'virus':list(),
'expression_time':list(),
'RS':list(),
'Rin':list(),
'AP_amplitude':list(),
'AP_halfwidth':list(),
'AP_threshold':list(),
'resting':list()}
cells = ephys_patch.Cell()
APs_needed = 50
AP_max_RS = 40
AP_max_baseline = -40 #mV
max_baseline_current = 100*10**-12 #A
max_v0 = -.04 #V
min_current = -40*10**-12 #A
min_pulse_time = 0.3 #s
integration_window = .04 #sec
for cell in cells:
if len((lab.Surgery()&'subject_id = {}'.format(cell['subject_id'])).fetch('start_time'))==1:
continue # if there is only one surgery, we don't care
expression_time = np.diff((lab.Surgery()&'subject_id = {}'.format(cell['subject_id'])).fetch('start_time'))[0].days
virus_id = (lab.Surgery.VirusInjection()&'subject_id = {}'.format(cell['subject_id'])).fetch('virus_id')[0]
if virus_id == 238:
virus = 'Voltron 1'
def make(self, key):
#%
#key = {'subject_id': 454597, 'session': 1, 'cell_number': 0, 'motion_correction_method': 'Matlab', 'roi_type': 'SpikePursuit', 'roi_number': 1}
#key = {'subject_id': 456462, 'session': 1, 'cell_number': 5, 'movie_number': 3, 'motion_correction_method': 'VolPy', 'roi_type': 'VolPy', 'roi_number': 1}
if len(
ROIEphysCorrelation & key
) > 0: # and key['roi_type'] == 'SpikePursuit' #only spikepursuit for now..
key_to_compare = key.copy()
del key_to_compare['roi_number']
#print(key)
#%
roinums = np.unique(
(ROIEphysCorrelation() & key_to_compare).fetch('roi_number'))
snratios_mean = list()
snratios_median = list()
snratios_first50 = list()
for roinum_now in roinums:
snratios = (ROIAPWave() & key_to_compare
& 'roi_number = {}'.format(roinum_now)
).fetch('apwave_snratio')
snratios_mean.append(np.mean(snratios))
snratios_median.append(np.median(snratios))
snratios_first50.append(np.mean(snratios[:50]))
#%%
if np.max(
(ROIEphysCorrelation() & key).fetch('roi_number')
) == roinums[np.argmax(
snratios_first50
)]: #np.max((imaging_gt.ROIEphysCorrelation()&key).fetch('roi_number')) == np.min((imaging_gt.ROIEphysCorrelation&key_to_compare).fetch('roi_number')):#np.max(np.abs((ROIEphysCorrelation&key).fetch('corr_coeff'))) == np.max(np.abs((ROIEphysCorrelation&key_to_compare).fetch('corr_coeff'))):
print('this is it')
print(key['roi_type'])
cellstarttime = (ephys_patch.Cell()
& key).fetch1('cell_recording_start')
sessionstarttime = (experiment.Session()
& key).fetch1('session_time')
aptimes = np.asarray(
(ephysanal.ActionPotential() & key).fetch('ap_max_time'),
float) + (cellstarttime -
sessionstarttime).total_seconds()
sweep_start_times, sweep_end_times = (ephys_patch.Sweep()
& key).fetch(
'sweep_start_time',
'sweep_end_time')
sweep_start_times = np.asarray(sweep_start_times, float) + (
cellstarttime - sessionstarttime).total_seconds()
sweep_end_times = np.asarray(sweep_end_times, float) + (
cellstarttime - sessionstarttime).total_seconds()
frame_timess, roi_spike_indicess = (
imaging.MovieFrameTimes() * imaging.Movie() *
imaging.ROI() & key).fetch('frame_times',
'roi_spike_indices')
movie_start_times = list()
movie_end_times = list()
roi_ap_times = list()
for frame_times, roi_spike_indices in zip(
frame_timess, roi_spike_indicess):
movie_start_times.append(frame_times[0])
movie_end_times.append(frame_times[-1])
roi_ap_times.append(frame_times[roi_spike_indices])
movie_start_times = np.sort(movie_start_times)
movie_end_times = np.sort(movie_end_times)
roi_ap_times = np.sort(np.concatenate(roi_ap_times))
#%
##delete spikes in optical traces where there was no ephys recording
for start_t, end_t in zip(
np.concatenate([sweep_start_times, [np.inf]]),
np.concatenate([[0], sweep_end_times])):
idxtodel = np.where((roi_ap_times > end_t)
& (roi_ap_times < start_t))[0]
if len(idxtodel) > 0:
roi_ap_times = np.delete(roi_ap_times, idxtodel)
##delete spikes in ephys traces where there was no imaging
for start_t, end_t in zip(
np.concatenate([movie_start_times, [np.inf]]),
np.concatenate([[0], movie_end_times])):
idxtodel = np.where((aptimes > end_t)
& (aptimes < start_t))[0]
if len(idxtodel) > 0:
#print(idxtodel)
aptimes = np.delete(aptimes, idxtodel)
#%
D = np.zeros([len(aptimes), len(roi_ap_times)])
for idx, apt in enumerate(aptimes):
D[idx, :] = (roi_ap_times - apt) * 1000
D_test = np.abs(D)
D_test[D_test > 15] = 1000
D_test[D < -1] = 1000
X = scipy.optimize.linear_sum_assignment(D_test)
#%
cost = D_test[X[0], X[1]]
unmatched = np.where(cost == 1000)[0]
X0_final = np.delete(X[0], unmatched)
X1_final = np.delete(X[1], unmatched)
ephys_ap_times = aptimes[X0_final]
ophys_ap_times = roi_ap_times[X1_final]
false_positive_time_imaging = list()
for roi_ap_time in roi_ap_times:
if roi_ap_time not in ophys_ap_times:
false_positive_time_imaging.append(roi_ap_time)
false_negative_time_ephys = list()
for aptime in aptimes:
if aptime not in ephys_ap_times:
false_negative_time_ephys.append(aptime)
key['ephys_matched_ap_times'] = ephys_ap_times
key['ophys_matched_ap_times'] = ophys_ap_times
key['ephys_unmatched_ap_times'] = false_negative_time_ephys
key['ophys_unmatched_ap_times'] = false_positive_time_imaging
#print(imaging.ROI()&key)
#print([len(aptimes),'vs',len(roi_ap_times)])
#%%
self.insert1(key, skip_duplicates=True)
def populateelphys():
#%%
df_subject_wr_sessions = pd.DataFrame(lab.WaterRestriction() *
experiment.Session() *
experiment.SessionDetails)
df_subject_ids = pd.DataFrame(lab.Subject())
if len(df_subject_wr_sessions) > 0:
subject_names = df_subject_wr_sessions[
'water_restriction_number'].unique()
subject_names.sort()
else:
subject_names = list()
subject_ids = df_subject_ids['subject_id'].unique()
#%
sumdata = list()
basedir = Path(dj.config['locations.elphysdata_acq4'])
for setup_dir in basedir.iterdir():
setup_name = setup_dir.name
sessions = np.sort(
os.listdir(setup_dir)
) #configfile.readConfigFile(setup_dir.joinpath('.index'))
for session_acq in sessions[::-1]: #.keys():
if session_acq != '.' and session_acq != 'log.txt':
session_dir = setup_dir.joinpath(session_acq)
try:
cells = configfile.readConfigFile(
session_dir.joinpath('.index'))
except: # if there is no file
cells = None
if cells and 'WR_name/ID' in cells['.'].keys(
): # it needs to have WRname
wrname_ephys = cells['.']['WR_name/ID']
wrname = None
for wrname_potential in subject_names: # look for water restriction number
if wrname_potential.lower() in wrname_ephys.lower():
wrname = wrname_potential
subject_id = (df_subject_wr_sessions.loc[
df_subject_wr_sessions[
'water_restriction_number'] == wrname,
'subject_id']).unique()[0]
if wrname == None: # look for animal identifier:
for wrname_potential in subject_ids: # look for water restriction number
if str(wrname_potential) in wrname_ephys.lower():
subject_id = wrname_potential
if len(df_subject_wr_sessions) > 0 and len(
(df_subject_wr_sessions.loc[
df_subject_wr_sessions['subject_id'] ==
subject_id, 'water_restriction_number']
).unique()) > 0:
wrname = (df_subject_wr_sessions.loc[
df_subject_wr_sessions['subject_id'] ==
subject_id, 'water_restriction_number']
).unique()[0]
else:
wrname = 'no water restriction number for this mouse'
if wrname:
session_date = (
session_acq[0:session_acq.find('_')]).replace(
'.', '-')
print('animal: ' + str(subject_id) + ' - ' +
wrname) ##
if setup_name == 'Voltage_rig_1P':
setupname = 'Voltage-Imaging-1p'
else:
print('unkwnown setup, please add')
timer.wait(1000)
if 'experimenter' in cells['.'].keys():
username = cells['.']['experimenter']
else:
username = '******'
print(
'username not specified in acq4 file, assuming rozsam'
)
### check if session already exists
sessiondata = {
'subject_id':
subject_id, #(lab.WaterRestriction() & 'water_restriction_number = "'+df_behavior_session['subject'][0]+'"').fetch()[0]['subject_id'],
'session': np.nan,
'session_date': session_date,
'session_time':
np.nan, #session_time.strftime('%H:%M:%S'),
'username': username,
'rig': setupname
}
for cell in cells.keys():
if cell != '.' and cell != 'log.txt':
ephisdata_cell = list()
sweepstarttimes = list()
cell_dir = session_dir.joinpath(cell)
serieses = configfile.readConfigFile(
cell_dir.joinpath('.index'))
cellstarttime = datetime.datetime.fromtimestamp(
serieses['.']['__timestamp__'])
for series in serieses.keys():
if series != '.' and series != 'log.txt':
series_dir = cell_dir.joinpath(series)
sweeps = configfile.readConfigFile(
series_dir.joinpath('.index'))
if 'Clamp1.ma' in sweeps.keys():
protocoltype = 'single sweep'
sweepkeys = ['']
else:
protocoltype = 'multiple sweeps'
sweepkeys = sweeps.keys()
for sweep in sweepkeys:
if sweep != '.' and '.txt' not in sweep and '.ma' not in sweep:
sweep_dir = series_dir.joinpath(
sweep)
sweepinfo = configfile.readConfigFile(
sweep_dir.joinpath(
'.index'))
if sweep == '':
sweep = '0'
for file in sweepinfo.keys():
if '.ma' in file:
try: # old file version
#print('new file version')
#%
ephysfile = h5.File(
sweep_dir.
joinpath(file),
"r")
data = ephysfile[
'data'][()]
metadata_h5 = ephysfile[
'info']
metadata = read_h5f_metadata(
metadata_h5)
daqchannels = list(
metadata[2]
['DAQ'].keys())
sweepstarttime = datetime.datetime.fromtimestamp(
metadata[2]
['DAQ']
[daqchannels[
0]]
['startTime'])
relativetime = (
sweepstarttime
- cellstarttime
).total_seconds()
if len(
ephisdata_cell
) > 0 and ephisdata_cell[
-1]['sweepstarttime'] == sweepstarttime:
ephisdata = ephisdata_cell.pop(
)
else:
ephisdata = dict(
)
if 'primary' in daqchannels: # ephys data
ephisdata[
'V'] = data[
1]
ephisdata[
'stim'] = data[
0]
ephisdata[
'data'] = data
ephisdata[
'metadata'] = metadata
ephisdata[
'time'] = metadata[
1]['values']
ephisdata[
'relativetime'] = relativetime
ephisdata[
'sweepstarttime'] = sweepstarttime
ephisdata[
'series'] = series
ephisdata[
'sweep'] = sweep
sweepstarttimes.append(
sweepstarttime
)
else: # other daq stuff
#%
for idx, channel in enumerate(
metadata[
0]
['cols']):
channelname = channel[
'name'].decode(
)
if channelname[
0] == 'u':
channelname = channelname[
2:
-1]
if channelname in [
'OrcaFlashExposure',
'Temperature',
'LED525',
'FrameCommand',
'NextFileTrigger'
]:
ephisdata[
channelname] = data[
idx]
#print('{} added'.format(channelname))
else:
print(
'waiting in the other daq'
)
timer.sleep(
1000
)
ephisdata_cell.append(
ephisdata)
#%
except: # new file version
print(
'old version')
ephysfile = MetaArray(
)
ephysfile.readFile(
sweep_dir.
joinpath(file))
data = ephysfile.asarray(
)
metadata = ephysfile.infoCopy(
)
sweepstarttime = datetime.datetime.fromtimestamp(
metadata[2]
['startTime'])
relativetime = (
sweepstarttime
- cellstarttime
).total_seconds()
ephisdata = dict()
ephisdata[
'V'] = data[1]
ephisdata[
'stim'] = data[
0]
ephisdata[
'data'] = data
ephisdata[
'metadata'] = metadata
ephisdata[
'time'] = metadata[
1]['values']
ephisdata[
'relativetime'] = relativetime
ephisdata[
'sweepstarttime'] = sweepstarttime
ephisdata[
'series'] = series
ephisdata[
'sweep'] = sweep
sweepstarttimes.append(
sweepstarttime)
ephisdata_cell.append(
ephisdata)
# ============================================================================
# if wrname == 'FOR04':
# =============================================================================
# add session to DJ if not present
if len(ephisdata_cell) > 0:
# =============================================================================
# print('waiting')
# timer.sleep(1000)
# =============================================================================
#%
if len(experiment.Session()
& 'subject_id = "' +
str(sessiondata['subject_id']) + '"'
& 'session_date = "' +
str(sessiondata['session_date']) +
'"') == 0:
if len(experiment.Session()
& 'subject_id = "' +
str(sessiondata['subject_id']) +
'"') == 0:
sessiondata['session'] = 1
else:
sessiondata['session'] = len(
(experiment.Session()
& 'subject_id = "' +
str(sessiondata['subject_id'])
+ '"').fetch()['session']) + 1
sessiondata['session_time'] = (
sweepstarttimes[0]
).strftime(
'%H:%M:%S'
) # the time of the first sweep will be the session time
experiment.Session().insert1(
sessiondata)
#%
session = (
experiment.Session()
& 'subject_id = "' +
str(sessiondata['subject_id']) + '"'
& 'session_date = "' +
str(sessiondata['session_date']) +
'"').fetch('session')[0]
cell_number = int(cell[cell.find('_') +
1:])
#add cell if not added already
celldata = {
'subject_id': subject_id,
'session': session,
'cell_number': cell_number,
}
#%
if len(ephys_patch.Cell() & celldata
) == 0 or len(ephys_patch.Cell() *
ephys_patch.Sweep()
& celldata) < len(
ephisdata_cell):
if len(ephys_patch.Cell() *
ephys_patch.Sweep() & celldata
) < len(ephisdata_cell):
print('finishing a recording:')
else:
print('adding new recording:')
print(celldata)
if 'type' in serieses['.'].keys():
if serieses['.'][
'type'] == 'interneuron':
celldata['cell_type'] = 'int'
elif serieses['.'][
'type'] == 'unknown' or serieses[
'.']['type'] == 'fail':
celldata[
'cell_type'] = 'unidentified'
else:
print('unhandled cell type!!')
timer.sleep(1000)
else:
celldata[
'cell_type'] = 'unidentified'
celldata['cell_recording_start'] = (
sweepstarttimes[0]
).strftime('%H:%M:%S')
if 'depth' in serieses['.'].keys(
) and len(serieses['.']['depth']) > 0:
celldata['depth'] = int(
serieses['.']['depth'])
else:
celldata['depth'] = -1
try:
ephys_patch.Cell().insert1(
celldata,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
if 'notes' in serieses['.'].keys():
cellnotes = serieses['.']['notes']
else:
cellnotes = ''
cellnotesdata = {
'subject_id': subject_id,
'session': session,
'cell_number': cell_number,
'notes': cellnotes
}
try:
ephys_patch.CellNotes().insert1(
cellnotesdata,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
#%
for i, ephisdata in enumerate(
ephisdata_cell):
#%
sweep_number = i
print('sweep {}'.format(
sweep_number))
sweep_data = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'sweep_start_time':
(ephisdata['sweepstarttime'] -
sweepstarttimes[0]
).total_seconds(),
'sweep_end_time':
(ephisdata['sweepstarttime'] -
sweepstarttimes[0]
).total_seconds() +
ephisdata['time'][-1],
'protocol_name':
ephisdata[
'series'], #[:ephisdata['series'].find('_')],
'protocol_sweep_number':
int(ephisdata['sweep'])
}
if 'mode' in ephisdata['metadata'][
2]['ClampState']: # old file version
recmode = ephisdata[
'metadata'][2][
'ClampState']['mode']
else:
recmode = ephisdata[
'metadata'][2]['Protocol'][
'mode']
if 'IC' in str(recmode):
recording_mode = 'current clamp'
else:
print(
'unhandled recording mode, please act..'
)
timer.sleep(10000)
channelnames = list()
channelunits = list()
for line_now in ephisdata[
'metadata'][0]['cols']:
if type(line_now['name']
) == bytes:
channelnames.append(
line_now['name'].
decode().strip("'"))
channelunits.append(
line_now['units'].
decode().strip("'"))
else:
channelnames.append(
line_now['name'])
channelunits.append(
line_now['units'])
commandidx = np.where(
np.array(channelnames) ==
'command')[0][0]
dataidx = np.where(
np.array(channelnames) ==
'primary')[0][0]
#%
clampparams_data = ephisdata[
'metadata'][2]['ClampState'][
'ClampParams'].copy()
clampparams_data_new = dict()
for clampparamkey in clampparams_data.keys(
): #6004 is true for some reason.. changing it back to 1
if type(clampparams_data[
clampparamkey]
) == np.int32:
if clampparams_data[
clampparamkey] > 0:
clampparams_data[
clampparamkey] = int(
1)
else:
clampparams_data[
clampparamkey] = int(
0)
else:
clampparams_data[
clampparamkey] = float(
clampparams_data[
clampparamkey])
clampparams_data_new[
clampparamkey.lower(
)] = clampparams_data[
clampparamkey]
#%
sweepmetadata_data = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'recording_mode':
recording_mode,
'sample_rate':
np.round(1 / np.median(
np.diff(
ephisdata['metadata']
[1]['values'])))
}
sweepmetadata_data.update(
clampparams_data_new)
sweepdata_data = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'response_trace':
ephisdata['data'][dataidx, :],
'response_units':
ephisdata['metadata'][0]
['cols'][dataidx]['units']
}
sweepstimulus_data = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'stimulus_trace':
ephisdata['data'][
commandidx, :],
'stimulus_units':
ephisdata['metadata'][0]
['cols'][commandidx]['units']
}
#print('waiting')
#timer.sleep(10000)
try:
ephys_patch.Sweep().insert1(
sweep_data,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
try: # maybe it's a duplicate..
ephys_patch.ClampParams(
).insert1(
clampparams_data_new,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
try:
ephys_patch.SweepMetadata(
).insert1(
sweepmetadata_data,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
try:
ephys_patch.SweepResponse(
).insert1(
sweepdata_data,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
try:
ephys_patch.SweepStimulus(
).insert1(
sweepstimulus_data,
allow_direct_insert=True)
except dj.errors.DuplicateError:
pass #already uploaded
#%
if 'OrcaFlashExposure' in ephisdata.keys(
):
sweepimagingexposuredata = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'imaging_exposure_trace':
ephisdata[
'OrcaFlashExposure']
}
try:
ephys_patch.SweepImagingExposure(
).insert1(
sweepimagingexposuredata,
allow_direct_insert=True
)
except dj.errors.DuplicateError:
pass #already uploaded
if 'Temperature' in ephisdata.keys(
):
sweeptemperaturedata = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'temperature_trace':
ephisdata['Temperature'] *
10,
'temperature_units':
'degC'
}
try:
ephys_patch.SweepTemperature(
).insert1(
sweeptemperaturedata,
allow_direct_insert=True
)
except dj.errors.DuplicateError:
pass #already uploaded
if 'LED525' in ephisdata.keys():
sweepLEDdata = {
'subject_id':
subject_id,
'session':
session,
'cell_number':
cell_number,
'sweep_number':
sweep_number,
'imaging_led_trace':
ephisdata['LED525']
}
try:
ephys_patch.SweepLED(
).insert1(
sweepLEDdata,
allow_direct_insert=True
)
except dj.errors.DuplicateError:
pass #already uploaded
plot_precision_recall(key_cell,binwidth = binwidth ,frbinwidth = frbinwidth,firing_rate_window = firing_rate_window)
# =============================================================================
# data = plot_ephys_ophys_trace(key_cell,time_to_plot=None,trace_window = 50,show_stimulus = True,show_e_ap_peaks = True,show_o_ap_peaks = True)
# data['figure_handle'].savefig('./figures/{}_cell_{}_roi_type_{}_long.png'.format(key_cell['subject_id'],key_cell['cell_number'],key_cell['roi_type']), bbox_inches = 'tight')
# print(cell)
# =============================================================================
#%% #%%
data = plot_ephys_ophys_trace(key_cell,time_to_plot=25,trace_window = 1,show_e_ap_peaks = True,show_o_ap_peaks = True)
#%%
session = 1
subject_id = 456462
cell_number = 5
roi_type = 'Spikepursuit'#'Spikepursuit'#'VolPy_denoised'#'SpikePursuit'#'VolPy_dexpF0'#'VolPy'#'SpikePursuit_dexpF0'#'VolPy_dexpF0'#''Spikepursuit'#'VolPy'#
key_cell = {'session':session,'subject_id':subject_id,'cell_number':cell_number,'roi_type':roi_type }
session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key_cell).fetch1('session_time','cell_recording_start')
first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_start_time'),float))
first_movie_start_time_real = first_movie_start_time + session_time.total_seconds()
threshold,apmaxtime = (imaging_gt.ROIAPWave()*ephysanal.ActionPotential()*ephysanal.ActionPotentialDetails()&key_cell&'ap_real=1').fetch('ap_threshold','ap_max_time')
threshold=np.asarray(threshold,float)
apmaxtime=np.asarray(apmaxtime,float)
# =============================================================================
# session_time_to_plot = time_to_plot+first_movie_start_time # time relative to session start
# cell_time_to_plot= session_time_to_plot + session_time.total_seconds() -cell_recording_start.total_seconds() # time relative to recording start
# =============================================================================
#%
time_to_plot = apmaxtime[np.argmin(threshold)]+cell_recording_start.total_seconds() - first_movie_start_time_real
data = plot_ephys_ophys_trace(key_cell,
time_to_plot=time_to_plot,
virus_name = (lab.Virus()&'virus_id = {}'.format(virus_id)).fetch1('virus_name')
if virus_name != 'syn-FLEX-Voltron-ST':
continue
if len(lab.Surgery()&'subject_id = {}'.format(subject_id))!=2:
print('only {} surgery for {}'.format(len(lab.Surgery()&'subject_id = {}'.format(subject_id)),subject_id))
continue
movie_num_list.append(len(imaging.Movie()&'subject_id = {}'.format(subject_id)))
date_of_birth,sex = (lab.Subject()&'subject_id = {}'.format(subject_id)).fetch1('date_of_birth','sex')
surgery_times = (lab.Surgery()&'subject_id = {}'.format(subject_id)).fetch('start_time')
ml,ap = (lab.Surgery.Procedure()&'subject_id = {}'.format(subject_id)).fetch1('ml_location','ap_location')
ml_list.append(int(ml))
ap_list.append(int(ap))
age_first_surgery_list.append((np.min(surgery_times).date()-date_of_birth).days)
expression_time_list.append(np.diff(surgery_times)[0].days)
sex_list.append(sex)
depth_list.extend((ephys_patch.Cell()&'subject_id = {}'.format(subject_id)).fetch('depth'))
depth_list = np.asarray(depth_list)
ap_list = np.asarray(ap_list)
ml_list = np.asarray(ml_list)
movie_num_list = np.asarray(movie_num_list)
sex_list = np.asarray(sex_list)
age_first_surgery_list = np.asarray(age_first_surgery_list)
expression_time_list = np.asarray(expression_time_list)
print('{} mice'.format(len(sex_list)))
print('{} males, {} females'.format(sum(sex_list=='M'), sum(sex_list=='F')))
print('ages at first surgery: {}'.format(np.sort(age_first_surgery_list)))
print('ml cranio locations: {}, ap cranio locations {}'.format(ml_list,ap_list))
print('exression times:{}'.format(np.sort(expression_time_list)))
print('cell depths:{}'.format(np.sort(depth_list)))
#break
time_to_plot=25,
trace_window=1,
show_e_ap_peaks=True,
show_o_ap_peaks=True)
#%%
data = plot_ephys_ophys_trace(key_cell,
time_to_plot=None,
trace_window=50,
show_stimulus=True,
show_e_ap_peaks=True,
show_o_ap_peaks=True)
#%%
#%%plot time offset between 1st ROI and real elphys
min_corr_coeff = .1
subject_ids = ephys_patch.Cell().fetch('subject_id')
sessions = ephys_patch.Cell().fetch('session')
cellnums = ephys_patch.Cell().fetch('cell_number')
roi_types = ['Spikepursuit', 'VolPy']
for roi_type_idx, roi_type in enumerate(roi_types):
fig = plt.figure()
axs_delay_sweep = list()
axs_delay_coeff = list()
axs_coeff_sweep = list()
axs_delay_sweep.append(fig.add_axes([0, -roi_type_idx, .8, .8]))
axs_coeff_sweep.append(fig.add_axes([1, -roi_type_idx, .8, .8]))
axs_delay_coeff.append(fig.add_axes([2, -roi_type_idx, .8, .8]))
for subject_id, session, cellnum in zip(subject_ids, sessions, cellnums):
key = {
'subject_id': subject_id,
'session': session,
for movie in movies:
print(movie)
if len(imaging_gt.GroundTruthROI()&movie&'roi_type = "VolPy"') == 0:
print('no groundtruth.. skipped')
else:
movie_dict = dict((imaging.Movie()&movie).fetch1())
for keynow in movie_dict.keys():
if type(movie_dict[keynow]) == decimal.Decimal:
movie_dict[keynow] = float(movie_dict[keynow])
save_dir_base = os.path.join(gt_package_directory,str(key['subject_id']),'Cell_{}'.format(key['cell_number']),movie_dict['movie_name'])
if ((os.path.isdir(save_dir_base) and (len(os.listdir(save_dir_base))>3 and not overwrite))):
print('already exported, skipped')
else:
sweep_numbers = movie['sweep_numbers']
del movie['sweep_numbers']
session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key).fetch1('session_time','cell_recording_start')
first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key).fetch('movie_start_time'),float))
first_movie_start_time_real = first_movie_start_time + session_time.total_seconds()
frame_times = (imaging.MovieFrameTimes()&movie).fetch1('frame_times') - cell_recording_start.total_seconds() + session_time.total_seconds()
movie_dict['movie_start_time'] = frame_times[0]
movie_files = list()
repositories , directories , fnames = (imaging.MovieFile() & movie).fetch('movie_file_repository','movie_file_directory','movie_file_name')
for repository,directory,fname in zip(repositories,directories,fnames):
movie_files.append(os.path.join(dj.config['locations.{}'.format(repository)],directory,fname))
sweepdata_out = list()
sweepmetadata_out = list()
for sweep_number in sweep_numbers:
#%
key_sweep = key.copy()
key_sweep['sweep_number'] = sweep_number
