def make(self, key): #% # key = { 'subject_id': 462149, 'session':1,'cell_number':1,'movie_number':11} session_time = (experiment.Session() & key).fetch('session_time')[0] cell_time = (ephys_patch.Cell() & key).fetch('cell_recording_start')[0] cell_sweep_start_times = (ephys_patch.Sweep() & key).fetch('sweep_start_time') cell_sweep_end_times = (ephys_patch.Sweep() & key).fetch('sweep_end_time') time_start = float( np.min(cell_sweep_start_times)) + cell_time.total_seconds( ) - session_time.total_seconds() time_end = float( np.max(cell_sweep_end_times)) + cell_time.total_seconds( ) - session_time.total_seconds() try: movie = (imaging.Movie()) & key & 'movie_start_time > ' + str( time_start) & 'movie_start_time < ' + str(time_end) sweep_start_times, sweep_end_times, sweep_nums = ( ephys_patch.Sweep() & key).fetch('sweep_start_time', 'sweep_end_time', 'sweep_number') sweep_start_times = np.asarray(sweep_start_times, float) + cell_time.total_seconds( ) - session_time.total_seconds() sweep_end_times = np.asarray(sweep_end_times, float) + cell_time.total_seconds( ) - session_time.total_seconds() #for movie in movies_now: frametimes = (imaging.MovieFrameTimes & movie).fetch1('frame_times') needed_start_time = frametimes[0] needed_end_time = frametimes[-1] sweep_nums_needed = sweep_nums[ ((sweep_start_times > needed_start_time) & (sweep_start_times < needed_end_time)) | ((sweep_end_times > needed_start_time) & (sweep_end_times < needed_end_time)) | ((sweep_end_times > needed_end_time) & (sweep_start_times < needed_start_time))] if len(sweep_nums_needed) > 0: key['sweep_numbers'] = sweep_nums_needed self.insert1(key, skip_duplicates=True) except: pass
def plot_precision_recall(key_cell,binwidth = 30,frbinwidth = 0.01,firing_rate_window = 3,save_figures = True,xlimits =None): #%% session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key_cell).fetch1('session_time','cell_recording_start') first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_start_time'),float)) first_movie_start_time_real = first_movie_start_time + session_time.total_seconds() #% fr_kernel = np.ones(int(firing_rate_window/frbinwidth))/(firing_rate_window/frbinwidth) first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_start_time'),float)) ephys_matched_ap_times,ephys_unmatched_ap_times,ophys_matched_ap_times,ophys_unmatched_ap_times = (imaging_gt.GroundTruthROI()&key_cell).fetch('ephys_matched_ap_times','ephys_unmatched_ap_times','ophys_matched_ap_times','ophys_unmatched_ap_times') #% ephys_matched_ap_times = np.concatenate(ephys_matched_ap_times) - first_movie_start_time ephys_unmatched_ap_times = np.concatenate(ephys_unmatched_ap_times) - first_movie_start_time all_ephys_ap_times = np.concatenate([ephys_matched_ap_times,ephys_unmatched_ap_times]) ophys_matched_ap_times = np.concatenate(ophys_matched_ap_times) - first_movie_start_time ophys_unmatched_ap_times = np.concatenate(ophys_unmatched_ap_times) - first_movie_start_time all_ophys_ap_times = np.concatenate([ophys_matched_ap_times,ophys_unmatched_ap_times]) all_times = np.concatenate([ephys_matched_ap_times,ephys_unmatched_ap_times,ophys_matched_ap_times,ophys_unmatched_ap_times]) maxtime = np.max(all_times) #% fr_bincenters = np.arange(frbinwidth/2,maxtime+frbinwidth,frbinwidth) fr_binedges = np.concatenate([fr_bincenters-frbinwidth/2,[fr_bincenters[-1]+frbinwidth/2]]) fr_e = np.histogram(all_ephys_ap_times,fr_binedges)[0]/frbinwidth fr_e = np.convolve(fr_e, fr_kernel,'same') fr_o = np.histogram(all_ophys_ap_times,fr_binedges)[0]/frbinwidth fr_o = np.convolve(fr_o, fr_kernel,'same') #% bincenters = np.arange(binwidth/2,maxtime+binwidth,binwidth) binedges = np.concatenate([bincenters-binwidth/2,[bincenters[-1]+binwidth/2]]) ephys_matched_ap_num_binned,tmp = np.histogram(ephys_matched_ap_times,binedges) ephys_unmatched_ap_num_binned,tmp = np.histogram(ephys_unmatched_ap_times,binedges) ophys_matched_ap_num_binned,tmp = np.histogram(ophys_matched_ap_times,binedges) ophys_unmatched_ap_num_binned,tmp = np.histogram(ophys_unmatched_ap_times,binedges) precision_binned = ophys_matched_ap_num_binned/(ophys_matched_ap_num_binned+ophys_unmatched_ap_num_binned) recall_binned = ephys_matched_ap_num_binned/(ephys_matched_ap_num_binned+ephys_unmatched_ap_num_binned) f1_binned = 2*precision_binned*recall_binned/(precision_binned+recall_binned) #% if xlimits == None: xlimits = [binedges[0],binedges[-1]] fig=plt.figure()#figsize=(50,0) ax_rates = fig.add_axes([0,1.4,2,.3]) ax_spikes = fig.add_axes([0,1,2,.3]) ax = fig.add_axes([0,0,2,.8]) ax_snratio = fig.add_axes([0,-1,2,.8]) ax_latency = fig.add_axes([0,-2,2,.8]) ax_latency_hist = fig.add_axes([.5,-3,1,.8]) ax.plot(bincenters,precision_binned,'go-',label = 'precision') ax.plot(bincenters,recall_binned,'ro-',label = 'recall') ax.plot(bincenters,f1_binned,'bo-',label = 'f1 score') ax.set_xlim([binedges[0],binedges[-1]]) ax.set_ylim([0,1]) ax.legend() ax_spikes.plot(ephys_unmatched_ap_times,np.zeros(len(ephys_unmatched_ap_times)),'r|', ms = 10) ax_spikes.plot(all_ephys_ap_times,np.zeros(len(all_ephys_ap_times))+.33,'k|', ms = 10) ax_spikes.plot(ophys_unmatched_ap_times,np.ones(len(ophys_unmatched_ap_times)),'r|',ms = 10) ax_spikes.plot(all_ophys_ap_times,np.ones(len(all_ophys_ap_times))-.33,'g|', ms = 10) ax_spikes.set_yticks([0,.33,.67,1]) ax_spikes.set_yticklabels(['false negative','ephys','ophys','false positive']) ax_spikes.set_ylim([-.2,1.2]) ax_spikes.set_xlim(xlimits) t,sn = (ephysanal.ActionPotential()*imaging_gt.GroundTruthROI()*imaging_gt.ROIAPWave()&key_cell).fetch('ap_max_time','apwave_snratio') t= np.asarray(t,float) + cell_recording_start.total_seconds() - first_movie_start_time_real ax_snratio.plot(t,sn,'ko') ax_snratio.set_ylabel('signal to noise ratio') ax_snratio.set_ylim([0,20]) ax_snratio.set_xlim(xlimits) ax_rates.plot(fr_bincenters,fr_e,'k-',label = 'ephys') ax_rates.plot(fr_bincenters,fr_o,'g-',label = 'ophys') ax_rates.legend() ax_rates.set_xlim(xlimits) ax_rates.set_ylabel('Firing rate (Hz)') ax_rates.set_title('subject_id: %d, cell number: %s' %(key_cell['subject_id'],key_cell['cell_number'])) ax_latency.plot(ephys_matched_ap_times,1000*(ophys_matched_ap_times-ephys_matched_ap_times),'ko') ax_latency.set_ylabel('ephys-ophys spike latency (ms)') ax_latency.set_ylim([0,10]) ax_latency.set_xlabel('time from first movie start (s)') ax_latency.set_xlim(xlimits) ax_latency_hist.hist(1000*(ophys_matched_ap_times-ephys_matched_ap_times),np.arange(-5,15,.1)) ax_latency_hist.set_xlabel('ephys-ophys spike latency (ms)') ax_latency_hist.set_ylabel('matched ap count') data = list() data.append(plot_ephys_ophys_trace(key_cell,ephys_matched_ap_times[0],trace_window = 1,show_e_ap_peaks = True,show_o_ap_peaks = True)) data.append(plot_ephys_ophys_trace(key_cell,ephys_unmatched_ap_times[0],trace_window = 1,show_e_ap_peaks = True,show_o_ap_peaks = True)) data.append(plot_ephys_ophys_trace(key_cell,ophys_unmatched_ap_times[0],trace_window = 1,show_e_ap_peaks = True,show_o_ap_peaks = True)) if save_figures: fig.savefig('./figures/{}_cell_{}_roi_type_{}.png'.format(key_cell['subject_id'],key_cell['cell_number'],key_cell['roi_type']), bbox_inches = 'tight') for data_now,fname in zip(data,['matched','false_negative','false_positive']): data_now['figure_handle'].savefig('./figures/{}_cell_{}_roi_type_{}_{}.png'.format(key_cell['subject_id'],key_cell['cell_number'],key_cell['roi_type'],fname), bbox_inches = 'tight')
def plot_ephys_ophys_trace(key_cell,time_to_plot=None,trace_window = 1,show_stimulus = False,show_e_ap_peaks = False, show_o_ap_peaks = False): #%% # ============================================================================= # key_cell = {'session': 1, # 'subject_id': 456462, # 'cell_number': 3, # 'motion_correction_method': 'VolPy', # 'roi_type': 'VolPy'} # time_to_plot=None # trace_window = 100 # show_stimulus = False # show_e_ap_peaks = True # show_o_ap_peaks = True # ============================================================================= fig=plt.figure() ax_ophys = fig.add_axes([0,0,2,.8]) ax_ephys = fig.add_axes([0,-1,2,.8]) if show_stimulus: ax_ephys_stim = fig.add_axes([0,-1.5,2,.4]) #% session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key_cell).fetch1('session_time','cell_recording_start') first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_start_time'),float)) first_movie_start_time_real = first_movie_start_time + session_time.total_seconds() if not time_to_plot: time_to_plot = trace_window/2#ephys_matched_ap_times[0] session_time_to_plot = time_to_plot+first_movie_start_time # time relative to session start cell_time_to_plot= session_time_to_plot + session_time.total_seconds() -cell_recording_start.total_seconds() # time relative to recording start #% sweep_start_times,sweep_end_times,sweep_nums = (ephys_patch.Sweep()&key_cell).fetch('sweep_start_time','sweep_end_time','sweep_number') needed_start_time = cell_time_to_plot - trace_window/2 needed_end_time = cell_time_to_plot + trace_window/2 #% sweep_nums = sweep_nums[((sweep_start_times > needed_start_time) & (sweep_start_times < needed_end_time)) | ((sweep_end_times > needed_start_time) & (sweep_end_times < needed_end_time)) | ((sweep_end_times > needed_end_time) & (sweep_start_times < needed_start_time)) ] ephys_traces = list() ephys_trace_times = list() ephys_sweep_start_times = list() ephys_traces_stim = list() for sweep_num in sweep_nums: sweep = ephys_patch.Sweep()&key_cell&'sweep_number = %d' % sweep_num trace,sr= (ephys_patch.SweepMetadata()*ephys_patch.SweepResponse()&sweep).fetch1('response_trace','sample_rate') trace = trace*1000 sweep_start_time = float(sweep.fetch('sweep_start_time')) trace_time = np.arange(len(trace))/sr + sweep_start_time + cell_recording_start.total_seconds() - first_movie_start_time_real trace_idx = (time_to_plot-trace_window/2 < trace_time) & (time_to_plot+trace_window/2 > trace_time) ax_ephys.plot(trace_time[trace_idx],trace[trace_idx],'k-') ephys_traces.append(trace) ephys_trace_times.append(trace_time) ephys_sweep_start_times.append(sweep_start_time) if show_e_ap_peaks: ap_max_index = (ephysanal.ActionPotential()&sweep).fetch('ap_max_index') aptimes = trace_time[np.asarray(ap_max_index,int)] apVs = trace[np.asarray(ap_max_index,int)] ap_needed = (time_to_plot-trace_window/2 < aptimes) & (time_to_plot+trace_window/2 > aptimes) aptimes = aptimes[ap_needed] apVs = apVs[ap_needed] ax_ephys.plot(aptimes,apVs,'ro') if show_stimulus: trace_stim= (ephys_patch.SweepMetadata()*ephys_patch.SweepStimulus()&sweep).fetch1('stimulus_trace') trace_stim = trace_stim*10**12 ax_ephys_stim.plot(trace_time[trace_idx],trace_stim[trace_idx],'k-') ephys_traces_stim.append(trace_stim) ephysdata = {'ephys_traces':ephys_traces,'ephys_trace_times':ephys_trace_times} if show_stimulus: ephysdata['ephys_traces_stimulus'] = ephys_traces_stim movie_nums, movie_start_times,movie_frame_rates,movie_frame_nums = ((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_number','movie_start_time','movie_frame_rate','movie_frame_num') movie_start_times = np.asarray(movie_start_times, float) movie_end_times = np.asarray(movie_start_times, float)+np.asarray(movie_frame_nums, float)/np.asarray(movie_frame_rates, float) needed_start_time = session_time_to_plot - trace_window/2 needed_end_time = session_time_to_plot + trace_window/2 movie_nums = movie_nums[((movie_start_times >= needed_start_time) & (movie_start_times <= needed_end_time)) | ((movie_end_times >= needed_start_time) & (movie_end_times <= needed_end_time)) | ((movie_end_times >= needed_end_time) & (movie_start_times <= needed_start_time)) ] dffs=list() frametimes = list() for movie_num in movie_nums: #movie_num = movie_nums[(session_time_to_plot>movie_start_times)&(session_time_to_plot<movie_end_times)][0] key_movie = key_cell.copy() key_movie['movie_number'] = movie_num dff,gt_roi_number = ((imaging.ROI()*imaging_gt.GroundTruthROI())&key_movie).fetch1('roi_dff','roi_number') dff_all,roi_number_all = (imaging.ROI()&key_movie).fetch('roi_dff','roi_number') dff_all =dff_all[roi_number_all!=gt_roi_number] frame_times = ((imaging.MovieFrameTimes()*imaging_gt.GroundTruthROI())&key_movie).fetch1('frame_times') + (session_time).total_seconds() - first_movie_start_time_real #modified to cell time frame_idx = (time_to_plot-trace_window/2 < frame_times) & (time_to_plot+trace_window/2 > frame_times) dff_list = [dff] dff_list.extend(dff_all) prevminval = 0 for dff_now,alpha_now in zip(dff_list,np.arange(1,1/(len(dff_list)+1),-1/(len(dff_list)+1))): dfftoplotnow = dff_now[frame_idx] + prevminval ax_ophys.plot(frame_times[frame_idx],dfftoplotnow,'g-',alpha=alpha_now) prevminval = np.min(dfftoplotnow) -.005 #ax_ophys.plot(frame_times[frame_idx],dff[frame_idx],'g-') dffs.append(dff) frametimes.append(frame_times) if show_o_ap_peaks: if 'raw' in key_cell['roi_type']: apidxes = ((imaging.ROI()*imaging_gt.GroundTruthROI())&key_movie).fetch1('roi_spike_indices') else: apidxes = ((imaging.ROI()*imaging_gt.GroundTruthROI())&key_movie).fetch1('roi_spike_indices')-1 oap_times = frame_times[apidxes] oap_vals = dff[apidxes] oap_needed = (time_to_plot-trace_window/2 < oap_times) & (time_to_plot+trace_window/2 > oap_times) oap_times = oap_times[oap_needed] oap_vals = oap_vals[oap_needed] ax_ophys.plot(oap_times,oap_vals,'ro') ophysdata = {'ophys_traces':dffs,'ophys_trace_times':frametimes} ax_ophys.autoscale(tight = True) # ============================================================================= # dfff = np.concatenate(dffs) # dfff[np.argmin(dfff)]=0 # ax_ophys.set_ylim([min(dfff),max(dfff)]) # ============================================================================= ax_ophys.set_xlim([time_to_plot-trace_window/2,time_to_plot+trace_window/2]) ax_ophys.set_ylabel('dF/F') ax_ophys.spines["top"].set_visible(False) ax_ophys.spines["right"].set_visible(False) ax_ophys.invert_yaxis() ax_ephys.autoscale(tight = True) ax_ephys.set_xlim([time_to_plot-trace_window/2,time_to_plot+trace_window/2]) ax_ephys.set_ylabel('Vm (mV))') ax_ephys.spines["top"].set_visible(False) ax_ephys.spines["right"].set_visible(False) if show_stimulus: # ax_ephys_stim.autoscale(tight = True) ax_ephys_stim.set_xlim([time_to_plot-trace_window/2,time_to_plot+trace_window/2]) ax_ephys_stim.set_ylabel('Injected current (pA))') ax_ephys_stim.set_xlabel('time from first movie start (s)') ax_ephys_stim.spines["top"].set_visible(False) ax_ephys_stim.spines["right"].set_visible(False) else: ax_ephys.set_xlabel('time from first movie start (s)') outdict = {'ephys':ephysdata,'ophys':ophysdata,'figure_handle':fig} #%% return outdict
matplotlib.rc('font', **font) from pathlib import Path homefolder = dj.config['locations.mr_share'] #%% ephys parameters at different expression times and different viruses ephys_data=dict() ephys_data = {'cell_dict':list(), 'virus':list(), 'expression_time':list(), 'RS':list(), 'Rin':list(), 'AP_amplitude':list(), 'AP_halfwidth':list(), 'AP_threshold':list(), 'resting':list()} cells = ephys_patch.Cell() APs_needed = 50 AP_max_RS = 40 AP_max_baseline = -40 #mV max_baseline_current = 100*10**-12 #A max_v0 = -.04 #V min_current = -40*10**-12 #A min_pulse_time = 0.3 #s integration_window = .04 #sec for cell in cells: if len((lab.Surgery()&'subject_id = {}'.format(cell['subject_id'])).fetch('start_time'))==1: continue # if there is only one surgery, we don't care expression_time = np.diff((lab.Surgery()&'subject_id = {}'.format(cell['subject_id'])).fetch('start_time'))[0].days virus_id = (lab.Surgery.VirusInjection()&'subject_id = {}'.format(cell['subject_id'])).fetch('virus_id')[0] if virus_id == 238: virus = 'Voltron 1'
def make(self, key): #% #key = {'subject_id': 454597, 'session': 1, 'cell_number': 0, 'motion_correction_method': 'Matlab', 'roi_type': 'SpikePursuit', 'roi_number': 1} #key = {'subject_id': 456462, 'session': 1, 'cell_number': 5, 'movie_number': 3, 'motion_correction_method': 'VolPy', 'roi_type': 'VolPy', 'roi_number': 1} if len( ROIEphysCorrelation & key ) > 0: # and key['roi_type'] == 'SpikePursuit' #only spikepursuit for now.. key_to_compare = key.copy() del key_to_compare['roi_number'] #print(key) #% roinums = np.unique( (ROIEphysCorrelation() & key_to_compare).fetch('roi_number')) snratios_mean = list() snratios_median = list() snratios_first50 = list() for roinum_now in roinums: snratios = (ROIAPWave() & key_to_compare & 'roi_number = {}'.format(roinum_now) ).fetch('apwave_snratio') snratios_mean.append(np.mean(snratios)) snratios_median.append(np.median(snratios)) snratios_first50.append(np.mean(snratios[:50])) #%% if np.max( (ROIEphysCorrelation() & key).fetch('roi_number') ) == roinums[np.argmax( snratios_first50 )]: #np.max((imaging_gt.ROIEphysCorrelation()&key).fetch('roi_number')) == np.min((imaging_gt.ROIEphysCorrelation&key_to_compare).fetch('roi_number')):#np.max(np.abs((ROIEphysCorrelation&key).fetch('corr_coeff'))) == np.max(np.abs((ROIEphysCorrelation&key_to_compare).fetch('corr_coeff'))): print('this is it') print(key['roi_type']) cellstarttime = (ephys_patch.Cell() & key).fetch1('cell_recording_start') sessionstarttime = (experiment.Session() & key).fetch1('session_time') aptimes = np.asarray( (ephysanal.ActionPotential() & key).fetch('ap_max_time'), float) + (cellstarttime - sessionstarttime).total_seconds() sweep_start_times, sweep_end_times = (ephys_patch.Sweep() & key).fetch( 'sweep_start_time', 'sweep_end_time') sweep_start_times = np.asarray(sweep_start_times, float) + ( cellstarttime - sessionstarttime).total_seconds() sweep_end_times = np.asarray(sweep_end_times, float) + ( cellstarttime - sessionstarttime).total_seconds() frame_timess, roi_spike_indicess = ( imaging.MovieFrameTimes() * imaging.Movie() * imaging.ROI() & key).fetch('frame_times', 'roi_spike_indices') movie_start_times = list() movie_end_times = list() roi_ap_times = list() for frame_times, roi_spike_indices in zip( frame_timess, roi_spike_indicess): movie_start_times.append(frame_times[0]) movie_end_times.append(frame_times[-1]) roi_ap_times.append(frame_times[roi_spike_indices]) movie_start_times = np.sort(movie_start_times) movie_end_times = np.sort(movie_end_times) roi_ap_times = np.sort(np.concatenate(roi_ap_times)) #% ##delete spikes in optical traces where there was no ephys recording for start_t, end_t in zip( np.concatenate([sweep_start_times, [np.inf]]), np.concatenate([[0], sweep_end_times])): idxtodel = np.where((roi_ap_times > end_t) & (roi_ap_times < start_t))[0] if len(idxtodel) > 0: roi_ap_times = np.delete(roi_ap_times, idxtodel) ##delete spikes in ephys traces where there was no imaging for start_t, end_t in zip( np.concatenate([movie_start_times, [np.inf]]), np.concatenate([[0], movie_end_times])): idxtodel = np.where((aptimes > end_t) & (aptimes < start_t))[0] if len(idxtodel) > 0: #print(idxtodel) aptimes = np.delete(aptimes, idxtodel) #% D = np.zeros([len(aptimes), len(roi_ap_times)]) for idx, apt in enumerate(aptimes): D[idx, :] = (roi_ap_times - apt) * 1000 D_test = np.abs(D) D_test[D_test > 15] = 1000 D_test[D < -1] = 1000 X = scipy.optimize.linear_sum_assignment(D_test) #% cost = D_test[X[0], X[1]] unmatched = np.where(cost == 1000)[0] X0_final = np.delete(X[0], unmatched) X1_final = np.delete(X[1], unmatched) ephys_ap_times = aptimes[X0_final] ophys_ap_times = roi_ap_times[X1_final] false_positive_time_imaging = list() for roi_ap_time in roi_ap_times: if roi_ap_time not in ophys_ap_times: false_positive_time_imaging.append(roi_ap_time) false_negative_time_ephys = list() for aptime in aptimes: if aptime not in ephys_ap_times: false_negative_time_ephys.append(aptime) key['ephys_matched_ap_times'] = ephys_ap_times key['ophys_matched_ap_times'] = ophys_ap_times key['ephys_unmatched_ap_times'] = false_negative_time_ephys key['ophys_unmatched_ap_times'] = false_positive_time_imaging #print(imaging.ROI()&key) #print([len(aptimes),'vs',len(roi_ap_times)]) #%% self.insert1(key, skip_duplicates=True)
def populateelphys(): #%% df_subject_wr_sessions = pd.DataFrame(lab.WaterRestriction() * experiment.Session() * experiment.SessionDetails) df_subject_ids = pd.DataFrame(lab.Subject()) if len(df_subject_wr_sessions) > 0: subject_names = df_subject_wr_sessions[ 'water_restriction_number'].unique() subject_names.sort() else: subject_names = list() subject_ids = df_subject_ids['subject_id'].unique() #% sumdata = list() basedir = Path(dj.config['locations.elphysdata_acq4']) for setup_dir in basedir.iterdir(): setup_name = setup_dir.name sessions = np.sort( os.listdir(setup_dir) ) #configfile.readConfigFile(setup_dir.joinpath('.index')) for session_acq in sessions[::-1]: #.keys(): if session_acq != '.' and session_acq != 'log.txt': session_dir = setup_dir.joinpath(session_acq) try: cells = configfile.readConfigFile( session_dir.joinpath('.index')) except: # if there is no file cells = None if cells and 'WR_name/ID' in cells['.'].keys( ): # it needs to have WRname wrname_ephys = cells['.']['WR_name/ID'] wrname = None for wrname_potential in subject_names: # look for water restriction number if wrname_potential.lower() in wrname_ephys.lower(): wrname = wrname_potential subject_id = (df_subject_wr_sessions.loc[ df_subject_wr_sessions[ 'water_restriction_number'] == wrname, 'subject_id']).unique()[0] if wrname == None: # look for animal identifier: for wrname_potential in subject_ids: # look for water restriction number if str(wrname_potential) in wrname_ephys.lower(): subject_id = wrname_potential if len(df_subject_wr_sessions) > 0 and len( (df_subject_wr_sessions.loc[ df_subject_wr_sessions['subject_id'] == subject_id, 'water_restriction_number'] ).unique()) > 0: wrname = (df_subject_wr_sessions.loc[ df_subject_wr_sessions['subject_id'] == subject_id, 'water_restriction_number'] ).unique()[0] else: wrname = 'no water restriction number for this mouse' if wrname: session_date = ( session_acq[0:session_acq.find('_')]).replace( '.', '-') print('animal: ' + str(subject_id) + ' - ' + wrname) ## if setup_name == 'Voltage_rig_1P': setupname = 'Voltage-Imaging-1p' else: print('unkwnown setup, please add') timer.wait(1000) if 'experimenter' in cells['.'].keys(): username = cells['.']['experimenter'] else: username = '******' print( 'username not specified in acq4 file, assuming rozsam' ) ### check if session already exists sessiondata = { 'subject_id': subject_id, #(lab.WaterRestriction() & 'water_restriction_number = "'+df_behavior_session['subject'][0]+'"').fetch()[0]['subject_id'], 'session': np.nan, 'session_date': session_date, 'session_time': np.nan, #session_time.strftime('%H:%M:%S'), 'username': username, 'rig': setupname } for cell in cells.keys(): if cell != '.' and cell != 'log.txt': ephisdata_cell = list() sweepstarttimes = list() cell_dir = session_dir.joinpath(cell) serieses = configfile.readConfigFile( cell_dir.joinpath('.index')) cellstarttime = datetime.datetime.fromtimestamp( serieses['.']['__timestamp__']) for series in serieses.keys(): if series != '.' and series != 'log.txt': series_dir = cell_dir.joinpath(series) sweeps = configfile.readConfigFile( series_dir.joinpath('.index')) if 'Clamp1.ma' in sweeps.keys(): protocoltype = 'single sweep' sweepkeys = [''] else: protocoltype = 'multiple sweeps' sweepkeys = sweeps.keys() for sweep in sweepkeys: if sweep != '.' and '.txt' not in sweep and '.ma' not in sweep: sweep_dir = series_dir.joinpath( sweep) sweepinfo = configfile.readConfigFile( sweep_dir.joinpath( '.index')) if sweep == '': sweep = '0' for file in sweepinfo.keys(): if '.ma' in file: try: # old file version #print('new file version') #% ephysfile = h5.File( sweep_dir. joinpath(file), "r") data = ephysfile[ 'data'][()] metadata_h5 = ephysfile[ 'info'] metadata = read_h5f_metadata( metadata_h5) daqchannels = list( metadata[2] ['DAQ'].keys()) sweepstarttime = datetime.datetime.fromtimestamp( metadata[2] ['DAQ'] [daqchannels[ 0]] ['startTime']) relativetime = ( sweepstarttime - cellstarttime ).total_seconds() if len( ephisdata_cell ) > 0 and ephisdata_cell[ -1]['sweepstarttime'] == sweepstarttime: ephisdata = ephisdata_cell.pop( ) else: ephisdata = dict( ) if 'primary' in daqchannels: # ephys data ephisdata[ 'V'] = data[ 1] ephisdata[ 'stim'] = data[ 0] ephisdata[ 'data'] = data ephisdata[ 'metadata'] = metadata ephisdata[ 'time'] = metadata[ 1]['values'] ephisdata[ 'relativetime'] = relativetime ephisdata[ 'sweepstarttime'] = sweepstarttime ephisdata[ 'series'] = series ephisdata[ 'sweep'] = sweep sweepstarttimes.append( sweepstarttime ) else: # other daq stuff #% for idx, channel in enumerate( metadata[ 0] ['cols']): channelname = channel[ 'name'].decode( ) if channelname[ 0] == 'u': channelname = channelname[ 2: -1] if channelname in [ 'OrcaFlashExposure', 'Temperature', 'LED525', 'FrameCommand', 'NextFileTrigger' ]: ephisdata[ channelname] = data[ idx] #print('{} added'.format(channelname)) else: print( 'waiting in the other daq' ) timer.sleep( 1000 ) ephisdata_cell.append( ephisdata) #% except: # new file version print( 'old version') ephysfile = MetaArray( ) ephysfile.readFile( sweep_dir. joinpath(file)) data = ephysfile.asarray( ) metadata = ephysfile.infoCopy( ) sweepstarttime = datetime.datetime.fromtimestamp( metadata[2] ['startTime']) relativetime = ( sweepstarttime - cellstarttime ).total_seconds() ephisdata = dict() ephisdata[ 'V'] = data[1] ephisdata[ 'stim'] = data[ 0] ephisdata[ 'data'] = data ephisdata[ 'metadata'] = metadata ephisdata[ 'time'] = metadata[ 1]['values'] ephisdata[ 'relativetime'] = relativetime ephisdata[ 'sweepstarttime'] = sweepstarttime ephisdata[ 'series'] = series ephisdata[ 'sweep'] = sweep sweepstarttimes.append( sweepstarttime) ephisdata_cell.append( ephisdata) # ============================================================================ # if wrname == 'FOR04': # ============================================================================= # add session to DJ if not present if len(ephisdata_cell) > 0: # ============================================================================= # print('waiting') # timer.sleep(1000) # ============================================================================= #% if len(experiment.Session() & 'subject_id = "' + str(sessiondata['subject_id']) + '"' & 'session_date = "' + str(sessiondata['session_date']) + '"') == 0: if len(experiment.Session() & 'subject_id = "' + str(sessiondata['subject_id']) + '"') == 0: sessiondata['session'] = 1 else: sessiondata['session'] = len( (experiment.Session() & 'subject_id = "' + str(sessiondata['subject_id']) + '"').fetch()['session']) + 1 sessiondata['session_time'] = ( sweepstarttimes[0] ).strftime( '%H:%M:%S' ) # the time of the first sweep will be the session time experiment.Session().insert1( sessiondata) #% session = ( experiment.Session() & 'subject_id = "' + str(sessiondata['subject_id']) + '"' & 'session_date = "' + str(sessiondata['session_date']) + '"').fetch('session')[0] cell_number = int(cell[cell.find('_') + 1:]) #add cell if not added already celldata = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, } #% if len(ephys_patch.Cell() & celldata ) == 0 or len(ephys_patch.Cell() * ephys_patch.Sweep() & celldata) < len( ephisdata_cell): if len(ephys_patch.Cell() * ephys_patch.Sweep() & celldata ) < len(ephisdata_cell): print('finishing a recording:') else: print('adding new recording:') print(celldata) if 'type' in serieses['.'].keys(): if serieses['.'][ 'type'] == 'interneuron': celldata['cell_type'] = 'int' elif serieses['.'][ 'type'] == 'unknown' or serieses[ '.']['type'] == 'fail': celldata[ 'cell_type'] = 'unidentified' else: print('unhandled cell type!!') timer.sleep(1000) else: celldata[ 'cell_type'] = 'unidentified' celldata['cell_recording_start'] = ( sweepstarttimes[0] ).strftime('%H:%M:%S') if 'depth' in serieses['.'].keys( ) and len(serieses['.']['depth']) > 0: celldata['depth'] = int( serieses['.']['depth']) else: celldata['depth'] = -1 try: ephys_patch.Cell().insert1( celldata, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded if 'notes' in serieses['.'].keys(): cellnotes = serieses['.']['notes'] else: cellnotes = '' cellnotesdata = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'notes': cellnotes } try: ephys_patch.CellNotes().insert1( cellnotesdata, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded #% for i, ephisdata in enumerate( ephisdata_cell): #% sweep_number = i print('sweep {}'.format( sweep_number)) sweep_data = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'sweep_start_time': (ephisdata['sweepstarttime'] - sweepstarttimes[0] ).total_seconds(), 'sweep_end_time': (ephisdata['sweepstarttime'] - sweepstarttimes[0] ).total_seconds() + ephisdata['time'][-1], 'protocol_name': ephisdata[ 'series'], #[:ephisdata['series'].find('_')], 'protocol_sweep_number': int(ephisdata['sweep']) } if 'mode' in ephisdata['metadata'][ 2]['ClampState']: # old file version recmode = ephisdata[ 'metadata'][2][ 'ClampState']['mode'] else: recmode = ephisdata[ 'metadata'][2]['Protocol'][ 'mode'] if 'IC' in str(recmode): recording_mode = 'current clamp' else: print( 'unhandled recording mode, please act..' ) timer.sleep(10000) channelnames = list() channelunits = list() for line_now in ephisdata[ 'metadata'][0]['cols']: if type(line_now['name'] ) == bytes: channelnames.append( line_now['name']. decode().strip("'")) channelunits.append( line_now['units']. decode().strip("'")) else: channelnames.append( line_now['name']) channelunits.append( line_now['units']) commandidx = np.where( np.array(channelnames) == 'command')[0][0] dataidx = np.where( np.array(channelnames) == 'primary')[0][0] #% clampparams_data = ephisdata[ 'metadata'][2]['ClampState'][ 'ClampParams'].copy() clampparams_data_new = dict() for clampparamkey in clampparams_data.keys( ): #6004 is true for some reason.. changing it back to 1 if type(clampparams_data[ clampparamkey] ) == np.int32: if clampparams_data[ clampparamkey] > 0: clampparams_data[ clampparamkey] = int( 1) else: clampparams_data[ clampparamkey] = int( 0) else: clampparams_data[ clampparamkey] = float( clampparams_data[ clampparamkey]) clampparams_data_new[ clampparamkey.lower( )] = clampparams_data[ clampparamkey] #% sweepmetadata_data = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'recording_mode': recording_mode, 'sample_rate': np.round(1 / np.median( np.diff( ephisdata['metadata'] [1]['values']))) } sweepmetadata_data.update( clampparams_data_new) sweepdata_data = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'response_trace': ephisdata['data'][dataidx, :], 'response_units': ephisdata['metadata'][0] ['cols'][dataidx]['units'] } sweepstimulus_data = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'stimulus_trace': ephisdata['data'][ commandidx, :], 'stimulus_units': ephisdata['metadata'][0] ['cols'][commandidx]['units'] } #print('waiting') #timer.sleep(10000) try: ephys_patch.Sweep().insert1( sweep_data, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded try: # maybe it's a duplicate.. ephys_patch.ClampParams( ).insert1( clampparams_data_new, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded try: ephys_patch.SweepMetadata( ).insert1( sweepmetadata_data, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded try: ephys_patch.SweepResponse( ).insert1( sweepdata_data, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded try: ephys_patch.SweepStimulus( ).insert1( sweepstimulus_data, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded #% if 'OrcaFlashExposure' in ephisdata.keys( ): sweepimagingexposuredata = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'imaging_exposure_trace': ephisdata[ 'OrcaFlashExposure'] } try: ephys_patch.SweepImagingExposure( ).insert1( sweepimagingexposuredata, allow_direct_insert=True ) except dj.errors.DuplicateError: pass #already uploaded if 'Temperature' in ephisdata.keys( ): sweeptemperaturedata = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'temperature_trace': ephisdata['Temperature'] * 10, 'temperature_units': 'degC' } try: ephys_patch.SweepTemperature( ).insert1( sweeptemperaturedata, allow_direct_insert=True ) except dj.errors.DuplicateError: pass #already uploaded if 'LED525' in ephisdata.keys(): sweepLEDdata = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'imaging_led_trace': ephisdata['LED525'] } try: ephys_patch.SweepLED( ).insert1( sweepLEDdata, allow_direct_insert=True ) except dj.errors.DuplicateError: pass #already uploaded
plot_precision_recall(key_cell,binwidth = binwidth ,frbinwidth = frbinwidth,firing_rate_window = firing_rate_window) # ============================================================================= # data = plot_ephys_ophys_trace(key_cell,time_to_plot=None,trace_window = 50,show_stimulus = True,show_e_ap_peaks = True,show_o_ap_peaks = True) # data['figure_handle'].savefig('./figures/{}_cell_{}_roi_type_{}_long.png'.format(key_cell['subject_id'],key_cell['cell_number'],key_cell['roi_type']), bbox_inches = 'tight') # print(cell) # ============================================================================= #%% #%% data = plot_ephys_ophys_trace(key_cell,time_to_plot=25,trace_window = 1,show_e_ap_peaks = True,show_o_ap_peaks = True) #%% session = 1 subject_id = 456462 cell_number = 5 roi_type = 'Spikepursuit'#'Spikepursuit'#'VolPy_denoised'#'SpikePursuit'#'VolPy_dexpF0'#'VolPy'#'SpikePursuit_dexpF0'#'VolPy_dexpF0'#''Spikepursuit'#'VolPy'# key_cell = {'session':session,'subject_id':subject_id,'cell_number':cell_number,'roi_type':roi_type } session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key_cell).fetch1('session_time','cell_recording_start') first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_start_time'),float)) first_movie_start_time_real = first_movie_start_time + session_time.total_seconds() threshold,apmaxtime = (imaging_gt.ROIAPWave()*ephysanal.ActionPotential()*ephysanal.ActionPotentialDetails()&key_cell&'ap_real=1').fetch('ap_threshold','ap_max_time') threshold=np.asarray(threshold,float) apmaxtime=np.asarray(apmaxtime,float) # ============================================================================= # session_time_to_plot = time_to_plot+first_movie_start_time # time relative to session start # cell_time_to_plot= session_time_to_plot + session_time.total_seconds() -cell_recording_start.total_seconds() # time relative to recording start # ============================================================================= #% time_to_plot = apmaxtime[np.argmin(threshold)]+cell_recording_start.total_seconds() - first_movie_start_time_real data = plot_ephys_ophys_trace(key_cell, time_to_plot=time_to_plot,
virus_name = (lab.Virus()&'virus_id = {}'.format(virus_id)).fetch1('virus_name') if virus_name != 'syn-FLEX-Voltron-ST': continue if len(lab.Surgery()&'subject_id = {}'.format(subject_id))!=2: print('only {} surgery for {}'.format(len(lab.Surgery()&'subject_id = {}'.format(subject_id)),subject_id)) continue movie_num_list.append(len(imaging.Movie()&'subject_id = {}'.format(subject_id))) date_of_birth,sex = (lab.Subject()&'subject_id = {}'.format(subject_id)).fetch1('date_of_birth','sex') surgery_times = (lab.Surgery()&'subject_id = {}'.format(subject_id)).fetch('start_time') ml,ap = (lab.Surgery.Procedure()&'subject_id = {}'.format(subject_id)).fetch1('ml_location','ap_location') ml_list.append(int(ml)) ap_list.append(int(ap)) age_first_surgery_list.append((np.min(surgery_times).date()-date_of_birth).days) expression_time_list.append(np.diff(surgery_times)[0].days) sex_list.append(sex) depth_list.extend((ephys_patch.Cell()&'subject_id = {}'.format(subject_id)).fetch('depth')) depth_list = np.asarray(depth_list) ap_list = np.asarray(ap_list) ml_list = np.asarray(ml_list) movie_num_list = np.asarray(movie_num_list) sex_list = np.asarray(sex_list) age_first_surgery_list = np.asarray(age_first_surgery_list) expression_time_list = np.asarray(expression_time_list) print('{} mice'.format(len(sex_list))) print('{} males, {} females'.format(sum(sex_list=='M'), sum(sex_list=='F'))) print('ages at first surgery: {}'.format(np.sort(age_first_surgery_list))) print('ml cranio locations: {}, ap cranio locations {}'.format(ml_list,ap_list)) print('exression times:{}'.format(np.sort(expression_time_list))) print('cell depths:{}'.format(np.sort(depth_list))) #break
time_to_plot=25, trace_window=1, show_e_ap_peaks=True, show_o_ap_peaks=True) #%% data = plot_ephys_ophys_trace(key_cell, time_to_plot=None, trace_window=50, show_stimulus=True, show_e_ap_peaks=True, show_o_ap_peaks=True) #%% #%%plot time offset between 1st ROI and real elphys min_corr_coeff = .1 subject_ids = ephys_patch.Cell().fetch('subject_id') sessions = ephys_patch.Cell().fetch('session') cellnums = ephys_patch.Cell().fetch('cell_number') roi_types = ['Spikepursuit', 'VolPy'] for roi_type_idx, roi_type in enumerate(roi_types): fig = plt.figure() axs_delay_sweep = list() axs_delay_coeff = list() axs_coeff_sweep = list() axs_delay_sweep.append(fig.add_axes([0, -roi_type_idx, .8, .8])) axs_coeff_sweep.append(fig.add_axes([1, -roi_type_idx, .8, .8])) axs_delay_coeff.append(fig.add_axes([2, -roi_type_idx, .8, .8])) for subject_id, session, cellnum in zip(subject_ids, sessions, cellnums): key = { 'subject_id': subject_id, 'session': session,
for movie in movies: print(movie) if len(imaging_gt.GroundTruthROI()&movie&'roi_type = "VolPy"') == 0: print('no groundtruth.. skipped') else: movie_dict = dict((imaging.Movie()&movie).fetch1()) for keynow in movie_dict.keys(): if type(movie_dict[keynow]) == decimal.Decimal: movie_dict[keynow] = float(movie_dict[keynow]) save_dir_base = os.path.join(gt_package_directory,str(key['subject_id']),'Cell_{}'.format(key['cell_number']),movie_dict['movie_name']) if ((os.path.isdir(save_dir_base) and (len(os.listdir(save_dir_base))>3 and not overwrite))): print('already exported, skipped') else: sweep_numbers = movie['sweep_numbers'] del movie['sweep_numbers'] session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key).fetch1('session_time','cell_recording_start') first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key).fetch('movie_start_time'),float)) first_movie_start_time_real = first_movie_start_time + session_time.total_seconds() frame_times = (imaging.MovieFrameTimes()&movie).fetch1('frame_times') - cell_recording_start.total_seconds() + session_time.total_seconds() movie_dict['movie_start_time'] = frame_times[0] movie_files = list() repositories , directories , fnames = (imaging.MovieFile() & movie).fetch('movie_file_repository','movie_file_directory','movie_file_name') for repository,directory,fname in zip(repositories,directories,fnames): movie_files.append(os.path.join(dj.config['locations.{}'.format(repository)],directory,fname)) sweepdata_out = list() sweepmetadata_out = list() for sweep_number in sweep_numbers: #% key_sweep = key.copy() key_sweep['sweep_number'] = sweep_number