def populatebehavior(paralel = True,drop_last_session_for_mice_in_training = True): print('adding behavior experiments') if paralel: #ray.init() result_ids = [] #%% IDs = {k: v for k, v in zip(*lab.WaterRestriction().fetch('water_restriction_number', 'subject_id'))} df_surgery = pd.read_csv(dj.config['locations.metadata_behavior']+'Surgery.csv') for subject_now,subject_id_now in zip(IDs.keys(),IDs.values()): # iterating over subjects and removing last session if subject_now in df_surgery['ID'].values and drop_last_session_for_mice_in_training == True and df_surgery['status'][df_surgery['ID']==subject_now].values[0] != 'sacrificed': # the last session is deleted only if the animal is still in training.. print(df_surgery['status'][df_surgery['ID']==subject_now].values[0]) if len((experiment.Session() & 'subject_id = "'+str(subject_id_now)+'"').fetch('session')) > 0: sessiontodel = np.max((experiment.Session() & 'subject_id = "'+str(subject_id_now)+'"').fetch('session')) session_todel = experiment.Session() & 'subject_id = "' + str(subject_id_now)+'"' & 'session = ' + str(sessiontodel) dj.config['safemode'] = False print('deleting last session of ' + subject_now) session_todel.delete() dj.config['safemode'] = True #%% for subject_now,subject_id_now in zip(IDs.keys(),IDs.values()): # iterating over subjects dict_now = dict() dict_now[subject_now] = subject_id_now result_ids.append(populatebehavior_core(dict_now)) #ray.get(result_ids) #ray.shutdown() else: arguments = {'display_progress' : True} populatebehavior_core(arguments)
def make(self, key): ''' Ephys .make() function ''' log.info('EphysIngest().make(): key: {k}'.format(k=key)) # # Find corresponding BehaviorIngest # # ... we are keying times, sessions, etc from behavior ingest; # so lookup behavior ingest for session id, quit with warning otherwise # try: behavior = (behavior_ingest.BehaviorIngest() & key).fetch1() except dj.DataJointError: log.warning('EphysIngest().make(): skip - behavior ingest error') return log.info('behavior for ephys: {b}'.format(b=behavior)) # # Find Ephys Recording # key = (experiment.Session & key).fetch1() sinfo = ((lab.WaterRestriction() * lab.Subject().proj() * experiment.Session()) & key).fetch1() rigpath = EphysDataPath().fetch1('data_path') h2o = sinfo['water_restriction_number'] date = key['session_date'].strftime('%Y%m%d') dpath = pathlib.Path(rigpath, h2o, date) dglob = '[0-9]/{}' # probe directory pattern v3spec = '{}_*_jrc.mat'.format(h2o) # old v3spec = '{}_g0_*.imec.ap_imec3_opt3_jrc.mat'.format(h2o) v3files = list(dpath.glob(dglob.format(v3spec))) v4spec = '{}_*.ap_res.mat'.format(h2o) # old v4spec = '{}_g0_*.imec?.ap_res.mat'.format(h2o) # TODO v4ify v4files = list(dpath.glob(dglob.format(v4spec))) if (v3files and v4files) or not (v3files or v4files): log.warning( 'Error - v3files ({}) + v4files ({}). Skipping.'.format( v3files, v4files)) return if v3files: files = v3files loader = self._load_v3 if v4files: files = v4files loader = self._load_v4 for f in files: self._load(loader(sinfo, rigpath, dpath, f.relative_to(dpath)))
def subject2water(subject, session_num): water = (lab.WaterRestriction & { 'subject_id': subject }).fetch('water_restriction_number') date = (experiment.Session() * lab.WaterRestriction & { 'subject_id': subject, 'session': session_num }).fetch('session_date') return water, date
def water2subject(water, date): subject_id = (lab.WaterRestriction & { 'water_restriction_number': water }).fetch('subject_id') session_num = (experiment.Session() * lab.WaterRestriction & { 'water_restriction_number': water, 'session_date': date }).fetch('session') return subject_id, session_num
def retrieve1(self, key): ''' retrieve related files for a given key ''' # >>> list(key.keys()) # ['subject_id', 'session', 'trial', 'electrode_group', 'globus_alia log.debug(key) lep, lep_sub, lep_dir = GlobusStorageLocation().local_endpoint log.info('local_endpoint: {}:{} -> {}'.format(lep, lep_sub, lep_dir)) # get session related information needed for filenames/records sinfo = ((lab.WaterRestriction * lab.Subject.proj() * experiment.Session() * experiment.SessionTrial) & key).fetch1() h2o = sinfo['water_restriction_number'] sdate = sinfo['session_date'] eg = key['electrode_group'] trial = key['trial'] # build file locations: # fpat: base file pattern for this sessions files # gbase: globus-url base path for this sessions files fpat = '{}_{}_{}_g0_t{}'.format(h2o, sdate, eg, trial) gbase = '/'.join((h2o, str(sdate), str(eg), fpat)) repname, rep, rep_sub = (GlobusStorageLocation() & key).fetch()[0] gsm = self.get_gsm() gsm.activate_endpoint(lep) # XXX: cache this / prevent duplicate RPC? gsm.activate_endpoint(rep) # XXX: cache this / prevent duplicate RPC? sfxmap = {'.imec.ap.bin': ArchivedRawEphysTrial.ArchivedApChannel, '.imec.ap.meta': ArchivedRawEphysTrial.ArchivedApMeta, '.imec.lf.bin': ArchivedRawEphysTrial.ArchivedLfChannel, '.imec.lf.meta': ArchivedRawEphysTrial.ArchivedLfMeta} for sfx, cls in sfxmap.items(): if cls & key: log.debug('record found for {} & {}'.format(cls.__name__, key)) gname = '{}{}'.format(gbase, sfx) srcp = '{}:/{}/{}'.format(rep, rep_sub, gname) dstp = '{}:/{}/{}'.format(lep, lep_sub, gname) log.info('transferring {} to {}'.format(srcp, dstp)) # XXX: check if exists 1st? (manually or via API copy-checksum) if not gsm.cp(srcp, dstp): emsg = "couldn't transfer {} to {}".format(srcp, dstp) log.error(emsg) raise dj.DataJointError(emsg)
def load_all_sessions(subject_id): loader = get_loader() # ---- parse data dir and load all sessions ---- """ For each session, return + subject_name + session_date + session_time + session_basename + session_files + username + rig """ try: sessions_to_ingest = loader.load_sessions(subject_id) except FileNotFoundError as e: print(str(e)) return # ---- work on each session ---- for sess in sessions_to_ingest: session_files = sess.pop('session_files') if experiment.Session & sess: log.info(f'Session {sess} already exists. Skipping...') continue # ---- synthesize session number ---- sess_num = (dj.U().aggr(experiment.Session() & { 'subject_id': subject_id }, n='max(session)').fetch1('n') or 0) + 1 # ---- insert ---- sess_key = {**sess, 'session': sess_num} with dj.conn().transaction: experiment.Session.insert1(sess_key) InsertedSession.insert1( { **sess_key, 'loader_method': loader.loader_name, 'sess_data_dir': session_files[0].parent.as_posix() }, allow_direct_insert=True, ignore_extra_fields=True) InsertedSession.SessionFile.insert( [{ **sess_key, 'filepath': f.as_posix() } for f in session_files], allow_direct_insert=True, ignore_extra_fields=True) log.info(f'Inserted new session: {sess}')
def notify(self, key): ts, temperatures = (self & key).fetch1('temp_time', 'temperatures') import matplotlib.pyplot as plt fig = plt.figure(figsize=(10, 5)) plt.plot(ts, temperatures) plt.ylabel('Temperature (C)') plt.xlabel('Seconds') img_filename = '/tmp/' + key_hash(key) + '.png' fig.savefig(img_filename) plt.close(fig) msg = 'temperature for {animal_id}-{session}-{scan_idx}'.format(**key) slack_user = notify.SlackUser() & (experiment.Session() & key) slack_user.notify(file=img_filename, file_title=msg)
def make(self, key): #% # key = { 'subject_id': 462149, 'session':1,'cell_number':1,'movie_number':11} session_time = (experiment.Session() & key).fetch('session_time')[0] cell_time = (ephys_patch.Cell() & key).fetch('cell_recording_start')[0] cell_sweep_start_times = (ephys_patch.Sweep() & key).fetch('sweep_start_time') cell_sweep_end_times = (ephys_patch.Sweep() & key).fetch('sweep_end_time') time_start = float( np.min(cell_sweep_start_times)) + cell_time.total_seconds( ) - session_time.total_seconds() time_end = float( np.max(cell_sweep_end_times)) + cell_time.total_seconds( ) - session_time.total_seconds() try: movie = (imaging.Movie()) & key & 'movie_start_time > ' + str( time_start) & 'movie_start_time < ' + str(time_end) sweep_start_times, sweep_end_times, sweep_nums = ( ephys_patch.Sweep() & key).fetch('sweep_start_time', 'sweep_end_time', 'sweep_number') sweep_start_times = np.asarray(sweep_start_times, float) + cell_time.total_seconds( ) - session_time.total_seconds() sweep_end_times = np.asarray(sweep_end_times, float) + cell_time.total_seconds( ) - session_time.total_seconds() #for movie in movies_now: frametimes = (imaging.MovieFrameTimes & movie).fetch1('frame_times') needed_start_time = frametimes[0] needed_end_time = frametimes[-1] sweep_nums_needed = sweep_nums[ ((sweep_start_times > needed_start_time) & (sweep_start_times < needed_end_time)) | ((sweep_end_times > needed_start_time) & (sweep_end_times < needed_end_time)) | ((sweep_end_times > needed_end_time) & (sweep_start_times < needed_start_time))] if len(sweep_nums_needed) > 0: key['sweep_numbers'] = sweep_nums_needed self.insert1(key, skip_duplicates=True) except: pass
def test_paths(): rel = experiment.Session() * experiment.Scan.EyeVideo( ) * experiment.Scan.BehaviorFile().proj(hdf_file='filename') path_info = random.choice(rel.fetch.as_dict()) tmp = path_info['hdf_file'].split('.') if '%d' in tmp[0]: # new version path_info['hdf_file'] = tmp[0][:-2] + '0.' + tmp[-1] else: path_info['hdf_file'] = tmp[0][:-1] + '0.' + tmp[-1] hdf_path = lab.Paths().get_local_path( '{behavior_path}/{hdf_file}'.format(**path_info)) avi_path = lab.Paths().get_local_path( '{behavior_path}/{filename}'.format(**path_info)) assert_true(os.path.isfile(avi_path) and os.path.isfile(hdf_path))
def session_plot(self): """ Do a plot of how temperature progress through a session""" import matplotlib.pyplot as plt import matplotlib.ticker as ticker # Check that plot is restricted to a single session session_key = self.fetch('KEY', limit=1)[0] session_key.pop('scan_idx') if len(self & session_key) != len(self): raise PipelineException( 'Plot can only be generated for one session at a ' 'time') # Get times and timestamps, scan_ts scan_indices, ts, temperatures = self.fetch('scan_idx', 'temp_time', 'temperatures', order_by='scan_idx') session_ts = (experiment.Session() & self).fetch1('session_ts') scan_ts = (experiment.Scan() & self).fetch('scan_ts', order_by='scan_idx') abs_ts = [(sts - session_ts).seconds + (t - t[0]) for sts, t in zip(scan_ts, ts)] # Plot fig = plt.figure(figsize=(10, 5)) for abs_ts_, temp_, scan_idx in zip(abs_ts, temperatures, scan_indices): plt.plot(abs_ts_ / 3600, temp_, label='Scan {}'.format(scan_idx)) # in hours plt.title( 'Temperature for {animal_id}-{session} starting at {session_ts}'. format(session_ts=session_ts, **session_key)) plt.ylabel('Temperature (Celsius)') plt.xlabel('Hour') plt.legend() # Plot formatting plt.gca().yaxis.set_major_locator(ticker.MultipleLocator(0.5)) plt.grid(linestyle='--', alpha=0.8) return fig
def _make_tuples(self, key): # Get behavior filename behavior_path = (experiment.Session() & key).fetch1('behavior_path') local_path = lab.Paths().get_local_path(behavior_path) filename = (experiment.Scan.BehaviorFile() & key).fetch1('filename') full_filename = os.path.join(local_path, filename) # Read file data = h5.read_behavior_file(full_filename) # Get counter timestamps and convert to seconds ts = h5.ts2sec(data['ts'], is_packeted=True) # Read temperature (if available) and invalidate points with unreliable timestamps temp_raw = data.get('temperature', None) if temp_raw is None: raise PipelineException( 'Scan {animal_id}-{session}-{scan_idx} does not have ' 'temperature data'.format(**key)) temp_raw[np.isnan(ts)] = float('nan') # Read temperature and smooth it temp_celsius = (temp_raw * 100 - 32) / 1.8 # F to C sampling_rate = int(round( 1 / np.nanmedian(np.diff(ts)))) # samples per second smooth_temp = signal.low_pass_filter(temp_celsius, sampling_rate, cutoff_freq=1, filter_size=2 * sampling_rate) # Resample at 1 Hz downsampled_ts = ts[::sampling_rate] downsampled_temp = smooth_temp[::sampling_rate] # Insert self.insert1({ **key, 'temp_time': downsampled_ts, 'temperatures': downsampled_temp, 'median_temperature': np.nanmedian(downsampled_temp) }) self.notify(key)
def plot_ephys_ophys_trace(key_cell,time_to_plot=None,trace_window = 1,show_stimulus = False,show_e_ap_peaks = False, show_o_ap_peaks = False): #%% # ============================================================================= # key_cell = {'session': 1, # 'subject_id': 456462, # 'cell_number': 3, # 'motion_correction_method': 'VolPy', # 'roi_type': 'VolPy'} # time_to_plot=None # trace_window = 100 # show_stimulus = False # show_e_ap_peaks = True # show_o_ap_peaks = True # ============================================================================= fig=plt.figure() ax_ophys = fig.add_axes([0,0,2,.8]) ax_ephys = fig.add_axes([0,-1,2,.8]) if show_stimulus: ax_ephys_stim = fig.add_axes([0,-1.5,2,.4]) #% session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key_cell).fetch1('session_time','cell_recording_start') first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_start_time'),float)) first_movie_start_time_real = first_movie_start_time + session_time.total_seconds() if not time_to_plot: time_to_plot = trace_window/2#ephys_matched_ap_times[0] session_time_to_plot = time_to_plot+first_movie_start_time # time relative to session start cell_time_to_plot= session_time_to_plot + session_time.total_seconds() -cell_recording_start.total_seconds() # time relative to recording start #% sweep_start_times,sweep_end_times,sweep_nums = (ephys_patch.Sweep()&key_cell).fetch('sweep_start_time','sweep_end_time','sweep_number') needed_start_time = cell_time_to_plot - trace_window/2 needed_end_time = cell_time_to_plot + trace_window/2 #% sweep_nums = sweep_nums[((sweep_start_times > needed_start_time) & (sweep_start_times < needed_end_time)) | ((sweep_end_times > needed_start_time) & (sweep_end_times < needed_end_time)) | ((sweep_end_times > needed_end_time) & (sweep_start_times < needed_start_time)) ] ephys_traces = list() ephys_trace_times = list() ephys_sweep_start_times = list() ephys_traces_stim = list() for sweep_num in sweep_nums: sweep = ephys_patch.Sweep()&key_cell&'sweep_number = %d' % sweep_num trace,sr= (ephys_patch.SweepMetadata()*ephys_patch.SweepResponse()&sweep).fetch1('response_trace','sample_rate') trace = trace*1000 sweep_start_time = float(sweep.fetch('sweep_start_time')) trace_time = np.arange(len(trace))/sr + sweep_start_time + cell_recording_start.total_seconds() - first_movie_start_time_real trace_idx = (time_to_plot-trace_window/2 < trace_time) & (time_to_plot+trace_window/2 > trace_time) ax_ephys.plot(trace_time[trace_idx],trace[trace_idx],'k-') ephys_traces.append(trace) ephys_trace_times.append(trace_time) ephys_sweep_start_times.append(sweep_start_time) if show_e_ap_peaks: ap_max_index = (ephysanal.ActionPotential()&sweep).fetch('ap_max_index') aptimes = trace_time[np.asarray(ap_max_index,int)] apVs = trace[np.asarray(ap_max_index,int)] ap_needed = (time_to_plot-trace_window/2 < aptimes) & (time_to_plot+trace_window/2 > aptimes) aptimes = aptimes[ap_needed] apVs = apVs[ap_needed] ax_ephys.plot(aptimes,apVs,'ro') if show_stimulus: trace_stim= (ephys_patch.SweepMetadata()*ephys_patch.SweepStimulus()&sweep).fetch1('stimulus_trace') trace_stim = trace_stim*10**12 ax_ephys_stim.plot(trace_time[trace_idx],trace_stim[trace_idx],'k-') ephys_traces_stim.append(trace_stim) ephysdata = {'ephys_traces':ephys_traces,'ephys_trace_times':ephys_trace_times} if show_stimulus: ephysdata['ephys_traces_stimulus'] = ephys_traces_stim movie_nums, movie_start_times,movie_frame_rates,movie_frame_nums = ((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_number','movie_start_time','movie_frame_rate','movie_frame_num') movie_start_times = np.asarray(movie_start_times, float) movie_end_times = np.asarray(movie_start_times, float)+np.asarray(movie_frame_nums, float)/np.asarray(movie_frame_rates, float) needed_start_time = session_time_to_plot - trace_window/2 needed_end_time = session_time_to_plot + trace_window/2 movie_nums = movie_nums[((movie_start_times >= needed_start_time) & (movie_start_times <= needed_end_time)) | ((movie_end_times >= needed_start_time) & (movie_end_times <= needed_end_time)) | ((movie_end_times >= needed_end_time) & (movie_start_times <= needed_start_time)) ] dffs=list() frametimes = list() for movie_num in movie_nums: #movie_num = movie_nums[(session_time_to_plot>movie_start_times)&(session_time_to_plot<movie_end_times)][0] key_movie = key_cell.copy() key_movie['movie_number'] = movie_num dff,gt_roi_number = ((imaging.ROI()*imaging_gt.GroundTruthROI())&key_movie).fetch1('roi_dff','roi_number') dff_all,roi_number_all = (imaging.ROI()&key_movie).fetch('roi_dff','roi_number') dff_all =dff_all[roi_number_all!=gt_roi_number] frame_times = ((imaging.MovieFrameTimes()*imaging_gt.GroundTruthROI())&key_movie).fetch1('frame_times') + (session_time).total_seconds() - first_movie_start_time_real #modified to cell time frame_idx = (time_to_plot-trace_window/2 < frame_times) & (time_to_plot+trace_window/2 > frame_times) dff_list = [dff] dff_list.extend(dff_all) prevminval = 0 for dff_now,alpha_now in zip(dff_list,np.arange(1,1/(len(dff_list)+1),-1/(len(dff_list)+1))): dfftoplotnow = dff_now[frame_idx] + prevminval ax_ophys.plot(frame_times[frame_idx],dfftoplotnow,'g-',alpha=alpha_now) prevminval = np.min(dfftoplotnow) -.005 #ax_ophys.plot(frame_times[frame_idx],dff[frame_idx],'g-') dffs.append(dff) frametimes.append(frame_times) if show_o_ap_peaks: if 'raw' in key_cell['roi_type']: apidxes = ((imaging.ROI()*imaging_gt.GroundTruthROI())&key_movie).fetch1('roi_spike_indices') else: apidxes = ((imaging.ROI()*imaging_gt.GroundTruthROI())&key_movie).fetch1('roi_spike_indices')-1 oap_times = frame_times[apidxes] oap_vals = dff[apidxes] oap_needed = (time_to_plot-trace_window/2 < oap_times) & (time_to_plot+trace_window/2 > oap_times) oap_times = oap_times[oap_needed] oap_vals = oap_vals[oap_needed] ax_ophys.plot(oap_times,oap_vals,'ro') ophysdata = {'ophys_traces':dffs,'ophys_trace_times':frametimes} ax_ophys.autoscale(tight = True) # ============================================================================= # dfff = np.concatenate(dffs) # dfff[np.argmin(dfff)]=0 # ax_ophys.set_ylim([min(dfff),max(dfff)]) # ============================================================================= ax_ophys.set_xlim([time_to_plot-trace_window/2,time_to_plot+trace_window/2]) ax_ophys.set_ylabel('dF/F') ax_ophys.spines["top"].set_visible(False) ax_ophys.spines["right"].set_visible(False) ax_ophys.invert_yaxis() ax_ephys.autoscale(tight = True) ax_ephys.set_xlim([time_to_plot-trace_window/2,time_to_plot+trace_window/2]) ax_ephys.set_ylabel('Vm (mV))') ax_ephys.spines["top"].set_visible(False) ax_ephys.spines["right"].set_visible(False) if show_stimulus: # ax_ephys_stim.autoscale(tight = True) ax_ephys_stim.set_xlim([time_to_plot-trace_window/2,time_to_plot+trace_window/2]) ax_ephys_stim.set_ylabel('Injected current (pA))') ax_ephys_stim.set_xlabel('time from first movie start (s)') ax_ephys_stim.spines["top"].set_visible(False) ax_ephys_stim.spines["right"].set_visible(False) else: ax_ephys.set_xlabel('time from first movie start (s)') outdict = {'ephys':ephysdata,'ophys':ophysdata,'figure_handle':fig} #%% return outdict
def make(self, key): #% #key = {'subject_id': 454597, 'session': 1, 'cell_number': 0, 'motion_correction_method': 'Matlab', 'roi_type': 'SpikePursuit', 'roi_number': 1} #key = {'subject_id': 456462, 'session': 1, 'cell_number': 5, 'movie_number': 3, 'motion_correction_method': 'VolPy', 'roi_type': 'VolPy', 'roi_number': 1} if len( ROIEphysCorrelation & key ) > 0: # and key['roi_type'] == 'SpikePursuit' #only spikepursuit for now.. key_to_compare = key.copy() del key_to_compare['roi_number'] #print(key) #% roinums = np.unique( (ROIEphysCorrelation() & key_to_compare).fetch('roi_number')) snratios_mean = list() snratios_median = list() snratios_first50 = list() for roinum_now in roinums: snratios = (ROIAPWave() & key_to_compare & 'roi_number = {}'.format(roinum_now) ).fetch('apwave_snratio') snratios_mean.append(np.mean(snratios)) snratios_median.append(np.median(snratios)) snratios_first50.append(np.mean(snratios[:50])) #%% if np.max( (ROIEphysCorrelation() & key).fetch('roi_number') ) == roinums[np.argmax( snratios_first50 )]: #np.max((imaging_gt.ROIEphysCorrelation()&key).fetch('roi_number')) == np.min((imaging_gt.ROIEphysCorrelation&key_to_compare).fetch('roi_number')):#np.max(np.abs((ROIEphysCorrelation&key).fetch('corr_coeff'))) == np.max(np.abs((ROIEphysCorrelation&key_to_compare).fetch('corr_coeff'))): print('this is it') print(key['roi_type']) cellstarttime = (ephys_patch.Cell() & key).fetch1('cell_recording_start') sessionstarttime = (experiment.Session() & key).fetch1('session_time') aptimes = np.asarray( (ephysanal.ActionPotential() & key).fetch('ap_max_time'), float) + (cellstarttime - sessionstarttime).total_seconds() sweep_start_times, sweep_end_times = (ephys_patch.Sweep() & key).fetch( 'sweep_start_time', 'sweep_end_time') sweep_start_times = np.asarray(sweep_start_times, float) + ( cellstarttime - sessionstarttime).total_seconds() sweep_end_times = np.asarray(sweep_end_times, float) + ( cellstarttime - sessionstarttime).total_seconds() frame_timess, roi_spike_indicess = ( imaging.MovieFrameTimes() * imaging.Movie() * imaging.ROI() & key).fetch('frame_times', 'roi_spike_indices') movie_start_times = list() movie_end_times = list() roi_ap_times = list() for frame_times, roi_spike_indices in zip( frame_timess, roi_spike_indicess): movie_start_times.append(frame_times[0]) movie_end_times.append(frame_times[-1]) roi_ap_times.append(frame_times[roi_spike_indices]) movie_start_times = np.sort(movie_start_times) movie_end_times = np.sort(movie_end_times) roi_ap_times = np.sort(np.concatenate(roi_ap_times)) #% ##delete spikes in optical traces where there was no ephys recording for start_t, end_t in zip( np.concatenate([sweep_start_times, [np.inf]]), np.concatenate([[0], sweep_end_times])): idxtodel = np.where((roi_ap_times > end_t) & (roi_ap_times < start_t))[0] if len(idxtodel) > 0: roi_ap_times = np.delete(roi_ap_times, idxtodel) ##delete spikes in ephys traces where there was no imaging for start_t, end_t in zip( np.concatenate([movie_start_times, [np.inf]]), np.concatenate([[0], movie_end_times])): idxtodel = np.where((aptimes > end_t) & (aptimes < start_t))[0] if len(idxtodel) > 0: #print(idxtodel) aptimes = np.delete(aptimes, idxtodel) #% D = np.zeros([len(aptimes), len(roi_ap_times)]) for idx, apt in enumerate(aptimes): D[idx, :] = (roi_ap_times - apt) * 1000 D_test = np.abs(D) D_test[D_test > 15] = 1000 D_test[D < -1] = 1000 X = scipy.optimize.linear_sum_assignment(D_test) #% cost = D_test[X[0], X[1]] unmatched = np.where(cost == 1000)[0] X0_final = np.delete(X[0], unmatched) X1_final = np.delete(X[1], unmatched) ephys_ap_times = aptimes[X0_final] ophys_ap_times = roi_ap_times[X1_final] false_positive_time_imaging = list() for roi_ap_time in roi_ap_times: if roi_ap_time not in ophys_ap_times: false_positive_time_imaging.append(roi_ap_time) false_negative_time_ephys = list() for aptime in aptimes: if aptime not in ephys_ap_times: false_negative_time_ephys.append(aptime) key['ephys_matched_ap_times'] = ephys_ap_times key['ophys_matched_ap_times'] = ophys_ap_times key['ephys_unmatched_ap_times'] = false_negative_time_ephys key['ophys_unmatched_ap_times'] = false_positive_time_imaging #print(imaging.ROI()&key) #print([len(aptimes),'vs',len(roi_ap_times)]) #%% self.insert1(key, skip_duplicates=True)
def make(self, key): water_res_num, sess_date = get_wr_sessdate(key) sess_dir = store_stage / water_res_num / sess_date sess_dir.mkdir(parents=True, exist_ok=True) # -- Plotting -- fig = plt.figure(figsize=(20, 12)) fig.suptitle(f'{water_res_num}, session {key["session"]}') gs = GridSpec(5, 1, wspace=0.4, hspace=0.4, bottom=0.07, top=0.95, left=0.1, right=0.9) ax1 = fig.add_subplot(gs[0:3, :]) ax2 = fig.add_subplot(gs[3, :]) ax3 = fig.add_subplot(gs[4, :]) ax1.get_shared_x_axes().join(ax1, ax2, ax3) # Plot settings plot_setting = {'left lick': 'red', 'right lick': 'blue'} # -- Get event times -- key_subject_id_session = ( experiment.Session() & (lab.WaterRestriction() & 'water_restriction_number="{}"'.format(water_res_num)) & 'session="{}"'.format(key['session'])).fetch1("KEY") go_cue_times = (experiment.TrialEvent() & key_subject_id_session & 'trial_event_type="go"').fetch( 'trial_event_time', order_by='trial').astype(float) lick_times = pd.DataFrame( (experiment.ActionEvent() & key_subject_id_session).fetch(order_by='trial')) trial_num = len(go_cue_times) all_trial_num = np.arange(1, trial_num + 1).tolist() all_trial_start = [[-x] for x in go_cue_times] all_lick = dict() for event_type in plot_setting: all_lick[event_type] = [] for i, trial_start in enumerate(all_trial_start): all_lick[event_type].append( (lick_times[(lick_times['trial'] == i + 1) & (lick_times['action_event_type'] == event_type )]['action_event_time'].values.astype(float) + trial_start).tolist()) # -- All licking events (Ordered by trials) -- ax1.plot([0, 0], [0, trial_num], 'k', lw=0.5) # Aligned by go cue ax1.set(ylabel='Trial number', xlim=(-3, 3), xticks=[]) # Batch plotting to speed up ax1.eventplot(lineoffsets=all_trial_num, positions=all_trial_start, color='k') # Aligned by go cue for event_type in plot_setting: ax1.eventplot(lineoffsets=all_trial_num, positions=all_lick[event_type], color=plot_setting[event_type], linewidth=2) # Trial start # -- Histogram of all licks -- for event_type in plot_setting: sns.histplot(np.hstack(all_lick[event_type]), binwidth=0.01, alpha=0.5, ax=ax2, color=plot_setting[event_type], label=event_type) # 10-ms window ymax_tmp = max(ax2.get_ylim()) sns.histplot(-go_cue_times, binwidth=0.01, color='k', ax=ax2, label='trial start') # 10-ms window ax2.axvline(x=0, color='k', lw=0.5) ax2.set(ylim=(0, ymax_tmp), xticks=[], title='All events') # Fix the ylim of left and right licks ax2.legend() # -- Histogram of reaction time (first lick after go cue) -- plot_setting = {'LEFT': 'red', 'RIGHT': 'blue'} for water_port in plot_setting: this_RT = (foraging_analysis.TrialStats() & key_subject_id_session & (experiment.WaterPortChoice() & 'water_port="{}"'.format(water_port)) ).fetch('reaction_time').astype(float) sns.histplot(this_RT, binwidth=0.01, alpha=0.5, ax=ax3, color=plot_setting[water_port], label=water_port) # 10-ms window ax3.axvline(x=0, color='k', lw=0.5) ax3.set(xlabel='Time to Go Cue (s)', title='First lick (reaction time)' ) # Fix the ylim of left and right licks ax3.legend() # ---- Save fig and insert ---- fn_prefix = f'{water_res_num}_{sess_date}_' fig_dict = save_figs((fig, ), ('session_foraging_licking_psth', ), sess_dir, fn_prefix) plt.close('all') self.insert1({**key, **fig_dict})
'sweep number from imaging start') axs_coeff_sweep[-1].set_ylabel('correlation coefficient') #%% photocurrent window = 3 #seconds roi_type = 'Spikepursuit' key = {'roi_type': roi_type} gtrois = (imaging_gt.GroundTruthROI() & key).fetch('subject_id', 'session', 'cell_number', 'movie_number', 'motion_correction_method', 'roi_type', 'roi_number', as_dict=True) for roi in gtrois: session_time = (experiment.Session() & roi).fetch('session_time')[0] cell_time = (ephys_patch.Cell() & roi).fetch('cell_recording_start')[0] movie_start_time = float( (imaging.Movie() & roi).fetch1('movie_start_time')) movie_start_time = session_time.total_seconds( ) + movie_start_time - cell_time.total_seconds() sweeps = (imaging_gt.ROIEphysCorrelation() & roi).fetch('sweep_number') sweep_now = ephys_patch.Sweep() & roi & 'sweep_number = ' + str(sweeps[0]) trace, sr = ((ephys_patch.SweepResponse() * ephys_patch.SweepMetadata()) & sweep_now).fetch1('response_trace', 'sample_rate') sweep_start_time = float(sweep_now.fetch1('sweep_start_time')) trace_time = np.arange(len(trace)) / sr + sweep_start_time neededidx = (trace_time > movie_start_time - window) & (trace_time < movie_start_time) fig = plt.figure()
print('no groundtruth.. skipped') else: movie_dict = dict((ephys_patch.Cell()*imaging_gt.CellMovieCorrespondance()*imaging.Movie()&movie).fetch1()) for keynow in movie_dict.keys(): if type(movie_dict[keynow]) == decimal.Decimal: movie_dict[keynow] = float(movie_dict[keynow]) elif type(movie_dict[keynow]) == datetime.timedelta: movie_dict[keynow] = str(movie_dict[keynow]) elif type(movie_dict[keynow]) == np.ndarray: movie_dict[keynow] = movie_dict[keynow].tolist() save_dir_base = os.path.join(gt_package_directory,str(key['subject_id']),'Cell_{}'.format(key['cell_number']),movie_dict['movie_name']) if ((os.path.isdir(save_dir_base) and (len(os.listdir(save_dir_base))>3 and not overwrite))): print('already exported, skipped') else: session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key).fetch1('session_time','cell_recording_start') first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key).fetch('movie_start_time'),float)) first_movie_start_time_real = first_movie_start_time + session_time.total_seconds() frame_times = (imaging.MovieFrameTimes()&movie).fetch1('frame_times') - cell_recording_start.total_seconds() + session_time.total_seconds() motion_corr_vectors = np.asarray((imaging.MotionCorrection()*imaging.RegisteredMovie()&movie&'motion_correction_method = "VolPy"'&'motion_corr_description= "rigid motion correction done with VolPy"').fetch1('motion_corr_vectors')) movie_dict['movie_start_time'] = frame_times[0] movie_files = list() repositories , directories , fnames = (imaging.MovieFile() & movie).fetch('movie_file_repository','movie_file_directory','movie_file_name') for repository,directory,fname in zip(repositories,directories,fnames): movie_files.append(os.path.join(dj.config['locations.{}'.format(repository)],directory,fname)) sweepdata_out = list() sweepmetadata_out = list() for sweep_number in sweep_numbers: #% key_sweep = key.copy() key_sweep['sweep_number'] = sweep_number
def plot_precision_recall(key_cell,binwidth = 30,frbinwidth = 0.01,firing_rate_window = 3,save_figures = True,xlimits =None): #%% session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key_cell).fetch1('session_time','cell_recording_start') first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_start_time'),float)) first_movie_start_time_real = first_movie_start_time + session_time.total_seconds() #% fr_kernel = np.ones(int(firing_rate_window/frbinwidth))/(firing_rate_window/frbinwidth) first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key_cell).fetch('movie_start_time'),float)) ephys_matched_ap_times,ephys_unmatched_ap_times,ophys_matched_ap_times,ophys_unmatched_ap_times = (imaging_gt.GroundTruthROI()&key_cell).fetch('ephys_matched_ap_times','ephys_unmatched_ap_times','ophys_matched_ap_times','ophys_unmatched_ap_times') #% ephys_matched_ap_times = np.concatenate(ephys_matched_ap_times) - first_movie_start_time ephys_unmatched_ap_times = np.concatenate(ephys_unmatched_ap_times) - first_movie_start_time all_ephys_ap_times = np.concatenate([ephys_matched_ap_times,ephys_unmatched_ap_times]) ophys_matched_ap_times = np.concatenate(ophys_matched_ap_times) - first_movie_start_time ophys_unmatched_ap_times = np.concatenate(ophys_unmatched_ap_times) - first_movie_start_time all_ophys_ap_times = np.concatenate([ophys_matched_ap_times,ophys_unmatched_ap_times]) all_times = np.concatenate([ephys_matched_ap_times,ephys_unmatched_ap_times,ophys_matched_ap_times,ophys_unmatched_ap_times]) maxtime = np.max(all_times) #% fr_bincenters = np.arange(frbinwidth/2,maxtime+frbinwidth,frbinwidth) fr_binedges = np.concatenate([fr_bincenters-frbinwidth/2,[fr_bincenters[-1]+frbinwidth/2]]) fr_e = np.histogram(all_ephys_ap_times,fr_binedges)[0]/frbinwidth fr_e = np.convolve(fr_e, fr_kernel,'same') fr_o = np.histogram(all_ophys_ap_times,fr_binedges)[0]/frbinwidth fr_o = np.convolve(fr_o, fr_kernel,'same') #% bincenters = np.arange(binwidth/2,maxtime+binwidth,binwidth) binedges = np.concatenate([bincenters-binwidth/2,[bincenters[-1]+binwidth/2]]) ephys_matched_ap_num_binned,tmp = np.histogram(ephys_matched_ap_times,binedges) ephys_unmatched_ap_num_binned,tmp = np.histogram(ephys_unmatched_ap_times,binedges) ophys_matched_ap_num_binned,tmp = np.histogram(ophys_matched_ap_times,binedges) ophys_unmatched_ap_num_binned,tmp = np.histogram(ophys_unmatched_ap_times,binedges) precision_binned = ophys_matched_ap_num_binned/(ophys_matched_ap_num_binned+ophys_unmatched_ap_num_binned) recall_binned = ephys_matched_ap_num_binned/(ephys_matched_ap_num_binned+ephys_unmatched_ap_num_binned) f1_binned = 2*precision_binned*recall_binned/(precision_binned+recall_binned) #% if xlimits == None: xlimits = [binedges[0],binedges[-1]] fig=plt.figure()#figsize=(50,0) ax_rates = fig.add_axes([0,1.4,2,.3]) ax_spikes = fig.add_axes([0,1,2,.3]) ax = fig.add_axes([0,0,2,.8]) ax_snratio = fig.add_axes([0,-1,2,.8]) ax_latency = fig.add_axes([0,-2,2,.8]) ax_latency_hist = fig.add_axes([.5,-3,1,.8]) ax.plot(bincenters,precision_binned,'go-',label = 'precision') ax.plot(bincenters,recall_binned,'ro-',label = 'recall') ax.plot(bincenters,f1_binned,'bo-',label = 'f1 score') ax.set_xlim([binedges[0],binedges[-1]]) ax.set_ylim([0,1]) ax.legend() ax_spikes.plot(ephys_unmatched_ap_times,np.zeros(len(ephys_unmatched_ap_times)),'r|', ms = 10) ax_spikes.plot(all_ephys_ap_times,np.zeros(len(all_ephys_ap_times))+.33,'k|', ms = 10) ax_spikes.plot(ophys_unmatched_ap_times,np.ones(len(ophys_unmatched_ap_times)),'r|',ms = 10) ax_spikes.plot(all_ophys_ap_times,np.ones(len(all_ophys_ap_times))-.33,'g|', ms = 10) ax_spikes.set_yticks([0,.33,.67,1]) ax_spikes.set_yticklabels(['false negative','ephys','ophys','false positive']) ax_spikes.set_ylim([-.2,1.2]) ax_spikes.set_xlim(xlimits) t,sn = (ephysanal.ActionPotential()*imaging_gt.GroundTruthROI()*imaging_gt.ROIAPWave()&key_cell).fetch('ap_max_time','apwave_snratio') t= np.asarray(t,float) + cell_recording_start.total_seconds() - first_movie_start_time_real ax_snratio.plot(t,sn,'ko') ax_snratio.set_ylabel('signal to noise ratio') ax_snratio.set_ylim([0,20]) ax_snratio.set_xlim(xlimits) ax_rates.plot(fr_bincenters,fr_e,'k-',label = 'ephys') ax_rates.plot(fr_bincenters,fr_o,'g-',label = 'ophys') ax_rates.legend() ax_rates.set_xlim(xlimits) ax_rates.set_ylabel('Firing rate (Hz)') ax_rates.set_title('subject_id: %d, cell number: %s' %(key_cell['subject_id'],key_cell['cell_number'])) ax_latency.plot(ephys_matched_ap_times,1000*(ophys_matched_ap_times-ephys_matched_ap_times),'ko') ax_latency.set_ylabel('ephys-ophys spike latency (ms)') ax_latency.set_ylim([0,10]) ax_latency.set_xlabel('time from first movie start (s)') ax_latency.set_xlim(xlimits) ax_latency_hist.hist(1000*(ophys_matched_ap_times-ephys_matched_ap_times),np.arange(-5,15,.1)) ax_latency_hist.set_xlabel('ephys-ophys spike latency (ms)') ax_latency_hist.set_ylabel('matched ap count') data = list() data.append(plot_ephys_ophys_trace(key_cell,ephys_matched_ap_times[0],trace_window = 1,show_e_ap_peaks = True,show_o_ap_peaks = True)) data.append(plot_ephys_ophys_trace(key_cell,ephys_unmatched_ap_times[0],trace_window = 1,show_e_ap_peaks = True,show_o_ap_peaks = True)) data.append(plot_ephys_ophys_trace(key_cell,ophys_unmatched_ap_times[0],trace_window = 1,show_e_ap_peaks = True,show_o_ap_peaks = True)) if save_figures: fig.savefig('./figures/{}_cell_{}_roi_type_{}.png'.format(key_cell['subject_id'],key_cell['cell_number'],key_cell['roi_type']), bbox_inches = 'tight') for data_now,fname in zip(data,['matched','false_negative','false_positive']): data_now['figure_handle'].savefig('./figures/{}_cell_{}_roi_type_{}_{}.png'.format(key_cell['subject_id'],key_cell['cell_number'],key_cell['roi_type'],fname), bbox_inches = 'tight')
def make(self, key): log.info('BehaviorIngest.make(): key: {key}'.format(key=key)) subject_id = key['subject_id'] h2o = (lab.WaterRestriction() & { 'subject_id': subject_id }).fetch1('water_restriction_number') ymd = key['session_date'] datestr = ymd.strftime('%Y%m%d') log.info('h2o: {h2o}, date: {d}'.format(h2o=h2o, d=datestr)) # session record key skey = {} skey['subject_id'] = subject_id skey['session_date'] = ymd skey['username'] = self.get_session_user() skey['rig'] = key['rig'] # File paths conform to the pattern: # dl7/TW_autoTrain/Session Data/dl7_TW_autoTrain_20180104_132813.mat # which is, more generally: # {h2o}/{training_protocol}/Session Data/{h2o}_{training protocol}_{YYYYMMDD}_{HHMMSS}.mat path = pathlib.Path(key['rig_data_path'], key['subpath']) if experiment.Session() & skey: log.info("note: session exists for {h2o} on {d}".format(h2o=h2o, d=ymd)) trial = namedtuple( # simple structure to track per-trial vars 'trial', ('ttype', 'stim', 'free', 'settings', 'state_times', 'state_names', 'state_data', 'event_data', 'event_times', 'trial_start')) if os.stat(path).st_size / 1024 < 1000: log.info('skipping file {} - too small'.format(path)) return log.debug('loading file {}'.format(path)) mat = spio.loadmat(path, squeeze_me=True) SessionData = mat['SessionData'].flatten() # parse session datetime session_datetime_str = str('').join( (str(SessionData['Info'][0]['SessionDate']), ' ', str(SessionData['Info'][0]['SessionStartTime_UTC']))) session_datetime = datetime.strptime(session_datetime_str, '%d-%b-%Y %H:%M:%S') AllTrialTypes = SessionData['TrialTypes'][0] AllTrialSettings = SessionData['TrialSettings'][0] AllTrialStarts = SessionData['TrialStartTimestamp'][0] AllTrialStarts = AllTrialStarts - AllTrialStarts[0] # real 1st trial RawData = SessionData['RawData'][0].flatten() AllStateNames = RawData['OriginalStateNamesByNumber'][0] AllStateData = RawData['OriginalStateData'][0] AllEventData = RawData['OriginalEventData'][0] AllStateTimestamps = RawData['OriginalStateTimestamps'][0] AllEventTimestamps = RawData['OriginalEventTimestamps'][0] # verify trial-related data arrays are all same length assert (all((x.shape[0] == AllStateTimestamps.shape[0] for x in (AllTrialTypes, AllTrialSettings, AllStateNames, AllStateData, AllEventData, AllEventTimestamps, AllTrialStarts, AllTrialStarts)))) # AllStimTrials optional special case if 'StimTrials' in SessionData.dtype.fields: log.debug('StimTrials detected in session - will include') AllStimTrials = SessionData['StimTrials'][0] assert (AllStimTrials.shape[0] == AllStateTimestamps.shape[0]) else: log.debug('StimTrials not detected in session - will skip') AllStimTrials = np.array( [None for _ in enumerate(range(AllStateTimestamps.shape[0]))]) # AllFreeTrials optional special case if 'FreeTrials' in SessionData.dtype.fields: log.debug('FreeTrials detected in session - will include') AllFreeTrials = SessionData['FreeTrials'][0] assert (AllFreeTrials.shape[0] == AllStateTimestamps.shape[0]) else: log.debug('FreeTrials not detected in session - synthesizing') AllFreeTrials = np.zeros(AllStateTimestamps.shape[0], dtype=np.uint8) trials = list( zip(AllTrialTypes, AllStimTrials, AllFreeTrials, AllTrialSettings, AllStateTimestamps, AllStateNames, AllStateData, AllEventData, AllEventTimestamps, AllTrialStarts)) if not trials: log.warning('skipping date {d}, no valid files'.format(d=date)) return # # Trial data seems valid; synthesize session id & add session record # XXX: note - later breaks can result in Sessions without valid trials # assert skey['session_date'] == session_datetime.date() skey['session_date'] = session_datetime.date() skey['session_time'] = session_datetime.time() log.debug('synthesizing session ID') session = (dj.U().aggr(experiment.Session() & { 'subject_id': subject_id }, n='max(session)').fetch1('n') or 0) + 1 log.info('generated session id: {session}'.format(session=session)) skey['session'] = session key = dict(key, **skey) # # Actually load the per-trial data # log.info('BehaviorIngest.make(): trial parsing phase') # lists of various records for batch-insert rows = { k: list() for k in ('trial', 'behavior_trial', 'trial_note', 'trial_event', 'corrected_trial_event', 'action_event', 'photostim', 'photostim_location', 'photostim_trial', 'photostim_trial_event') } i = 0 # trial numbering starts at 1 for t in trials: # # Misc # t = trial(*t) # convert list of items to a 'trial' structure i += 1 # increment trial counter log.debug('BehaviorIngest.make(): parsing trial {i}'.format(i=i)) # covert state data names into a lookup dictionary # # names (seem to be? are?): # # Trigtrialstart, PreSamplePeriod, SamplePeriod, DelayPeriod # EarlyLickDelay, EarlyLickSample, ResponseCue, GiveLeftDrop # GiveRightDrop, GiveLeftDropShort, GiveRightDropShort # AnswerPeriod, Reward, RewardConsumption, NoResponse # TimeOut, StopLicking, StopLickingReturn, TrialEnd # states = {k: (v + 1) for v, k in enumerate(t.state_names)} required_states = ('PreSamplePeriod', 'SamplePeriod', 'DelayPeriod', 'ResponseCue', 'StopLicking', 'TrialEnd') missing = list(k for k in required_states if k not in states) if len(missing): log.warning('skipping trial {i}; missing {m}'.format( i=i, m=missing)) continue gui = t.settings['GUI'].flatten() # ProtocolType - only ingest protocol >= 3 # # 1 Water-Valve-Calibration 2 Licking 3 Autoassist # 4 No autoassist 5 DelayEnforce 6 SampleEnforce 7 Fixed # if 'ProtocolType' not in gui.dtype.names: log.warning( 'skipping trial {i}; protocol undefined'.format(i=i)) continue protocol_type = gui['ProtocolType'][0] if gui['ProtocolType'][0] < 3: log.warning('skipping trial {i}; protocol {n} < 3'.format( i=i, n=gui['ProtocolType'][0])) continue # # Top-level 'Trial' record # tkey = dict(skey) startindex = np.where(t.state_data == states['PreSamplePeriod'])[0] # should be only end of 1st StopLicking; # rest of data is irrelevant w/r/t separately ingested ephys endindex = np.where(t.state_data == states['StopLicking'])[0] log.debug('states\n' + str(states)) log.debug('state_data\n' + str(t.state_data)) log.debug('startindex\n' + str(startindex)) log.debug('endindex\n' + str(endindex)) if not (len(startindex) and len(endindex)): log.warning('skipping {}: start/end mismatch: {}/{}'.format( i, str(startindex), str(endindex))) continue try: tkey['trial'] = i tkey['trial_uid'] = i tkey['start_time'] = t.trial_start tkey['stop_time'] = t.trial_start + t.state_times[endindex][0] except IndexError: log.warning('skipping {}: IndexError: {}/{} -> {}'.format( i, str(startindex), str(endindex), str(t.state_times))) continue log.debug('tkey' + str(tkey)) rows['trial'].append(tkey) # # Specific BehaviorTrial information for this trial # bkey = dict(tkey) bkey['task'] = 'audio delay' # hard-coded here bkey['task_protocol'] = 1 # hard-coded here # determine trial instruction trial_instruction = 'left' # hard-coded here if gui['Reversal'][0] == 1: if t.ttype == 1: trial_instruction = 'left' elif t.ttype == 0: trial_instruction = 'right' elif gui['Reversal'][0] == 2: if t.ttype == 1: trial_instruction = 'right' elif t.ttype == 0: trial_instruction = 'left' bkey['trial_instruction'] = trial_instruction # determine early lick early_lick = 'no early' if (protocol_type >= 5 and 'EarlyLickDelay' in states and np.any(t.state_data == states['EarlyLickDelay'])): early_lick = 'early' if (protocol_type >= 5 and ('EarlyLickSample' in states and np.any(t.state_data == states['EarlyLickSample']))): early_lick = 'early' bkey['early_lick'] = early_lick # determine outcome outcome = 'ignore' if ('Reward' in states and np.any(t.state_data == states['Reward'])): outcome = 'hit' elif ('TimeOut' in states and np.any(t.state_data == states['TimeOut'])): outcome = 'miss' elif ('NoResponse' in states and np.any(t.state_data == states['NoResponse'])): outcome = 'ignore' bkey['outcome'] = outcome # Determine free/autowater (Autowater 1 == enabled, 2 == disabled) bkey['auto_water'] = True if gui['Autowater'][0] == 1 else False bkey['free_water'] = t.free rows['behavior_trial'].append(bkey) # # Add 'protocol' note # nkey = dict(tkey) nkey['trial_note_type'] = 'protocol #' nkey['trial_note'] = str(protocol_type) rows['trial_note'].append(nkey) # # Add 'autolearn' note # nkey = dict(tkey) nkey['trial_note_type'] = 'autolearn' nkey['trial_note'] = str(gui['Autolearn'][0]) rows['trial_note'].append(nkey) # # Add 'bitcode' note # if 'randomID' in gui.dtype.names: nkey = dict(tkey) nkey['trial_note_type'] = 'bitcode' nkey['trial_note'] = str(gui['randomID'][0]) rows['trial_note'].append(nkey) # # Add presample event # log.debug('BehaviorIngest.make(): presample') ekey = dict(tkey) sampleindex = np.where(t.state_data == states['SamplePeriod'])[0] ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'presample' ekey['trial_event_time'] = t.state_times[startindex][0] ekey['duration'] = (t.state_times[sampleindex[0]] - t.state_times[startindex])[0] if math.isnan(ekey['duration']): log.debug('BehaviorIngest.make(): fixing presample duration') ekey['duration'] = 0.0 # FIXDUR: lookup from previous trial rows['trial_event'].append(ekey) # # Add other 'sample' events # log.debug('BehaviorIngest.make(): sample events') last_dur = None for s in sampleindex: # in protocol > 6 ~-> n>1 # todo: batch events ekey = dict(tkey) ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'sample' ekey['trial_event_time'] = t.state_times[s] ekey['duration'] = gui['SamplePeriod'][0] if math.isnan(ekey['duration']) and last_dur is None: log.warning( '... trial {} bad duration, no last_edur'.format( i, last_dur)) ekey['duration'] = 0.0 # FIXDUR: cross-trial check rows['corrected_trial_event'].append(ekey) elif math.isnan(ekey['duration']) and last_dur is not None: log.warning( '... trial {} duration using last_edur {}'.format( i, last_dur)) ekey['duration'] = last_dur rows['corrected_trial_event'].append(ekey) else: last_dur = ekey['duration'] # only track 'good' values. rows['trial_event'].append(ekey) # # Add 'delay' events # log.debug('BehaviorIngest.make(): delay events') last_dur = None delayindex = np.where(t.state_data == states['DelayPeriod'])[0] for d in delayindex: # protocol > 6 ~-> n>1 ekey = dict(tkey) ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'delay' ekey['trial_event_time'] = t.state_times[d] ekey['duration'] = gui['DelayPeriod'][0] if math.isnan(ekey['duration']) and last_dur is None: log.warning('... {} bad duration, no last_edur'.format( i, last_dur)) ekey['duration'] = 0.0 # FIXDUR: cross-trial check rows['corrected_trial_event'].append(ekey) elif math.isnan(ekey['duration']) and last_dur is not None: log.warning('... {} duration using last_edur {}'.format( i, last_dur)) ekey['duration'] = last_dur rows['corrected_trial_event'].append(ekey) else: last_dur = ekey['duration'] # only track 'good' values. log.debug('delay event duration: {}'.format(ekey['duration'])) rows['trial_event'].append(ekey) # # Add 'go' event # log.debug('BehaviorIngest.make(): go') ekey = dict(tkey) responseindex = np.where(t.state_data == states['ResponseCue'])[0] ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'go' ekey['trial_event_time'] = t.state_times[responseindex][0] ekey['duration'] = gui['AnswerPeriod'][0] if math.isnan(ekey['duration']): log.debug('BehaviorIngest.make(): fixing go duration') ekey['duration'] = 0.0 # FIXDUR: lookup from previous trials rows['corrected_trial_event'].append(ekey) rows['trial_event'].append(ekey) # # Add 'trialEnd' events # log.debug('BehaviorIngest.make(): trialend events') last_dur = None trialendindex = np.where(t.state_data == states['TrialEnd'])[0] ekey = dict(tkey) ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'trialend' ekey['trial_event_time'] = t.state_times[trialendindex][0] ekey['duration'] = 0.0 rows['trial_event'].append(ekey) # # Add lick events # lickleft = np.where(t.event_data == 69)[0] log.debug('... lickleft: {r}'.format(r=str(lickleft))) action_event_count = len(rows['action_event']) if len(lickleft): [ rows['action_event'].append( dict(tkey, action_event_id=action_event_count + idx, action_event_type='left lick', action_event_time=t.event_times[l])) for idx, l in enumerate(lickleft) ] lickright = np.where(t.event_data == 71)[0] log.debug('... lickright: {r}'.format(r=str(lickright))) action_event_count = len(rows['action_event']) if len(lickright): [ rows['action_event'].append( dict(tkey, action_event_id=action_event_count + idx, action_event_type='right lick', action_event_time=t.event_times[r])) for idx, r in enumerate(lickright) ] # # Photostim Events # if t.stim: log.debug('BehaviorIngest.make(): t.stim == {}'.format(t.stim)) rows['photostim_trial'].append(tkey) delay_period_idx = np.where( t.state_data == states['DelayPeriod'])[0][0] rows['photostim_trial_event'].append( dict(tkey, photo_stim=t.stim, photostim_event_id=len(rows['photostim_trial_event']), photostim_event_time=t.state_times[delay_period_idx], power=5.5)) # end of trial loop. # Session Insertion log.info('BehaviorIngest.make(): adding session record') experiment.Session().insert1(skey) # Behavior Insertion log.info('BehaviorIngest.make(): bulk insert phase') log.info('BehaviorIngest.make(): saving ingest {d}'.format(d=key)) self.insert1(key, ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.Session.Trial') experiment.SessionTrial().insert(rows['trial'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.BehaviorTrial') experiment.BehaviorTrial().insert(rows['behavior_trial'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.TrialNote') experiment.TrialNote().insert(rows['trial_note'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.TrialEvent') experiment.TrialEvent().insert(rows['trial_event'], ignore_extra_fields=True, allow_direct_insert=True, skip_duplicates=True) log.info('BehaviorIngest.make(): ... CorrectedTrialEvents') BehaviorIngest().CorrectedTrialEvents().insert( rows['corrected_trial_event'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.ActionEvent') experiment.ActionEvent().insert(rows['action_event'], ignore_extra_fields=True, allow_direct_insert=True) # Photostim Insertion photostim_ids = np.unique( [r['photo_stim'] for r in rows['photostim_trial_event']]) unknown_photostims = np.setdiff1d(photostim_ids, list(photostims.keys())) if unknown_photostims: raise ValueError( 'Unknown photostim protocol: {}'.format(unknown_photostims)) if photostim_ids.size > 0: log.info('BehaviorIngest.make(): ... experiment.Photostim') for stim in photostim_ids: experiment.Photostim.insert1(dict(skey, **photostims[stim]), ignore_extra_fields=True) experiment.Photostim.PhotostimLocation.insert( (dict( skey, **loc, photo_stim=photostims[stim]['photo_stim']) for loc in photostims[stim]['locations']), ignore_extra_fields=True) log.info('BehaviorIngest.make(): ... experiment.PhotostimTrial') experiment.PhotostimTrial.insert(rows['photostim_trial'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.PhotostimTrialEvent') experiment.PhotostimEvent.insert(rows['photostim_trial_event'], ignore_extra_fields=True, allow_direct_insert=True) # Behavior Ingest Insertion log.info('BehaviorIngest.make(): ... BehaviorIngest.BehaviorFile') BehaviorIngest.BehaviorFile().insert1(dict( key, behavior_file=os.path.basename(key['subpath'])), ignore_extra_fields=True, allow_direct_insert=True)
def make(self, key): log.info('BehaviorIngest.make(): key: {key}'.format(key=key)) subject_id = key['subject_id'] h2o = (lab.WaterRestriction() & { 'subject_id': subject_id }).fetch1('water_restriction_number') date = key['session_date'] datestr = date.strftime('%Y%m%d') log.info('h2o: {h2o}, date: {d}'.format(h2o=h2o, d=datestr)) # session record key skey = {} skey['subject_id'] = subject_id skey['session_date'] = date skey['username'] = self.get_session_user() # File paths conform to the pattern: # dl7/TW_autoTrain/Session Data/dl7_TW_autoTrain_20180104_132813.mat # which is, more generally: # {h2o}/{training_protocol}/Session Data/{h2o}_{training protocol}_{YYYYMMDD}_{HHMMSS}.mat root = pathlib.Path(key['rig_data_path'], os.path.dirname(key['subpath'])) path = root / '{h2o}_*_{d}*.mat'.format(h2o=h2o, d=datestr) log.info('rigpath {p}'.format(p=path)) matches = sorted( root.glob('{h2o}_*_{d}*.mat'.format(h2o=h2o, d=datestr))) if matches: log.info('found files: {}, this is the rig'.format(matches)) skey['rig'] = key['rig'] else: log.info('no file matches found in {p}'.format(p=path)) if not len(matches): log.warning('no file matches found for {h2o} / {d}'.format( h2o=h2o, d=datestr)) return # # Find files & Check for split files # XXX: not checking rig.. 2+ sessions on 2+ rigs possible for date? # if len(matches) > 1: log.warning( 'split session case detected for {h2o} on {date}'.format( h2o=h2o, date=date)) # session:date relationship is 1:1; skip if we have a session if experiment.Session() & skey: log.warning("Warning! session exists for {h2o} on {d}".format( h2o=h2o, d=date)) return # # Prepare PhotoStim # photosti_duration = 0.5 # (s) Hard-coded here photostims = { 4: { 'photo_stim': 4, 'photostim_device': 'OBIS470', 'brain_location_name': 'left_alm', 'duration': photosti_duration }, 5: { 'photo_stim': 5, 'photostim_device': 'OBIS470', 'brain_location_name': 'right_alm', 'duration': photosti_duration }, 6: { 'photo_stim': 6, 'photostim_device': 'OBIS470', 'brain_location_name': 'both_alm', 'duration': photosti_duration } } # # Extract trial data from file(s) & prepare trial loop # trials = zip() trial = namedtuple( # simple structure to track per-trial vars 'trial', ('ttype', 'stim', 'settings', 'state_times', 'state_names', 'state_data', 'event_data', 'event_times')) for f in matches: if os.stat(f).st_size / 1024 < 1000: log.info('skipping file {f} - too small'.format(f=f)) continue log.debug('loading file {}'.format(f)) mat = spio.loadmat(f, squeeze_me=True) SessionData = mat['SessionData'].flatten() AllTrialTypes = SessionData['TrialTypes'][0] AllTrialSettings = SessionData['TrialSettings'][0] RawData = SessionData['RawData'][0].flatten() AllStateNames = RawData['OriginalStateNamesByNumber'][0] AllStateData = RawData['OriginalStateData'][0] AllEventData = RawData['OriginalEventData'][0] AllStateTimestamps = RawData['OriginalStateTimestamps'][0] AllEventTimestamps = RawData['OriginalEventTimestamps'][0] # verify trial-related data arrays are all same length assert (all( (x.shape[0] == AllStateTimestamps.shape[0] for x in (AllTrialTypes, AllTrialSettings, AllStateNames, AllStateData, AllEventData, AllEventTimestamps)))) if 'StimTrials' in SessionData.dtype.fields: log.debug('StimTrials detected in session - will include') AllStimTrials = SessionData['StimTrials'][0] assert (AllStimTrials.shape[0] == AllStateTimestamps.shape[0]) else: log.debug('StimTrials not detected in session - will skip') AllStimTrials = np.array([ None for i in enumerate(range(AllStateTimestamps.shape[0])) ]) z = zip(AllTrialTypes, AllStimTrials, AllTrialSettings, AllStateTimestamps, AllStateNames, AllStateData, AllEventData, AllEventTimestamps) trials = chain(trials, z) # concatenate the files trials = list(trials) # all files were internally invalid or size < 100k if not trials: log.warning('skipping date {d}, no valid files'.format(d=date)) return # # Trial data seems valid; synthesize session id & add session record # XXX: note - later breaks can result in Sessions without valid trials # log.debug('synthesizing session ID') session = (dj.U().aggr(experiment.Session() & { 'subject_id': subject_id }, n='max(session)').fetch1('n') or 0) + 1 log.info('generated session id: {session}'.format(session=session)) skey['session'] = session key = dict(key, **skey) # # Actually load the per-trial data # log.info('BehaviorIngest.make(): trial parsing phase') # lists of various records for batch-insert rows = { k: list() for k in ('trial', 'behavior_trial', 'trial_note', 'trial_event', 'corrected_trial_event', 'action_event', 'photostim', 'photostim_location', 'photostim_trial', 'photostim_trial_event') } i = -1 for t in trials: # # Misc # t = trial(*t) # convert list of items to a 'trial' structure i += 1 # increment trial counter log.debug('BehaviorIngest.make(): parsing trial {i}'.format(i=i)) # covert state data names into a lookup dictionary # # names (seem to be? are?): # # Trigtrialstart # PreSamplePeriod # SamplePeriod # DelayPeriod # EarlyLickDelay # EarlyLickSample # ResponseCue # GiveLeftDrop # GiveRightDrop # GiveLeftDropShort # GiveRightDropShort # AnswerPeriod # Reward # RewardConsumption # NoResponse # TimeOut # StopLicking # StopLickingReturn # TrialEnd states = {k: (v + 1) for v, k in enumerate(t.state_names)} required_states = ('PreSamplePeriod', 'SamplePeriod', 'DelayPeriod', 'ResponseCue', 'StopLicking', 'TrialEnd') missing = list(k for k in required_states if k not in states) if len(missing): log.warning('skipping trial {i}; missing {m}'.format( i=i, m=missing)) continue gui = t.settings['GUI'].flatten() # ProtocolType - only ingest protocol >= 3 # # 1 Water-Valve-Calibration 2 Licking 3 Autoassist # 4 No autoassist 5 DelayEnforce 6 SampleEnforce 7 Fixed # if 'ProtocolType' not in gui.dtype.names: log.warning( 'skipping trial {i}; protocol undefined'.format(i=i)) continue protocol_type = gui['ProtocolType'][0] if gui['ProtocolType'][0] < 3: log.warning('skipping trial {i}; protocol {n} < 3'.format( i=i, n=gui['ProtocolType'][0])) continue # # Top-level 'Trial' record # tkey = dict(skey) startindex = np.where(t.state_data == states['PreSamplePeriod'])[0] # should be only end of 1st StopLicking; # rest of data is irrelevant w/r/t separately ingested ephys endindex = np.where(t.state_data == states['StopLicking'])[0] log.debug('states\n' + str(states)) log.debug('state_data\n' + str(t.state_data)) log.debug('startindex\n' + str(startindex)) log.debug('endindex\n' + str(endindex)) if not (len(startindex) and len(endindex)): log.warning( 'skipping trial {i}: start/end index error: {s}/{e}'. format(i=i, s=str(startindex), e=str(endindex))) continue try: tkey['trial'] = i tkey[ 'trial_uid'] = i # Arseny has unique id to identify some trials tkey['start_time'] = t.state_times[startindex][0] tkey['stop_time'] = t.state_times[endindex][0] except IndexError: log.warning( 'skipping trial {i}: error indexing {s}/{e} into {t}'. format(i=i, s=str(startindex), e=str(endindex), t=str(t.state_times))) continue log.debug('BehaviorIngest.make(): Trial().insert1') # TODO msg log.debug('tkey' + str(tkey)) rows['trial'].append(tkey) # # Specific BehaviorTrial information for this trial # bkey = dict(tkey) bkey['task'] = 'audio delay' # hard-coded here bkey['task_protocol'] = 1 # hard-coded here # determine trial instruction trial_instruction = 'left' # hard-coded here if gui['Reversal'][0] == 1: if t.ttype == 1: trial_instruction = 'left' elif t.ttype == 0: trial_instruction = 'right' elif gui['Reversal'][0] == 2: if t.ttype == 1: trial_instruction = 'right' elif t.ttype == 0: trial_instruction = 'left' bkey['trial_instruction'] = trial_instruction # determine early lick early_lick = 'no early' if (protocol_type >= 5 and 'EarlyLickDelay' in states and np.any(t.state_data == states['EarlyLickDelay'])): early_lick = 'early' if (protocol_type > 5 and ('EarlyLickSample' in states and np.any(t.state_data == states['EarlyLickSample']))): early_lick = 'early' bkey['early_lick'] = early_lick # determine outcome outcome = 'ignore' if ('Reward' in states and np.any(t.state_data == states['Reward'])): outcome = 'hit' elif ('TimeOut' in states and np.any(t.state_data == states['TimeOut'])): outcome = 'miss' elif ('NoResponse' in states and np.any(t.state_data == states['NoResponse'])): outcome = 'ignore' bkey['outcome'] = outcome rows['behavior_trial'].append(bkey) # # Add 'protocol' note # nkey = dict(tkey) nkey['trial_note_type'] = 'protocol #' nkey['trial_note'] = str(protocol_type) rows['trial_note'].append(nkey) # # Add 'autolearn' note # nkey = dict(tkey) nkey['trial_note_type'] = 'autolearn' nkey['trial_note'] = str(gui['Autolearn'][0]) rows['trial_note'].append(nkey) # # Add 'bitcode' note # if 'randomID' in gui.dtype.names: nkey = dict(tkey) nkey['trial_note_type'] = 'bitcode' nkey['trial_note'] = str(gui['randomID'][0]) rows['trial_note'].append(nkey) # # Add presample event # log.debug('BehaviorIngest.make(): presample') ekey = dict(tkey) sampleindex = np.where(t.state_data == states['SamplePeriod'])[0] ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'presample' ekey['trial_event_time'] = t.state_times[startindex][0] ekey['duration'] = (t.state_times[sampleindex[0]] - t.state_times[startindex])[0] if math.isnan(ekey['duration']): log.debug('BehaviorIngest.make(): fixing presample duration') ekey['duration'] = 0.0 # FIXDUR: lookup from previous trial rows['trial_event'].append(ekey) # # Add other 'sample' events # log.debug('BehaviorIngest.make(): sample events') last_dur = None for s in sampleindex: # in protocol > 6 ~-> n>1 # todo: batch events ekey = dict(tkey) ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'sample' ekey['trial_event_time'] = t.state_times[s] ekey['duration'] = gui['SamplePeriod'][0] if math.isnan(ekey['duration']) and last_dur is None: log.warning( '... trial {} bad duration, no last_edur'.format( i, last_dur)) ekey['duration'] = 0.0 # FIXDUR: cross-trial check rows['corrected_trial_event'].append(ekey) elif math.isnan(ekey['duration']) and last_dur is not None: log.warning( '... trial {} duration using last_edur {}'.format( i, last_dur)) ekey['duration'] = last_dur rows['corrected_trial_event'].append(ekey) else: last_dur = ekey['duration'] # only track 'good' values. rows['trial_event'].append(ekey) # # Add 'delay' events # log.debug('BehaviorIngest.make(): delay events') last_dur = None delayindex = np.where(t.state_data == states['DelayPeriod'])[0] for d in delayindex: # protocol > 6 ~-> n>1 ekey = dict(tkey) ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'delay' ekey['trial_event_time'] = t.state_times[d] ekey['duration'] = gui['DelayPeriod'][0] if math.isnan(ekey['duration']) and last_dur is None: log.warning('... {} bad duration, no last_edur'.format( i, last_dur)) ekey['duration'] = 0.0 # FIXDUR: cross-trial check rows['corrected_trial_event'].append(ekey) elif math.isnan(ekey['duration']) and last_dur is not None: log.warning('... {} duration using last_edur {}'.format( i, last_dur)) ekey['duration'] = last_dur rows['corrected_trial_event'].append(ekey) else: last_dur = ekey['duration'] # only track 'good' values. log.debug('delay event duration: {}'.format(ekey['duration'])) rows['trial_event'].append(ekey) # # Add 'go' event # log.debug('BehaviorIngest.make(): go') ekey = dict(tkey) responseindex = np.where(t.state_data == states['ResponseCue'])[0] ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'go' ekey['trial_event_time'] = t.state_times[responseindex][0] ekey['duration'] = gui['AnswerPeriod'][0] if math.isnan(ekey['duration']): log.debug('BehaviorIngest.make(): fixing go duration') ekey['duration'] = 0.0 # FIXDUR: lookup from previous trials rows['corrected_trial_event'].append(ekey) rows['trial_event'].append(ekey) # # Add 'trialEnd' events # log.debug('BehaviorIngest.make(): trialend events') last_dur = None trialendindex = np.where(t.state_data == states['TrialEnd'])[0] ekey = dict(tkey) ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'trialend' ekey['trial_event_time'] = t.state_times[trialendindex][0] ekey['duration'] = 0.0 rows['trial_event'].append(ekey) # # Add lick events # lickleft = np.where(t.event_data == 69)[0] log.debug('... lickleft: {r}'.format(r=str(lickleft))) action_event_count = len(rows['action_event']) if len(lickleft): [ rows['action_event'].append( dict(tkey, action_event_id=action_event_count + idx, action_event_type='left lick', action_event_time=t.event_times[l])) for idx, l in enumerate(lickleft) ] lickright = np.where(t.event_data == 71)[0] log.debug('... lickright: {r}'.format(r=str(lickright))) action_event_count = len(rows['action_event']) if len(lickright): [ rows['action_event'].append( dict(tkey, action_event_id=action_event_count + idx, action_event_type='right lick', action_event_time=t.event_times[r])) for idx, r in enumerate(lickright) ] # Photostim Events # # TODO: # # - base stimulation parameters: # # - should be loaded elsewhere - where # - actual ccf locations - cannot be known apriori apparently? # - Photostim.Profile: what is? fix/add # # - stim data # # - how retrieve power from file (didn't see) or should # be statically coded here? # - how encode stim type 6? # - we have hemisphere as boolean or # - but adding an event 4 and event 5 means querying # is less straightforwrard (e.g. sessions with 5 & 6) if t.stim: log.info('BehaviorIngest.make(): t.stim == {}'.format(t.stim)) rows['photostim_trial'].append(tkey) delay_period_idx = np.where( t.state_data == states['DelayPeriod'])[0][0] rows['photostim_trial_event'].append( dict(tkey, **photostims[t.stim], photostim_event_id=len(rows['photostim_trial_event']), photostim_event_time=t.state_times[delay_period_idx], power=5.5)) # end of trial loop. # Session Insertion log.info('BehaviorIngest.make(): adding session record') experiment.Session().insert1(skey) # Behavior Insertion log.info('BehaviorIngest.make(): bulk insert phase') log.info('BehaviorIngest.make(): saving ingest {d}'.format(d=key)) self.insert1(key, ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.Session.Trial') experiment.SessionTrial().insert(rows['trial'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.BehaviorTrial') experiment.BehaviorTrial().insert(rows['behavior_trial'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.TrialNote') experiment.TrialNote().insert(rows['trial_note'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.TrialEvent') experiment.TrialEvent().insert(rows['trial_event'], ignore_extra_fields=True, allow_direct_insert=True, skip_duplicates=True) log.info('BehaviorIngest.make(): ... CorrectedTrialEvents') BehaviorIngest().CorrectedTrialEvents().insert( rows['corrected_trial_event'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.ActionEvent') experiment.ActionEvent().insert(rows['action_event'], ignore_extra_fields=True, allow_direct_insert=True) BehaviorIngest.BehaviorFile().insert( (dict(key, behavior_file=f.name) for f in matches), ignore_extra_fields=True, allow_direct_insert=True) # Photostim Insertion photostim_ids = set( [r['photo_stim'] for r in rows['photostim_trial_event']]) if photostim_ids: log.info('BehaviorIngest.make(): ... experiment.Photostim') experiment.Photostim.insert( (dict(skey, **photostims[stim]) for stim in photostim_ids), ignore_extra_fields=True) log.info('BehaviorIngest.make(): ... experiment.PhotostimTrial') experiment.PhotostimTrial.insert(rows['photostim_trial'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.PhotostimTrialEvent') experiment.PhotostimEvent.insert(rows['photostim_trial_event'], ignore_extra_fields=True, allow_direct_insert=True)
def make(self, key): log.info('BehaviorIngest.make(): key: {key}'.format(key=key)) rigpaths = [ p for p in RigDataPath().fetch(order_by='rig_data_path') if 'RRig' in p['rig'] ] # change between TRig and RRig subject_id = key['subject_id'] h2o = (lab.WaterRestriction() & { 'subject_id': subject_id }).fetch1('water_restriction_number') date = key['session_date'] datestr = date.strftime('%Y%m%d') log.debug('h2o: {h2o}, date: {d}'.format(h2o=h2o, d=datestr)) # session record key skey = {} skey['subject_id'] = subject_id skey['session_date'] = date skey['username'] = '******' # username has to be changed # e.g: dl7/TW_autoTrain/Session Data/dl7_TW_autoTrain_20180104_132813.mat # # p.split('/foo/bar')[1] for rp in rigpaths: root = rp['rig_data_path'] path = root path = os.path.join(path, h2o) # path = os.path.join(path, 'TW_autoTrain') path = os.path.join(path, 'tw2') path = os.path.join(path, 'Session Data') path = os.path.join( # path, '{h2o}_TW_autoTrain_{d}*.mat'.format(h2o=h2o, d=datestr)) # earlier program protocol path, '{h2o}_tw2_{d}*.mat'.format( h2o=h2o, d=datestr)) # later program protocol log.debug('rigpath {p}'.format(p=path)) matches = glob.glob(path) if len(matches): log.debug('found files, this is the rig') skey['rig'] = rp['rig'] break else: log.info('no file matches found in {p}'.format(p=path)) if not len(matches): log.warning('no file matches found for {h2o} / {d}'.format( h2o=h2o, d=datestr)) return # # Find files & Check for split files # XXX: not checking rig.. 2+ sessions on 2+ rigs possible for date? # if len(matches) > 1: log.warning( 'split session case detected for {h2o} on {date}'.format( h2o=h2o, date=date)) # session:date relationship is 1:1; skip if we have a session if experiment.Session() & skey: log.warning("Warning! session exists for {h2o} on {d}".format( h2o=h2o, d=date)) return # # Extract trial data from file(s) & prepare trial loop # trials = zip() trial = namedtuple( # simple structure to track per-trial vars 'trial', ('ttype', 'settings', 'state_times', 'state_names', 'state_data', 'event_data', 'event_times')) for f in matches: if os.stat(f).st_size / 1024 < 100: log.info('skipping file {f} - too small'.format(f=f)) continue mat = spio.loadmat(f, squeeze_me=True) SessionData = mat['SessionData'].flatten() AllTrialTypes = SessionData['TrialTypes'][0] AllTrialSettings = SessionData['TrialSettings'][0] RawData = SessionData['RawData'][0].flatten() AllStateNames = RawData['OriginalStateNamesByNumber'][0] AllStateData = RawData['OriginalStateData'][0] AllEventData = RawData['OriginalEventData'][0] AllStateTimestamps = RawData['OriginalStateTimestamps'][0] AllEventTimestamps = RawData['OriginalEventTimestamps'][0] # verify trial-related data arrays are all same length assert (all( (x.shape[0] == AllStateTimestamps.shape[0] for x in (AllTrialTypes, AllTrialSettings, AllStateNames, AllStateData, AllEventData, AllEventTimestamps)))) z = zip(AllTrialTypes, AllTrialSettings, AllStateTimestamps, AllStateNames, AllStateData, AllEventData, AllEventTimestamps) trials = chain(trials, z) # concatenate the files trials = list(trials) # all files were internally invalid or size < 100k if not trials: log.warning('skipping date {d}, no valid files'.format(d=date)) # # Trial data seems valid; synthesize session id & add session record # XXX: note - later breaks can result in Sessions without valid trials # log.debug('synthesizing session ID') session = (dj.U().aggr(experiment.Session() & { 'subject_id': subject_id }, n='max(session)').fetch1('n') or 0) + 1 log.info('generated session id: {session}'.format(session=session)) skey['session'] = session key = dict(key, **skey) log.debug('BehaviorIngest.make(): adding session record') experiment.Session().insert1(skey) # # Actually load the per-trial data # log.info('BehaviorIngest.make(): trial parsing phase') # lists of various records for batch-insert rows = { k: list() for k in ('trial', 'behavior_trial', 'trial_note', 'trial_event', 'action_event') } i = -1 for t in trials: # # Misc # t = trial(*t) # convert list of items to a 'trial' structure i += 1 # increment trial counter log.info('BehaviorIngest.make(): parsing trial {i}'.format(i=i)) # covert state data names into a lookup dictionary # # names (seem to be? are?): # # Trigtrialstart # PreSamplePeriod # SamplePeriod # DelayPeriod # EarlyLickDelay # EarlyLickSample # ResponseCue # GiveLeftDrop # GiveRightDrop # GiveLeftDropShort # GiveRightDropShort # AnswerPeriod # Reward # RewardConsumption # NoResponse # TimeOut # StopLicking # StopLickingReturn # TrialEnd states = {k: (v + 1) for v, k in enumerate(t.state_names)} required_states = ('PreSamplePeriod', 'SamplePeriod', 'DelayPeriod', 'ResponseCue', 'StopLicking', 'TrialEnd') missing = list(k for k in required_states if k not in states) if len(missing): log.info('skipping trial {i}; missing {m}'.format(i=i, m=missing)) continue gui = t.settings['GUI'].flatten() # ProtocolType - only ingest protocol >= 3 # # 1 Water-Valve-Calibration 2 Licking 3 Autoassist # 4 No autoassist 5 DelayEnforce 6 SampleEnforce 7 Fixed # if 'ProtocolType' not in gui.dtype.names: log.info('skipping trial {i}; protocol undefined'.format(i=i)) continue protocol_type = gui['ProtocolType'][0] if gui['ProtocolType'][0] < 3: log.info('skipping trial {i}; protocol {n} < 3'.format( i=i, n=gui['ProtocolType'][0])) continue # # Top-level 'Trial' record # tkey = dict(skey) startindex = np.where(t.state_data == states['PreSamplePeriod'])[0] # should be only end of 1st StopLicking; # rest of data is irrelevant w/r/t separately ingested ephys endindex = np.where(t.state_data == states['StopLicking'])[0] log.debug('states\n' + str(states)) log.debug('state_data\n' + str(t.state_data)) log.debug('startindex\n' + str(startindex)) log.debug('endendex\n' + str(endindex)) if not (len(startindex) and len(endindex)): log.info('skipping trial {i}: start/end index error: {s}/{e}'. format(i=i, s=str(startindex), e=str(endindex))) continue try: tkey['trial'] = i tkey['trial_uid'] = i tkey['start_time'] = t.state_times[startindex][0] except IndexError: log.info('skipping trial {i}: error indexing {s}/{e} into {t}'. format(i=i, s=str(startindex), e=str(endindex), t=str(t.state_times))) continue log.debug('BehaviorIngest.make(): Trial().insert1') # TODO msg log.debug('tkey' + str(tkey)) rows['trial'].append(tkey) # # Specific BehaviorTrial information for this trial # bkey = dict(tkey) bkey['task'] = 'audio delay' bkey['task_protocol'] = 1 # determine trial instruction trial_instruction = 'left' if gui['Reversal'][0] == 1: if t.ttype == 1: trial_instruction = 'left' elif t.ttype == 0: trial_instruction = 'right' elif gui['Reversal'][0] == 2: if t.ttype == 1: trial_instruction = 'right' elif t.ttype == 0: trial_instruction = 'left' bkey['trial_instruction'] = trial_instruction # determine early lick early_lick = 'no early' if (protocol_type >= 5 and 'EarlyLickDelay' in states and np.any(t.state_data == states['EarlyLickDelay'])): early_lick = 'early' if (protocol_type > 5 and ('EarlyLickSample' in states and np.any(t.state_data == states['EarlyLickSample']))): early_lick = 'early' bkey['early_lick'] = early_lick # determine outcome outcome = 'ignore' if ('Reward' in states and np.any(t.state_data == states['Reward'])): outcome = 'hit' elif ('TimeOut' in states and np.any(t.state_data == states['TimeOut'])): outcome = 'miss' elif ('NoResponse' in states and np.any(t.state_data == states['NoResponse'])): outcome = 'ignore' bkey['outcome'] = outcome # add behavior record log.debug('BehaviorIngest.make(): BehaviorTrial()') rows['behavior_trial'].append(bkey) # # Add 'protocol' note # nkey = dict(tkey) nkey['trial_note_type'] = 'protocol #' nkey['trial_note'] = str(protocol_type) log.debug('BehaviorIngest.make(): TrialNote().insert1') rows['trial_note'].append(nkey) # # Add 'autolearn' note # nkey = dict(tkey) nkey['trial_note_type'] = 'autolearn' nkey['trial_note'] = str(gui['Autolearn'][0]) rows['trial_note'].append(nkey) #pdb.set_trace() # # Add 'bitcode' note # if 'randomID' in gui.dtype.names: nkey = dict(tkey) nkey['trial_note_type'] = 'bitcode' nkey['trial_note'] = str(gui['randomID'][0]) rows['trial_note'].append(nkey) # # Add presample event # ekey = dict(tkey) sampleindex = np.where(t.state_data == states['SamplePeriod'])[0] ekey['trial_event_type'] = 'presample' ekey['trial_event_time'] = t.state_times[startindex][0] ekey['duration'] = (t.state_times[sampleindex[0]] - t.state_times[startindex])[0] log.debug('BehaviorIngest.make(): presample') rows['trial_event'].append(ekey) # # Add 'go' event # ekey = dict(tkey) responseindex = np.where(t.state_data == states['ResponseCue'])[0] ekey['trial_event_type'] = 'go' ekey['trial_event_time'] = t.state_times[responseindex][0] ekey['duration'] = gui['AnswerPeriod'][0] log.debug('BehaviorIngest.make(): go') rows['trial_event'].append(ekey) # # Add other 'sample' events # log.debug('BehaviorIngest.make(): sample events') for s in sampleindex: # in protocol > 6 ~-> n>1 # todo: batch events ekey = dict(tkey) ekey['trial_event_type'] = 'sample' ekey['trial_event_time'] = t.state_times[s] ekey['duration'] = gui['SamplePeriod'][0] rows['trial_event'].append(ekey) # # Add 'delay' events # delayindex = np.where(t.state_data == states['DelayPeriod'])[0] log.debug('BehaviorIngest.make(): delay events') for d in delayindex: # protocol > 6 ~-> n>1 # todo: batch events ekey = dict(tkey) ekey['trial_event_type'] = 'delay' ekey['trial_event_time'] = t.state_times[d] ekey['duration'] = gui['DelayPeriod'][0] rows['trial_event'].append(ekey) # # Add lick events # lickleft = np.where(t.event_data == 69)[0] log.debug('... lickleft: {r}'.format(r=str(lickleft))) if len(lickleft): [ rows['action_event'].append( dict(**tkey, action_event_type='left lick', action_event_time=t.event_times[l])) for l in lickleft ] lickright = np.where(t.event_data == 70)[0] log.debug('... lickright: {r}'.format(r=str(lickright))) if len(lickright): [ rows['action_event'].append( dict(**tkey, action_event_type='right lick', action_event_time=t.event_times[r])) for r in lickright ] # end of trial loop. log.info('BehaviorIngest.make(): bulk insert phase') log.info('BehaviorIngest.make(): ... experiment.Session.Trial') experiment.SessionTrial().insert(rows['trial'], ignore_extra_fields=True) log.info('BehaviorIngest.make(): ... experiment.BehaviorTrial') experiment.BehaviorTrial().insert(rows['behavior_trial'], ignore_extra_fields=True) log.info('BehaviorIngest.make(): ... experiment.TrialNote') experiment.TrialNote().insert(rows['trial_note'], ignore_extra_fields=True) log.info('BehaviorIngest.make(): ... experiment.TrialEvent') experiment.TrialEvent().insert(rows['trial_event'], ignore_extra_fields=True) log.info('BehaviorIngest.make(): ... experiment.ActionEvent') experiment.ActionEvent().insert(rows['action_event'], ignore_extra_fields=True) log.info('BehaviorIngest.make(): saving ingest {d}'.format(d=key)) self.insert1(key, ignore_extra_fields=True) BehaviorIngest.BehaviorFile().insert( (dict(key, behavior_file=f.split(root)[1]) for f in matches), ignore_extra_fields=True)
def populatebehavior_core(IDs = None): if IDs: print('subject started:') print(IDs.keys()) print(IDs.values()) rigpath_1 = 'E:/Projects/Ablation/datajoint/Behavior' #df_surgery = pd.read_csv(dj.config['locations.metadata']+'Surgery.csv') if IDs == None: IDs = {k: v for k, v in zip(*lab.WaterRestriction().fetch('water_restriction_number', 'subject_id'))} for subject_now,subject_id_now in zip(IDs.keys(),IDs.values()): # iterating over subjects print('subject: ',subject_now) # ============================================================================= # if drop_last_session_for_mice_in_training: # delete_last_session_before_upload = True # else: # delete_last_session_before_upload = False # #df_wr = online_notebook.fetch_water_restriction_metadata(subject_now) # ============================================================================= try: df_wr = pd.read_csv(dj.config['locations.metadata_behavior']+subject_now+'.csv') except: print(subject_now + ' has no metadata available') df_wr = pd.DataFrame() for df_wr_row in df_wr.iterrows(): date_now = df_wr_row[1].Date.replace('-','') print('subject: ',subject_now,' date: ',date_now) session_date = datetime(int(date_now[0:4]),int(date_now[4:6]),int(date_now[6:8])) if len(experiment.Session() & 'subject_id = "'+str(subject_id_now)+'"' & 'session_date > "'+str(session_date)+'"') != 0: # if it is not the last print('session already imported, skipping: ' + str(session_date)) dotheupload = False elif len(experiment.Session() & 'subject_id = "'+str(subject_id_now)+'"' & 'session_date = "'+str(session_date)+'"') != 0: # if it is the last dotheupload = False else: # reuploading new session that is not present on the server dotheupload = True # if dotheupload is True, meaning that there are new mat file hasn't been uploaded # => needs to find which mat file hasn't been uploaded if dotheupload: found = set() rigpath_2 = subject_now rigpath_3 = rigpath_1 + '/' + rigpath_2 rigpath = pathlib.Path(rigpath_3) def buildrec(rigpath, root, f): try: fullpath = pathlib.Path(root, f) subpath = fullpath.relative_to(rigpath) fsplit = subpath.stem.split('_') h2o = fsplit[0] ymd = fsplit[-2:-1][0] animal = IDs[h2o] if ymd == date_now: return { 'subject_id': animal, 'session_date': date(int(ymd[0:4]), int(ymd[4:6]), int(ymd[6:8])), 'rig_data_path': rigpath.as_posix(), 'subpath': subpath.as_posix(), } except: pass for root, dirs, files in os.walk(rigpath): for f in files: r = buildrec(rigpath, root, f) if r: found.add(r['subpath']) file = r # now start insert data path = pathlib.Path(file['rig_data_path'], file['subpath']) mat = spio.loadmat(path, squeeze_me=True) SessionData = mat['SessionData'].flatten() # session record key skey = {} skey['subject_id'] = file['subject_id'] skey['session_date'] = file['session_date'] skey['username'] = '******' #skey['rig'] = key['rig'] trial = namedtuple( # simple structure to track per-trial vars 'trial', ('ttype', 'settings', 'state_times', 'state_names', 'state_data', 'event_data', 'event_times', 'trial_start')) # parse session datetime session_datetime_str = str('').join((str(SessionData['Info'][0]['SessionDate']),' ', str(SessionData['Info'][0]['SessionStartTime_UTC']))) session_datetime = datetime.strptime(session_datetime_str, '%d-%b-%Y %H:%M:%S') AllTrialTypes = SessionData['TrialTypes'][0] AllTrialSettings = SessionData['TrialSettings'][0] AllTrialStarts = SessionData['TrialStartTimestamp'][0] AllTrialStarts = AllTrialStarts - AllTrialStarts[0] RawData = SessionData['RawData'][0].flatten() AllStateNames = RawData['OriginalStateNamesByNumber'][0] AllStateData = RawData['OriginalStateData'][0] AllEventData = RawData['OriginalEventData'][0] AllStateTimestamps = RawData['OriginalStateTimestamps'][0] AllEventTimestamps = RawData['OriginalEventTimestamps'][0] trials = list(zip(AllTrialTypes, AllTrialSettings, AllStateTimestamps, AllStateNames, AllStateData, AllEventData, AllEventTimestamps, AllTrialStarts)) if not trials: log.warning('skipping date {d}, no valid files'.format(d=date)) return # # Trial data seems valid; synthesize session id & add session record # XXX: note - later breaks can result in Sessions without valid trials # assert skey['session_date'] == session_datetime.date() skey['session_date'] = session_datetime.date() #skey['session_time'] = session_datetime.time() if len(experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"' & 'session_date = "'+str(file['session_date'])+'"') == 0: if len(experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"') == 0: skey['session'] = 1 else: skey['session'] = len((experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"').fetch()['session']) + 1 # # Actually load the per-trial data # log.info('BehaviorIngest.make(): trial parsing phase') # lists of various records for batch-insert rows = {k: list() for k in ('trial', 'behavior_trial', 'trial_note', 'trial_event', 'corrected_trial_event', 'action_event')} #, 'photostim', #'photostim_location', 'photostim_trial', #'photostim_trial_event')} i = 0 # trial numbering starts at 1 for t in trials: t = trial(*t) # convert list of items to a 'trial' structure i += 1 # increment trial counter log.debug('BehaviorIngest.make(): parsing trial {i}'.format(i=i)) states = {k: (v+1) for v, k in enumerate(t.state_names)} required_states = ('PreSamplePeriod', 'SamplePeriod', 'DelayPeriod', 'ResponseCue', 'StopLicking', 'TrialEnd') missing = list(k for k in required_states if k not in states) if len(missing) and missing =='PreSamplePeriod': log.warning('skipping trial {i}; missing {m}'.format(i=i, m=missing)) continue gui = t.settings['GUI'].flatten() if len(experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"' & 'session_date = "'+str(file['session_date'])+'"') == 0: if len(experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"') == 0: skey['session'] = 1 else: skey['session'] = len((experiment.Session() & 'subject_id = "'+str(file['subject_id'])+'"').fetch()['session']) + 1 # # Top-level 'Trial' record # protocol_type = gui['ProtocolType'][0] tkey = dict(skey) has_presample = 1 try: startindex = np.where(t.state_data == states['PreSamplePeriod'])[0] has_presample = 1 except: startindex = np.where(t.state_data == states['SamplePeriod'])[0] has_presample = 0 # should be only end of 1st StopLicking; # rest of data is irrelevant w/r/t separately ingested ephys endindex = np.where(t.state_data == states['StopLicking'])[0] log.debug('states\n' + str(states)) log.debug('state_data\n' + str(t.state_data)) log.debug('startindex\n' + str(startindex)) log.debug('endindex\n' + str(endindex)) if not(len(startindex) and len(endindex)): log.warning('skipping {}: start/end mismatch: {}/{}'.format(i, str(startindex), str(endindex))) continue try: tkey['trial'] = i tkey['trial_uid'] = i tkey['trial_start_time'] = t.trial_start tkey['trial_stop_time'] = t.trial_start + t.state_times[endindex][0] except IndexError: log.warning('skipping {}: IndexError: {}/{} -> {}'.format(i, str(startindex), str(endindex), str(t.state_times))) continue log.debug('tkey' + str(tkey)) rows['trial'].append(tkey) # # Specific BehaviorTrial information for this trial # bkey = dict(tkey) bkey['task'] = 'audio delay' # hard-coded here bkey['task_protocol'] = 1 # hard-coded here # determine trial instruction trial_instruction = 'left' # hard-coded here if gui['Reversal'][0] == 1: if t.ttype == 1: trial_instruction = 'left' elif t.ttype == 0: trial_instruction = 'right' elif t.ttype == 2: trial_instruction = 'catch_right_autowater' elif t.ttype == 3: trial_instruction = 'catch_left_autowater' elif t.ttype == 4: trial_instruction = 'catch_right_noDelay' elif t.ttype == 5: trial_instruction = 'catch_left_noDelay' elif gui['Reversal'][0] == 2: if t.ttype == 1: trial_instruction = 'right' elif t.ttype == 0: trial_instruction = 'left' elif t.ttype == 2: trial_instruction = 'catch_left_autowater' elif t.ttype == 3: trial_instruction = 'catch_right_autowater' elif t.ttype == 4: trial_instruction = 'catch_left_noDelay' elif t.ttype == 5: trial_instruction = 'catch_right_noDelay' bkey['trial_instruction'] = trial_instruction # determine early lick early_lick = 'no early' if (protocol_type >= 5 and 'EarlyLickDelay' in states and np.any(t.state_data == states['EarlyLickDelay'])): early_lick = 'early' if (protocol_type >= 5 and ('EarlyLickSample' in states and np.any(t.state_data == states['EarlyLickSample']))): early_lick = 'early' bkey['early_lick'] = early_lick # determine outcome outcome = 'ignore' if ('Reward' in states and np.any(t.state_data == states['Reward'])): outcome = 'hit' elif ('TimeOut' in states and np.any(t.state_data == states['TimeOut'])): outcome = 'miss' elif ('NoResponse' in states and np.any(t.state_data == states['NoResponse'])): outcome = 'ignore' bkey['outcome'] = outcome rows['behavior_trial'].append(bkey) # # Add 'protocol' note # nkey = dict(tkey) nkey['trial_note_type'] = 'protocol #' nkey['trial_note'] = str(protocol_type) rows['trial_note'].append(nkey) # # Add 'autolearn' note # nkey = dict(tkey) nkey['trial_note_type'] = 'autolearn' nkey['trial_note'] = str(gui['Autolearn'][0]) rows['trial_note'].append(nkey) # # Add 'bitcode' note # if 'randomID' in gui.dtype.names: nkey = dict(tkey) nkey['trial_note_type'] = 'bitcode' nkey['trial_note'] = str(gui['randomID'][0]) rows['trial_note'].append(nkey) # # Add presample event # sampleindex = np.where(t.state_data == states['SamplePeriod'])[0] if has_presample == 1: log.debug('BehaviorIngest.make(): presample') ekey = dict(tkey) ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'presample' ekey['trial_event_time'] = t.state_times[startindex][0] ekey['duration'] = (t.state_times[sampleindex[0]]- t.state_times[startindex])[0] if math.isnan(ekey['duration']): log.debug('BehaviorIngest.make(): fixing presample duration') ekey['duration'] = 0.0 # FIXDUR: lookup from previous trial rows['trial_event'].append(ekey) # # Add other 'sample' events # log.debug('BehaviorIngest.make(): sample events') last_dur = None for s in sampleindex: # in protocol > 6 ~-> n>1 # todo: batch events ekey = dict(tkey) ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'sample' ekey['trial_event_time'] = t.state_times[s] ekey['duration'] = gui['SamplePeriod'][0] if math.isnan(ekey['duration']) and last_dur is None: log.warning('... trial {} bad duration, no last_edur'.format(i, last_dur)) ekey['duration'] = 0.0 # FIXDUR: cross-trial check rows['corrected_trial_event'].append(ekey) elif math.isnan(ekey['duration']) and last_dur is not None: log.warning('... trial {} duration using last_edur {}'.format(i, last_dur)) ekey['duration'] = last_dur rows['corrected_trial_event'].append(ekey) else: last_dur = ekey['duration'] # only track 'good' values. rows['trial_event'].append(ekey) # # Add 'delay' events # log.debug('BehaviorIngest.make(): delay events') last_dur = None delayindex = np.where(t.state_data == states['DelayPeriod'])[0] for d in delayindex: # protocol > 6 ~-> n>1 ekey = dict(tkey) ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'delay' ekey['trial_event_time'] = t.state_times[d] ekey['duration'] = gui['DelayPeriod'][0] if math.isnan(ekey['duration']) and last_dur is None: log.warning('... {} bad duration, no last_edur'.format(i, last_dur)) ekey['duration'] = 0.0 # FIXDUR: cross-trial check rows['corrected_trial_event'].append(ekey) elif math.isnan(ekey['duration']) and last_dur is not None: log.warning('... {} duration using last_edur {}'.format(i, last_dur)) ekey['duration'] = last_dur rows['corrected_trial_event'].append(ekey) else: last_dur = ekey['duration'] # only track 'good' values. log.debug('delay event duration: {}'.format(ekey['duration'])) rows['trial_event'].append(ekey) # # Add 'go' event # log.debug('BehaviorIngest.make(): go') ekey = dict(tkey) responseindex = np.where(t.state_data == states['ResponseCue'])[0] ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'go' ekey['trial_event_time'] = t.state_times[responseindex][0] ekey['duration'] = gui['AnswerPeriod'][0] if math.isnan(ekey['duration']): log.debug('BehaviorIngest.make(): fixing go duration') ekey['duration'] = 0.0 # FIXDUR: lookup from previous trials rows['corrected_trial_event'].append(ekey) rows['trial_event'].append(ekey) # # Add 'trialEnd' events # log.debug('BehaviorIngest.make(): trialend events') last_dur = None trialendindex = np.where(t.state_data == states['TrialEnd'])[0] ekey = dict(tkey) ekey['trial_event_id'] = len(rows['trial_event']) ekey['trial_event_type'] = 'trialend' ekey['trial_event_time'] = t.state_times[trialendindex][0] ekey['duration'] = 0.0 rows['trial_event'].append(ekey) # # Add lick events # lickleft = np.where(t.event_data == 69)[0] log.debug('... lickleft: {r}'.format(r=str(lickleft))) action_event_count = len(rows['action_event']) if len(lickleft): [rows['action_event'].append( dict(tkey, action_event_id=action_event_count+idx, action_event_type='left lick', action_event_time=t.event_times[l])) for idx, l in enumerate(lickleft)] lickright = np.where(t.event_data == 71)[0] log.debug('... lickright: {r}'.format(r=str(lickright))) action_event_count = len(rows['action_event']) if len(lickright): [rows['action_event'].append( dict(tkey, action_event_id=action_event_count+idx, action_event_type='right lick', action_event_time=t.event_times[r])) for idx, r in enumerate(lickright)] # end of trial loop.. # Session Insertion log.info('BehaviorIngest.make(): adding session record') skey['session_date'] = df_wr_row[1].Date skey['rig'] = 'Old Recording rig' skey['username'] = '******' experiment.Session().insert1(skey,skip_duplicates=True) # Behavior Insertion log.info('BehaviorIngest.make(): ... experiment.Session.Trial') experiment.SessionTrial().insert( rows['trial'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.BehaviorTrial') experiment.BehaviorTrial().insert( rows['behavior_trial'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.TrialNote') experiment.TrialNote().insert( rows['trial_note'], ignore_extra_fields=True, allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.TrialEvent') experiment.TrialEvent().insert( rows['trial_event'], ignore_extra_fields=True, allow_direct_insert=True, skip_duplicates=True) # log.info('BehaviorIngest.make(): ... CorrectedTrialEvents') # BehaviorIngest().CorrectedTrialEvents().insert( # rows['corrected_trial_event'], ignore_extra_fields=True, # allow_direct_insert=True) log.info('BehaviorIngest.make(): ... experiment.ActionEvent') experiment.ActionEvent().insert( rows['action_event'], ignore_extra_fields=True, allow_direct_insert=True) #%% for ingest tracking if IDs: print('subject started:') print(IDs.keys()) print(IDs.values()) rigpath_tracking_1 = 'E:/Projects/Ablation/datajoint/video/' rigpath_tracking_2 = subject_now VideoDate1 = str(df_wr_row[1].VideoDate) if len(VideoDate1)==5: VideoDate = '0'+ VideoDate1 elif len(VideoDate1)==7: VideoDate = '0'+ VideoDate1 rigpath_tracking_3 = rigpath_tracking_1 + rigpath_tracking_2 + '/' + rigpath_tracking_2 + '_'+ VideoDate + '_front' rigpath_tracking = pathlib.Path(rigpath_tracking_3) #df_surgery = pd.read_csv(dj.config['locations.metadata']+'Surgery.csv') if IDs == None: IDs = {k: v for k, v in zip(*lab.WaterRestriction().fetch('water_restriction_number', 'subject_id'))} h2o = subject_now session = df_wr_row[1].Date trials = (experiment.SessionTrial() & session).fetch('trial') log.info('got session: {} ({} trials)'.format(session, len(trials))) #sdate = session['session_date'] #sdate_sml = date_now #"{}{:02d}{:02d}".format(sdate.year, sdate.month, sdate.day) paths = rigpath_tracking devices = tracking.TrackingDevice().fetch(as_dict=True) # paths like: <root>/<h2o>/YYYY-MM-DD/tracking tracking_files = [] for d in (d for d in devices): tdev = d['tracking_device'] tpos = d['tracking_position'] tdat = paths log.info('checking {} for tracking data'.format(tdat)) # if not tpath.exists(): # log.warning('tracking path {} n/a - skipping'.format(tpath)) # continue # # camtrial = '{}_{}_{}.txt'.format(h2o, sdate_sml, tpos) # campath = tpath / camtrial # # log.info('trying camera position trial map: {}'.format(campath)) # # if not campath.exists(): # log.info('skipping {} - does not exist'.format(campath)) # continue # # tmap = load_campath(campath) # file:trial # n_tmap = len(tmap) # log.info('loading tracking data for {} trials'.format(n_tmap)) i = 0 VideoTrialNum = df_wr_row[1].VideoTrialNum #tpath = pathlib.Path(tdat, h2o, VideoDate, 'tracking') ppp = list(range(0,VideoTrialNum)) for tt in reversed(range(VideoTrialNum)): # load tracking for trial i += 1 # if i % 50 == 0: # log.info('item {}/{}, trial #{} ({:.2f}%)' # .format(i, n_tmap, t, (i/n_tmap)*100)) # else: # log.debug('item {}/{}, trial #{} ({:.2f}%)' # .format(i, n_tmap, t, (i/n_tmap)*100)) # ex: dl59_side_1-0000.csv / h2o_position_tn-0000.csv tfile = '{}_{}_{}_{}-*.csv'.format(h2o, VideoDate ,tpos, tt) tfull = list(tdat.glob(tfile)) if not tfull or len(tfull) > 1: log.info('file mismatch: file: {} trial: ({})'.format( tt, tfull)) continue tfull = tfull[-1] trk = load_tracking(tfull) recs = {} #key_source = experiment.Session - tracking.Tracking rec_base = dict(trial=ppp[tt], tracking_device=tdev) #print(rec_base) for k in trk: if k == 'samples': recs['tracking'] = { 'subject_id' : skey['subject_id'], 'session' : skey['session'], **rec_base, 'tracking_samples': len(trk['samples']['ts']), } else: rec = dict(rec_base) for attr in trk[k]: rec_key = '{}_{}'.format(k, attr) rec[rec_key] = np.array(trk[k][attr]) recs[k] = rec tracking.Tracking.insert1( recs['tracking'], allow_direct_insert=True) #if len(recs['nose']) > 3000: #continue recs['nose'] = { 'subject_id' : skey['subject_id'], 'session' : skey['session'], **recs['nose'], } #print(recs['nose']['nose_x']) if 'nose' in recs: tracking.Tracking.NoseTracking.insert1( recs['nose'], allow_direct_insert=True) recs['tongue_mid'] = { 'subject_id' : skey['subject_id'], 'session' : skey['session'], **recs['tongue_mid'], } if 'tongue_mid' in recs: tracking.Tracking.TongueTracking.insert1( recs['tongue_mid'], allow_direct_insert=True) recs['jaw'] = { 'subject_id' : skey['subject_id'], 'session' : skey['session'], **recs['jaw'], } if 'jaw' in recs: tracking.Tracking.JawTracking.insert1( recs['jaw'], allow_direct_insert=True) recs['tongue_left'] = { 'subject_id' : skey['subject_id'], 'session' : skey['session'], **recs['tongue_left'], } if 'tongue_left' in recs: tracking.Tracking.LeftTongueTracking.insert1( recs['tongue_left'], allow_direct_insert=True) recs['tongue_right'] = { 'subject_id' : skey['subject_id'], 'session' : skey['session'], **recs['tongue_right'], } if 'tongue_right' in recs: tracking.Tracking.RightTongueTracking.insert1( recs['tongue_right'], allow_direct_insert=True) # fmap = {'paw_left_x': 'left_paw_x', # remap field names # 'paw_left_y': 'left_paw_y', # 'paw_left_likelihood': 'left_paw_likelihood'} # tracking.Tracking.LeftPawTracking.insert1({ # **{k: v for k, v in recs['paw_left'].items() # if k not in fmap}, # **{fmap[k]: v for k, v in recs['paw_left'].items() # if k in fmap}}, allow_direct_insert=True) recs['right_lickport'] = { 'subject_id' : skey['subject_id'], 'session' : skey['session'], **recs['right_lickport'], } if 'right_lickport' in recs: tracking.Tracking.RightLickPortTracking.insert1( recs['right_lickport'], allow_direct_insert=True) # fmap = {'paw_right_x': 'right_paw_x', # remap field names # 'paw_right_y': 'right_paw_y', # 'paw_right_likelihood': 'right_paw_likelihood'} # # tracking.Tracking.RightPawTracking.insert1({ # **{k: v for k, v in recs['paw_right'].items() # if k not in fmap}, # **{fmap[k]: v for k, v in recs['paw_right'].items() # if k in fmap}}, allow_direct_insert=True) recs['left_lickport'] = { 'subject_id' : skey['subject_id'], 'session' : skey['session'], **recs['left_lickport'], } if 'left_lickport' in recs: tracking.Tracking.LeftLickPortTracking.insert1( recs['left_lickport'], allow_direct_insert=True) # tracking_files.append({**key, 'trial': tmap[t], 'tracking_device': tdev, # 'tracking_file': str(tfull.relative_to(tdat))}) # # log.info('... completed {}/{} items.'.format(i, n_tmap)) # # self.insert1(key) # self.TrackingFile.insert(tracking_files) # tracking.VideoFiducialsTrial.populate() bottom_tongue.Camera_pixels.populate() print('start!') bottom_tongue.VideoTongueTrial.populate() sessiontrialdata={ 'subject_id':skey['subject_id'], 'session':skey['session'], 'trial': tt } if len(bottom_tongue.VideoTongueTrial* experiment.Session & experiment.BehaviorTrial & 'session_date = "'+str(file['session_date'])+'"' &{'trial':tt})==0: print('trial couldn''t be exported, deleting trial') print(tt) dj.config['safemode'] = False (experiment.SessionTrial()&sessiontrialdata).delete() dj.config['safemode'] = True log.info('... done.')
def make(self, key): ''' TrackingIngest .make() function ''' log.info('TrackingIngest().make(): key: {k}'.format(k=key)) h2o = (lab.WaterRestriction() & key).fetch1('water_restriction_number') session = (experiment.Session() & key).fetch1() trials = (experiment.SessionTrial() & session).fetch('trial') log.info('got session: {} ({} trials)'.format(session, len(trials))) sdate = session['session_date'] sdate_iso = sdate.isoformat() # YYYY-MM-DD sdate_sml = "{}{:02d}{:02d}".format(sdate.year, sdate.month, sdate.day) paths = TrackingDataPath.fetch(as_dict=True) devices = tracking.TrackingDevice().fetch(as_dict=True) # paths like: <root>/<h2o>/YYYY-MM-DD/tracking for p, d in ((p, d) for d in devices for p in paths): tdev = d['tracking_device'] tpos = d['tracking_position'] tdat = p['tracking_data_path'] log.info('checking {} for tracking data'.format(tdat)) tpath = pathlib.Path(tdat, h2o, sdate_iso, 'tracking') if not tpath.exists(): log.warning('tracking path {} n/a - skipping'.format(tpath)) continue camtrial = '{}_{}_{}.txt'.format(h2o, sdate_sml, tpos) campath = tpath / camtrial log.info('trying camera position trial map: {}'.format(campath)) if not campath.exists(): log.info('skipping {} - does not exist'.format(campath)) continue tmap = self.load_campath(campath) n_tmap = len(tmap) log.info('loading tracking data for {} trials'.format(n_tmap)) i = 0 for t in tmap: # load tracking for trial if tmap[t] not in trials: log.warning('nonexistant trial {}.. skipping'.format(t)) continue i += 1 if i % 50 == 0: log.info('item {}/{}, trial #{} ({:.2f}%)'.format( i, n_tmap, t, (i / n_tmap) * 100)) else: log.debug('item {}/{}, trial #{} ({:.2f}%)'.format( i, n_tmap, t, (i / n_tmap) * 100)) # ex: dl59_side_1-0000.csv / h2o_position_tn-0000.csv tfile = '{}_{}_{}-*.csv'.format(h2o, tpos, t) tfull = list(tpath.glob(tfile)) if not tfull or len(tfull) > 1: log.info('tracking file {} mismatch'.format(tfull)) continue tfull = tfull[-1] trk = self.load_tracking(tfull) recs = {} rec_base = dict(key, trial=tmap[t], tracking_device=tdev) for k in trk: if k == 'samples': recs['tracking'] = { **rec_base, 'tracking_samples': len(trk['samples']['ts']), } else: rec = dict(rec_base) for attr in trk[k]: rec_key = '{}_{}'.format(k, attr) rec[rec_key] = np.array(trk[k][attr]) recs[k] = rec tracking.Tracking.insert1(recs['tracking'], allow_direct_insert=True) tracking.Tracking.NoseTracking.insert1( recs['nose'], allow_direct_insert=True) tracking.Tracking.TongueTracking.insert1( recs['tongue'], allow_direct_insert=True) tracking.Tracking.JawTracking.insert1(recs['jaw'], allow_direct_insert=True) log.info('... completed {}/{} items.'.format(i, n_tmap)) log.info('... saving load record') self.insert1(key) log.info('... done.')
def populateelphys(): #%% df_subject_wr_sessions = pd.DataFrame(lab.WaterRestriction() * experiment.Session() * experiment.SessionDetails) df_subject_ids = pd.DataFrame(lab.Subject()) if len(df_subject_wr_sessions) > 0: subject_names = df_subject_wr_sessions[ 'water_restriction_number'].unique() subject_names.sort() else: subject_names = list() subject_ids = df_subject_ids['subject_id'].unique() #% sumdata = list() basedir = Path(dj.config['locations.elphysdata_acq4']) for setup_dir in basedir.iterdir(): setup_name = setup_dir.name sessions = np.sort( os.listdir(setup_dir) ) #configfile.readConfigFile(setup_dir.joinpath('.index')) for session_acq in sessions[::-1]: #.keys(): if session_acq != '.' and session_acq != 'log.txt': session_dir = setup_dir.joinpath(session_acq) try: cells = configfile.readConfigFile( session_dir.joinpath('.index')) except: # if there is no file cells = None if cells and 'WR_name/ID' in cells['.'].keys( ): # it needs to have WRname wrname_ephys = cells['.']['WR_name/ID'] wrname = None for wrname_potential in subject_names: # look for water restriction number if wrname_potential.lower() in wrname_ephys.lower(): wrname = wrname_potential subject_id = (df_subject_wr_sessions.loc[ df_subject_wr_sessions[ 'water_restriction_number'] == wrname, 'subject_id']).unique()[0] if wrname == None: # look for animal identifier: for wrname_potential in subject_ids: # look for water restriction number if str(wrname_potential) in wrname_ephys.lower(): subject_id = wrname_potential if len(df_subject_wr_sessions) > 0 and len( (df_subject_wr_sessions.loc[ df_subject_wr_sessions['subject_id'] == subject_id, 'water_restriction_number'] ).unique()) > 0: wrname = (df_subject_wr_sessions.loc[ df_subject_wr_sessions['subject_id'] == subject_id, 'water_restriction_number'] ).unique()[0] else: wrname = 'no water restriction number for this mouse' if wrname: session_date = ( session_acq[0:session_acq.find('_')]).replace( '.', '-') print('animal: ' + str(subject_id) + ' - ' + wrname) ## if setup_name == 'Voltage_rig_1P': setupname = 'Voltage-Imaging-1p' else: print('unkwnown setup, please add') timer.wait(1000) if 'experimenter' in cells['.'].keys(): username = cells['.']['experimenter'] else: username = '******' print( 'username not specified in acq4 file, assuming rozsam' ) ### check if session already exists sessiondata = { 'subject_id': subject_id, #(lab.WaterRestriction() & 'water_restriction_number = "'+df_behavior_session['subject'][0]+'"').fetch()[0]['subject_id'], 'session': np.nan, 'session_date': session_date, 'session_time': np.nan, #session_time.strftime('%H:%M:%S'), 'username': username, 'rig': setupname } for cell in cells.keys(): if cell != '.' and cell != 'log.txt': ephisdata_cell = list() sweepstarttimes = list() cell_dir = session_dir.joinpath(cell) serieses = configfile.readConfigFile( cell_dir.joinpath('.index')) cellstarttime = datetime.datetime.fromtimestamp( serieses['.']['__timestamp__']) for series in serieses.keys(): if series != '.' and series != 'log.txt': series_dir = cell_dir.joinpath(series) sweeps = configfile.readConfigFile( series_dir.joinpath('.index')) if 'Clamp1.ma' in sweeps.keys(): protocoltype = 'single sweep' sweepkeys = [''] else: protocoltype = 'multiple sweeps' sweepkeys = sweeps.keys() for sweep in sweepkeys: if sweep != '.' and '.txt' not in sweep and '.ma' not in sweep: sweep_dir = series_dir.joinpath( sweep) sweepinfo = configfile.readConfigFile( sweep_dir.joinpath( '.index')) if sweep == '': sweep = '0' for file in sweepinfo.keys(): if '.ma' in file: try: # old file version #print('new file version') #% ephysfile = h5.File( sweep_dir. joinpath(file), "r") data = ephysfile[ 'data'][()] metadata_h5 = ephysfile[ 'info'] metadata = read_h5f_metadata( metadata_h5) daqchannels = list( metadata[2] ['DAQ'].keys()) sweepstarttime = datetime.datetime.fromtimestamp( metadata[2] ['DAQ'] [daqchannels[ 0]] ['startTime']) relativetime = ( sweepstarttime - cellstarttime ).total_seconds() if len( ephisdata_cell ) > 0 and ephisdata_cell[ -1]['sweepstarttime'] == sweepstarttime: ephisdata = ephisdata_cell.pop( ) else: ephisdata = dict( ) if 'primary' in daqchannels: # ephys data ephisdata[ 'V'] = data[ 1] ephisdata[ 'stim'] = data[ 0] ephisdata[ 'data'] = data ephisdata[ 'metadata'] = metadata ephisdata[ 'time'] = metadata[ 1]['values'] ephisdata[ 'relativetime'] = relativetime ephisdata[ 'sweepstarttime'] = sweepstarttime ephisdata[ 'series'] = series ephisdata[ 'sweep'] = sweep sweepstarttimes.append( sweepstarttime ) else: # other daq stuff #% for idx, channel in enumerate( metadata[ 0] ['cols']): channelname = channel[ 'name'].decode( ) if channelname[ 0] == 'u': channelname = channelname[ 2: -1] if channelname in [ 'OrcaFlashExposure', 'Temperature', 'LED525', 'FrameCommand', 'NextFileTrigger' ]: ephisdata[ channelname] = data[ idx] #print('{} added'.format(channelname)) else: print( 'waiting in the other daq' ) timer.sleep( 1000 ) ephisdata_cell.append( ephisdata) #% except: # new file version print( 'old version') ephysfile = MetaArray( ) ephysfile.readFile( sweep_dir. joinpath(file)) data = ephysfile.asarray( ) metadata = ephysfile.infoCopy( ) sweepstarttime = datetime.datetime.fromtimestamp( metadata[2] ['startTime']) relativetime = ( sweepstarttime - cellstarttime ).total_seconds() ephisdata = dict() ephisdata[ 'V'] = data[1] ephisdata[ 'stim'] = data[ 0] ephisdata[ 'data'] = data ephisdata[ 'metadata'] = metadata ephisdata[ 'time'] = metadata[ 1]['values'] ephisdata[ 'relativetime'] = relativetime ephisdata[ 'sweepstarttime'] = sweepstarttime ephisdata[ 'series'] = series ephisdata[ 'sweep'] = sweep sweepstarttimes.append( sweepstarttime) ephisdata_cell.append( ephisdata) # ============================================================================ # if wrname == 'FOR04': # ============================================================================= # add session to DJ if not present if len(ephisdata_cell) > 0: # ============================================================================= # print('waiting') # timer.sleep(1000) # ============================================================================= #% if len(experiment.Session() & 'subject_id = "' + str(sessiondata['subject_id']) + '"' & 'session_date = "' + str(sessiondata['session_date']) + '"') == 0: if len(experiment.Session() & 'subject_id = "' + str(sessiondata['subject_id']) + '"') == 0: sessiondata['session'] = 1 else: sessiondata['session'] = len( (experiment.Session() & 'subject_id = "' + str(sessiondata['subject_id']) + '"').fetch()['session']) + 1 sessiondata['session_time'] = ( sweepstarttimes[0] ).strftime( '%H:%M:%S' ) # the time of the first sweep will be the session time experiment.Session().insert1( sessiondata) #% session = ( experiment.Session() & 'subject_id = "' + str(sessiondata['subject_id']) + '"' & 'session_date = "' + str(sessiondata['session_date']) + '"').fetch('session')[0] cell_number = int(cell[cell.find('_') + 1:]) #add cell if not added already celldata = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, } #% if len(ephys_patch.Cell() & celldata ) == 0 or len(ephys_patch.Cell() * ephys_patch.Sweep() & celldata) < len( ephisdata_cell): if len(ephys_patch.Cell() * ephys_patch.Sweep() & celldata ) < len(ephisdata_cell): print('finishing a recording:') else: print('adding new recording:') print(celldata) if 'type' in serieses['.'].keys(): if serieses['.'][ 'type'] == 'interneuron': celldata['cell_type'] = 'int' elif serieses['.'][ 'type'] == 'unknown' or serieses[ '.']['type'] == 'fail': celldata[ 'cell_type'] = 'unidentified' else: print('unhandled cell type!!') timer.sleep(1000) else: celldata[ 'cell_type'] = 'unidentified' celldata['cell_recording_start'] = ( sweepstarttimes[0] ).strftime('%H:%M:%S') if 'depth' in serieses['.'].keys( ) and len(serieses['.']['depth']) > 0: celldata['depth'] = int( serieses['.']['depth']) else: celldata['depth'] = -1 try: ephys_patch.Cell().insert1( celldata, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded if 'notes' in serieses['.'].keys(): cellnotes = serieses['.']['notes'] else: cellnotes = '' cellnotesdata = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'notes': cellnotes } try: ephys_patch.CellNotes().insert1( cellnotesdata, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded #% for i, ephisdata in enumerate( ephisdata_cell): #% sweep_number = i print('sweep {}'.format( sweep_number)) sweep_data = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'sweep_start_time': (ephisdata['sweepstarttime'] - sweepstarttimes[0] ).total_seconds(), 'sweep_end_time': (ephisdata['sweepstarttime'] - sweepstarttimes[0] ).total_seconds() + ephisdata['time'][-1], 'protocol_name': ephisdata[ 'series'], #[:ephisdata['series'].find('_')], 'protocol_sweep_number': int(ephisdata['sweep']) } if 'mode' in ephisdata['metadata'][ 2]['ClampState']: # old file version recmode = ephisdata[ 'metadata'][2][ 'ClampState']['mode'] else: recmode = ephisdata[ 'metadata'][2]['Protocol'][ 'mode'] if 'IC' in str(recmode): recording_mode = 'current clamp' else: print( 'unhandled recording mode, please act..' ) timer.sleep(10000) channelnames = list() channelunits = list() for line_now in ephisdata[ 'metadata'][0]['cols']: if type(line_now['name'] ) == bytes: channelnames.append( line_now['name']. decode().strip("'")) channelunits.append( line_now['units']. decode().strip("'")) else: channelnames.append( line_now['name']) channelunits.append( line_now['units']) commandidx = np.where( np.array(channelnames) == 'command')[0][0] dataidx = np.where( np.array(channelnames) == 'primary')[0][0] #% clampparams_data = ephisdata[ 'metadata'][2]['ClampState'][ 'ClampParams'].copy() clampparams_data_new = dict() for clampparamkey in clampparams_data.keys( ): #6004 is true for some reason.. changing it back to 1 if type(clampparams_data[ clampparamkey] ) == np.int32: if clampparams_data[ clampparamkey] > 0: clampparams_data[ clampparamkey] = int( 1) else: clampparams_data[ clampparamkey] = int( 0) else: clampparams_data[ clampparamkey] = float( clampparams_data[ clampparamkey]) clampparams_data_new[ clampparamkey.lower( )] = clampparams_data[ clampparamkey] #% sweepmetadata_data = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'recording_mode': recording_mode, 'sample_rate': np.round(1 / np.median( np.diff( ephisdata['metadata'] [1]['values']))) } sweepmetadata_data.update( clampparams_data_new) sweepdata_data = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'response_trace': ephisdata['data'][dataidx, :], 'response_units': ephisdata['metadata'][0] ['cols'][dataidx]['units'] } sweepstimulus_data = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'stimulus_trace': ephisdata['data'][ commandidx, :], 'stimulus_units': ephisdata['metadata'][0] ['cols'][commandidx]['units'] } #print('waiting') #timer.sleep(10000) try: ephys_patch.Sweep().insert1( sweep_data, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded try: # maybe it's a duplicate.. ephys_patch.ClampParams( ).insert1( clampparams_data_new, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded try: ephys_patch.SweepMetadata( ).insert1( sweepmetadata_data, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded try: ephys_patch.SweepResponse( ).insert1( sweepdata_data, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded try: ephys_patch.SweepStimulus( ).insert1( sweepstimulus_data, allow_direct_insert=True) except dj.errors.DuplicateError: pass #already uploaded #% if 'OrcaFlashExposure' in ephisdata.keys( ): sweepimagingexposuredata = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'imaging_exposure_trace': ephisdata[ 'OrcaFlashExposure'] } try: ephys_patch.SweepImagingExposure( ).insert1( sweepimagingexposuredata, allow_direct_insert=True ) except dj.errors.DuplicateError: pass #already uploaded if 'Temperature' in ephisdata.keys( ): sweeptemperaturedata = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'temperature_trace': ephisdata['Temperature'] * 10, 'temperature_units': 'degC' } try: ephys_patch.SweepTemperature( ).insert1( sweeptemperaturedata, allow_direct_insert=True ) except dj.errors.DuplicateError: pass #already uploaded if 'LED525' in ephisdata.keys(): sweepLEDdata = { 'subject_id': subject_id, 'session': session, 'cell_number': cell_number, 'sweep_number': sweep_number, 'imaging_led_trace': ephisdata['LED525'] } try: ephys_patch.SweepLED( ).insert1( sweepLEDdata, allow_direct_insert=True ) except dj.errors.DuplicateError: pass #already uploaded
def model_and_populate_a_session(subject_now, subject_id, session): print('session ' + str(session)) p_reward_L, p_reward_R, n_trials = foraging_model.generate_block_structure( n_trials_base=80, n_trials_sd=10, blocknum=8, reward_ratio_pairs=np.array([[.4, .05], [.3857, .0643], [.3375, .1125], [.225, .225]])) if subject_now == 'leaky3t5it30h': rewards, choices = foraging_model.run_task( p_reward_L, p_reward_R, n_trials, unchosen_rewards_to_keep=1, subject='clever', min_rewardnum=30, filter_tau_fast=3, filter_tau_slow=100, filter_tau_slow_amplitude=00.00, softmax_temperature=5, plot=False) elif subject_now == 'leaky3t3it30h': rewards, choices = foraging_model.run_task(p_reward_L, p_reward_R, n_trials, unchosen_rewards_to_keep=1, subject='clever', min_rewardnum=30, filter_tau_fast=3, filter_tau_slow=100, filter_tau_slow_amplitude=0, softmax_temperature=3, plot=False) elif subject_now == 'W-St-L-Sw': rewards, choices = foraging_model.run_task( p_reward_L, p_reward_R, n_trials, unchosen_rewards_to_keep=1, subject='win_stay-loose_switch', min_rewardnum=3, filter_tau_fast=3, filter_tau_slow=100, filter_tau_slow_amplitude=00.01, plot=False) elif subject_now == 'W-St-L-Rnd': rewards, choices = foraging_model.run_task( p_reward_L, p_reward_R, n_trials, unchosen_rewards_to_keep=1, subject='win_stay-loose_random', min_rewardnum=3, filter_tau_fast=3, filter_tau_slow=100, filter_tau_slow_amplitude=00.01, filter_constant=.05, plot=False) elif subject_now == 'leaky3t.05c15h': rewards, choices = foraging_model.run_task( p_reward_L, p_reward_R, n_trials, unchosen_rewards_to_keep=1, subject='clever', min_rewardnum=15, filter_tau_fast=3, filter_tau_slow=100, filter_tau_slow_amplitude=00.0, filter_constant=.05, plot=False) elif subject_now == 'leaky3t.05c5h': rewards, choices = foraging_model.run_task( p_reward_L, p_reward_R, n_trials, unchosen_rewards_to_keep=1, subject='clever', min_rewardnum=5, filter_tau_fast=3, filter_tau_slow=100, filter_tau_slow_amplitude=00.0, filter_constant=.05, plot=False) elif subject_now == 'leaky3t.05c30h': rewards, choices = foraging_model.run_task( p_reward_L, p_reward_R, n_trials, unchosen_rewards_to_keep=1, subject='clever', min_rewardnum=30, filter_tau_fast=3, filter_tau_slow=100, filter_tau_slow_amplitude=00.0, filter_constant=.05, plot=False) elif subject_now == 'cheater': rewards, choices = foraging_model.run_task( p_reward_L, p_reward_R, n_trials, unchosen_rewards_to_keep=1, subject='perfect', min_rewardnum=30, filter_tau_fast=3, filter_tau_slow=100, filter_tau_slow_amplitude=00.0, filter_constant=.05, plot=False) else: print('unknown model') sessiondata = { 'subject_id': subject_id, 'session': session, 'session_date': datetime.now().strftime('%Y-%m-%d'), 'session_time': datetime.now().strftime('%H:%M:%S'), 'username': experimenter, 'rig': setupname } experiment.Session().insert1(sessiondata) trialssofar = 0 columns = [ 'subject_id', 'session', 'block', 'block_uid', 'block_start_time', 'p_reward_left', 'p_reward_right' ] df_sessionblockdata = pd.DataFrame(data=np.zeros( (len(p_reward_L), len(columns))), columns=columns) for blocknum, (p_L, p_R, trialnum) in enumerate( zip(p_reward_L, p_reward_R, n_trials), 1): df_sessionblockdata.loc[blocknum - 1, 'subject_id'] = subject_id df_sessionblockdata.loc[blocknum - 1, 'session'] = session df_sessionblockdata.loc[blocknum - 1, 'block'] = blocknum df_sessionblockdata.loc[blocknum - 1, 'block_uid'] = blocknum df_sessionblockdata.loc[blocknum - 1, 'block_start_time'] = trialssofar df_sessionblockdata.loc[blocknum - 1, 'p_reward_left'] = p_L df_sessionblockdata.loc[blocknum - 1, 'p_reward_right'] = p_R trialssofar += trialnum experiment.SessionBlock().insert( df_sessionblockdata.to_records(index=False), allow_direct_insert=True) columns_sessiontrial = [ 'subject_id', 'session', 'trial', 'trial_uid', 'trial_start_time', 'trial_stop_time' ] df_sessiontrialdata = pd.DataFrame(data=np.zeros( (len(rewards), len(columns_sessiontrial))), columns=columns_sessiontrial) columns_behaviortrial = [ 'subject_id', 'session', 'trial', 'task', 'task_protocol', 'trial_choice', 'early_lick', 'outcome', 'block' ] df_behaviortrialdata = pd.DataFrame(data=np.zeros( (len(rewards), len(columns_behaviortrial))), columns=columns_behaviortrial) for trialnum, (reward, choice) in enumerate(zip(rewards, choices), 1): df_sessiontrialdata.loc[trialnum - 1, 'subject_id'] = subject_id df_sessiontrialdata.loc[trialnum - 1, 'session'] = session df_sessiontrialdata.loc[trialnum - 1, 'trial'] = trialnum df_sessiontrialdata.loc[trialnum - 1, 'trial_uid'] = trialnum df_sessiontrialdata.loc[trialnum - 1, 'trial_start_time'] = trialnum - .9 df_sessiontrialdata.loc[trialnum - 1, 'trial_stop_time'] = trialnum - .1 #% outcome if reward: outcome = 'hit' else: outcome = 'miss' if choice == 1: trial_choice = 'right' else: trial_choice = 'left' task = 'foraging' task_protocol = 10 df_behaviortrialdata.loc[trialnum - 1, 'subject_id'] = subject_id df_behaviortrialdata.loc[trialnum - 1, 'session'] = session df_behaviortrialdata.loc[trialnum - 1, 'trial'] = trialnum df_behaviortrialdata.loc[trialnum - 1, 'task'] = task df_behaviortrialdata.loc[trialnum - 1, 'task_protocol'] = task_protocol df_behaviortrialdata.loc[trialnum - 1, 'trial_choice'] = trial_choice df_behaviortrialdata.loc[trialnum - 1, 'early_lick'] = 'no early' df_behaviortrialdata.loc[trialnum - 1, 'outcome'] = outcome df_behaviortrialdata.loc[trialnum - 1, 'block'] = np.argmax( np.cumsum(n_trials) >= trialnum) + 1 experiment.SessionTrial().insert( df_sessiontrialdata.to_records(index=False), allow_direct_insert=True) experiment.BehaviorTrial().insert( df_behaviortrialdata.to_records(index=False), allow_direct_insert=True)
def make(self, key): ''' TrackingIngest .make() function ''' log.info('TrackingIngest().make(): key: {k}'.format(k=key)) h2o = (lab.WaterRestriction() & key).fetch1('water_restriction_number') session = (experiment.Session() & key).fetch1() trials = (experiment.SessionTrial() & session).fetch('trial') log.info('got session: {} ({} trials)'.format(session, len(trials))) sdate = session['session_date'] sdate_sml = "{}{:02d}{:02d}".format(sdate.year, sdate.month, sdate.day) paths = get_tracking_paths() devices = tracking.TrackingDevice().fetch(as_dict=True) # paths like: <root>/<h2o>/YYYY-MM-DD/tracking tracking_files = [] for p, d in ((p, d) for d in devices for p in paths): tdev = d['tracking_device'] tpos = d['tracking_position'] tdat = p[-1] log.info('checking {} for tracking data'.format(tdat)) tpath = pathlib.Path(tdat, h2o, sdate.strftime('%Y%m%d'), 'tracking') if not tpath.exists(): log.warning('tracking path {} n/a - skipping'.format(tpath)) continue camtrial = '{}_{}_{}.txt'.format(h2o, sdate_sml, tpos) campath = tpath / camtrial log.info('trying camera position trial map: {}'.format(campath)) if not campath.exists(): log.info('skipping {} - does not exist'.format(campath)) continue tmap = self.load_campath(campath) # file:trial n_tmap = len(tmap) log.info('loading tracking data for {} trials'.format(n_tmap)) i = 0 for t in tmap: # load tracking for trial if tmap[t] not in trials: log.warning('nonexistant trial {}.. skipping'.format(t)) continue i += 1 if i % 50 == 0: log.info('item {}/{}, trial #{} ({:.2f}%)'.format( i, n_tmap, t, (i / n_tmap) * 100)) else: log.debug('item {}/{}, trial #{} ({:.2f}%)'.format( i, n_tmap, t, (i / n_tmap) * 100)) # ex: dl59_side_1-0000.csv / h2o_position_tn-0000.csv tfile = '{}_{}_{}-*.csv'.format(h2o, tpos, t) tfull = list(tpath.glob(tfile)) if not tfull or len(tfull) > 1: log.info('file mismatch: file: {} trial: {} ({})'.format( t, tmap[t], tfull)) continue tfull = tfull[-1] trk = self.load_tracking(tfull) recs = {} rec_base = dict(key, trial=tmap[t], tracking_device=tdev) for k in trk: if k == 'samples': recs['tracking'] = { **rec_base, 'tracking_samples': len(trk['samples']['ts']), } else: rec = dict(rec_base) for attr in trk[k]: rec_key = '{}_{}'.format(k, attr) rec[rec_key] = np.array(trk[k][attr]) recs[k] = rec tracking.Tracking.insert1(recs['tracking'], allow_direct_insert=True) if 'nose' in recs: tracking.Tracking.NoseTracking.insert1( recs['nose'], allow_direct_insert=True) if 'tongue' in recs: tracking.Tracking.TongueTracking.insert1( recs['tongue'], allow_direct_insert=True) if 'jaw' in recs: tracking.Tracking.JawTracking.insert1( recs['jaw'], allow_direct_insert=True) if 'paw_left' in recs: fmap = { 'paw_left_x': 'left_paw_x', # remap field names 'paw_left_y': 'left_paw_y', 'paw_left_likelihood': 'left_paw_likelihood' } tracking.Tracking.LeftPawTracking.insert1( { **{ k: v for k, v in recs['paw_left'].items() if k not in fmap }, **{ fmap[k]: v for k, v in recs['paw_left'].items() if k in fmap } }, allow_direct_insert=True) if 'paw_right' in recs: fmap = { 'paw_right_x': 'right_paw_x', # remap field names 'paw_right_y': 'right_paw_y', 'paw_right_likelihood': 'right_paw_likelihood' } tracking.Tracking.RightPawTracking.insert1( { **{ k: v for k, v in recs['paw_right'].items() if k not in fmap }, **{ fmap[k]: v for k, v in recs['paw_right'].items() if k in fmap } }, allow_direct_insert=True) tracking_files.append({ **key, 'trial': tmap[t], 'tracking_device': tdev, 'tracking_file': str(tfull.relative_to(tdat)) }) log.info('... completed {}/{} items.'.format(i, n_tmap)) self.insert1(key) self.TrackingFile.insert(tracking_files) log.info('... done.')
key ={'subject_id':466771} key['movie_number'] = 1 #3 key ={'subject_id':456462} key['movie_number'] = 3 #3 key = (imaging_gt.CellMovieCorrespondance()&key).fetch(as_dict = True)[0] key['roi_type'] = "VolPy" sweep_numbers = key['sweep_numbers'] del key['sweep_numbers'] #% motion_corr_vectors = np.asarray((imaging.MotionCorrection()*imaging.RegisteredMovie()&key&'motion_correction_method = "VolPy"'&'motion_corr_description= "rigid motion correction done with VolPy"').fetch1('motion_corr_vectors')) #% ophys related stuff session_time, cell_recording_start = (experiment.Session()*ephys_patch.Cell()&key).fetch1('session_time','cell_recording_start') first_movie_start_time = np.min(np.asarray(((imaging.Movie()*imaging_gt.GroundTruthROI())&key).fetch('movie_start_time'),float)) first_movie_start_time_real = first_movie_start_time + session_time.total_seconds() frame_times = (imaging.MovieFrameTimes()&key).fetch1('frame_times') - cell_recording_start.total_seconds() + session_time.total_seconds() roi_dff,roi_f0,roi_spike_indices,framerate = (imaging.Movie()*imaging.ROI*imaging_gt.GroundTruthROI()&key).fetch1('roi_dff','roi_f0','roi_spike_indices','movie_frame_rate') roi_spike_indices = roi_spike_indices-1 roi_f = (roi_dff*roi_f0)+roi_f0 xvals = frame_times-frame_times[0] yvals = roi_f out = scipy.optimize.curve_fit(lambda t,a,b,c,d,e: a*np.exp(-t/b) + c + d*np.exp(-t/e), xvals, yvals,bounds=((0,0,-np.inf,0,0),(np.inf,np.inf,np.inf,np.inf,np.inf))) f0_fit_f = out[0][0]*np.exp(-xvals/out[0][1])+out[0][2] +out[0][3]*np.exp(-xvals/out[0][4]) dff_fit = (roi_f-f0_fit_f)/f0_fit_f
def make(self, key): ''' determine available files from local endpoint and publish (create database records and transfer to globus) ''' # >>> list(key.keys()) # ['subject_id', 'session', 'trial', 'electrode_group', 'globus_alias'] log.debug(key) lep, lep_sub, lep_dir = GlobusStorageLocation().local_endpoint log.info('local_endpoint: {}:{} -> {}'.format(lep, lep_sub, lep_dir)) # get session related information needed for filenames/records sinfo = ((lab.WaterRestriction * lab.Subject.proj() * experiment.Session() * experiment.SessionTrial) & key).fetch1() h2o = sinfo['water_restriction_number'] sdate = sinfo['session_date'] eg = key['electrode_group'] trial = key['trial'] # build file locations: # subdir - common subdirectory for globus/native filesystem # fpat: base file pattern for this sessions files # fbase: filesystem base path for this sessions files # gbase: globus-url base path for this sessions files subdir = os.path.join(h2o, str(sdate), str(eg)) fpat = '{}_{}_{}_g0_t{}'.format(h2o, sdate, eg, trial) fbase = os.path.join(lep_dir, subdir, fpat) gbase = '/'.join((h2o, str(sdate), str(eg), fpat)) # check for existence of actual files & use to build xfer list log.debug('checking {}'.format(fbase)) ffound = [] ftypes = RawEphysFileTypes.contents for ft in ftypes: fname = '{}{}'.format(fbase, ft[1]) gname = '{}{}'.format(gbase, ft[1]) if not os.path.exists(fname): log.debug('... {}: not found'.format(fname)) continue log.debug('... {}: found'.format(fname)) ffound.append((ft, gname,)) # if files are found, transfer and create publication schema records if not len(ffound): log.info('no files found for key') return log.info('found files for key: {}'.format([f[1] for f in ffound])) repname, rep, rep_sub = (GlobusStorageLocation() & key).fetch()[0] gsm = self.get_gsm() gsm.activate_endpoint(lep) # XXX: cache this / prevent duplicate RPC? gsm.activate_endpoint(rep) # XXX: cache this / prevent duplicate RPC? if not ArchivedRawEphysTrial & key: log.info('ArchivedRawEphysTrial.insert1()') ArchivedRawEphysTrial.insert1(key) ftmap = {'ap-30kHz': ArchivedRawEphysTrial.ArchivedApChannel, 'ap-30kHz-meta': ArchivedRawEphysTrial.ArchivedApMeta, 'lf-2.5kHz': ArchivedRawEphysTrial.ArchivedLfChannel, 'lf-2.5kHz-meta': ArchivedRawEphysTrial.ArchivedLfMeta} for ft, gname in ffound: # XXX: transfer/insert could be batched ft_class = ftmap[ft[0]] if not ft_class & key: srcp = '{}:/{}/{}'.format(lep, lep_sub, gname) dstp = '{}:/{}/{}'.format(rep, rep_sub, gname) log.info('transferring {} to {}'.format(srcp, dstp)) # XXX: check if exists 1st? (manually or via API copy-checksum) if not gsm.cp(srcp, dstp): emsg = "couldn't transfer {} to {}".format(srcp, dstp) log.error(emsg) raise dj.DataJointError(emsg) log.info('ArchivedRawEphysTrial.{}.insert1()' .format(ft_class.__name__)) ft_class.insert1(key)