def plot_barcode_interval_hist(probe_dirs, syncDataset, FIG_SAVE_DIR, prefix=''): for ip, probe in enumerate(probe_dirs): p_name = probe.split('_')[-2][-1] base = os.path.join(probe, r'events\\Neuropix-PXI-100.0\\TTL_1') channel_states_file = glob_file(base, 'channel_states.npy') event_times_file = glob_file(base, 'event_timestamps.npy') if channel_states_file and event_times_file: #get barcode intervals from probe events file channel_states = np.load(channel_states_file) event_times = np.load(event_times_file) beRising = event_times[channel_states > 0] / 30000. beFalling = event_times[channel_states < 0] / 30000. be_t, be = ecephys.extract_barcodes_from_times(beRising, beFalling) barcode_intervals = np.diff(be_t) #get intervals from sync file for comparison bRising, bFalling = get_sync_line_data(syncDataset, channel=0) bs_t, bs = ecephys.extract_barcodes_from_times(bRising, bFalling) sync_barcode_intervals = np.diff(bs_t) fig, ax = plt.subplots(1, 2) fig.set_size_inches([8, 4]) bins = np.arange( np.min([barcode_intervals.min(), sync_barcode_intervals.min()]) - 1, np.max([barcode_intervals.max(), sync_barcode_intervals.max()]) + 1) ax[0].hist(barcode_intervals, bins) ax[1].hist(sync_barcode_intervals, bins) ax[0].axhline(len(be_t) - 1) ax[1].axhline(len(bs_t) - 1) ax[0].set_ylim([0.1, len(be_t)]) ax[1].set_ylim([0.1, len(bs_t)]) [a.set_yscale('log') for a in ax] ax[0].set_title('Probe {} ephys barcode intervals'.format(p_name)) ax[1].set_title('Probe {} sync barcode intervals'.format(p_name)) save_figure( fig, os.path.join( FIG_SAVE_DIR, prefix + 'Probe_{}_barcode_interval_hist.png'.format(p_name)))
def _load_sync_data(self): self.syncDataset = sync_dataset(self.SYNC_FILE) try: vr, self.vf = probeSync.get_sync_line_data(self.syncDataset, channel=2) MONITOR_LAG = analysis.get_monitor_lag(self.syncDataset) if MONITOR_LAG > 0.06: self.errors.append( ('vsync', 'abnormal monitor lag {}, using default {}'.format( MONITOR_LAG, 0.036))) MONITOR_LAG = 0.036 #self.FRAME_APPEAR_TIMES = self.vf + MONITOR_LAG self.FRAME_APPEAR_TIMES = probeSync.get_experiment_frame_times( self.syncDataset) #trying new vsync method self.MONITOR_LAG = MONITOR_LAG self.vsync_times = np.copy(self.vf) except: print('error getting vsync times') self.data_stream_status['sync'][0] = True
MAPPING_PKL = paths['mapping_pkl'] try: syncDataset = sync_dataset(SYNC_FILE) except Exception as e: logging.error('Error reading sync file: {}'.format(e)) try: mapping_data = pd.read_pickle(MAPPING_PKL) except Exception as e: logging.error('Error reading mapping pkl file: {}'.format(e)) # replay_data = pd.read_pickle(REPLAY_PKL) ### PLOT FRAME INTERVALS ### vr, vf = probeSync.get_sync_line_data(syncDataset, channel=2) # behavior_frame_count = behavior_data['items']['behavior']['intervalsms'].size + 1 mapping_frame_count = mapping_data['intervalsms'].size + 1 # replay_frame_count = replay_data['intervalsms'].size + 1 MONITOR_LAG = 0.036 FRAME_APPEAR_TIMES = vf + MONITOR_LAG ### CHECK THAT NO FRAMES WERE DROPPED FROM SYNC ### # total_pkl_frames = (behavior_frame_count + # mapping_frame_count + # replay_frame_count) # print('frames in pkl files: {}'.format(total_pkl_frames)) # print('frames in sync file: {}'.format(len(vf)))
def run_qc(exp_id, save_root): identifier = exp_id if identifier.find('_') >= 0: d = data_getters.local_data_getter(base_dir=identifier) else: d = data_getters.lims_data_getter(exp_id=identifier) paths = d.data_dict FIG_SAVE_DIR = os.path.join( save_root, paths['es_id'] + '_' + paths['external_specimen_name'] + '_' + paths['datestring']) if not os.path.exists(FIG_SAVE_DIR): os.mkdir(FIG_SAVE_DIR) figure_prefix = paths['external_specimen_name'] + '_' + paths[ 'datestring'] + '_' ### GET FILE PATHS TO SYNC AND PKL FILES ### SYNC_FILE = paths['sync_file'] BEHAVIOR_PKL = paths['behavior_pkl'] REPLAY_PKL = paths['replay_pkl'] MAPPING_PKL = paths['mapping_pkl'] for f, s in zip([SYNC_FILE, BEHAVIOR_PKL, REPLAY_PKL, MAPPING_PKL], ['sync: ', 'behavior: ', 'replay: ', 'mapping: ']): print(s + f) ### GET MAIN DATA STREAMS ### syncDataset = sync_dataset(SYNC_FILE) behavior_data = pd.read_pickle(BEHAVIOR_PKL) mapping_data = pd.read_pickle(MAPPING_PKL) replay_data = pd.read_pickle(REPLAY_PKL) ### Behavior Analysis ### behavior_plot_dir = os.path.join(FIG_SAVE_DIR, 'behavior') trials = behavior_analysis.get_trials_df(behavior_data) behavior_analysis.plot_behavior(trials, behavior_plot_dir, prefix=figure_prefix) trial_types, counts = behavior_analysis.get_trial_counts(trials) behavior_analysis.plot_trial_type_pie(counts, trial_types, behavior_plot_dir, prefix=figure_prefix) ### CHECK FRAME COUNTS ### vr, vf = probeSync.get_sync_line_data(syncDataset, channel=2) behavior_frame_count = behavior_data['items']['behavior'][ 'intervalsms'].size + 1 mapping_frame_count = mapping_data['intervalsms'].size + 1 replay_frame_count = replay_data['intervalsms'].size + 1 total_pkl_frames = (behavior_frame_count + mapping_frame_count + replay_frame_count) ### CHECK THAT NO FRAMES WERE DROPPED FROM SYNC ### print('frames in pkl files: {}'.format(total_pkl_frames)) print('frames in sync file: {}'.format(len(vf))) #assert(total_pkl_frames==len(vf)) ### CHECK THAT REPLAY AND BEHAVIOR HAVE SAME FRAME COUNT ### print('frames in behavior stim: {}'.format(behavior_frame_count)) print('frames in replay stim: {}'.format(replay_frame_count)) #assert(behavior_frame_count==replay_frame_count) # look for potential frame offsets from aborted stims (behavior_start_frame, mapping_start_frame, replay_start_frame) = probeSync.get_frame_offsets( syncDataset, [behavior_frame_count, mapping_frame_count, replay_frame_count]) behavior_end_frame = behavior_start_frame + behavior_frame_count - 1 mapping_end_frame = mapping_start_frame + mapping_frame_count - 1 replay_end_frame = replay_start_frame + replay_frame_count - 1 MONITOR_LAG = 0.036 #TO DO: don't hardcode this... FRAME_APPEAR_TIMES = vf + MONITOR_LAG behavior_start_time, mapping_start_time, replay_start_time = [ FRAME_APPEAR_TIMES[f] for f in [behavior_start_frame, mapping_start_frame, replay_start_frame] ] behavior_end_time, mapping_end_time, replay_end_time = [ FRAME_APPEAR_TIMES[f] for f in [behavior_end_frame, mapping_end_frame, replay_end_frame] ] ### Plot vsync info ### vsync_save_dir = os.path.join(FIG_SAVE_DIR, 'vsyncs') analysis.plot_frame_intervals(vf, behavior_frame_count, mapping_frame_count, behavior_start_frame, mapping_start_frame, replay_start_frame, vsync_save_dir, prefix=figure_prefix) analysis.plot_vsync_interval_histogram(vf, vsync_save_dir, prefix=figure_prefix) analysis.vsync_report(vf, total_pkl_frames, vsync_save_dir, prefix=figure_prefix) ### BUILD UNIT TABLE #### probe_dict = probeSync.build_unit_table(paths['data_probes'], paths, syncDataset) ### Plot Probe Yield QC ### probe_yield_dir = os.path.join(FIG_SAVE_DIR, 'probe_yield') probe_dirs = [paths['probe' + pid] for pid in paths['data_probes']] analysis.plot_unit_quality_hist(probe_dict, probe_yield_dir, prefix=figure_prefix) analysis.plot_unit_distribution_along_probe(probe_dict, probe_yield_dir, prefix=figure_prefix) analysis.plot_all_spike_hist(probe_dict, probe_yield_dir, prefix=figure_prefix + 'good') analysis.copy_probe_depth_images(paths, probe_yield_dir, prefix=figure_prefix) ### Unit Metrics ### unit_metrics_dir = os.path.join(FIG_SAVE_DIR, 'unit_metrics') analysis.plot_unit_metrics(paths, unit_metrics_dir, prefix=figure_prefix) ### Probe/Sync alignment probeSyncDir = os.path.join(FIG_SAVE_DIR, 'probeSyncAlignment') analysis.plot_barcode_interval_hist(probe_dirs, syncDataset, probeSyncDir, prefix=figure_prefix) analysis.plot_barcode_intervals(probe_dirs, syncDataset, probeSyncDir, prefix=figure_prefix) analysis.probe_sync_report(probe_dirs, syncDataset, probeSyncDir, prefix=figure_prefix) analysis.plot_barcode_matches(probe_dirs, syncDataset, probeSyncDir, prefix=figure_prefix) ### Plot visual responses get_RFs(probe_dict, mapping_data, mapping_start_frame, FRAME_APPEAR_TIMES, os.path.join(FIG_SAVE_DIR, 'receptive_fields'), prefix=figure_prefix) analysis.plot_population_change_response(probe_dict, behavior_frame_count, mapping_frame_count, trials, FRAME_APPEAR_TIMES, os.path.join( FIG_SAVE_DIR, 'change_response'), ctx_units_percentile=66, prefix=figure_prefix) ### Plot running ### analysis.plot_running_wheel(behavior_data, mapping_data, replay_data, behavior_plot_dir, prefix=figure_prefix) ### LFP ### lfp_save_dir = os.path.join(FIG_SAVE_DIR, 'LFP') lick_times = analysis.get_rewarded_lick_times( probeSync.get_lick_times(syncDataset), FRAME_APPEAR_TIMES, trials, min_inter_lick_time=0.5) lfp_dict = probeSync.build_lfp_dict(probe_dirs, syncDataset) analysis.plot_lick_triggered_LFP(lfp_dict, lick_times, lfp_save_dir, prefix=figure_prefix, agarChRange=None, num_licks=20, windowBefore=0.5, windowAfter=1.5, min_inter_lick_time=0.5, behavior_duration=3600) ### VIDEOS ### video_dir = os.path.join(FIG_SAVE_DIR, 'videos') analysis.lost_camera_frame_report(paths, video_dir, prefix=figure_prefix) analysis.camera_frame_grabs( paths, syncDataset, video_dir, [behavior_start_time, mapping_start_time, replay_start_time], [behavior_end_time, mapping_end_time, replay_end_time], epoch_frame_nums=[2, 2, 2], prefix=figure_prefix)
def __init__(self, exp_id, save_root, modules_to_run='all'): self.modules_to_run = modules_to_run identifier = exp_id if identifier.find('_')>=0: d = data_getters.local_data_getter(base_dir=identifier) else: d = data_getters.lims_data_getter(exp_id=identifier) self.paths = d.data_dict self.FIG_SAVE_DIR = os.path.join(save_root, self.paths['es_id']+'_'+ self.paths['external_specimen_name']+'_'+ self.paths['datestring']) if not os.path.exists(self.FIG_SAVE_DIR): os.mkdir(self.FIG_SAVE_DIR) self.figure_prefix = self.paths['external_specimen_name'] + '_' + self.paths['datestring'] + '_' ### GET FILE PATHS TO SYNC AND PKL FILES ### self.SYNC_FILE = self.paths['sync_file'] self.BEHAVIOR_PKL = self.paths['behavior_pkl'] self.REPLAY_PKL = self.paths['replay_pkl'] self.MAPPING_PKL = self.paths['mapping_pkl'] self.OPTO_PKL = self.paths['opto_pkl'] for f,s in zip([self.SYNC_FILE, self.BEHAVIOR_PKL, self.REPLAY_PKL, self.MAPPING_PKL], ['sync: ', 'behavior: ', 'replay: ', 'mapping: ']): print(s + f) ### GET MAIN DATA STREAMS ### self.syncDataset = sync_dataset(self.SYNC_FILE) self.behavior_data = pd.read_pickle(self.BEHAVIOR_PKL) self.mapping_data = pd.read_pickle(self.MAPPING_PKL) self.replay_data = pd.read_pickle(self.REPLAY_PKL) self.opto_data = pd.read_pickle(self.OPTO_PKL) self.trials = behavior_analysis.get_trials_df(self.behavior_data) ### CHECK FRAME COUNTS ### vr, self.vf = probeSync.get_sync_line_data(self.syncDataset, channel=2) self.behavior_frame_count = self.behavior_data['items']['behavior']['intervalsms'].size + 1 self.mapping_frame_count = self.mapping_data['intervalsms'].size + 1 self.replay_frame_count = self.replay_data['intervalsms'].size + 1 self.total_pkl_frames = (self.behavior_frame_count + self.mapping_frame_count + self.replay_frame_count) ### CHECK THAT NO FRAMES WERE DROPPED FROM SYNC ### print('frames in pkl files: {}'.format(self.total_pkl_frames)) print('frames in sync file: {}'.format(len(self.vf))) #assert(total_pkl_frames==len(vf)) ### CHECK THAT REPLAY AND BEHAVIOR HAVE SAME FRAME COUNT ### print('frames in behavior stim: {}'.format(self.behavior_frame_count)) print('frames in replay stim: {}'.format(self.replay_frame_count)) #assert(behavior_frame_count==replay_frame_count) # look for potential frame offsets from aborted stims (self.behavior_start_frame, self.mapping_start_frame, self.replay_start_frame) = probeSync.get_frame_offsets( self.syncDataset, [self.behavior_frame_count, self.mapping_frame_count, self.replay_frame_count]) self.behavior_end_frame = self.behavior_start_frame + self.behavior_frame_count - 1 self.mapping_end_frame = self.mapping_start_frame + self.mapping_frame_count - 1 self.replay_end_frame = self.replay_start_frame + self.replay_frame_count - 1 MONITOR_LAG = 0.036 #TO DO: don't hardcode this... self.FRAME_APPEAR_TIMES = self.vf + MONITOR_LAG self.behavior_start_time, self.mapping_start_time, self.replay_start_time = [self.FRAME_APPEAR_TIMES[f] for f in [self.behavior_start_frame, self.mapping_start_frame, self.replay_start_frame]] self.behavior_end_time, self.mapping_end_time, self.replay_end_time = [self.FRAME_APPEAR_TIMES[f] for f in [self.behavior_end_frame, self.mapping_end_frame, self.replay_end_frame]] self.probe_dirs = [self.paths['probe'+pid] for pid in self.paths['data_probes']] self.probe_dict = None self.lfp_dict = None self._run_modules()