def plot_barcode_interval_hist(probe_dirs,
syncDataset,
FIG_SAVE_DIR,
prefix=''):
for ip, probe in enumerate(probe_dirs):
p_name = probe.split('_')[-2][-1]
base = os.path.join(probe, r'events\\Neuropix-PXI-100.0\\TTL_1')
channel_states_file = glob_file(base, 'channel_states.npy')
event_times_file = glob_file(base, 'event_timestamps.npy')
if channel_states_file and event_times_file:
#get barcode intervals from probe events file
channel_states = np.load(channel_states_file)
event_times = np.load(event_times_file)
beRising = event_times[channel_states > 0] / 30000.
beFalling = event_times[channel_states < 0] / 30000.
be_t, be = ecephys.extract_barcodes_from_times(beRising, beFalling)
barcode_intervals = np.diff(be_t)
#get intervals from sync file for comparison
bRising, bFalling = get_sync_line_data(syncDataset, channel=0)
bs_t, bs = ecephys.extract_barcodes_from_times(bRising, bFalling)
sync_barcode_intervals = np.diff(bs_t)
fig, ax = plt.subplots(1, 2)
fig.set_size_inches([8, 4])
bins = np.arange(
np.min([barcode_intervals.min(),
sync_barcode_intervals.min()]) - 1,
np.max([barcode_intervals.max(),
sync_barcode_intervals.max()]) + 1)
ax[0].hist(barcode_intervals, bins)
ax[1].hist(sync_barcode_intervals, bins)
ax[0].axhline(len(be_t) - 1)
ax[1].axhline(len(bs_t) - 1)
ax[0].set_ylim([0.1, len(be_t)])
ax[1].set_ylim([0.1, len(bs_t)])
[a.set_yscale('log') for a in ax]
ax[0].set_title('Probe {} ephys barcode intervals'.format(p_name))
ax[1].set_title('Probe {} sync barcode intervals'.format(p_name))
save_figure(
fig,
os.path.join(
FIG_SAVE_DIR, prefix +
'Probe_{}_barcode_interval_hist.png'.format(p_name)))
def _load_sync_data(self):
self.syncDataset = sync_dataset(self.SYNC_FILE)
try:
vr, self.vf = probeSync.get_sync_line_data(self.syncDataset,
channel=2)
MONITOR_LAG = analysis.get_monitor_lag(self.syncDataset)
if MONITOR_LAG > 0.06:
self.errors.append(
('vsync',
'abnormal monitor lag {}, using default {}'.format(
MONITOR_LAG, 0.036)))
MONITOR_LAG = 0.036
#self.FRAME_APPEAR_TIMES = self.vf + MONITOR_LAG
self.FRAME_APPEAR_TIMES = probeSync.get_experiment_frame_times(
self.syncDataset) #trying new vsync method
self.MONITOR_LAG = MONITOR_LAG
self.vsync_times = np.copy(self.vf)
except:
print('error getting vsync times')
self.data_stream_status['sync'][0] = True
MAPPING_PKL = paths['mapping_pkl']
try:
syncDataset = sync_dataset(SYNC_FILE)
except Exception as e:
logging.error('Error reading sync file: {}'.format(e))
try:
mapping_data = pd.read_pickle(MAPPING_PKL)
except Exception as e:
logging.error('Error reading mapping pkl file: {}'.format(e))
# replay_data = pd.read_pickle(REPLAY_PKL)
### PLOT FRAME INTERVALS ###
vr, vf = probeSync.get_sync_line_data(syncDataset, channel=2)
# behavior_frame_count = behavior_data['items']['behavior']['intervalsms'].size + 1
mapping_frame_count = mapping_data['intervalsms'].size + 1
# replay_frame_count = replay_data['intervalsms'].size + 1
MONITOR_LAG = 0.036
FRAME_APPEAR_TIMES = vf + MONITOR_LAG
### CHECK THAT NO FRAMES WERE DROPPED FROM SYNC ###
# total_pkl_frames = (behavior_frame_count +
# mapping_frame_count +
# replay_frame_count)
# print('frames in pkl files: {}'.format(total_pkl_frames))
# print('frames in sync file: {}'.format(len(vf)))
def run_qc(exp_id, save_root):
identifier = exp_id
if identifier.find('_') >= 0:
d = data_getters.local_data_getter(base_dir=identifier)
else:
d = data_getters.lims_data_getter(exp_id=identifier)
paths = d.data_dict
FIG_SAVE_DIR = os.path.join(
save_root, paths['es_id'] + '_' + paths['external_specimen_name'] +
'_' + paths['datestring'])
if not os.path.exists(FIG_SAVE_DIR):
os.mkdir(FIG_SAVE_DIR)
figure_prefix = paths['external_specimen_name'] + '_' + paths[
'datestring'] + '_'
### GET FILE PATHS TO SYNC AND PKL FILES ###
SYNC_FILE = paths['sync_file']
BEHAVIOR_PKL = paths['behavior_pkl']
REPLAY_PKL = paths['replay_pkl']
MAPPING_PKL = paths['mapping_pkl']
for f, s in zip([SYNC_FILE, BEHAVIOR_PKL, REPLAY_PKL, MAPPING_PKL],
['sync: ', 'behavior: ', 'replay: ', 'mapping: ']):
print(s + f)
### GET MAIN DATA STREAMS ###
syncDataset = sync_dataset(SYNC_FILE)
behavior_data = pd.read_pickle(BEHAVIOR_PKL)
mapping_data = pd.read_pickle(MAPPING_PKL)
replay_data = pd.read_pickle(REPLAY_PKL)
### Behavior Analysis ###
behavior_plot_dir = os.path.join(FIG_SAVE_DIR, 'behavior')
trials = behavior_analysis.get_trials_df(behavior_data)
behavior_analysis.plot_behavior(trials,
behavior_plot_dir,
prefix=figure_prefix)
trial_types, counts = behavior_analysis.get_trial_counts(trials)
behavior_analysis.plot_trial_type_pie(counts,
trial_types,
behavior_plot_dir,
prefix=figure_prefix)
### CHECK FRAME COUNTS ###
vr, vf = probeSync.get_sync_line_data(syncDataset, channel=2)
behavior_frame_count = behavior_data['items']['behavior'][
'intervalsms'].size + 1
mapping_frame_count = mapping_data['intervalsms'].size + 1
replay_frame_count = replay_data['intervalsms'].size + 1
total_pkl_frames = (behavior_frame_count + mapping_frame_count +
replay_frame_count)
### CHECK THAT NO FRAMES WERE DROPPED FROM SYNC ###
print('frames in pkl files: {}'.format(total_pkl_frames))
print('frames in sync file: {}'.format(len(vf)))
#assert(total_pkl_frames==len(vf))
### CHECK THAT REPLAY AND BEHAVIOR HAVE SAME FRAME COUNT ###
print('frames in behavior stim: {}'.format(behavior_frame_count))
print('frames in replay stim: {}'.format(replay_frame_count))
#assert(behavior_frame_count==replay_frame_count)
# look for potential frame offsets from aborted stims
(behavior_start_frame,
mapping_start_frame, replay_start_frame) = probeSync.get_frame_offsets(
syncDataset,
[behavior_frame_count, mapping_frame_count, replay_frame_count])
behavior_end_frame = behavior_start_frame + behavior_frame_count - 1
mapping_end_frame = mapping_start_frame + mapping_frame_count - 1
replay_end_frame = replay_start_frame + replay_frame_count - 1
MONITOR_LAG = 0.036 #TO DO: don't hardcode this...
FRAME_APPEAR_TIMES = vf + MONITOR_LAG
behavior_start_time, mapping_start_time, replay_start_time = [
FRAME_APPEAR_TIMES[f] for f in
[behavior_start_frame, mapping_start_frame, replay_start_frame]
]
behavior_end_time, mapping_end_time, replay_end_time = [
FRAME_APPEAR_TIMES[f]
for f in [behavior_end_frame, mapping_end_frame, replay_end_frame]
]
### Plot vsync info ###
vsync_save_dir = os.path.join(FIG_SAVE_DIR, 'vsyncs')
analysis.plot_frame_intervals(vf,
behavior_frame_count,
mapping_frame_count,
behavior_start_frame,
mapping_start_frame,
replay_start_frame,
vsync_save_dir,
prefix=figure_prefix)
analysis.plot_vsync_interval_histogram(vf,
vsync_save_dir,
prefix=figure_prefix)
analysis.vsync_report(vf,
total_pkl_frames,
vsync_save_dir,
prefix=figure_prefix)
### BUILD UNIT TABLE ####
probe_dict = probeSync.build_unit_table(paths['data_probes'], paths,
syncDataset)
### Plot Probe Yield QC ###
probe_yield_dir = os.path.join(FIG_SAVE_DIR, 'probe_yield')
probe_dirs = [paths['probe' + pid] for pid in paths['data_probes']]
analysis.plot_unit_quality_hist(probe_dict,
probe_yield_dir,
prefix=figure_prefix)
analysis.plot_unit_distribution_along_probe(probe_dict,
probe_yield_dir,
prefix=figure_prefix)
analysis.plot_all_spike_hist(probe_dict,
probe_yield_dir,
prefix=figure_prefix + 'good')
analysis.copy_probe_depth_images(paths,
probe_yield_dir,
prefix=figure_prefix)
### Unit Metrics ###
unit_metrics_dir = os.path.join(FIG_SAVE_DIR, 'unit_metrics')
analysis.plot_unit_metrics(paths, unit_metrics_dir, prefix=figure_prefix)
### Probe/Sync alignment
probeSyncDir = os.path.join(FIG_SAVE_DIR, 'probeSyncAlignment')
analysis.plot_barcode_interval_hist(probe_dirs,
syncDataset,
probeSyncDir,
prefix=figure_prefix)
analysis.plot_barcode_intervals(probe_dirs,
syncDataset,
probeSyncDir,
prefix=figure_prefix)
analysis.probe_sync_report(probe_dirs,
syncDataset,
probeSyncDir,
prefix=figure_prefix)
analysis.plot_barcode_matches(probe_dirs,
syncDataset,
probeSyncDir,
prefix=figure_prefix)
### Plot visual responses
get_RFs(probe_dict,
mapping_data,
mapping_start_frame,
FRAME_APPEAR_TIMES,
os.path.join(FIG_SAVE_DIR, 'receptive_fields'),
prefix=figure_prefix)
analysis.plot_population_change_response(probe_dict,
behavior_frame_count,
mapping_frame_count,
trials,
FRAME_APPEAR_TIMES,
os.path.join(
FIG_SAVE_DIR,
'change_response'),
ctx_units_percentile=66,
prefix=figure_prefix)
### Plot running ###
analysis.plot_running_wheel(behavior_data,
mapping_data,
replay_data,
behavior_plot_dir,
prefix=figure_prefix)
### LFP ###
lfp_save_dir = os.path.join(FIG_SAVE_DIR, 'LFP')
lick_times = analysis.get_rewarded_lick_times(
probeSync.get_lick_times(syncDataset),
FRAME_APPEAR_TIMES,
trials,
min_inter_lick_time=0.5)
lfp_dict = probeSync.build_lfp_dict(probe_dirs, syncDataset)
analysis.plot_lick_triggered_LFP(lfp_dict,
lick_times,
lfp_save_dir,
prefix=figure_prefix,
agarChRange=None,
num_licks=20,
windowBefore=0.5,
windowAfter=1.5,
min_inter_lick_time=0.5,
behavior_duration=3600)
### VIDEOS ###
video_dir = os.path.join(FIG_SAVE_DIR, 'videos')
analysis.lost_camera_frame_report(paths, video_dir, prefix=figure_prefix)
analysis.camera_frame_grabs(
paths,
syncDataset,
video_dir,
[behavior_start_time, mapping_start_time, replay_start_time],
[behavior_end_time, mapping_end_time, replay_end_time],
epoch_frame_nums=[2, 2, 2],
prefix=figure_prefix)
def __init__(self, exp_id, save_root, modules_to_run='all'):
self.modules_to_run = modules_to_run
identifier = exp_id
if identifier.find('_')>=0:
d = data_getters.local_data_getter(base_dir=identifier)
else:
d = data_getters.lims_data_getter(exp_id=identifier)
self.paths = d.data_dict
self.FIG_SAVE_DIR = os.path.join(save_root, self.paths['es_id']+'_'+ self.paths['external_specimen_name']+'_'+ self.paths['datestring'])
if not os.path.exists(self.FIG_SAVE_DIR):
os.mkdir(self.FIG_SAVE_DIR)
self.figure_prefix = self.paths['external_specimen_name'] + '_' + self.paths['datestring'] + '_'
### GET FILE PATHS TO SYNC AND PKL FILES ###
self.SYNC_FILE = self.paths['sync_file']
self.BEHAVIOR_PKL = self.paths['behavior_pkl']
self.REPLAY_PKL = self.paths['replay_pkl']
self.MAPPING_PKL = self.paths['mapping_pkl']
self.OPTO_PKL = self.paths['opto_pkl']
for f,s in zip([self.SYNC_FILE, self.BEHAVIOR_PKL, self.REPLAY_PKL, self.MAPPING_PKL], ['sync: ', 'behavior: ', 'replay: ', 'mapping: ']):
print(s + f)
### GET MAIN DATA STREAMS ###
self.syncDataset = sync_dataset(self.SYNC_FILE)
self.behavior_data = pd.read_pickle(self.BEHAVIOR_PKL)
self.mapping_data = pd.read_pickle(self.MAPPING_PKL)
self.replay_data = pd.read_pickle(self.REPLAY_PKL)
self.opto_data = pd.read_pickle(self.OPTO_PKL)
self.trials = behavior_analysis.get_trials_df(self.behavior_data)
### CHECK FRAME COUNTS ###
vr, self.vf = probeSync.get_sync_line_data(self.syncDataset, channel=2)
self.behavior_frame_count = self.behavior_data['items']['behavior']['intervalsms'].size + 1
self.mapping_frame_count = self.mapping_data['intervalsms'].size + 1
self.replay_frame_count = self.replay_data['intervalsms'].size + 1
self.total_pkl_frames = (self.behavior_frame_count +
self.mapping_frame_count +
self.replay_frame_count)
### CHECK THAT NO FRAMES WERE DROPPED FROM SYNC ###
print('frames in pkl files: {}'.format(self.total_pkl_frames))
print('frames in sync file: {}'.format(len(self.vf)))
#assert(total_pkl_frames==len(vf))
### CHECK THAT REPLAY AND BEHAVIOR HAVE SAME FRAME COUNT ###
print('frames in behavior stim: {}'.format(self.behavior_frame_count))
print('frames in replay stim: {}'.format(self.replay_frame_count))
#assert(behavior_frame_count==replay_frame_count)
# look for potential frame offsets from aborted stims
(self.behavior_start_frame, self.mapping_start_frame, self.replay_start_frame) = probeSync.get_frame_offsets(
self.syncDataset,
[self.behavior_frame_count,
self.mapping_frame_count,
self.replay_frame_count])
self.behavior_end_frame = self.behavior_start_frame + self.behavior_frame_count - 1
self.mapping_end_frame = self.mapping_start_frame + self.mapping_frame_count - 1
self.replay_end_frame = self.replay_start_frame + self.replay_frame_count - 1
MONITOR_LAG = 0.036 #TO DO: don't hardcode this...
self.FRAME_APPEAR_TIMES = self.vf + MONITOR_LAG
self.behavior_start_time, self.mapping_start_time, self.replay_start_time = [self.FRAME_APPEAR_TIMES[f] for f in
[self.behavior_start_frame, self.mapping_start_frame, self.replay_start_frame]]
self.behavior_end_time, self.mapping_end_time, self.replay_end_time = [self.FRAME_APPEAR_TIMES[f] for f in
[self.behavior_end_frame, self.mapping_end_frame, self.replay_end_frame]]
self.probe_dirs = [self.paths['probe'+pid] for pid in self.paths['data_probes']]
self.probe_dict = None
self.lfp_dict = None
self._run_modules()