def make(self, key):
print(f'Populating trials for {key}')
# Get olfactory h5 path and filename
olfactory_path = (OdorSession & key).fetch1('odor_path')
local_path = lab.Paths().get_local_path(olfactory_path)
filename_base = (OdorRecording & key).fetch1('filename')
digital_filename = os.path.join(local_path, filename_base + '_D_%d.h5')
# Load olfactory data
digital_data = h5.read_digital_olfaction_file(digital_filename)
# Check valve data ends with all valves closed
if digital_data['valves'][-1] != 0:
msg = f'Error: Final valve state is open! Ending time cannot be calculated for {key}.'
raise PipelineException(msg)
valve_open_idx = np.where(digital_data['valves'] > 0)[0]
trial_valve_states = digital_data['valves'][valve_open_idx]
trial_start_times = h5.ts2sec(digital_data['ts'][valve_open_idx])
# Shift start indices by one to get end indices
trial_end_times = h5.ts2sec(digital_data['ts'][valve_open_idx + 1])
# All keys are appended to a list and inserted at the end to prevent errors from halting mid-calculation
all_trial_keys = []
# Find all trials and insert a key for each channel open during each trial
for trial_num, (state, start, stop) in enumerate(
zip(trial_valve_states, trial_start_times, trial_end_times)):
valve_array = OdorTrials.convert_valves(state)
for valve_num in np.where(
valve_array
)[0]: # Valve array is already a boolean, look for all true values
# We start counting valves at 1, not 0 like python indices
valve_num = valve_num + 1
trial_key = [
key['animal_id'], key['odor_session'],
key['recording_idx'], trial_num, valve_num, start, stop
]
all_trial_keys.append(trial_key)
self.insert(all_trial_keys)
print(
f'{valve_open_idx.shape[0]} odor trials found and inserted for {key}.\n'
)
def make(self, key):
print(f'Populating Sync for {key}')
# Get olfactory h5 path and filename
olfactory_path = (OdorSession & key).fetch1('odor_path')
local_path = lab.Paths().get_local_path(olfactory_path)
filename_base = (OdorRecording & key).fetch1('filename')
analog_filename = os.path.join(local_path, filename_base + '_%d.h5')
# Load olfactory data
analog_data = h5.read_analog_olfaction_file(analog_filename)
scan_times = h5.ts2sec(analog_data['ts'], is_packeted=True)
binarized_signal = analog_data['scanImage'] > 2.7 # TTL voltage low/high threshold
rising_edges = np.where(np.diff(binarized_signal.astype(int)) > 0)[0]
frame_times = scan_times[rising_edges]
# Correct NaN gaps in timestamps (mistimed or dropped packets during recording)
if np.any(np.isnan(frame_times)):
# Raise exception if first or last frame pulse was recorded in mistimed packet
if np.isnan(frame_times[0]) or np.isnan(frame_times[-1]):
msg = ('First or last frame happened during misstamped packets. Pulses '
'could have been missed: start/end of scanning is unknown.')
raise PipelineException(msg)
# Fill each gap of nan values with correct number of timepoints
frame_period = np.nanmedian(np.diff(frame_times)) # approx
nan_limits = np.where(np.diff(np.isnan(frame_times)))[0]
nan_limits[1::2] += 1 # limits are indices of the last valid point before the nan gap and first after it
correct_fts = []
for i, (start, stop) in enumerate(zip(nan_limits[::2], nan_limits[1::2])):
correct_fts.extend(frame_times[0 if i == 0 else nan_limits[2 * i - 1]: start + 1])
num_missing_points = int(round((frame_times[stop] - frame_times[start]) /
frame_period - 1))
correct_fts.extend(np.linspace(frame_times[start], frame_times[stop],
num_missing_points + 2)[1:-1])
correct_fts.extend(frame_times[nan_limits[-1]:])
frame_times = np.array(correct_fts)
# Check that frame times occur at the same period
frame_intervals = np.diff(frame_times)
frame_period = np.median(frame_intervals)
if np.any(abs(frame_intervals - frame_period) > 0.15 * frame_period):
raise PipelineException('Frame time period is irregular')
self.insert1({**key, 'signal_start_time': frame_times[0],
'signal_duration': frame_times[-1] - frame_times[0],
'frame_times': frame_times})
print(f'ScanImage sync added for animal {key["animal_id"]}, '
f'olfactory session {key["odor_session"]}, '
f'recording {key["recording_idx"]}\n')
def make(self, key):
print(f'Populating Respiration for {key}')
# Get olfactory h5 path and filename
olfactory_path = (OdorSession & key).fetch1('odor_path')
local_path = lab.Paths().get_local_path(olfactory_path)
filename_base = (OdorRecording & key).fetch1('filename')
analog_filename = os.path.join(local_path, filename_base + '_%d.h5')
# Load olfactory data
analog_data = h5.read_analog_olfaction_file(analog_filename)
breath_times = h5.ts2sec(analog_data['ts'], is_packeted=True)
breath_trace = analog_data['breath']
# Correct NaN gaps in timestamps (mistimed or dropped packets during recording)
if np.any(np.isnan(breath_times)):
# Raise exception if first or last frame pulse was recorded in mistimed packet
if np.isnan(breath_times[0]) or np.isnan(breath_times[-1]):
msg = (
'First or last breath happened during misstamped packets. Pulses '
'could have been missed: start/end of collection is unknown.'
)
raise PipelineException(msg)
# Linear interpolate between nans
nans_idx = np.where(np.isnan(breath_times))[0]
non_nans_idx = np.where(~np.isnan(breath_times))[0]
breath_times[nans_idx] = np.interp(nans_idx, non_nans_idx,
breath_times[non_nans_idx])
print(
f'Largest NaN gap found: {np.max(np.abs(np.diff(breath_times[non_nans_idx])))} seconds'
)
# Check that frame times occur at the same period
breath_intervals = np.diff(breath_times)
breath_period = np.median(breath_intervals)
if np.any(
abs(breath_intervals - breath_period) > 0.15 * breath_period):
raise PipelineException('Breath time period is irregular')
# Error check tracing and timing match
if breath_trace.shape[0] != breath_times.shape[0]:
raise PipelineException('Breath timing and trace mismatch!')
breath_key = {**key, 'trace': breath_trace, 'times': breath_times}
self.insert1(breath_key)
print(f'Respiration data for {key} successfully inserted.\n')
def make(self, key):
""" Read ephys data and insert into table """
import h5py
# Read the scan
print('Reading file...')
vreso_path, filename_base = (PatchSession *
(Recording() & key)).fetch1(
'recording_path', 'file_name')
local_path = lab.Paths().get_local_path(vreso_path)
filename = os.path.join(local_path, filename_base + '_%d.h5')
with h5py.File(filename, 'r', driver='family', memb_size=0) as f:
# Load timing info
ANALOG_PACKET_LEN = f.attrs['waveform Frame Size'][0]
# Get counter timestamps and convert to seconds
patch_times = h5.ts2sec(f['waveform'][10, :], is_packeted=True)
# Detect rising edges in scanimage clock signal (start of each frame)
binarized_signal = f['waveform'][
9, :] > 2.7 # TTL voltage low/high threshold
rising_edges = np.where(
np.diff(binarized_signal.astype(int)) > 0)[0]
frame_times = patch_times[rising_edges]
# Correct NaN gaps in timestamps (mistimed or dropped packets during recording)
if np.any(np.isnan(frame_times)):
# Raise exception if first or last frame pulse was recorded in mistimed packet
if np.isnan(frame_times[0]) or np.isnan(frame_times[-1]):
msg = (
'First or last frame happened during misstamped packets. Pulses '
'could have been missed: start/end of scanning is unknown.'
)
raise PipelineException(msg)
# Fill each gap of nan values with correct number of timepoints
frame_period = np.nanmedian(np.diff(frame_times)) # approx
nan_limits = np.where(np.diff(np.isnan(frame_times)))[0]
nan_limits[
1::
2] += 1 # limits are indices of the last valid point before the nan gap and first after it
correct_fts = []
for i, (start, stop) in enumerate(
zip(nan_limits[::2], nan_limits[1::2])):
correct_fts.extend(
frame_times[0 if i == 0 else nan_limits[2 * i -
1]:start + 1])
num_missing_points = int(
round((frame_times[stop] - frame_times[start]) /
frame_period - 1))
correct_fts.extend(
np.linspace(frame_times[start], frame_times[stop],
num_missing_points + 2)[1:-1])
correct_fts.extend(frame_times[nan_limits[-1]:])
frame_times = np.array(correct_fts)
# Record the NaN fix
num_gaps = int(len(nan_limits) / 2)
nan_length = sum(nan_limits[1::2] -
nan_limits[::2]) * frame_period # secs
####### WARNING: FRAME INTERVALS NOT ERROR CHECKED - TEMP CODE #######
# Check that frame times occur at the same period
frame_intervals = np.diff(frame_times)
frame_period = np.median(frame_intervals)
#if np.any(abs(frame_intervals - frame_period) > 0.15 * frame_period):
# raise PipelineException('Frame time period is irregular')
# Drop last frame time if scan crashed or was stopped before completion
valid_times = ~np.isnan(
patch_times[rising_edges[0]:rising_edges[-1]]
) # restricted to scan period
binarized_valid = binarized_signal[
rising_edges[0]:rising_edges[-1]][valid_times]
frame_duration = np.mean(binarized_valid) * frame_period
falling_edges = np.where(
np.diff(binarized_signal.astype(int)) < 0)[0]
last_frame_duration = patch_times[
falling_edges[-1]] - frame_times[-1]
if (np.isnan(last_frame_duration) or last_frame_duration < 0
or abs(last_frame_duration - frame_duration) >
0.15 * frame_duration):
frame_times = frame_times[:-1]
####### WARNING: NO CORRECTION APPLIED - TEMP CODE #######
voltage = np.array(f['waveform'][1, :], dtype='float32')
current = np.array(f['waveform'][0, :], dtype='float32')
command = np.array(f['waveform'][5, :], dtype='float32')
####### WARNING: DUMMY VARIABLES - TEMP CODE #######
vgain = 0
igain = 0
command_gain = 0
self.insert1({
**key, 'voltage': voltage,
'current': current,
'command': command,
'patch_times': patch_times,
'frame_times': frame_times,
'vgain': vgain,
'igain': igain,
'command_gain': command_gain
})