def grab_timestamps_and_frames(self, key, n_sample_frames=16):
import cv2
rel = experiment.Session() * experiment.Scan.EyeVideo() * experiment.Scan.BehaviorFile().proj(
hdf_file='filename')
info = (rel & key).fetch1()
avi_path = lab.Paths().get_local_path("{behavior_path}/{filename}".format(**info))
# replace number by %d for hdf-file reader
tmp = info['hdf_file'].split('.')
if not '%d' in tmp[0]:
info['hdf_file'] = tmp[0][:-1] + '%d.' + tmp[-1]
hdf_path = lab.Paths().get_local_path("{behavior_path}/{hdf_file}".format(**info))
data = read_video_hdf5(hdf_path)
packet_length = data['analogPacketLen']
dat_time, _ = ts2sec(data['ts'], packet_length)
if float(data['version']) == 2.:
cam_key = 'eyecam_ts'
eye_time, _ = ts2sec(data[cam_key][0])
else:
cam_key = 'cam1ts' if info['rig'] == '2P3' else 'cam2ts'
eye_time, _ = ts2sec(data[cam_key])
total_frames = len(eye_time)
frame_idx = np.floor(np.linspace(0, total_frames - 1, n_sample_frames))
cap = cv2.VideoCapture(avi_path)
no_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
if total_frames != no_frames:
warn("{total_frames} timestamps, but {no_frames} movie frames.".format(total_frames=total_frames,
no_frames=no_frames))
if total_frames > no_frames and total_frames and no_frames:
total_frames = no_frames
eye_time = eye_time[:total_frames]
frame_idx = np.round(np.linspace(0, total_frames - 1, n_sample_frames)).astype(int)
else:
raise PipelineException('Can not reconcile frame count', key)
frames = []
for frame_pos in frame_idx:
cap.set(cv2.CAP_PROP_POS_FRAMES, frame_pos)
ret, frame = cap.read()
frames.append(np.asarray(frame, dtype=float)[..., 0])
frames = np.stack(frames, axis=2)
return eye_time, frames, total_frames
def local_filenames_as_wildcard(self):
"""Returns the local filename for all parts of this scan (ends in *.tif)."""
scan_path = (Session() & self).fetch1('scan_path')
local_path = lab.Paths().get_local_path(scan_path)
scan_name = (self.__class__() & self).fetch1('filename')
local_filename = os.path.join(local_path, scan_name) + '*.tif' # all parts
return local_filename
def _make_tuples(self, key):
# Get behavior filename
behavior_path = (experiment.Session() & key).fetch1('behavior_path')
local_path = lab.Paths().get_local_path(behavior_path)
filename = (experiment.Scan.BehaviorFile() & key).fetch1('filename')
full_filename = os.path.join(local_path, filename)
# Read file
data = h5.read_behavior_file(full_filename)
# Read counter timestamps and convert to seconds
timestamps_in_secs = h5.ts2sec(data['posture_ts'][0])
ts = h5.ts2sec(data['ts'], is_packeted=True)
# edge case when ts and eye ts start in different sides of the master clock max value 2 **32
if abs(ts[0] - timestamps_in_secs[0]) > 2 ** 31:
timestamps_in_secs += (2 ** 32 if ts[0] > timestamps_in_secs[0] else -2 ** 32)
# Fill with NaNs for out-of-range data or mistimed packets (NaNs in ts)
timestamps_in_secs[timestamps_in_secs < ts[0]] = float('nan')
timestamps_in_secs[timestamps_in_secs > ts[-1]] = float('nan')
nan_limits = np.where(np.diff([0, *np.isnan(ts), 0]))[0]
for start, stop in zip(nan_limits[::2], nan_limits[1::2]):
lower_ts = float('-inf') if start == 0 else ts[start - 1]
upper_ts = float('inf') if stop == len(ts) else ts[stop]
timestamps_in_secs[np.logical_and(timestamps_in_secs > lower_ts,
timestamps_in_secs < upper_ts)] = float('nan')
# Read video
filename = (experiment.Scan.PostureVideo() & key).fetch1('filename')
full_filename = os.path.join(local_path, filename)
video = cv2.VideoCapture(full_filename)
# Fix inconsistent num_video_frames vs num_timestamps
num_video_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
num_timestamps = len(timestamps_in_secs)
if num_timestamps != num_video_frames:
if abs(num_timestamps - num_video_frames) > 1:
msg = ('Number of movie frames and timestamps differ: {} frames vs {} '
'timestamps'). format(num_video_frames, num_timestamps)
raise PipelineException(msg)
elif num_timestamps > num_video_frames: # cut timestamps to match video frames
timestamps_in_secs = timestamps_in_secs[:-1]
else: # fill with NaNs
timestamps_in_secs[-1] = float('nan')
# Get 16 sample frames
frames = []
for frame_idx in np.round(np.linspace(0, num_video_frames - 1, 16)).astype(int):
video.set(cv2.CAP_PROP_POS_FRAMES, frame_idx)
_, frame = video.read()
frames.append(np.asarray(frame, dtype=float)[..., 0])
frames = np.stack(frames, axis=-1)
# Insert
self.insert1({**key, 'posture_time': timestamps_in_secs,
'total_frames': len(timestamps_in_secs), 'preview_frames': frames})
self.notify(key, frames)
def test_paths():
rel = experiment.Session() * experiment.Scan.EyeVideo(
) * experiment.Scan.BehaviorFile().proj(hdf_file='filename')
path_info = random.choice(rel.fetch.as_dict())
tmp = path_info['hdf_file'].split('.')
if '%d' in tmp[0]:
# new version
path_info['hdf_file'] = tmp[0][:-2] + '0.' + tmp[-1]
else:
path_info['hdf_file'] = tmp[0][:-1] + '0.' + tmp[-1]
hdf_path = lab.Paths().get_local_path(
'{behavior_path}/{hdf_file}'.format(**path_info))
avi_path = lab.Paths().get_local_path(
'{behavior_path}/{filename}'.format(**path_info))
assert_true(os.path.isfile(avi_path) and os.path.isfile(hdf_path))
def _make_tuples(self, key):
# Get behavior filename
behavior_path = (experiment.Session() & key).fetch1('behavior_path')
local_path = lab.Paths().get_local_path(behavior_path)
filename = (experiment.Scan.BehaviorFile() & key).fetch1('filename')
full_filename = os.path.join(local_path, filename)
# Read file
data = h5.read_behavior_file(full_filename)
# Get counter timestamps and convert to seconds
timestamps_in_secs = h5.ts2sec(data['wheel'][1])
ts = h5.ts2sec(data['ts'], is_packeted=True)
# edge case when ts and wheel ts start in different sides of the master clock max value 2 **32
if abs(ts[0] - timestamps_in_secs[0]) > 2 ** 31:
timestamps_in_secs += (2 ** 32 if ts[0] > timestamps_in_secs[0] else -2 ** 32)
# Read wheel position counter and fix wrap around at 2 ** 32
wheel_position = data['wheel'][0]
wheel_diffs = np.diff(wheel_position)
for wrap_idx in np.where(abs(wheel_diffs) > 2 ** 31)[0]:
wheel_position[wrap_idx + 1:] += (2 ** 32 if wheel_diffs[wrap_idx] < 0 else -2 ** 32)
wheel_position -= wheel_position[0] # start counts at zero
# Compute wheel velocity
num_samples = int(round((timestamps_in_secs[-1] - timestamps_in_secs[0]) * 10)) # every 100 msecs
sample_times = np.linspace(timestamps_in_secs[0], timestamps_in_secs[-1], num_samples)
sample_position = np.interp(sample_times, timestamps_in_secs, wheel_position)
counter_velocity = np.gradient(sample_position) * 10 # counts / sec
# Transform velocity from counts/sec to cm/sec
wheel_specs = experiment.TreadmillSpecs() * experiment.Session() & key
diameter, counts_per_rev = wheel_specs.fetch1('diameter', 'counts_per_revolution')
wheel_perimeter = np.pi * diameter # 1 rev = xx cms
velocity = (counter_velocity / counts_per_rev) * wheel_perimeter # cm /sec
# Resample at initial timestamps
velocity = np.interp(timestamps_in_secs, sample_times, velocity)
# Fill with NaNs for out-of-range data or mistimed packets
velocity[timestamps_in_secs < ts[0]] = float('nan')
velocity[timestamps_in_secs > ts[-1]] = float('nan')
nan_limits = np.where(np.diff([0, *np.isnan(ts), 0]))[0]
for start, stop in zip(nan_limits[::2], nan_limits[1::2]):
lower_ts = float('-inf') if start == 0 else ts[start - 1]
upper_ts = float('inf') if stop == len(ts) else ts[stop]
velocity[np.logical_and(timestamps_in_secs > lower_ts,
timestamps_in_secs < upper_ts)] = float('nan')
timestamps_in_secs[np.isnan(velocity)] = float('nan')
# Insert
self.insert1({**key, 'treadmill_time': timestamps_in_secs,
'treadmill_raw': data['wheel'][0], 'treadmill_vel': velocity})
self.notify(key)
def make(self, key):
print(f'Populating Sync for {key}')
# Get olfactory h5 path and filename
olfactory_path = (OdorSession & key).fetch1('odor_path')
local_path = lab.Paths().get_local_path(olfactory_path)
filename_base = (OdorRecording & key).fetch1('filename')
analog_filename = os.path.join(local_path, filename_base + '_%d.h5')
# Load olfactory data
analog_data = h5.read_analog_olfaction_file(analog_filename)
scan_times = h5.ts2sec(analog_data['ts'], is_packeted=True)
binarized_signal = analog_data['scanImage'] > 2.7 # TTL voltage low/high threshold
rising_edges = np.where(np.diff(binarized_signal.astype(int)) > 0)[0]
frame_times = scan_times[rising_edges]
# Correct NaN gaps in timestamps (mistimed or dropped packets during recording)
if np.any(np.isnan(frame_times)):
# Raise exception if first or last frame pulse was recorded in mistimed packet
if np.isnan(frame_times[0]) or np.isnan(frame_times[-1]):
msg = ('First or last frame happened during misstamped packets. Pulses '
'could have been missed: start/end of scanning is unknown.')
raise PipelineException(msg)
# Fill each gap of nan values with correct number of timepoints
frame_period = np.nanmedian(np.diff(frame_times)) # approx
nan_limits = np.where(np.diff(np.isnan(frame_times)))[0]
nan_limits[1::2] += 1 # limits are indices of the last valid point before the nan gap and first after it
correct_fts = []
for i, (start, stop) in enumerate(zip(nan_limits[::2], nan_limits[1::2])):
correct_fts.extend(frame_times[0 if i == 0 else nan_limits[2 * i - 1]: start + 1])
num_missing_points = int(round((frame_times[stop] - frame_times[start]) /
frame_period - 1))
correct_fts.extend(np.linspace(frame_times[start], frame_times[stop],
num_missing_points + 2)[1:-1])
correct_fts.extend(frame_times[nan_limits[-1]:])
frame_times = np.array(correct_fts)
# Check that frame times occur at the same period
frame_intervals = np.diff(frame_times)
frame_period = np.median(frame_intervals)
if np.any(abs(frame_intervals - frame_period) > 0.15 * frame_period):
raise PipelineException('Frame time period is irregular')
self.insert1({**key, 'signal_start_time': frame_times[0],
'signal_duration': frame_times[-1] - frame_times[0],
'frame_times': frame_times})
print(f'ScanImage sync added for animal {key["animal_id"]}, '
f'olfactory session {key["odor_session"]}, '
f'recording {key["recording_idx"]}\n')
def make(self, key):
print(f'Populating trials for {key}')
# Get olfactory h5 path and filename
olfactory_path = (OdorSession & key).fetch1('odor_path')
local_path = lab.Paths().get_local_path(olfactory_path)
filename_base = (OdorRecording & key).fetch1('filename')
digital_filename = os.path.join(local_path, filename_base + '_D_%d.h5')
# Load olfactory data
digital_data = h5.read_digital_olfaction_file(digital_filename)
# Check valve data ends with all valves closed
if digital_data['valves'][-1] != 0:
msg = f'Error: Final valve state is open! Ending time cannot be calculated for {key}.'
raise PipelineException(msg)
valve_open_idx = np.where(digital_data['valves'] > 0)[0]
trial_valve_states = digital_data['valves'][valve_open_idx]
trial_start_times = h5.ts2sec(digital_data['ts'][valve_open_idx])
# Shift start indices by one to get end indices
trial_end_times = h5.ts2sec(digital_data['ts'][valve_open_idx + 1])
# All keys are appended to a list and inserted at the end to prevent errors from halting mid-calculation
all_trial_keys = []
# Find all trials and insert a key for each channel open during each trial
for trial_num, (state, start, stop) in enumerate(
zip(trial_valve_states, trial_start_times, trial_end_times)):
valve_array = OdorTrials.convert_valves(state)
for valve_num in np.where(
valve_array
)[0]: # Valve array is already a boolean, look for all true values
# We start counting valves at 1, not 0 like python indices
valve_num = valve_num + 1
trial_key = [
key['animal_id'], key['odor_session'],
key['recording_idx'], trial_num, valve_num, start, stop
]
all_trial_keys.append(trial_key)
self.insert(all_trial_keys)
print(
f'{valve_open_idx.shape[0]} odor trials found and inserted for {key}.\n'
)
def make(self, key):
print(f'Populating Respiration for {key}')
# Get olfactory h5 path and filename
olfactory_path = (OdorSession & key).fetch1('odor_path')
local_path = lab.Paths().get_local_path(olfactory_path)
filename_base = (OdorRecording & key).fetch1('filename')
analog_filename = os.path.join(local_path, filename_base + '_%d.h5')
# Load olfactory data
analog_data = h5.read_analog_olfaction_file(analog_filename)
breath_times = h5.ts2sec(analog_data['ts'], is_packeted=True)
breath_trace = analog_data['breath']
# Correct NaN gaps in timestamps (mistimed or dropped packets during recording)
if np.any(np.isnan(breath_times)):
# Raise exception if first or last frame pulse was recorded in mistimed packet
if np.isnan(breath_times[0]) or np.isnan(breath_times[-1]):
msg = (
'First or last breath happened during misstamped packets. Pulses '
'could have been missed: start/end of collection is unknown.'
)
raise PipelineException(msg)
# Linear interpolate between nans
nans_idx = np.where(np.isnan(breath_times))[0]
non_nans_idx = np.where(~np.isnan(breath_times))[0]
breath_times[nans_idx] = np.interp(nans_idx, non_nans_idx,
breath_times[non_nans_idx])
print(
f'Largest NaN gap found: {np.max(np.abs(np.diff(breath_times[non_nans_idx])))} seconds'
)
# Check that frame times occur at the same period
breath_intervals = np.diff(breath_times)
breath_period = np.median(breath_intervals)
if np.any(
abs(breath_intervals - breath_period) > 0.15 * breath_period):
raise PipelineException('Breath time period is irregular')
# Error check tracing and timing match
if breath_trace.shape[0] != breath_times.shape[0]:
raise PipelineException('Breath timing and trace mismatch!')
breath_key = {**key, 'trace': breath_trace, 'times': breath_times}
self.insert1(breath_key)
print(f'Respiration data for {key} successfully inserted.\n')
def _make_tuples(self, key):
#pull filename for key
rel = experiment.Session() * experiment.Scan.BehaviorFile().proj(
hdf_file='filename')
info = (rel & key).fetch1()
# replace number by %d for hdf-file reader
tmp = info['hdf_file'].split('.')
if not '%d' in tmp[0]:
info['hdf_file'] = tmp[0][:-1] + '%d.' + tmp[-1]
#read hdf file for ball data
hdf_path = lab.Paths().get_local_path(
"{behavior_path}/{hdf_file}".format(**info))
data = read_video_hdf5(hdf_path)
#read out counter time stamp and convert to seconds
packet_length = data['analogPacketLen']
ball_time, _ = ts2sec(data['ball'].transpose()[1], packet_length)
#read out raw ball counts and integrate by 100ms intervals
ball_raw = data['ball'].transpose()[0]
ball_time_to_raw = interp1d(ball_time, ball_raw - ball_raw[0])
bin_times = np.arange(ball_time[0], ball_time[-1], .1)
bin_times[-1] = ball_time[-1]
ball_counts = np.append([0], np.diff(ball_time_to_raw(bin_times)))
#pull Treadmill specs, warn if more than one Treadmill fits session key
diam, counts_per_revolution = (
experiment.TreadmillSpecs() * experiment.Session() & key
& 'treadmill_start_date <= session_date').fetch(
'diameter', 'counts_per_revolution')
if len(diam) != 1:
raise PipelineException('Unclear which treadmill fits session key')
#convert ball counts to cm/s for each ball time point
cmPerCount = np.pi * diam[-1] / counts_per_revolution[-1]
ball_time_to_vel = interp1d(bin_times, ball_counts * cmPerCount * 10)
ball_vel = ball_time_to_vel(ball_time)
#assign calculated properties to key
key['treadmill_time'] = ball_time
key['treadmill_raw'] = ball_raw
key['treadmill_vel'] = ball_vel
#insert and notify user
self.insert1(key)
self.notify({k: key[k] for k in self.heading.primary_key})
def _make_tuples(self, key):
# Get behavior filename
behavior_path = (experiment.Session() & key).fetch1('behavior_path')
local_path = lab.Paths().get_local_path(behavior_path)
filename = (experiment.Scan.BehaviorFile() & key).fetch1('filename')
full_filename = os.path.join(local_path, filename)
# Read file
data = h5.read_behavior_file(full_filename)
# Get counter timestamps and convert to seconds
ts = h5.ts2sec(data['ts'], is_packeted=True)
# Read temperature (if available) and invalidate points with unreliable timestamps
temp_raw = data.get('temperature', None)
if temp_raw is None:
raise PipelineException(
'Scan {animal_id}-{session}-{scan_idx} does not have '
'temperature data'.format(**key))
temp_raw[np.isnan(ts)] = float('nan')
# Read temperature and smooth it
temp_celsius = (temp_raw * 100 - 32) / 1.8 # F to C
sampling_rate = int(round(
1 / np.nanmedian(np.diff(ts)))) # samples per second
smooth_temp = signal.low_pass_filter(temp_celsius,
sampling_rate,
cutoff_freq=1,
filter_size=2 * sampling_rate)
# Resample at 1 Hz
downsampled_ts = ts[::sampling_rate]
downsampled_temp = smooth_temp[::sampling_rate]
# Insert
self.insert1({
**key, 'temp_time': downsampled_ts,
'temperatures': downsampled_temp,
'median_temperature': np.nanmedian(downsampled_temp)
})
self.notify(key)
def _make_tuples(self, key):
rel = experiment.Session() * experiment.Scan.BehaviorFile().proj(
hdf_file='filename')
info = (rel & key).fetch1()
# replace number by %d for hdf-file reader
tmp = info['hdf_file'].split('.')
if not '%d' in tmp[0]:
info['hdf_file'] = tmp[0][:-1] + '%d.' + tmp[-1]
hdf_path = lab.Paths().get_local_path(
"{behavior_path}/{hdf_file}".format(**info))
data = read_video_hdf5(hdf_path)
packet_length = data['analogPacketLen']
dat_time, _ = ts2sec(data['ts'], packet_length)
dat_fs = 1. / np.median(np.diff(dat_time))
n = int(np.ceil(0.0002 * dat_fs))
k = np.hamming(2 * n)
k /= -k.sum()
k[:n] = -k[:n]
pulses = np.convolve(
data['scanImage'], k,
mode='full')[n:-n + 1] # mode='same' with MATLAB compatibility
peaks = spaced_max(pulses, 0.005 * dat_fs)
peaks = peaks[pulses[peaks] > 0.1 * np.percentile(pulses[peaks], 90)]
peaks = longest_contiguous_block(peaks)
self.insert1(dict(key, frame_times=dat_time[peaks]))
self.notify(key)
def get_video_path(self):
video_info = (experiment.Session() * experiment.Scan.PostureVideo()
& self).fetch1()
video_path = lab.Paths().get_local_path(
"{behavior_path}/{filename}".format(**video_info))
return video_path
def make(self, key):
""" Read ephys data and insert into table """
import h5py
# Read the scan
print('Reading file...')
vreso_path, filename_base = (PatchSession *
(Recording() & key)).fetch1(
'recording_path', 'file_name')
local_path = lab.Paths().get_local_path(vreso_path)
filename = os.path.join(local_path, filename_base + '_%d.h5')
with h5py.File(filename, 'r', driver='family', memb_size=0) as f:
# Load timing info
ANALOG_PACKET_LEN = f.attrs['waveform Frame Size'][0]
# Get counter timestamps and convert to seconds
patch_times = h5.ts2sec(f['waveform'][10, :], is_packeted=True)
# Detect rising edges in scanimage clock signal (start of each frame)
binarized_signal = f['waveform'][
9, :] > 2.7 # TTL voltage low/high threshold
rising_edges = np.where(
np.diff(binarized_signal.astype(int)) > 0)[0]
frame_times = patch_times[rising_edges]
# Correct NaN gaps in timestamps (mistimed or dropped packets during recording)
if np.any(np.isnan(frame_times)):
# Raise exception if first or last frame pulse was recorded in mistimed packet
if np.isnan(frame_times[0]) or np.isnan(frame_times[-1]):
msg = (
'First or last frame happened during misstamped packets. Pulses '
'could have been missed: start/end of scanning is unknown.'
)
raise PipelineException(msg)
# Fill each gap of nan values with correct number of timepoints
frame_period = np.nanmedian(np.diff(frame_times)) # approx
nan_limits = np.where(np.diff(np.isnan(frame_times)))[0]
nan_limits[
1::
2] += 1 # limits are indices of the last valid point before the nan gap and first after it
correct_fts = []
for i, (start, stop) in enumerate(
zip(nan_limits[::2], nan_limits[1::2])):
correct_fts.extend(
frame_times[0 if i == 0 else nan_limits[2 * i -
1]:start + 1])
num_missing_points = int(
round((frame_times[stop] - frame_times[start]) /
frame_period - 1))
correct_fts.extend(
np.linspace(frame_times[start], frame_times[stop],
num_missing_points + 2)[1:-1])
correct_fts.extend(frame_times[nan_limits[-1]:])
frame_times = np.array(correct_fts)
# Record the NaN fix
num_gaps = int(len(nan_limits) / 2)
nan_length = sum(nan_limits[1::2] -
nan_limits[::2]) * frame_period # secs
####### WARNING: FRAME INTERVALS NOT ERROR CHECKED - TEMP CODE #######
# Check that frame times occur at the same period
frame_intervals = np.diff(frame_times)
frame_period = np.median(frame_intervals)
#if np.any(abs(frame_intervals - frame_period) > 0.15 * frame_period):
# raise PipelineException('Frame time period is irregular')
# Drop last frame time if scan crashed or was stopped before completion
valid_times = ~np.isnan(
patch_times[rising_edges[0]:rising_edges[-1]]
) # restricted to scan period
binarized_valid = binarized_signal[
rising_edges[0]:rising_edges[-1]][valid_times]
frame_duration = np.mean(binarized_valid) * frame_period
falling_edges = np.where(
np.diff(binarized_signal.astype(int)) < 0)[0]
last_frame_duration = patch_times[
falling_edges[-1]] - frame_times[-1]
if (np.isnan(last_frame_duration) or last_frame_duration < 0
or abs(last_frame_duration - frame_duration) >
0.15 * frame_duration):
frame_times = frame_times[:-1]
####### WARNING: NO CORRECTION APPLIED - TEMP CODE #######
voltage = np.array(f['waveform'][1, :], dtype='float32')
current = np.array(f['waveform'][0, :], dtype='float32')
command = np.array(f['waveform'][5, :], dtype='float32')
####### WARNING: DUMMY VARIABLES - TEMP CODE #######
vgain = 0
igain = 0
command_gain = 0
self.insert1({
**key, 'voltage': voltage,
'current': current,
'command': command,
'patch_times': patch_times,
'frame_times': frame_times,
'vgain': vgain,
'igain': igain,
'command_gain': command_gain
})
def _make_tuples(self, key):
# Get behavior filename
behavior_path = (experiment.Session() & key).fetch1('behavior_path')
local_path = lab.Paths().get_local_path(behavior_path)
filename = (experiment.Scan.BehaviorFile() & key).fetch1('filename')
full_filename = os.path.join(local_path, filename)
# Read file
data = h5.read_behavior_file(full_filename)
# Read counter timestamps and convert to seconds
timestamps_in_secs = h5.ts2sec(data['ts'], is_packeted=True)
# Detect rising edges in scanimage clock signal (start of each frame)
binarized_signal = data[
'scanImage'] > 2.7 # TTL voltage low/high threshold
rising_edges = np.where(np.diff(binarized_signal.astype(int)) > 0)[0]
frame_times = timestamps_in_secs[rising_edges]
# Correct NaN gaps in timestamps (mistimed or dropped packets during recording)
if np.any(np.isnan(frame_times)):
# Raise exception if first or last frame pulse was recorded in mistimed packet
if np.isnan(frame_times[0]) or np.isnan(frame_times[-1]):
msg = (
'First or last frame happened during misstamped packets. Pulses '
'could have been missed: start/end of scanning is unknown.'
)
raise PipelineException(msg)
# Fill each gap of nan values with correct number of timepoints
frame_period = np.nanmedian(np.diff(frame_times)) # approx
nan_limits = np.where(np.diff(np.isnan(frame_times)))[0]
nan_limits[
1::
2] += 1 # limits are indices of the last valid point before the nan gap and first after it
correct_fts = []
for i, (start,
stop) in enumerate(zip(nan_limits[::2], nan_limits[1::2])):
correct_fts.extend(
frame_times[0 if i == 0 else nan_limits[2 * i - 1]:start +
1])
num_missing_points = int(
round((frame_times[stop] - frame_times[start]) /
frame_period - 1))
correct_fts.extend(
np.linspace(frame_times[start], frame_times[stop],
num_missing_points + 2)[1:-1])
correct_fts.extend(frame_times[nan_limits[-1]:])
frame_times = correct_fts
# Record the NaN fix
num_gaps = int(len(nan_limits) / 2)
nan_length = sum(nan_limits[1::2] -
nan_limits[::2]) * frame_period # secs
experiment.Fixes.insert1(key, skip_duplicates=True)
experiment.Fixes.IrregularTimestamps.insert1({
**key, 'num_gaps': num_gaps,
'num_secs': nan_length
})
# Check that frame times occur at the same period
frame_intervals = np.diff(frame_times)
frame_period = np.median(frame_intervals)
if np.any(abs(frame_intervals - frame_period) > 0.15 * frame_period):
raise PipelineException('Frame time period is irregular')
# Drop last frame time if scan crashed or was stopped before completion
valid_times = ~np.isnan(
timestamps_in_secs[rising_edges[0]:rising_edges[-1]]
) # restricted to scan period
binarized_valid = binarized_signal[rising_edges[0]:rising_edges[-1]][
valid_times]
frame_duration = np.mean(binarized_valid) * frame_period
falling_edges = np.where(np.diff(binarized_signal.astype(int)) < 0)[0]
last_frame_duration = timestamps_in_secs[
falling_edges[-1]] - frame_times[-1]
if (np.isnan(last_frame_duration) or last_frame_duration < 0
or abs(last_frame_duration - frame_duration) >
0.15 * frame_duration):
frame_times = frame_times[:-1]
self.insert1({**key, 'frame_times': frame_times})
self.notify(key)
