def filter_from_df(df, p_cutoff=0.95, smoothing=('raw',np.nan), features=('centroid','l_ear','r_ear'),
frame_rate=20.0):
resnet_name = df.keys()[0][0]
output = dict()
for feature in features:
feature_details = dict()
feature_x = df[(resnet_name,feature,'x')]
feature_y = df[(resnet_name,feature,'y')]
feature_p = df[(resnet_name,feature,'likelihood')]
# remove the unreliable
feature_x.loc[feature_p<p_cutoff] = np.nan
feature_y.loc[feature_p<p_cutoff] = np.nan
# smoothing
if smoothing[0]=='raw':
# do nothing to the distance
pass
elif smoothing[0]=='median_filter':
# median filter
filter_duration = smoothing[1] # 200 ms
filter_dur_in_frames = int(filter_duration*frame_rate)
if not is_odd(filter_dur_in_frames):filter_dur_in_frames = filter_dur_in_frames+1
feature_x = medfilt(feature_x,size=filter_dur_in_frames)
feature_y = medfilt(feature_y,size=filter_dur_in_frames)
# now add to the data
feature_details['x'] = feature_x
feature_details['y'] = feature_y
output[feature] = feature_details
return output
def get_animal_speed(df,bodypart='centroid',p_cutoff=0.95,strategy='raw',frame_rate=20.0,arena_size_in_mm=450,time_filter=None):
resnet_name = df.keys()[0][0]
t_l_corner = np.asarray([df[resnet_name,'box_tl','x'].mean(),-df[resnet_name,'box_tl','y'].mean()])
t_r_corner = np.asarray([df[resnet_name,'box_tr','x'].mean(),-df[resnet_name,'box_tl','y'].mean()])
arena_length_in_pix = euclidean_dist(t_l_corner,t_r_corner)
centroid_x = df[(resnet_name,'centroid','x')]
centroid_y = df[(resnet_name,'centroid','y')]
centroid_p = df[(resnet_name,'centroid','likelihood')]
good_data = (centroid_p>p_cutoff)
# remove data from the initial section - there is no animal and estimates are terrible
# criterion is atleast 1 second of reliable data
remove_until = 0
for i in range(1,len(good_data)):
if np.any(good_data[:i]) and np.all(good_data[i:i+int(frame_rate)]):
remove_until=i
break
body_location = [(k,l) for k,l in zip(centroid_x,centroid_y)]
body_location[:remove_until] = np.full((remove_until,1),np.nan)
body_speed = np.full_like(body_location,np.nan)
for i in range(remove_until+1,len(good_data)):
body_speed[i] = euclidean_dist(body_location[i-1],body_location[i])
body_speed = body_speed.astype(np.float64)
if time_filter:
n_to_keep = int(frame_rate*time_filter)
body_speed = body_speed[:n_to_keep]
if strategy=='raw':
# do nothing to the distance
pass
elif strategy=='raw+outlier_removal':
# do nothing to the distance
body_speed[body_speed>np.nanquantile(body_speed,0.999)] = np.nan
elif strategy=='median_filter':
# median filter
filter_duration = 0.1 # 200 ms
filter_dur_in_frames = int(filter_duration*frame_rate)
if not is_odd(filter_dur_in_frames):filter_dur_in_frames = filter_dur_in_frames+1
body_speed = medfilt(body_speed,size=filter_dur_in_frames)
elif strategy=='median_filter+outlier_removal':
# median filter
filter_duration = 0.1 # 200 ms
filter_dur_in_frames = int(filter_duration*frame_rate)
if not is_odd(filter_dur_in_frames):filter_dur_in_frames = filter_dur_in_frames+1
body_speed = medfilt(body_speed,size=filter_dur_in_frames)
body_speed[body_speed>np.nanquantile(body_speed,0.999)] = np.nan
# body_speed is in delta(pix)/frame. Convert to delta(mm)/s
body_speed = body_speed*(arena_size_in_mm/arena_length_in_pix)*frame_rate
return body_speed
def load_running_speed(data, smooth=False, time=None):
if time is None:
print('`time` not passed. using vsync from pkl file')
time = load_time(data)
dx_raw = np.array(data['dx'])
dx = medfilt(dx_raw, size=5) # remove big, single frame spikes in encoder values
dx = np.cumsum(dx) # wheel rotations
time = time[:len(dx)]
speed = calc_deriv(dx, time) # speed is in deg/s
speed = deg_to_dist(speed) # converts speed to cm/s
if smooth:
# running_speed_cm_per_sec = pd.rolling_mean(running_speed_cm_per_sec, window=6)
raise NotImplementedError
# accel = calc_deriv(speed, time)
# jerk = calc_deriv(accel, time)
running_speed = pd.DataFrame({
'time': time,
'frame': range(len(time)),
'speed': speed,
'dx': dx_raw,
'v_sig': data['vsig'],
'v_in': data['vin'],
# 'acceleration (cm/s^2)': accel,
# 'jerk (cm/s^3)': jerk,
})
return running_speed
def compute_running_speed(dx_raw, time, v_sig, v_in, smooth=False):
"""Calculate running speed
Parameters
----------
dx_raw: numpy.ndarray
dx values for each stimulus frame
time: numpy.ndarray
timestamps for each stimulus frame
v_sig: numpy.ndarray
v_sig for each stimulus frame: currently unused
v_in: numpy.ndarray
v_in for each stimulus frame: currently unused
smooth: boolean, default=False
flag to smooth output: not implemented
Returns
-------
numpy.ndarray
Running speed (cm/s)
"""
dx = medfilt(dx_raw,
size=5) # remove big, single frame spikes in encoder values
dx = np.cumsum(dx) # wheel rotations
speed = calc_deriv(dx, time) # speed in degrees/s
speed = deg_to_dist(speed)
if smooth:
raise NotImplementedError
return speed
def get_body_length(df,
p_cutoff=0.95,
strategy='median_filter',
frame_rate=20.0):
resnet_name = df.keys()[0][0]
centroid_x = df[(resnet_name, 'centroid', 'x')]
centroid_y = df[(resnet_name, 'centroid', 'y')]
centroid_p = df[(resnet_name, 'centroid', 'likelihood')]
l_ear_x = df[(resnet_name, 'l_ear', 'x')]
l_ear_y = df[(resnet_name, 'l_ear', 'y')]
l_ear_p = df[(resnet_name, 'l_ear', 'likelihood')]
r_ear_x = df[(resnet_name, 'r_ear', 'x')]
r_ear_y = df[(resnet_name, 'r_ear', 'y')]
r_ear_p = df[(resnet_name, 'r_ear', 'likelihood')]
good_data = (centroid_p > p_cutoff) & (l_ear_p > p_cutoff) & (r_ear_p >
p_cutoff)
# remove data from the initial section - there is no animal and estimates are terrible
# criterion is atleast 1 second of reliable data
remove_until = 0
for i in range(1, len(good_data)):
if np.any(good_data[:i]) and np.all(good_data[i:i + int(frame_rate)]):
remove_until = i
break
body_length = np.full((df.shape[0], 1), np.nan)
body_length_arrow_start = [(k, l)
for k, l in zip(body_length, body_length)]
body_length_arrow_end = [(k, l) for k, l in zip(body_length, body_length)]
# import pdb
# pdb.set_trace()
# get the distance for the other data
for i in range(remove_until, len(good_data)):
body_length_arrow_start[i] = (int(centroid_x[i]), int(centroid_y[i]))
body_length_arrow_end[i] = (int(
(l_ear_x[i] + r_ear_x[i]) / 2), int((l_ear_y[i] + r_ear_y[i]) / 2))
body_length[i] = euclidean_dist(body_length_arrow_start[i],
body_length_arrow_end[i])
if strategy == 'raw':
# do nothing to the distance
pass
elif strategy == 'median_filter':
filter_duration = 0.2 # 200 ms
filter_dur_in_frames = int(filter_duration * frame_rate)
if not is_odd(filter_dur_in_frames):
filter_dur_in_frames = filter_dur_in_frames + 1
body_length = medfilt(body_length, size=filter_dur_in_frames)
else:
raise ValueError(
"Unknownstrategy= {0}. Must be one of 'raw' ,'median_filter'".
format(strategy))
df['body_length'] = body_length
df['bl_arrow_start'] = body_length_arrow_start
df['bl_arrow_end'] = body_length_arrow_end
return df
def validate_encoder_voltage(core_data, range_threshold=3, wrap_threshold=2):
'''
check for potentially anomolous encoder voltage traces
Two failure modes we're checking here:
1) voltage range is less than the range_threshold.
A single rotation of the encoder will give a voltage range of ~5V, so a small range indicates
that the encoder makes less than 1 full rotation.
2) The ratio of forward wraps to backward wraps is less than the wrap_threshold:
When the encoder spinning clockwise (rotation direction for forward motion by the mouse),
the encoder transitions (wraps) from 5V to 0V once per rotation. If the encoder spinning CCW,
the transition will be from 0V to 5V. So assuming that the mouse is walking forward, there should be
far more forward wraps (5V-to-0V) than backward wraps (0V-to-5V). If the ratio is close to 1, this likely indicates
an encoder that is oscillating back and forth across the zero-point due to noise
Note that both failure modes can result from a stationary mouse (or a test session with no mouse).
'''
running = core_data['running']
# filter out voltage artifacts
filtered_vsig = medfilt(running['v_sig'], size=5)
v_sig_range = filtered_vsig.max() - filtered_vsig.min()
def get_wrap_ratio(forward_wraps, backward_wraps):
if backward_wraps == 0:
return np.inf
else:
return forward_wraps / backward_wraps
wrap_ratio = get_wrap_ratio(count_wraps(running, 'forward'),
count_wraps(running, 'backward'))
if v_sig_range < range_threshold or wrap_ratio < wrap_threshold:
return False
else:
return True
def save_core_data_components(core_data, lims_data, stimulus_timestamps):
rewards = core_data['rewards']
save_dataframe_as_h5(rewards, 'rewards', get_analysis_dir(lims_data))
running = core_data['running']
running_speed = running.rename(columns={'speed': 'running_speed'})
# filter to get rid of encoder spikes
running_speed['running_speed'] = medfilt(
running_speed.running_speed.values, size=5)
save_dataframe_as_h5(running_speed, 'running_speed',
get_analysis_dir(lims_data))
licks = core_data['licks']
save_dataframe_as_h5(licks, 'licks', get_analysis_dir(lims_data))
stimulus_table = core_data['visual_stimuli'][:
-10] # ignore last 10 flashes
if 'omitted_stimuli' in core_data.keys():
if len(core_data['omitted_stimuli']
) > 0: # sometimes there is a key but empty values
omitted_flash = core_data['omitted_stimuli'].copy()
omitted_flash = omitted_flash[['frame']]
omitted_flash['omitted'] = True
flashes = stimulus_table.merge(omitted_flash,
how='outer',
on='frame')
flashes['omitted'] = [
True if omitted is True else False
for omitted in flashes.omitted.values
]
flashes = flashes.sort_values(by='frame').reset_index().drop(
columns=['index']).fillna(method='ffill')
flashes = flashes[[
'frame', 'end_frame', 'time', 'image_category', 'image_name',
'omitted'
]]
flashes = flashes.reset_index()
flashes.image_name = [
'omitted' if flashes.iloc[row].omitted == True else
flashes.iloc[row].image_name for row in range(len(flashes))
]
# infer end time for omitted flashes as 16 frames after start frame (250ms*60Hz stim frame rate)
flashes['end_frame'] = [
flashes.loc[idx, 'end_frame'] if flashes.loc[idx, 'omitted']
== False else flashes.loc[idx, 'frame'] + 16
for idx in flashes.index.values
]
stimulus_table = flashes.copy()
else:
stimulus_table['omitted'] = False
else:
stimulus_table['omitted'] = False
if np.isnan(
stimulus_table.loc[0, 'end_frame']
): # exception for cases where the first flash in the session is omitted
stimulus_table = stimulus_table.drop(index=0)
# workaround to rename columns to harmonize with visual coding and rebase timestamps to sync time
stimulus_table.insert(loc=0,
column='flash_number',
value=np.arange(0, len(stimulus_table)))
stimulus_table = stimulus_table.rename(columns={
'frame': 'start_frame',
'time': 'start_time'
})
start_time = [
stimulus_timestamps[start_frame]
for start_frame in stimulus_table.start_frame.values
]
stimulus_table.start_time = start_time
end_time = [
stimulus_timestamps[int(end_frame)]
for end_frame in stimulus_table.end_frame.values
]
# end_time = [stimulus_timestamps[int(end_frame)] if np.isnan(end_frame) is False else np.nan()
# for end_frame in stimulus_table.end_frame.values]
stimulus_table.insert(loc=4, column='end_time', value=end_time)
if 'level_0' in stimulus_table.keys():
stimulus_table.drop(columns=['level_0'])
save_dataframe_as_h5(stimulus_table, 'stimulus_table',
get_analysis_dir(lims_data))
task_parameters = get_task_parameters(core_data)
save_dataframe_as_h5(task_parameters, 'task_parameters',
get_analysis_dir(lims_data))