def detect_epoch_ripples(epoch_key,
animals,
sampling_frequency,
brain_areas=_BRAIN_AREAS):
'''Returns a list of tuples containing the start and end times of
ripples. Candidate ripples are computed via the ripple detection
function and then filtered to exclude ripples where the animal was
still moving.
'''
logger.info('Detecting ripples')
tetrode_info = make_tetrode_dataframe(animals).xs(epoch_key,
drop_level=False)
brain_areas = [brain_areas] if isinstance(brain_areas,
str) else brain_areas
is_brain_areas = tetrode_info.area.isin(brain_areas)
if 'CA1' in brain_areas:
is_brain_areas = is_brain_areas & (tetrode_info.descrip.isin(
['riptet']) | tetrode_info.validripple)
logger.debug(
tetrode_info[is_brain_areas].loc[:, ['area', 'depth', 'descrip']])
tetrode_keys = tetrode_info[is_brain_areas].index.tolist()
LFPs = get_LFPs(tetrode_keys, animals)
speed = get_interpolated_position_dataframe(epoch_key,
animals,
max_distance_from_well=5).speed
not_null = np.any(pd.notnull(LFPs), axis=1) & pd.notnull(speed)
return Kay_ripple_detector(LFPs.index[not_null],
LFPs.values[not_null],
speed.values[not_null],
sampling_frequency,
minimum_duration=pd.Timedelta(milliseconds=15),
zscore_threshold=3)
def estimate_theta_power(time, tetrode_info, multitaper_params=None):
if multitaper_params is None:
multitaper_params = MULTITAPER_PARAMETERS['4Hz']
is_brain_areas = (
tetrode_info.area.astype(str).str.upper().isin(BRAIN_AREAS))
tetrode_keys = tetrode_info.loc[is_brain_areas].index
lfps = get_LFPs(tetrode_keys, ANIMALS)
m = Multitaper(lfps.values,
**multitaper_params,
start_time=lfps.index[0].total_seconds())
c = Connectivity.from_multitaper(m)
coordinates = {
'time': pd.to_timedelta(c.time, unit='s'),
'frequency': c.frequencies,
'tetrode': lfps.columns,
}
dimension_names = ['time', 'frequency', 'tetrode']
data_vars = {'power': (dimension_names, c.power())}
theta_power = (xr.Dataset(data_vars, coords=coordinates).sel(
frequency=slice(4, 12)).mean('frequency').dropna('tetrode').interp(
time=time).to_dataframe().unstack(level=0))
theta_power_change = theta_power.transform(lambda df: df / df.mean())
theta_power_zscore = theta_power.transform(lambda df:
(df - df.mean()) / df.std())
return dict(
theta_power=theta_power,
theta_power_change=theta_power_change,
theta_power_zscore=theta_power_zscore,
)
def get_theta_times(epoch_key, sampling_frequency=1500):
THETA_BAND = (6, 12)
TIME_WINDOW_STEP = TIME_WINDOW_DURATION = 0.300
TIME_HALFBANDWIDTH_PRODUCT = 1
position_info = (get_interpolated_position_dataframe(
epoch_key, ANIMALS).dropna(subset=['linear_position', 'speed']))
time = position_info.index
tetrode_info = make_tetrode_dataframe(ANIMALS, epoch_key=epoch_key)
tetrode_keys = tetrode_info.loc[tetrode_info.area == 'Reference'].index
lfps = get_LFPs(tetrode_keys, ANIMALS).reindex(time)
multitaper_params = dict(
time_halfbandwidth_product=TIME_HALFBANDWIDTH_PRODUCT,
time_window_duration=TIME_WINDOW_DURATION,
time_window_step=TIME_WINDOW_STEP,
start_time=(time.values / np.timedelta64(1, 's')).min(),
)
df, model = detect_spectral_rhythm(time=time.values /
np.timedelta64(1, 's'),
lfps=lfps.values,
sampling_frequency=sampling_frequency,
multitaper_params=multitaper_params,
frequency_band=THETA_BAND)
return df.is_spectral_rhythm
def load_data(epoch_key, brain_areas=None):
if brain_areas is None:
brain_areas = BRAIN_AREAS
time = get_trial_time(epoch_key, ANIMALS)
time = (pd.Series(np.ones_like(time, dtype=np.float),
index=time).resample('2ms').mean().index)
def _time_function(*args, **kwargs):
return time
position_info = (get_interpolated_position_dataframe(
epoch_key, ANIMALS,
_time_function).dropna(subset=['linear_position', 'speed']))
time = position_info.index
tetrode_info = make_tetrode_dataframe(ANIMALS, epoch_key=epoch_key)
is_brain_areas = (
tetrode_info.area.astype(str).str.upper().isin(brain_areas))
tetrode_keys = tetrode_info.loc[is_brain_areas].index
lfps = get_LFPs(tetrode_keys, ANIMALS)
lfps = lfps.resample('2ms').mean().fillna(method='pad').reindex(time)
try:
neuron_info = make_neuron_dataframe(ANIMALS).xs(epoch_key,
drop_level=False)
neuron_info = neuron_info.loc[(neuron_info.numspikes > 100)
& neuron_info.area.isin(brain_areas) &
(neuron_info.type == 'principal')]
spikes = get_all_spike_indicators(neuron_info.index, ANIMALS,
_time_function).reindex(time)
except KeyError:
spikes = None
tetrode_info = tetrode_info.loc[is_brain_areas]
multiunit = (get_all_multiunit_indicators(
tetrode_info.index, ANIMALS,
_time_function).sel(features=_MARKS).reindex({'time': time}))
multiunit_spikes = (np.any(~np.isnan(multiunit.values),
axis=1)).astype(np.float)
multiunit_firing_rate = pd.DataFrame(get_multiunit_population_firing_rate(
multiunit_spikes, SAMPLING_FREQUENCY, smoothing_sigma=0.020),
index=time,
columns=['firing_rate'])
return {
'position_info': position_info,
'spikes': spikes,
'multiunit': multiunit,
'lfps': lfps,
'tetrode_info': tetrode_info,
'multiunit_firing_rate': multiunit_firing_rate,
'sampling_frequency': SAMPLING_FREQUENCY,
}
def _load_lfp_from_filterframework(ntrode_keys, animal_dict):
print('loading lfp from ff')
print(ntrode_keys)
lfp = lfdp.get_LFPs(ntrode_keys, animal_dict)
# broadcast indices from cols and reshape df to be: (an,day,ep,td):(ntrode)
lfp_mi_list = [(str(a), int(d), int(e), int(n)) for col in lfp.columns.tolist() for (a, d, e, n) in [col.split('_')]]
lfp_mi = pd.MultiIndex.from_tuples(lfp_mi_list, names=['animal', 'day', 'epoch', 'ntrode'])
df = pd.DataFrame(lfp.T.values, index=lfp_mi).T #grab lfp values to avoid index merging
df['timedelta'] = lfp.index
df = df.set_index('timedelta').stack(level=['animal', 'day', 'epoch']).reorder_levels(['animal', 'day', 'epoch', 'timedelta'])
return df
def get_adhoc_ripple(epoch_key, tetrode_info, position_time):
LFP_SAMPLING_FREQUENCY = 1500
position_info = (
get_interpolated_position_dataframe(epoch_key, ANIMALS)
.dropna(subset=['linear_position', 'speed']))
speed = position_info['speed']
time = position_info.index
if ~np.all(np.isnan(tetrode_info.validripple.astype(float))):
tetrode_keys = tetrode_info.loc[
(tetrode_info.validripple == 1)].index
else:
is_brain_areas = (
tetrode_info.area.astype(str).str.upper().isin(BRAIN_AREAS))
tetrode_keys = tetrode_info.loc[is_brain_areas].index
ripple_lfps = get_LFPs(tetrode_keys, ANIMALS).reindex(time)
ripple_filtered_lfps = pd.DataFrame(
np.stack([filter_ripple_band(
ripple_lfps.values[:, ind], sampling_frequency=1500)
for ind in np.arange(ripple_lfps.shape[1])], axis=1),
index=ripple_lfps.index)
ripple_times = Kay_ripple_detector(
time, ripple_lfps.values, speed.values, LFP_SAMPLING_FREQUENCY,
zscore_threshold=2.0, close_ripple_threshold=np.timedelta64(0, 'ms'),
minimum_duration=np.timedelta64(15, 'ms'))
ripple_times.index = ripple_times.index.rename('replay_number')
ripple_labels = get_labels(ripple_times, position_time)
is_ripple = ripple_labels > 0
ripple_times = ripple_times.assign(
duration=lambda df: (df.end_time - df.start_time).dt.total_seconds())
ripple_power = estimate_ripple_band_power(
ripple_lfps, LFP_SAMPLING_FREQUENCY)
interpolated_ripple_power = ripple_power.interpolate()
ripple_power_change = interpolated_ripple_power.transform(
lambda df: df / df.mean())
ripple_power_zscore = np.log(interpolated_ripple_power).transform(
lambda df: (df - df.mean()) / df.std())
return dict(ripple_times=ripple_times,
ripple_labels=ripple_labels,
ripple_filtered_lfps=ripple_filtered_lfps,
ripple_power=ripple_power,
ripple_lfps=ripple_lfps,
ripple_power_change=ripple_power_change,
ripple_power_zscore=ripple_power_zscore,
is_ripple=is_ripple)
def get_ripple_times(epoch_key, sampling_frequency=1500,
brain_areas=['CA1', 'CA2', 'CA3']):
position_info = (
get_interpolated_position_dataframe(epoch_key, ANIMALS)
.dropna(subset=['linear_position', 'speed']))
speed = position_info['speed']
time = position_info.index
tetrode_info = make_tetrode_dataframe(ANIMALS).xs(
epoch_key, drop_level=False)
if ~np.all(np.isnan(tetrode_info.validripple.astype(float))):
tetrode_keys = tetrode_info.loc[
(tetrode_info.validripple == 1)].index
else:
is_brain_areas = (
tetrode_info.area.astype(str).str.upper().isin(brain_areas))
tetrode_keys = tetrode_info.loc[is_brain_areas].index
ripple_lfps = get_LFPs(tetrode_keys, ANIMALS).reindex(time)
ripple_filtered_lfps = pd.DataFrame(
filter_ripple_band(np.asarray(ripple_lfps)),
index=ripple_lfps.index)
ripple_times = Kay_ripple_detector(
time, ripple_lfps.values, speed.values, sampling_frequency,
zscore_threshold=2.0, close_ripple_threshold=np.timedelta64(0, 'ms'),
minimum_duration=np.timedelta64(15, 'ms'))
ripple_consensus_trace = pd.DataFrame(
get_ripple_consensus_trace(
ripple_filtered_lfps, sampling_frequency),
index=ripple_filtered_lfps.index,
columns=['ripple_consensus_trace'])
ripple_consensus_trace_zscore = pd.DataFrame(
zscore(ripple_consensus_trace, nan_policy='omit'),
index=ripple_filtered_lfps.index,
columns=['ripple_consensus_trace_zscore'])
return (ripple_times, ripple_filtered_lfps, ripple_lfps,
ripple_consensus_trace_zscore)
def get_ripple_times(epoch_key,
sampling_frequency=1500,
brain_areas=BRAIN_AREAS):
position_info = (get_interpolated_position_dataframe(
epoch_key, ANIMALS).dropna(subset=['linear_position', 'speed']))
speed = position_info['speed']
time = position_info.index
tetrode_info = make_tetrode_dataframe(ANIMALS, epoch_key=epoch_key)
if ~np.all(np.isnan(tetrode_info.validripple.astype(float))):
tetrode_keys = tetrode_info.loc[(tetrode_info.validripple == 1)].index
else:
is_brain_areas = (
tetrode_info.area.astype(str).str.upper().isin(brain_areas))
tetrode_keys = tetrode_info.loc[is_brain_areas].index
lfps = get_LFPs(tetrode_keys, ANIMALS).reindex(time)
return Kay_ripple_detector(time,
lfps.values,
speed.values,
sampling_frequency,
zscore_threshold=2.0,
close_ripple_threshold=np.timedelta64(0, 'ms'),
minimum_duration=np.timedelta64(15, 'ms'))
def get_ripple_times2(epoch_key,
sampling_frequency=1500,
brain_areas=BRAIN_AREAS):
RIPPLE_BAND = (150, 250)
TIME_WINDOW_STEP = TIME_WINDOW_DURATION = 0.020
TIME_HALFBANDWIDTH_PRODUCT = 1
position_info = (get_interpolated_position_dataframe(
epoch_key, ANIMALS).dropna(subset=['linear_distance', 'linear_speed']))
time = position_info.index
tetrode_info = make_tetrode_dataframe(ANIMALS).xs(epoch_key,
drop_level=False)
if ~np.all(np.isnan(tetrode_info.validripple.astype(float))):
tetrode_keys = tetrode_info.loc[(tetrode_info.validripple == 1)].index
else:
is_brain_areas = (
tetrode_info.area.astype(str).str.upper().isin(brain_areas))
tetrode_keys = tetrode_info.loc[is_brain_areas].index
lfps = get_LFPs(tetrode_keys, ANIMALS).reindex(time)
multitaper_params = dict(
time_halfbandwidth_product=TIME_HALFBANDWIDTH_PRODUCT,
time_window_duration=TIME_WINDOW_DURATION,
time_window_step=TIME_WINDOW_STEP,
start_time=(time.values / np.timedelta64(1, 's')).min(),
)
df, model = detect_spectral_rhythm(time=time.values /
np.timedelta64(1, 's'),
lfps=lfps.values,
sampling_frequency=sampling_frequency,
multitaper_params=multitaper_params,
frequency_band=RIPPLE_BAND)
return df.is_spectral_rhythm
def get_adhoc_ripple(epoch_key, tetrode_info, position_time):
LFP_SAMPLING_FREQUENCY = 1500
if ~np.all(np.isnan(tetrode_info.validripple.astype(float))):
tetrode_keys = tetrode_info.loc[(tetrode_info.validripple == 1)].index
else:
is_brain_areas = (
tetrode_info.area.astype(str).str.upper().isin(BRAIN_AREAS))
tetrode_keys = tetrode_info.loc[is_brain_areas].index
ripple_lfps = get_LFPs(tetrode_keys, ANIMALS)
ripple_filtered_lfps = pd.DataFrame(filter_ripple_band(
np.asarray(ripple_lfps)),
index=ripple_lfps.index)
position_df = get_position_dataframe(epoch_key, ANIMALS, use_hmm=False)
new_index = pd.Index(np.unique(
np.concatenate((position_df.index, ripple_filtered_lfps.index))),
name='time')
position_df = (position_df.reindex(index=new_index).interpolate(
method='linear').reindex(index=ripple_filtered_lfps.index))
ripple_times = Kay_ripple_detector(
time=ripple_filtered_lfps.index,
filtered_lfps=ripple_filtered_lfps.values,
speed=position_df.speed.values,
sampling_frequency=LFP_SAMPLING_FREQUENCY,
zscore_threshold=2.0,
close_ripple_threshold=np.timedelta64(0, 'ms'),
minimum_duration=np.timedelta64(15, 'ms'))
ripple_times.index = ripple_times.index.rename('replay_number')
ripple_labels = get_labels(ripple_times, position_time)
is_ripple = ripple_labels > 0
ripple_times = ripple_times.assign(
duration=lambda df: (df.end_time - df.start_time).dt.total_seconds())
ripple_consensus_trace = pd.DataFrame(get_ripple_consensus_trace(
ripple_filtered_lfps, LFP_SAMPLING_FREQUENCY),
index=ripple_filtered_lfps.index,
columns=['ripple_consensus_trace'])
ripple_consensus_trace_zscore = pd.DataFrame(
zscore(ripple_consensus_trace, nan_policy='omit'),
index=ripple_filtered_lfps.index,
columns=['ripple_consensus_trace_zscore'])
instantaneous_ripple_power = np.full_like(ripple_filtered_lfps, np.nan)
not_null = np.all(pd.notnull(ripple_filtered_lfps), axis=1)
instantaneous_ripple_power[not_null] = get_envelope(
np.asarray(ripple_filtered_lfps)[not_null])**2
instantaneous_ripple_power_change = np.nanmedian(
instantaneous_ripple_power /
np.nanmean(instantaneous_ripple_power, axis=0),
axis=1)
instantaneous_ripple_power_change = pd.DataFrame(
instantaneous_ripple_power_change,
index=ripple_filtered_lfps.index,
columns=['instantaneous_ripple_power_change'])
return dict(
ripple_times=ripple_times,
ripple_labels=ripple_labels,
ripple_filtered_lfps=ripple_filtered_lfps,
ripple_consensus_trace=ripple_consensus_trace,
ripple_lfps=ripple_lfps,
ripple_consensus_trace_zscore=ripple_consensus_trace_zscore,
instantaneous_ripple_power_change=instantaneous_ripple_power_change,
is_ripple=is_ripple)
def get_adhoc_ripple(epoch_key, tetrode_info, position_time,
position_to_linearize):
LFP_SAMPLING_FREQUENCY = 1500
# Get speed in terms of the LFP time
time = get_trial_time(epoch_key, ANIMALS)
position_df = get_position_info(epoch_key, skip_linearization=True)
new_index = pd.Index(np.unique(np.concatenate((position_df.index, time))),
name='time')
position_df = (position_df.reindex(index=new_index).interpolate(
method='linear').reindex(index=time))
speed_feature = position_to_linearize[0].split('_')[0]
speed = position_df[f'{speed_feature}_vel']
# Load LFPs
tetrode_keys = tetrode_info.loc[tetrode_info.area.isin(['ca1R',
'ca1L'])].index
ripple_lfps = get_LFPs(tetrode_keys, ANIMALS)
# Get ripple filtered LFPs
ripple_filtered_lfps = pd.DataFrame(filter_ripple_band(
np.asarray(ripple_lfps)),
index=ripple_lfps.index)
# Get Ripple Times
ripple_times = Kay_ripple_detector(
time=ripple_filtered_lfps.index,
filtered_lfps=ripple_filtered_lfps.values,
speed=speed.values,
sampling_frequency=LFP_SAMPLING_FREQUENCY,
zscore_threshold=2.0,
close_ripple_threshold=np.timedelta64(0, 'ms'),
minimum_duration=np.timedelta64(15, 'ms'))
ripple_times.index = ripple_times.index.rename('replay_number')
ripple_labels = get_labels(ripple_times, position_time)
is_ripple = ripple_labels > 0
ripple_times = ripple_times.assign(
duration=lambda df: (df.end_time - df.start_time).dt.total_seconds())
# Estmate ripple band power and change
ripple_consensus_trace = pd.DataFrame(get_ripple_consensus_trace(
ripple_filtered_lfps, LFP_SAMPLING_FREQUENCY),
index=ripple_filtered_lfps.index,
columns=['ripple_consensus_trace'])
ripple_consensus_trace_zscore = pd.DataFrame(
zscore(ripple_consensus_trace, nan_policy='omit'),
index=ripple_filtered_lfps.index,
columns=['ripple_consensus_trace_zscore'])
instantaneous_ripple_power = np.full_like(ripple_filtered_lfps, np.nan)
not_null = np.all(pd.notnull(ripple_filtered_lfps), axis=1)
instantaneous_ripple_power[not_null] = get_envelope(
np.asarray(ripple_filtered_lfps)[not_null])**2
instantaneous_ripple_power_change = np.nanmedian(
instantaneous_ripple_power /
np.nanmean(instantaneous_ripple_power, axis=0),
axis=1)
instantaneous_ripple_power_change = pd.DataFrame(
instantaneous_ripple_power_change,
index=ripple_filtered_lfps.index,
columns=['instantaneous_ripple_power_change'])
return dict(
ripple_times=ripple_times,
ripple_labels=ripple_labels,
ripple_filtered_lfps=ripple_filtered_lfps,
ripple_consensus_trace=ripple_consensus_trace,
ripple_lfps=ripple_lfps,
ripple_consensus_trace_zscore=ripple_consensus_trace_zscore,
instantaneous_ripple_power_change=instantaneous_ripple_power_change,
is_ripple=is_ripple)
def load_data(epoch_key, brain_areas=['CA1', 'CA2', 'CA3']):
time = get_trial_time(epoch_key, ANIMALS)
time = (pd.Series(np.ones_like(time, dtype=np.float), index=time)
.resample('2ms').mean()
.index)
def _time_function(*args, **kwargs):
return time
logger.info('Loading position info...')
position_info = (
get_interpolated_position_dataframe(
epoch_key, ANIMALS, _time_function)
.dropna(subset=['linear_position', 'speed']))
time = position_info.index
tetrode_info = make_tetrode_dataframe(ANIMALS, epoch_key=epoch_key)
is_brain_areas = (
tetrode_info.area.astype(str).str.upper().isin(brain_areas))
tetrode_keys = tetrode_info.loc[is_brain_areas].index
lfps = get_LFPs(tetrode_keys, ANIMALS)
lfps = lfps.resample('2ms').mean().fillna(method='pad').reindex(time)
logger.info('Loading spikes...')
try:
neuron_info = make_neuron_dataframe(ANIMALS).xs(
epoch_key, drop_level=False)
neuron_info = neuron_info.loc[
(neuron_info.numspikes > 100) &
neuron_info.area.isin(brain_areas) &
(neuron_info.type == 'principal')]
spikes = get_all_spike_indicators(
neuron_info.index, ANIMALS, _time_function).reindex(time)
except KeyError:
spikes = None
logger.info('Loading multiunit...')
tetrode_info = tetrode_info.loc[is_brain_areas]
multiunit = (get_all_multiunit_indicators(
tetrode_info.index, ANIMALS, _time_function)
.reindex({'time': time}))
multiunit = multiunit.sel(features=MARKS)
multiunit_spikes = (np.any(~np.isnan(multiunit.values), axis=1)
).astype(np.float)
multiunit_firing_rate = pd.DataFrame(
get_multiunit_population_firing_rate(
multiunit_spikes, SAMPLING_FREQUENCY), index=time,
columns=['firing_rate'])
logger.info('Finding ripple times...')
(ripple_times, ripple_filtered_lfps, ripple_lfps,
ripple_consensus_trace_zscore) = get_ripple_times(epoch_key)
ripple_times = ripple_times.assign(
duration=lambda df: (df.end_time - df.start_time).dt.total_seconds())
return {
'position_info': position_info,
'ripple_times': ripple_times,
'spikes': spikes,
'multiunit': multiunit,
'lfps': lfps,
'tetrode_info': tetrode_info,
'ripple_filtered_lfps': ripple_filtered_lfps,
'ripple_lfps': ripple_lfps,
'ripple_consensus_trace_zscore': ripple_consensus_trace_zscore,
'multiunit_firing_rate': multiunit_firing_rate,
'sampling_frequency': SAMPLING_FREQUENCY,
}