def detect_epoch_ripples(epoch_key, animals, sampling_frequency, brain_areas=_BRAIN_AREAS): '''Returns a list of tuples containing the start and end times of ripples. Candidate ripples are computed via the ripple detection function and then filtered to exclude ripples where the animal was still moving. ''' logger.info('Detecting ripples') tetrode_info = make_tetrode_dataframe(animals).xs(epoch_key, drop_level=False) brain_areas = [brain_areas] if isinstance(brain_areas, str) else brain_areas is_brain_areas = tetrode_info.area.isin(brain_areas) if 'CA1' in brain_areas: is_brain_areas = is_brain_areas & (tetrode_info.descrip.isin( ['riptet']) | tetrode_info.validripple) logger.debug( tetrode_info[is_brain_areas].loc[:, ['area', 'depth', 'descrip']]) tetrode_keys = tetrode_info[is_brain_areas].index.tolist() LFPs = get_LFPs(tetrode_keys, animals) speed = get_interpolated_position_dataframe(epoch_key, animals, max_distance_from_well=5).speed not_null = np.any(pd.notnull(LFPs), axis=1) & pd.notnull(speed) return Kay_ripple_detector(LFPs.index[not_null], LFPs.values[not_null], speed.values[not_null], sampling_frequency, minimum_duration=pd.Timedelta(milliseconds=15), zscore_threshold=3)
def estimate_theta_power(time, tetrode_info, multitaper_params=None): if multitaper_params is None: multitaper_params = MULTITAPER_PARAMETERS['4Hz'] is_brain_areas = ( tetrode_info.area.astype(str).str.upper().isin(BRAIN_AREAS)) tetrode_keys = tetrode_info.loc[is_brain_areas].index lfps = get_LFPs(tetrode_keys, ANIMALS) m = Multitaper(lfps.values, **multitaper_params, start_time=lfps.index[0].total_seconds()) c = Connectivity.from_multitaper(m) coordinates = { 'time': pd.to_timedelta(c.time, unit='s'), 'frequency': c.frequencies, 'tetrode': lfps.columns, } dimension_names = ['time', 'frequency', 'tetrode'] data_vars = {'power': (dimension_names, c.power())} theta_power = (xr.Dataset(data_vars, coords=coordinates).sel( frequency=slice(4, 12)).mean('frequency').dropna('tetrode').interp( time=time).to_dataframe().unstack(level=0)) theta_power_change = theta_power.transform(lambda df: df / df.mean()) theta_power_zscore = theta_power.transform(lambda df: (df - df.mean()) / df.std()) return dict( theta_power=theta_power, theta_power_change=theta_power_change, theta_power_zscore=theta_power_zscore, )
def get_theta_times(epoch_key, sampling_frequency=1500): THETA_BAND = (6, 12) TIME_WINDOW_STEP = TIME_WINDOW_DURATION = 0.300 TIME_HALFBANDWIDTH_PRODUCT = 1 position_info = (get_interpolated_position_dataframe( epoch_key, ANIMALS).dropna(subset=['linear_position', 'speed'])) time = position_info.index tetrode_info = make_tetrode_dataframe(ANIMALS, epoch_key=epoch_key) tetrode_keys = tetrode_info.loc[tetrode_info.area == 'Reference'].index lfps = get_LFPs(tetrode_keys, ANIMALS).reindex(time) multitaper_params = dict( time_halfbandwidth_product=TIME_HALFBANDWIDTH_PRODUCT, time_window_duration=TIME_WINDOW_DURATION, time_window_step=TIME_WINDOW_STEP, start_time=(time.values / np.timedelta64(1, 's')).min(), ) df, model = detect_spectral_rhythm(time=time.values / np.timedelta64(1, 's'), lfps=lfps.values, sampling_frequency=sampling_frequency, multitaper_params=multitaper_params, frequency_band=THETA_BAND) return df.is_spectral_rhythm
def load_data(epoch_key, brain_areas=None): if brain_areas is None: brain_areas = BRAIN_AREAS time = get_trial_time(epoch_key, ANIMALS) time = (pd.Series(np.ones_like(time, dtype=np.float), index=time).resample('2ms').mean().index) def _time_function(*args, **kwargs): return time position_info = (get_interpolated_position_dataframe( epoch_key, ANIMALS, _time_function).dropna(subset=['linear_position', 'speed'])) time = position_info.index tetrode_info = make_tetrode_dataframe(ANIMALS, epoch_key=epoch_key) is_brain_areas = ( tetrode_info.area.astype(str).str.upper().isin(brain_areas)) tetrode_keys = tetrode_info.loc[is_brain_areas].index lfps = get_LFPs(tetrode_keys, ANIMALS) lfps = lfps.resample('2ms').mean().fillna(method='pad').reindex(time) try: neuron_info = make_neuron_dataframe(ANIMALS).xs(epoch_key, drop_level=False) neuron_info = neuron_info.loc[(neuron_info.numspikes > 100) & neuron_info.area.isin(brain_areas) & (neuron_info.type == 'principal')] spikes = get_all_spike_indicators(neuron_info.index, ANIMALS, _time_function).reindex(time) except KeyError: spikes = None tetrode_info = tetrode_info.loc[is_brain_areas] multiunit = (get_all_multiunit_indicators( tetrode_info.index, ANIMALS, _time_function).sel(features=_MARKS).reindex({'time': time})) multiunit_spikes = (np.any(~np.isnan(multiunit.values), axis=1)).astype(np.float) multiunit_firing_rate = pd.DataFrame(get_multiunit_population_firing_rate( multiunit_spikes, SAMPLING_FREQUENCY, smoothing_sigma=0.020), index=time, columns=['firing_rate']) return { 'position_info': position_info, 'spikes': spikes, 'multiunit': multiunit, 'lfps': lfps, 'tetrode_info': tetrode_info, 'multiunit_firing_rate': multiunit_firing_rate, 'sampling_frequency': SAMPLING_FREQUENCY, }
def _load_lfp_from_filterframework(ntrode_keys, animal_dict): print('loading lfp from ff') print(ntrode_keys) lfp = lfdp.get_LFPs(ntrode_keys, animal_dict) # broadcast indices from cols and reshape df to be: (an,day,ep,td):(ntrode) lfp_mi_list = [(str(a), int(d), int(e), int(n)) for col in lfp.columns.tolist() for (a, d, e, n) in [col.split('_')]] lfp_mi = pd.MultiIndex.from_tuples(lfp_mi_list, names=['animal', 'day', 'epoch', 'ntrode']) df = pd.DataFrame(lfp.T.values, index=lfp_mi).T #grab lfp values to avoid index merging df['timedelta'] = lfp.index df = df.set_index('timedelta').stack(level=['animal', 'day', 'epoch']).reorder_levels(['animal', 'day', 'epoch', 'timedelta']) return df
def get_adhoc_ripple(epoch_key, tetrode_info, position_time): LFP_SAMPLING_FREQUENCY = 1500 position_info = ( get_interpolated_position_dataframe(epoch_key, ANIMALS) .dropna(subset=['linear_position', 'speed'])) speed = position_info['speed'] time = position_info.index if ~np.all(np.isnan(tetrode_info.validripple.astype(float))): tetrode_keys = tetrode_info.loc[ (tetrode_info.validripple == 1)].index else: is_brain_areas = ( tetrode_info.area.astype(str).str.upper().isin(BRAIN_AREAS)) tetrode_keys = tetrode_info.loc[is_brain_areas].index ripple_lfps = get_LFPs(tetrode_keys, ANIMALS).reindex(time) ripple_filtered_lfps = pd.DataFrame( np.stack([filter_ripple_band( ripple_lfps.values[:, ind], sampling_frequency=1500) for ind in np.arange(ripple_lfps.shape[1])], axis=1), index=ripple_lfps.index) ripple_times = Kay_ripple_detector( time, ripple_lfps.values, speed.values, LFP_SAMPLING_FREQUENCY, zscore_threshold=2.0, close_ripple_threshold=np.timedelta64(0, 'ms'), minimum_duration=np.timedelta64(15, 'ms')) ripple_times.index = ripple_times.index.rename('replay_number') ripple_labels = get_labels(ripple_times, position_time) is_ripple = ripple_labels > 0 ripple_times = ripple_times.assign( duration=lambda df: (df.end_time - df.start_time).dt.total_seconds()) ripple_power = estimate_ripple_band_power( ripple_lfps, LFP_SAMPLING_FREQUENCY) interpolated_ripple_power = ripple_power.interpolate() ripple_power_change = interpolated_ripple_power.transform( lambda df: df / df.mean()) ripple_power_zscore = np.log(interpolated_ripple_power).transform( lambda df: (df - df.mean()) / df.std()) return dict(ripple_times=ripple_times, ripple_labels=ripple_labels, ripple_filtered_lfps=ripple_filtered_lfps, ripple_power=ripple_power, ripple_lfps=ripple_lfps, ripple_power_change=ripple_power_change, ripple_power_zscore=ripple_power_zscore, is_ripple=is_ripple)
def get_ripple_times(epoch_key, sampling_frequency=1500, brain_areas=['CA1', 'CA2', 'CA3']): position_info = ( get_interpolated_position_dataframe(epoch_key, ANIMALS) .dropna(subset=['linear_position', 'speed'])) speed = position_info['speed'] time = position_info.index tetrode_info = make_tetrode_dataframe(ANIMALS).xs( epoch_key, drop_level=False) if ~np.all(np.isnan(tetrode_info.validripple.astype(float))): tetrode_keys = tetrode_info.loc[ (tetrode_info.validripple == 1)].index else: is_brain_areas = ( tetrode_info.area.astype(str).str.upper().isin(brain_areas)) tetrode_keys = tetrode_info.loc[is_brain_areas].index ripple_lfps = get_LFPs(tetrode_keys, ANIMALS).reindex(time) ripple_filtered_lfps = pd.DataFrame( filter_ripple_band(np.asarray(ripple_lfps)), index=ripple_lfps.index) ripple_times = Kay_ripple_detector( time, ripple_lfps.values, speed.values, sampling_frequency, zscore_threshold=2.0, close_ripple_threshold=np.timedelta64(0, 'ms'), minimum_duration=np.timedelta64(15, 'ms')) ripple_consensus_trace = pd.DataFrame( get_ripple_consensus_trace( ripple_filtered_lfps, sampling_frequency), index=ripple_filtered_lfps.index, columns=['ripple_consensus_trace']) ripple_consensus_trace_zscore = pd.DataFrame( zscore(ripple_consensus_trace, nan_policy='omit'), index=ripple_filtered_lfps.index, columns=['ripple_consensus_trace_zscore']) return (ripple_times, ripple_filtered_lfps, ripple_lfps, ripple_consensus_trace_zscore)
def get_ripple_times(epoch_key, sampling_frequency=1500, brain_areas=BRAIN_AREAS): position_info = (get_interpolated_position_dataframe( epoch_key, ANIMALS).dropna(subset=['linear_position', 'speed'])) speed = position_info['speed'] time = position_info.index tetrode_info = make_tetrode_dataframe(ANIMALS, epoch_key=epoch_key) if ~np.all(np.isnan(tetrode_info.validripple.astype(float))): tetrode_keys = tetrode_info.loc[(tetrode_info.validripple == 1)].index else: is_brain_areas = ( tetrode_info.area.astype(str).str.upper().isin(brain_areas)) tetrode_keys = tetrode_info.loc[is_brain_areas].index lfps = get_LFPs(tetrode_keys, ANIMALS).reindex(time) return Kay_ripple_detector(time, lfps.values, speed.values, sampling_frequency, zscore_threshold=2.0, close_ripple_threshold=np.timedelta64(0, 'ms'), minimum_duration=np.timedelta64(15, 'ms'))
def get_ripple_times2(epoch_key, sampling_frequency=1500, brain_areas=BRAIN_AREAS): RIPPLE_BAND = (150, 250) TIME_WINDOW_STEP = TIME_WINDOW_DURATION = 0.020 TIME_HALFBANDWIDTH_PRODUCT = 1 position_info = (get_interpolated_position_dataframe( epoch_key, ANIMALS).dropna(subset=['linear_distance', 'linear_speed'])) time = position_info.index tetrode_info = make_tetrode_dataframe(ANIMALS).xs(epoch_key, drop_level=False) if ~np.all(np.isnan(tetrode_info.validripple.astype(float))): tetrode_keys = tetrode_info.loc[(tetrode_info.validripple == 1)].index else: is_brain_areas = ( tetrode_info.area.astype(str).str.upper().isin(brain_areas)) tetrode_keys = tetrode_info.loc[is_brain_areas].index lfps = get_LFPs(tetrode_keys, ANIMALS).reindex(time) multitaper_params = dict( time_halfbandwidth_product=TIME_HALFBANDWIDTH_PRODUCT, time_window_duration=TIME_WINDOW_DURATION, time_window_step=TIME_WINDOW_STEP, start_time=(time.values / np.timedelta64(1, 's')).min(), ) df, model = detect_spectral_rhythm(time=time.values / np.timedelta64(1, 's'), lfps=lfps.values, sampling_frequency=sampling_frequency, multitaper_params=multitaper_params, frequency_band=RIPPLE_BAND) return df.is_spectral_rhythm
def get_adhoc_ripple(epoch_key, tetrode_info, position_time): LFP_SAMPLING_FREQUENCY = 1500 if ~np.all(np.isnan(tetrode_info.validripple.astype(float))): tetrode_keys = tetrode_info.loc[(tetrode_info.validripple == 1)].index else: is_brain_areas = ( tetrode_info.area.astype(str).str.upper().isin(BRAIN_AREAS)) tetrode_keys = tetrode_info.loc[is_brain_areas].index ripple_lfps = get_LFPs(tetrode_keys, ANIMALS) ripple_filtered_lfps = pd.DataFrame(filter_ripple_band( np.asarray(ripple_lfps)), index=ripple_lfps.index) position_df = get_position_dataframe(epoch_key, ANIMALS, use_hmm=False) new_index = pd.Index(np.unique( np.concatenate((position_df.index, ripple_filtered_lfps.index))), name='time') position_df = (position_df.reindex(index=new_index).interpolate( method='linear').reindex(index=ripple_filtered_lfps.index)) ripple_times = Kay_ripple_detector( time=ripple_filtered_lfps.index, filtered_lfps=ripple_filtered_lfps.values, speed=position_df.speed.values, sampling_frequency=LFP_SAMPLING_FREQUENCY, zscore_threshold=2.0, close_ripple_threshold=np.timedelta64(0, 'ms'), minimum_duration=np.timedelta64(15, 'ms')) ripple_times.index = ripple_times.index.rename('replay_number') ripple_labels = get_labels(ripple_times, position_time) is_ripple = ripple_labels > 0 ripple_times = ripple_times.assign( duration=lambda df: (df.end_time - df.start_time).dt.total_seconds()) ripple_consensus_trace = pd.DataFrame(get_ripple_consensus_trace( ripple_filtered_lfps, LFP_SAMPLING_FREQUENCY), index=ripple_filtered_lfps.index, columns=['ripple_consensus_trace']) ripple_consensus_trace_zscore = pd.DataFrame( zscore(ripple_consensus_trace, nan_policy='omit'), index=ripple_filtered_lfps.index, columns=['ripple_consensus_trace_zscore']) instantaneous_ripple_power = np.full_like(ripple_filtered_lfps, np.nan) not_null = np.all(pd.notnull(ripple_filtered_lfps), axis=1) instantaneous_ripple_power[not_null] = get_envelope( np.asarray(ripple_filtered_lfps)[not_null])**2 instantaneous_ripple_power_change = np.nanmedian( instantaneous_ripple_power / np.nanmean(instantaneous_ripple_power, axis=0), axis=1) instantaneous_ripple_power_change = pd.DataFrame( instantaneous_ripple_power_change, index=ripple_filtered_lfps.index, columns=['instantaneous_ripple_power_change']) return dict( ripple_times=ripple_times, ripple_labels=ripple_labels, ripple_filtered_lfps=ripple_filtered_lfps, ripple_consensus_trace=ripple_consensus_trace, ripple_lfps=ripple_lfps, ripple_consensus_trace_zscore=ripple_consensus_trace_zscore, instantaneous_ripple_power_change=instantaneous_ripple_power_change, is_ripple=is_ripple)
def get_adhoc_ripple(epoch_key, tetrode_info, position_time, position_to_linearize): LFP_SAMPLING_FREQUENCY = 1500 # Get speed in terms of the LFP time time = get_trial_time(epoch_key, ANIMALS) position_df = get_position_info(epoch_key, skip_linearization=True) new_index = pd.Index(np.unique(np.concatenate((position_df.index, time))), name='time') position_df = (position_df.reindex(index=new_index).interpolate( method='linear').reindex(index=time)) speed_feature = position_to_linearize[0].split('_')[0] speed = position_df[f'{speed_feature}_vel'] # Load LFPs tetrode_keys = tetrode_info.loc[tetrode_info.area.isin(['ca1R', 'ca1L'])].index ripple_lfps = get_LFPs(tetrode_keys, ANIMALS) # Get ripple filtered LFPs ripple_filtered_lfps = pd.DataFrame(filter_ripple_band( np.asarray(ripple_lfps)), index=ripple_lfps.index) # Get Ripple Times ripple_times = Kay_ripple_detector( time=ripple_filtered_lfps.index, filtered_lfps=ripple_filtered_lfps.values, speed=speed.values, sampling_frequency=LFP_SAMPLING_FREQUENCY, zscore_threshold=2.0, close_ripple_threshold=np.timedelta64(0, 'ms'), minimum_duration=np.timedelta64(15, 'ms')) ripple_times.index = ripple_times.index.rename('replay_number') ripple_labels = get_labels(ripple_times, position_time) is_ripple = ripple_labels > 0 ripple_times = ripple_times.assign( duration=lambda df: (df.end_time - df.start_time).dt.total_seconds()) # Estmate ripple band power and change ripple_consensus_trace = pd.DataFrame(get_ripple_consensus_trace( ripple_filtered_lfps, LFP_SAMPLING_FREQUENCY), index=ripple_filtered_lfps.index, columns=['ripple_consensus_trace']) ripple_consensus_trace_zscore = pd.DataFrame( zscore(ripple_consensus_trace, nan_policy='omit'), index=ripple_filtered_lfps.index, columns=['ripple_consensus_trace_zscore']) instantaneous_ripple_power = np.full_like(ripple_filtered_lfps, np.nan) not_null = np.all(pd.notnull(ripple_filtered_lfps), axis=1) instantaneous_ripple_power[not_null] = get_envelope( np.asarray(ripple_filtered_lfps)[not_null])**2 instantaneous_ripple_power_change = np.nanmedian( instantaneous_ripple_power / np.nanmean(instantaneous_ripple_power, axis=0), axis=1) instantaneous_ripple_power_change = pd.DataFrame( instantaneous_ripple_power_change, index=ripple_filtered_lfps.index, columns=['instantaneous_ripple_power_change']) return dict( ripple_times=ripple_times, ripple_labels=ripple_labels, ripple_filtered_lfps=ripple_filtered_lfps, ripple_consensus_trace=ripple_consensus_trace, ripple_lfps=ripple_lfps, ripple_consensus_trace_zscore=ripple_consensus_trace_zscore, instantaneous_ripple_power_change=instantaneous_ripple_power_change, is_ripple=is_ripple)
def load_data(epoch_key, brain_areas=['CA1', 'CA2', 'CA3']): time = get_trial_time(epoch_key, ANIMALS) time = (pd.Series(np.ones_like(time, dtype=np.float), index=time) .resample('2ms').mean() .index) def _time_function(*args, **kwargs): return time logger.info('Loading position info...') position_info = ( get_interpolated_position_dataframe( epoch_key, ANIMALS, _time_function) .dropna(subset=['linear_position', 'speed'])) time = position_info.index tetrode_info = make_tetrode_dataframe(ANIMALS, epoch_key=epoch_key) is_brain_areas = ( tetrode_info.area.astype(str).str.upper().isin(brain_areas)) tetrode_keys = tetrode_info.loc[is_brain_areas].index lfps = get_LFPs(tetrode_keys, ANIMALS) lfps = lfps.resample('2ms').mean().fillna(method='pad').reindex(time) logger.info('Loading spikes...') try: neuron_info = make_neuron_dataframe(ANIMALS).xs( epoch_key, drop_level=False) neuron_info = neuron_info.loc[ (neuron_info.numspikes > 100) & neuron_info.area.isin(brain_areas) & (neuron_info.type == 'principal')] spikes = get_all_spike_indicators( neuron_info.index, ANIMALS, _time_function).reindex(time) except KeyError: spikes = None logger.info('Loading multiunit...') tetrode_info = tetrode_info.loc[is_brain_areas] multiunit = (get_all_multiunit_indicators( tetrode_info.index, ANIMALS, _time_function) .reindex({'time': time})) multiunit = multiunit.sel(features=MARKS) multiunit_spikes = (np.any(~np.isnan(multiunit.values), axis=1) ).astype(np.float) multiunit_firing_rate = pd.DataFrame( get_multiunit_population_firing_rate( multiunit_spikes, SAMPLING_FREQUENCY), index=time, columns=['firing_rate']) logger.info('Finding ripple times...') (ripple_times, ripple_filtered_lfps, ripple_lfps, ripple_consensus_trace_zscore) = get_ripple_times(epoch_key) ripple_times = ripple_times.assign( duration=lambda df: (df.end_time - df.start_time).dt.total_seconds()) return { 'position_info': position_info, 'ripple_times': ripple_times, 'spikes': spikes, 'multiunit': multiunit, 'lfps': lfps, 'tetrode_info': tetrode_info, 'ripple_filtered_lfps': ripple_filtered_lfps, 'ripple_lfps': ripple_lfps, 'ripple_consensus_trace_zscore': ripple_consensus_trace_zscore, 'multiunit_firing_rate': multiunit_firing_rate, 'sampling_frequency': SAMPLING_FREQUENCY, }