def test_resample_ones():
chs = [2, 32, 100]
ts = [999, 1000, 1001, 5077]
rate = 200
fracs = [.5, 1, 1.5, 2]
for ch in chs:
for t in ts:
for fr in fracs:
X = np.ones((t, ch))
Xp = resample(X, rate * fr, rate)
assert_allclose(Xp, 1., atol=1e-3)
def test_pipeline():
"""Test that the NWB pipeline gives equal results to running preprocessing functions
by hand.
"""
num_channels = 64
duration = 10. # seconds
sample_rate = 10000. # Hz
new_sample_rate = 500. # hz
nwbfile, device, electrode_group, electrodes = generate_nwbfile()
neural_data = generate_synthetic_data(duration, num_channels, sample_rate)
ECoG_ts = ElectricalSeries('ECoG_data',
neural_data,
electrodes,
starting_time=0.,
rate=sample_rate)
nwbfile.add_acquisition(ECoG_ts)
electrical_series = nwbfile.acquisition['ECoG_data']
nwbfile.create_processing_module(name='preprocessing',
description='Preprocessing.')
# Resample
rs_data_nwb, rs_series = store_resample(
electrical_series, nwbfile.processing['preprocessing'],
new_sample_rate)
rs_data = resample(neural_data * 1e6, new_sample_rate, sample_rate)
assert_array_equal(rs_data_nwb, rs_data)
assert_array_equal(rs_series.data[:], rs_data)
# Linenoise and CAR
car_data_nwb, car_series = store_linenoise_notch_CAR(
rs_series, nwbfile.processing['preprocessing'])
nth_data = apply_linenoise_notch(rs_data, new_sample_rate)
car_data = subtract_CAR(nth_data)
assert_array_equal(car_data_nwb, car_data)
assert_array_equal(car_series.data[:], car_data)
# Wavelet transform
tf_data_nwb, tf_series = store_wavelet_transform(
car_series,
nwbfile.processing['preprocessing'],
filters='rat',
hg_only=True,
abs_only=False)
tf_data, _, _, _ = wavelet_transform(car_data,
new_sample_rate,
filters='rat',
hg_only=True)
assert_array_equal(tf_data_nwb, tf_data)
assert_array_equal(tf_series[0].data[:], abs(tf_data))
assert_array_equal(tf_series[1].data[:], np.angle(tf_data))
def resample(self, data):
# only resample if rate is not at nearest kHz
rate = self.hardware_rate
if (rate / 1000 % 1) > 0:
new_freq = (rate // 1000) * 1000
logger.info(f' - resampling from {rate} Hz to {new_freq} Hz')
new_data = resample(data, new_freq, rate)
self.resample_rate = new_freq
return new_data
else:
# no need to resample, already at nearest kHz
logger.info(' - already at integer kHz; no need to resample')
self.resample_flag = False
return data
def test_resample_low_freqs():
"""Resampling should now impact low frequencies.
"""
dt = 40. # seconds
rate = 400. # Hz
t = np.linspace(0, dt, int(dt * rate))
t = np.tile(t[:, np.newaxis], (1, 5))
freqs = np.linspace(1, 5.33, 20)
X = np.zeros_like(t)
for f in freqs:
X += np.sin(2 * np.pi * f * t)
new_rate = 211. # Hz
t = np.linspace(0, dt, int(dt * new_rate))
t = np.tile(t[:, np.newaxis], (1, 5))
X_new_rate = np.zeros_like(t)
for f in freqs:
X_new_rate += np.sin(2 * np.pi * f * t)
Xds = resample(X, new_rate, rate)
assert_allclose(cosine(Xds.ravel(), X_new_rate.ravel()), 0., atol=1e-3)
assert_allclose(X.mean(), Xds.mean(), atol=1e-3)
assert_allclose(X.std(), Xds.std(), atol=1e-3)
def store_wavelet_transform(elec_series, processing, filters='rat', hg_only=True, X_fft_h=None,
abs_only=True, npad=1000, post_resample_rate=None):
"""Apply a wavelet transform using a prespecified set of filters. Results are stored in the
NWB file as a `DecompositionSeries`.
Calculates the center frequencies and bandwidths for the wavelets and applies them along with
a heavyside function to the fft of the signal before performing an inverse fft. The center
frequencies and bandwidths are also stored in the NWB file.
Parameters
----------
elec_series : ElectricalSeries
ElectricalSeries to process.
processing : Processing module
NWB Processing module to save processed data.
filters : str (optional)
Which type of filters to use. Options are
'rat': center frequencies spanning 2-1200 Hz, constant Q, 54 bands
'human': center frequencies spanning 4-200 Hz, constant Q, 40 bands
'changlab': center frequencies spanning 4-200 Hz, variable Q, 40 bands
hg_only : bool
If True, only the amplitudes in the high gamma range [70-150 Hz] is computed.
X_fft_h : ndarray (n_time, n_channels)
Precomputed product of X_fft and heavyside.
abs_only : bool
If True, only the amplitude is stored.
npad : int
Padding to add to beginning and end of timeseries. Default 1000.
post_resample_rate : float
If not `None`, resample the computed wavelet amplitudes to this rate.
Returns
-------
X_wvlt : ndarray, complex
Complex wavelet coefficients.
series : list of DecompositionSeries
List of NWB objects.
"""
X = elec_series.data[:]
rate = elec_series.rate
X_wvlt, _, cfs, sds = wavelet_transform(X, rate, filters=filters, X_fft_h=X_fft_h,
hg_only=hg_only, npad=npad)
amplitude = abs(X_wvlt)
if post_resample_rate is not None:
amplitude = resample(amplitude, post_resample_rate, rate)
rate = post_resample_rate
elec_series_wvlt_amp = DecompositionSeries('wvlt_amp_' + elec_series.name,
abs(X_wvlt),
metric='amplitude',
source_timeseries=elec_series,
starting_time=elec_series.starting_time,
rate=rate,
description=('Wavlet: ' +
elec_series.description))
series = [elec_series_wvlt_amp]
if not abs_only:
if post_resample_rate is not None:
raise ValueError('Wavelet phase should not be resampled.')
elec_series_wvlt_phase = DecompositionSeries('wvlt_phase_' + elec_series.name,
np.angle(X_wvlt),
metric='phase',
source_timeseries=elec_series,
starting_time=elec_series.starting_time,
rate=rate,
description=('Wavlet: ' +
elec_series.description))
series.append(elec_series_wvlt_phase)
for es in series:
for ii, (cf, sd) in enumerate(zip(cfs, sds)):
es.add_band(band_name=str(ii), band_mean=cf,
band_stdev=sd, band_limits=(-1, -1))
processing.add(es)
return X_wvlt, series
def preprocess_raw_data(block_path, config):
"""
Takes raw data and runs:
1) CAR
2) notch filters
3) Downsampling
Parameters
----------
block_path : str
subject file path
config : dictionary
'referencing' - tuple specifying electrode referencing (type, options)
('CAR', N_channels_per_group)
('CMR', N_channels_per_group)
('bipolar', INCLUDE_OBLIQUE_NBHD)
'Notch' - Main frequency (Hz) for notch filters (default=60)
'Downsample' - Downsampling frequency (Hz, default= 400)
Returns
-------
Saves preprocessed signals (LFP) in the current NWB file.
Only if containers for these data do not exist in the current file.
"""
subj_path, block_name = os.path.split(block_path)
block_name = os.path.splitext(block_path)[0]
start = time.time()
with NWBHDF5IO(block_path, 'r+', load_namespaces=True) as io:
nwb = io.read()
# Storage of processed signals on NWB file ----------------------------
if 'ecephys' in nwb.processing:
ecephys_module = nwb.processing['ecephys']
else: # creates ecephys ProcessingModule
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.')
# Add module to NWB file
nwb.add_processing_module(ecephys_module)
print('Created ecephys')
# LFP: Downsampled and power line signal removed ----------------------
if 'LFP' in nwb.processing['ecephys'].data_interfaces:
warnings.warn(
'LFP data already exists in the nwb file. Skipping preprocessing.'
)
else: # creates LFP data interface container
lfp = LFP()
# Data source
source_list = [
acq for acq in nwb.acquisition.values()
if type(acq) == ElectricalSeries
]
assert len(source_list) == 1, (
'Not precisely one ElectricalSeries in acquisition!')
source = source_list[0]
nChannels = source.data.shape[1]
# Downsampling
if config['Downsample'] is not None:
print("Downsampling signals to " + str(config['Downsample']) +
" Hz.")
print("Please wait...")
start = time.time()
# Note: zero padding the signal to make the length
# a power of 2 won't help, since resample will further pad it
# (breaking the power of 2)
nBins = source.data.shape[0]
rate = config['Downsample']
# malloc
T = int(np.ceil(nBins * rate / source.rate))
X = np.zeros((source.data.shape[1], T))
# One channel at a time, to improve memory usage for long signals
for ch in np.arange(nChannels):
# 1e6 scaling helps with numerical accuracy
Xch = source.data[:, ch] * 1e6
X[ch, :] = resample(Xch, rate, source.rate)
print(
'Downsampling finished in {} seconds'.format(time.time() -
start))
else: # No downsample
rate = source.rate
X = source.data[()].T * 1e6
# re-reference the (scaled by 1e6!) data
electrodes = source.electrodes
if config['referencing'] is not None:
if config['referencing'][0] == 'CAR':
print(
"Computing and subtracting Common Average Reference in "
+ str(config['referencing'][1]) + " channel blocks.")
start = time.time()
X = subtract_CAR(X, b_size=config['referencing'][1])
print('CAR subtract time for {}: {} seconds'.format(
block_name,
time.time() - start))
elif config['referencing'][0] == 'bipolar':
X, bipolarTable, electrodes = get_bipolar_referenced_electrodes(
X, electrodes, rate, grid_step=1)
# add data interface for the metadata for saving
ecephys_module.add_data_interface(bipolarTable)
print('bipolarElectrodes stored for saving in ' +
block_path)
else:
print('UNRECOGNIZED REFERENCING SCHEME; ', end='')
print('SKIPPING REFERENCING!')
# Apply Notch filters
if config['Notch'] is not None:
print("Applying notch filtering of " + str(config['Notch']) +
" Hz")
# Note: zero padding the signal to make the length a power
# of 2 won't help, since notch filtering will further pad it
start = time.time()
for ch in np.arange(nChannels):
# NOTE: apply_linenoise_notch takes a signal that is
# (n_timePoints, n_channels). The documentation may be wrong
Xch = X[ch, :].reshape(-1, 1)
Xch = apply_linenoise_notch(Xch, rate)
X[ch, :] = Xch[:, 0]
print('Notch filter time for {}: {} seconds'.format(
block_name,
time.time() - start))
X = X.astype('float32') # signal (nChannels,nSamples)
X /= 1e6 # Scales signals back to volts
# Add preprocessed downsampled signals as an electrical_series
referencing = 'None' if config['referencing'] is None else config[
'referencing'][0]
notch = 'None' if config['Notch'] is None else str(config['Notch'])
downs = 'No' if config['Downsample'] is None else 'Yes'
config_comment = ('referencing:' + referencing + ', Notch:' +
notch + ', Downsampled:' + downs)
# create an electrical series for the LFP and store it in lfp
lfp.create_electrical_series(name='preprocessed',
data=X.T,
electrodes=electrodes,
rate=rate,
description='',
comments=config_comment)
ecephys_module.add_data_interface(lfp)
# Write LFP to NWB file
io.write(nwb)
print('LFP saved in ' + block_path)
def test_resample_shape():
X = np.random.randn(2000, 32)
Xp = resample(X, 100, 200)
assert Xp.shape == (1000, 32)
def detect_events(speaker_data,
mic_data=None,
interval=None,
dfact=30,
smooth_width=0.4,
speaker_threshold=0.05,
mic_threshold=0.05,
direction='both'):
"""
Automatically detects events in audio signals.
Parameters
----------
speaker_data : 'pynwb.base.TimeSeries' object
Object containing speaker data.
mic_data : 'pynwb.base.TimeSeries' object
Object containing microphone data.
interval : list of floats
Interval to be used [Start_bin, End_bin]. If 'None', the whole
signal is used.
dfact : float
Downsampling factor. Default 30.
smooth_width: float
Width scale for median smoothing filter (default = .4, decent for CVs).
speaker_threshold : float
Sets threshold level for speaker.
mic_threshold : float
Sets threshold level for mic.
direction : str
'Up' detects events start times. 'Down' detects events stop times.
'Both'
detects both start and stop times.
Returns
-------
speakerDS : 1D array of floats
Downsampled speaker signal.
speakerEventDS : 1D array of floats
Event times for speaker signal.
speakerFilt : 1D array of floats
Filtered speaker signal.
micDS : 1D array of floats
Downsampled microphone signal.
micEventDS : 1D array of floats
Event times for microphone signal.
micFilt : 1D array of floats
Filtered microphone signal.
"""
# Downsampling Speaker ---------------------------------------------------
speakerDS, speakerEventDS, speakerFilt = None, None, None
if speaker_data is not None:
if interval is None:
X = speaker_data.data[:]
else:
X = speaker_data.data[interval[0]:interval[1]]
fs = speaker_data.rate # sampling rate
ds = fs / dfact
# Pad zeros to make signal length a power of 2, improves performance
nBins = X.shape[0]
extraBins = 2**(np.ceil(np.log2(nBins)).astype('int')) - nBins
extraZeros = np.zeros(extraBins)
X = np.append(X, extraZeros)
speakerDS = resample(X, ds, fs)
# Remove excess bins (because of zero padding on previous step)
excessBins = int(np.ceil(extraBins * ds / fs))
speakerDS = speakerDS[0:-excessBins]
# Kernel size must be an odd number
speakerFilt = sgn.medfilt(
volume=np.diff(np.append(speakerDS, speakerDS[-1]))**2,
kernel_size=int((smooth_width * ds // 2) * 2 + 1))
# Normalize the filtered signal.
speakerFilt /= np.max(np.abs(speakerFilt))
# Find threshold crossing times
stimBinsDS = threshcross(speakerFilt, speaker_threshold, direction)
# Remove events that have a duration less than 0.1 s.
speaker_events = stimBinsDS.reshape((-1, 2))
rem_ind = np.where(
(speaker_events[:, 1] - speaker_events[:, 0]) < ds * 0.1)[0]
speaker_events = np.delete(speaker_events, rem_ind, axis=0)
stimBinsDS = speaker_events.reshape((-1))
# Transform bins to time
if interval is None:
speakerEventDS = (stimBinsDS / ds)
else:
speakerEventDS = (stimBinsDS / ds) + (interval[0] / fs)
# Downsampling Mic -------------------------------------------------------
micDS, micEventDS, micFilt = None, None, None
if mic_data is not None:
if interval is None:
X = mic_data.data[:]
else:
X = mic_data.data[interval[0]:interval[1]]
fs = mic_data.rate # sampling rate
ds = fs / dfact
# Pad zeros to make signal length a power of 2, improves performance
nBins = X.shape[0]
extraBins = 2**(np.ceil(np.log2(nBins)).astype('int')) - nBins
extraZeros = np.zeros(extraBins)
X = np.append(X, extraZeros)
micDS = resample(X, ds, fs)
# Remove excess bins (because of zero padding on previous step)
excessBins = int(np.ceil(extraBins * ds / fs))
micDS = micDS[0:-excessBins]
# Remove mic response to speaker
micDS[np.where(speakerFilt > speaker_threshold)[0]] = 0
micFilt = sgn.medfilt(volume=np.diff(np.append(micDS, micDS[-1]))**2,
kernel_size=int((smooth_width * ds // 2) * 2 +
1))
# Normalize the filtered signal.
micFilt /= np.max(np.abs(micFilt))
# Find threshold crossing times
micBinsDS = threshcross(micFilt, mic_threshold, direction)
# Remove events that have a duration less than 0.1 s.
mic_events = micBinsDS.reshape((-1, 2))
rem_ind = np.where((mic_events[:, 1] - mic_events[:, 0]) < ds * 0.1)[0]
mic_events = np.delete(mic_events, rem_ind, axis=0)
micBinsDS = mic_events.reshape((-1))
# Transform bins to time
if interval is None:
micEventDS = (micBinsDS / ds)
else:
micEventDS = (micBinsDS / ds) + (interval[0] / fs)
return speakerDS, speakerEventDS, speakerFilt, micDS, micEventDS, micFilt
plt.xlabel('Time (s)')
plt.ylabel('Amplitude (au)')
_ = plt.title('One channel of neural data')
# %%
# Resampling the data
# -------------------
# Often times, the raw data (voltages) are recorded at a much higher sampling
# rate than is needed for a particular analysis. Here, the raw data is sampled
# at 10,000 Hz but the high gamma range only goes up to 150 Hz. Resampling the
# will make many downstream computations much faster.
# %%
#
rs_data = resample(neural_data, new_sample_rate, sample_rate)
t = np.linspace(0, duration, rs_data.shape[0])
plt.plot(t[:500], rs_data[:500, 0])
plt.xlabel('Time (s)')
plt.ylabel('Amplitude (au)')
_ = plt.title('One channel of neural data after resampling')
# %%
# Notch filtering
# ----------------
# Notch filtering is used to remove the 60 Hz line noise and harmonics on all
# channels.
nth_data = apply_linenoise_notch(rs_data, new_sample_rate)