def test_frequency_specificity():
"""Test that multiples of 60 Hz are removed and other frequencies are not
highly filtered.
"""
dt = 50. # seconds
rate = 400. # Hz
t = np.linspace(0, dt, int(dt * rate))
t = np.tile(t[:, np.newaxis], (1, 5))
n_harmonics = int((rate / 2.) // 60)
X = np.zeros_like(t)
for ii in range(n_harmonics):
X += np.sin(2 * np.pi * (ii + 1) * 60. * t)
Xp = apply_linenoise_notch(X, rate)
assert np.linalg.norm(Xp) < np.linalg.norm(X) / 1000.
# Offset signals by 2 Hz
X = np.zeros_like(t)
for ii in range(n_harmonics):
X += np.sin(2 * np.pi * (ii + 1) * (60. + 2) * t)
Xp = apply_linenoise_notch(X, rate)
assert np.allclose(np.linalg.norm(Xp), np.linalg.norm(X))
def test_linenoise_notch_return():
"""Test the return shape.
"""
X = np.random.randn(1000, 32)
rate = 200
Xh = apply_linenoise_notch(X, rate)
assert Xh.shape == X.shape
def test_pipeline():
"""Test that the NWB pipeline gives equal results to running preprocessing functions
by hand.
"""
num_channels = 64
duration = 10. # seconds
sample_rate = 10000. # Hz
new_sample_rate = 500. # hz
nwbfile, device, electrode_group, electrodes = generate_nwbfile()
neural_data = generate_synthetic_data(duration, num_channels, sample_rate)
ECoG_ts = ElectricalSeries('ECoG_data',
neural_data,
electrodes,
starting_time=0.,
rate=sample_rate)
nwbfile.add_acquisition(ECoG_ts)
electrical_series = nwbfile.acquisition['ECoG_data']
nwbfile.create_processing_module(name='preprocessing',
description='Preprocessing.')
# Resample
rs_data_nwb, rs_series = store_resample(
electrical_series, nwbfile.processing['preprocessing'],
new_sample_rate)
rs_data = resample(neural_data * 1e6, new_sample_rate, sample_rate)
assert_array_equal(rs_data_nwb, rs_data)
assert_array_equal(rs_series.data[:], rs_data)
# Linenoise and CAR
car_data_nwb, car_series = store_linenoise_notch_CAR(
rs_series, nwbfile.processing['preprocessing'])
nth_data = apply_linenoise_notch(rs_data, new_sample_rate)
car_data = subtract_CAR(nth_data)
assert_array_equal(car_data_nwb, car_data)
assert_array_equal(car_series.data[:], car_data)
# Wavelet transform
tf_data_nwb, tf_series = store_wavelet_transform(
car_series,
nwbfile.processing['preprocessing'],
filters='rat',
hg_only=True,
abs_only=False)
tf_data, _, _, _ = wavelet_transform(car_data,
new_sample_rate,
filters='rat',
hg_only=True)
assert_array_equal(tf_data_nwb, tf_data)
assert_array_equal(tf_series[0].data[:], abs(tf_data))
assert_array_equal(tf_series[1].data[:], np.angle(tf_data))
def preprocess_raw_data(block_path, config):
"""
Takes raw data and runs:
1) CAR
2) notch filters
3) Downsampling
Parameters
----------
block_path : str
subject file path
config : dictionary
'referencing' - tuple specifying electrode referencing (type, options)
('CAR', N_channels_per_group)
('CMR', N_channels_per_group)
('bipolar', INCLUDE_OBLIQUE_NBHD)
'Notch' - Main frequency (Hz) for notch filters (default=60)
'Downsample' - Downsampling frequency (Hz, default= 400)
Returns
-------
Saves preprocessed signals (LFP) in the current NWB file.
Only if containers for these data do not exist in the current file.
"""
subj_path, block_name = os.path.split(block_path)
block_name = os.path.splitext(block_path)[0]
start = time.time()
with NWBHDF5IO(block_path, 'r+', load_namespaces=True) as io:
nwb = io.read()
# Storage of processed signals on NWB file ----------------------------
if 'ecephys' in nwb.processing:
ecephys_module = nwb.processing['ecephys']
else: # creates ecephys ProcessingModule
ecephys_module = ProcessingModule(
name='ecephys',
description='Extracellular electrophysiology data.')
# Add module to NWB file
nwb.add_processing_module(ecephys_module)
print('Created ecephys')
# LFP: Downsampled and power line signal removed ----------------------
if 'LFP' in nwb.processing['ecephys'].data_interfaces:
warnings.warn(
'LFP data already exists in the nwb file. Skipping preprocessing.'
)
else: # creates LFP data interface container
lfp = LFP()
# Data source
source_list = [
acq for acq in nwb.acquisition.values()
if type(acq) == ElectricalSeries
]
assert len(source_list) == 1, (
'Not precisely one ElectricalSeries in acquisition!')
source = source_list[0]
nChannels = source.data.shape[1]
# Downsampling
if config['Downsample'] is not None:
print("Downsampling signals to " + str(config['Downsample']) +
" Hz.")
print("Please wait...")
start = time.time()
# Note: zero padding the signal to make the length
# a power of 2 won't help, since resample will further pad it
# (breaking the power of 2)
nBins = source.data.shape[0]
rate = config['Downsample']
# malloc
T = int(np.ceil(nBins * rate / source.rate))
X = np.zeros((source.data.shape[1], T))
# One channel at a time, to improve memory usage for long signals
for ch in np.arange(nChannels):
# 1e6 scaling helps with numerical accuracy
Xch = source.data[:, ch] * 1e6
X[ch, :] = resample(Xch, rate, source.rate)
print(
'Downsampling finished in {} seconds'.format(time.time() -
start))
else: # No downsample
rate = source.rate
X = source.data[()].T * 1e6
# re-reference the (scaled by 1e6!) data
electrodes = source.electrodes
if config['referencing'] is not None:
if config['referencing'][0] == 'CAR':
print(
"Computing and subtracting Common Average Reference in "
+ str(config['referencing'][1]) + " channel blocks.")
start = time.time()
X = subtract_CAR(X, b_size=config['referencing'][1])
print('CAR subtract time for {}: {} seconds'.format(
block_name,
time.time() - start))
elif config['referencing'][0] == 'bipolar':
X, bipolarTable, electrodes = get_bipolar_referenced_electrodes(
X, electrodes, rate, grid_step=1)
# add data interface for the metadata for saving
ecephys_module.add_data_interface(bipolarTable)
print('bipolarElectrodes stored for saving in ' +
block_path)
else:
print('UNRECOGNIZED REFERENCING SCHEME; ', end='')
print('SKIPPING REFERENCING!')
# Apply Notch filters
if config['Notch'] is not None:
print("Applying notch filtering of " + str(config['Notch']) +
" Hz")
# Note: zero padding the signal to make the length a power
# of 2 won't help, since notch filtering will further pad it
start = time.time()
for ch in np.arange(nChannels):
# NOTE: apply_linenoise_notch takes a signal that is
# (n_timePoints, n_channels). The documentation may be wrong
Xch = X[ch, :].reshape(-1, 1)
Xch = apply_linenoise_notch(Xch, rate)
X[ch, :] = Xch[:, 0]
print('Notch filter time for {}: {} seconds'.format(
block_name,
time.time() - start))
X = X.astype('float32') # signal (nChannels,nSamples)
X /= 1e6 # Scales signals back to volts
# Add preprocessed downsampled signals as an electrical_series
referencing = 'None' if config['referencing'] is None else config[
'referencing'][0]
notch = 'None' if config['Notch'] is None else str(config['Notch'])
downs = 'No' if config['Downsample'] is None else 'Yes'
config_comment = ('referencing:' + referencing + ', Notch:' +
notch + ', Downsampled:' + downs)
# create an electrical series for the LFP and store it in lfp
lfp.create_electrical_series(name='preprocessed',
data=X.T,
electrodes=electrodes,
rate=rate,
description='',
comments=config_comment)
ecephys_module.add_data_interface(lfp)
# Write LFP to NWB file
io.write(nwb)
print('LFP saved in ' + block_path)
rs_data = resample(neural_data, new_sample_rate, sample_rate)
t = np.linspace(0, duration, rs_data.shape[0])
plt.plot(t[:500], rs_data[:500, 0])
plt.xlabel('Time (s)')
plt.ylabel('Amplitude (au)')
_ = plt.title('One channel of neural data after resampling')
# %%
# Notch filtering
# ----------------
# Notch filtering is used to remove the 60 Hz line noise and harmonics on all
# channels.
nth_data = apply_linenoise_notch(rs_data, new_sample_rate)
freq, car_pwr = welch(rs_data[:, 0], fs=new_sample_rate, nperseg=1024)
_, nth_pwr = welch(nth_data[:, 0], fs=new_sample_rate, nperseg=1024)
fig, axs = plt.subplots(1, 2, figsize=(10, 4), sharey=True, sharex=True)
axs[0].semilogy(freq, car_pwr)
axs[0].set_xlabel('Frequency (Hz)')
axs[0].set_ylabel('Power density (V^2/Hz)')
axs[0].set_xlim([1, 150])
axs[0].set_title('Pre notch filtering')
axs[1].semilogy(freq, nth_pwr)
axs[1].set_xlabel('Frequency (Hz)')
axs[1].set_xlim([1, 150])
axs[1].set_title('Post notch filtering')