def _hilbert_transform(X, rate, cfs, sds):
Y = np.zeros(shape=(len(cfs), X.shape[0], X.shape[1]), dtype=np.complex)
for i, (cf, sd) in enumerate(
tqdm(zip(cfs, sds), 'Applying Hilbert transform', total=len(cfs))):
kernel = gaussian(X, rate, cf, sd)
Y[i] = hilbert_transform(X, rate, kernel)
return Y
def test_hilbert_return():
"""
Test the return shape and dtype.
"""
X = np.random.randn(32, 1000)
rate = 200
filters = [gaussian(X, rate, 100, 5), hamming(X, rate, 60, 70)]
Xh = hilbert_transform(X, rate, filters)
assert Xh.shape == (len(filters), X.shape[0], X.shape[1])
assert Xh.dtype == np.complex
Xh = hilbert_transform(X, rate)
assert Xh.shape == X.shape
assert Xh.dtype == np.complex
def transform(block_path,
rate=400.,
cfs=None,
sds=None,
srf=1e4,
neuro=False,
suffix=None,
total_channels=256):
"""
Takes raw LFP data and does the standard hilb algorithm:
1) CAR
2) notch filters
3) Hilbert transform on different bands
...
Saves to os.path.join(block_path, subject + '_B' + block + '_Hilb.h5')
Parameters
----------
block_path
rate
cfs: filter center frequencies. If None, use Chang lab defaults
sds: filer standard deviations. If None, use Chang lab defaults
srf: htk multiple
takes about 20 minutes to run on 1 10-min block
"""
if neuro:
band_names = bands.neuro['bands']
min_freqs = bands.neuro['min_freqs']
max_freqs = bands.neuro['max_freqs']
else:
bands.neuro = bands.chang_lab
if cfs is None:
cfs = bands.neuro['cfs']
else:
cfs = np.array(cfs)
if sds is None:
sds = bands.neuro['cfs']
else:
sds = np.array(sds)
subj_path, block_name = os.path.split(block_path)
start = time.time()
h5_ecog_path = os.path.join(block_path, 'ecog400', 'ecog.h5')
h5_ecog_tmp_path = os.path.join(block_path, 'ecog400', 'ecog_tmp.h5')
mat_ecog_path = os.path.join(block_path, 'ecog400', 'ecog.mat')
try:
# HDF5 format
with h5py.File(h5_ecog_path, 'r') as f:
X = f['ecogDS']['data'].value
fs = f['ecogDS']['sampFreq'].value
print('Load time for h5 {}: {} seconds'.format(block_name,
time.time() - start))
print('rates {}: {} {}'.format(block_name, rate, fs))
if not np.allclose(rate, fs):
assert rate < fs
X = resample(X, rate, fs)
except IOError:
try:
# HDF5 .mat format
with h5py.File(mat_ecog_path, 'r') as f:
X = f['ecogDS']['data'][:].T
fs = f['ecogDS']['sampFreq'][0]
print('Load time for h5.mat {}:' +
' {} seconds'.format(block_name,
time.time() - start))
except IOError:
try:
# Old .mat format
X = None
fs = None
data = loadmat(mat_ecog_path)['ecogDS']
for ii, dtn in enumerate(data.dtype.names):
if dtn == 'data':
X = data.item()[ii]
elif dtn == 'sampFreq':
fs = data.item()[ii][0]
assert X is not None
assert fs is not None
print('Load time for mat {}:' +
' {} seconds'.format(block_name,
time.time() - start))
except IOError:
# Load raw HTK files
rd_path = os.path.join(block_path, 'RawHTK')
HTKoutR = HTK.read_HTKs(rd_path)
X = HTKoutR['data']
fs = HTKoutR['sampling_rate'] / srf
print('Load time for htk {}:' +
' {} seconds'.format(block_name,
time.time() - start))
try:
os.mkdir(os.path.join(block_path, 'ecog400'))
except OSError:
pass
if not np.allclose(rate, fs):
assert rate < fs
X = resample(X, rate, fs)
if np.allclose(rate, 400.):
start = time.time()
with h5py.File(h5_ecog_tmp_path, 'w') as f:
g = f.create_group('ecogDS')
g.create_dataset('data', data=X)
g.create_dataset('sampFreq', data=rate)
os.rename(h5_ecog_tmp_path, h5_ecog_path)
print('Save time for {}: {} seconds'.format(
block_name,
time.time() - start))
assert X.shape[0] == total_channels, (block_name, X.shape)
bad_elects = load_bad_electrodes(block_path)
if len(bad_elects) > 0:
X[bad_elects] = np.nan
# Subtract CAR
start = time.time()
X = subtract_CAR(X)
print('CAR subtract time for {}: {} seconds'.format(
block_name,
time.time() - start))
# Apply Notch filters
start = time.time()
X = linenoise_notch(X, rate)
print('Notch filter time for {}: {} seconds'.format(
block_name,
time.time() - start))
# Apply Hilbert transform and store
if suffix is None:
suffix_str = ''
else:
suffix_str = '_{}'.format(suffix)
if neuro:
hilb_path = os.path.join(
block_path, '{}_neuro_Hilb{}.h5'.format(block_name, suffix_str))
else:
hilb_path = os.path.join(block_path,
'{}_Hilb{}.h5'.format(block_name, suffix_str))
tmp_path = os.path.join(block_path, '{}_tmp.h5'.format(block_name))
with h5py.File(tmp_path, 'w') as f:
note = 'applying Hilbert transform'
if neuro:
dset = f.create_dataset('X',
(len(band_names), X.shape[0], X.shape[1]),
np.complex,
compression="gzip")
kernels = []
for ii, (min_f, max_f) in enumerate(
tqdm(zip(min_freqs, max_freqs), note,
total=len(min_freqs))):
kernels.append(hamming(X, rate, min_f, max_f))
dset[ii] = hilbert_transform(X, rate, kernels)
dset.dims[0].label = 'band'
dset.dims[1].label = 'channel'
dset.dims[2].label = 'time'
for val, name in ((min_freqs, 'min frequency'), (max_freqs,
'max frequency')):
if name not in f.keys():
f[name] = val
dset.dims.create_scale(f[name], name)
dset.dims[0].attach_scale(f[name])
else:
dset = f.create_dataset('X', (len(cfs), X.shape[0], X.shape[1]),
np.complex,
compression="gzip")
for ii, (cf,
sd) in enumerate(tqdm(zip(cfs, sds), note,
total=len(cfs))):
kernel = gaussian(X, rate, cf, sd)
dset[ii] = hilbert_transform(X, rate, kernel)
dset.dims[0].label = 'filter'
dset.dims[1].label = 'channel'
dset.dims[2].label = 'time'
for val, name in ((cfs, 'filter_center'), (sds, 'filter_sigma')):
if name not in f.keys():
f[name] = val
dset.dims.create_scale(f[name], name)
dset.dims[0].attach_scale(f[name])
f.attrs['sampling_rate'] = rate
os.rename(tmp_path, hilb_path)
print('{} finished'.format(block_name))
def transform(block_path, filter='default', bands_vals=None):
"""
Takes raw LFP data and does the standard Hilbert algorithm:
1) CAR
2) notch filters
3) Hilbert transform on different bands
Takes about 20 minutes to run on 1 10-min block.
Parameters
----------
block_path : str
subject file path
filter: str, optional
Frequency bands to filter the signal.
'default' for Chang lab default values (Gaussian filters)
'custom' for user defined (Gaussian filters)
bands_vals: 2D array, necessary only if filter='custom'
[2,nBands] numpy array with Gaussian filter parameters, where:
bands_vals[0,:] = filter centers [Hz]
bands_vals[1,:] = filter sigmas [Hz]
Returns
-------
Saves preprocessed signals (LFP) and spectral power (DecompositionSeries) in
the current NWB file. Only if containers for these data do not exist in the
file.
"""
write_file = 1
rate = 400.
# Define filter parameters
if filter == 'default':
band_param_0 = bands.chang_lab['cfs']
band_param_1 = bands.chang_lab['sds']
elif filter == 'high_gamma':
band_param_0 = bands.chang_lab['cfs'][(bands.chang_lab['cfs'] > 70)
& (bands.chang_lab['cfs'] < 150)]
band_param_1 = bands.chang_lab['sds'][(bands.chang_lab['cfs'] > 70)
& (bands.chang_lab['cfs'] < 150)]
#band_param_0 = [ bands.neuro['min_freqs'][-1] ] #for hamming window filter
#band_param_1 = [ bands.neuro['max_freqs'][-1] ]
#band_param_0 = bands.chang_lab['cfs'][29:37] #for average of gaussian filters
#band_param_1 = bands.chang_lab['sds'][29:37]
elif filter == 'custom':
band_param_0 = bands_vals[0, :]
band_param_1 = bands_vals[1, :]
block_name = os.path.splitext(block_path)[0]
start = time.time()
with NWBHDF5IO(block_path, 'a') as io:
nwb = io.read()
# Storage of processed signals on NWB file -----------------------------
if 'ecephys' not in nwb.modules:
# Add module to NWB file
nwb.create_processing_module(
name='ecephys',
description='Extracellular electrophysiology data.')
ecephys_module = nwb.modules['ecephys']
# LFP: Downsampled and power line signal removed
if 'LFP' in nwb.modules['ecephys'].data_interfaces:
lfp_ts = nwb.modules['ecephys'].data_interfaces[
'LFP'].electrical_series['preprocessed']
X = lfp_ts.data[:].T
rate = lfp_ts.rate
else:
# 1e6 scaling helps with numerical accuracy
X = nwb.acquisition['ECoG'].data[:].T * 1e6
fs = nwb.acquisition['ECoG'].rate
bad_elects = load_bad_electrodes(nwb)
print('Load time for h5 {}: {} seconds'.format(
block_name,
time.time() - start))
print('rates {}: {} {}'.format(block_name, rate, fs))
if not np.allclose(rate, fs):
assert rate < fs
start = time.time()
X = resample(X, rate, fs)
print('resample time for {}: {} seconds'.format(
block_name,
time.time() - start))
if bad_elects.sum() > 0:
X[bad_elects] = np.nan
# Subtract CAR
start = time.time()
X = subtract_CAR(X)
print('CAR subtract time for {}: {} seconds'.format(
block_name,
time.time() - start))
# Apply Notch filters
start = time.time()
X = linenoise_notch(X, rate)
print('Notch filter time for {}: {} seconds'.format(
block_name,
time.time() - start))
lfp = LFP()
# Add preprocessed downsampled signals as an electrical_series
lfp_ts = lfp.create_electrical_series(
name='preprocessed',
data=X.T,
electrodes=nwb.acquisition['ECoG'].electrodes,
rate=rate,
description='')
ecephys_module.add_data_interface(lfp)
# Spectral band power
if 'Bandpower_' + filter not in nwb.modules['ecephys'].data_interfaces:
# Apply Hilbert transform
X = X.astype('float32') # signal (nChannels,nSamples)
nChannels = X.shape[0]
nSamples = X.shape[1]
nBands = len(band_param_0)
Xp = np.zeros((nBands, nChannels,
nSamples)) # power (nBands,nChannels,nSamples)
X_fft_h = None
for ii, (bp0, bp1) in enumerate(zip(band_param_0, band_param_1)):
# if filter=='high_gamma':
# kernel = hamming(X, rate, bp0, bp1)
# else:
kernel = gaussian(X, rate, bp0, bp1)
X_analytic, X_fft_h = hilbert_transform(X,
rate,
kernel,
phase=None,
X_fft_h=X_fft_h)
Xp[ii] = abs(X_analytic).astype('float32')
# Scales signals back to Volt
X /= 1e6
band_param_0V = VectorData(
name='filter_param_0',
description='frequencies for bandpass filters',
data=band_param_0)
band_param_1V = VectorData(
name='filter_param_1',
description='frequencies for bandpass filters',
data=band_param_1)
bandsTable = DynamicTable(
name='bands',
description='Series of filters used for Hilbert transform.',
columns=[band_param_0V, band_param_1V],
colnames=['filter_param_0', 'filter_param_1'])
# data: (ndarray) dims: num_times * num_channels * num_bands
Xp = np.swapaxes(Xp, 0, 2)
decs = DecompositionSeries(
name='Bandpower_' + filter,
data=Xp,
description='Band power estimated with Hilbert transform.',
metric='power',
unit='V**2/Hz',
bands=bandsTable,
rate=rate,
source_timeseries=lfp_ts)
ecephys_module.add_data_interface(decs)
io.write(nwb)
print('done', flush=True)
def transform(block_path,
suffix=None,
phase=False,
total_channels=256,
seed=20180928):
"""
Takes raw LFP data and does the standard hilb algorithm:
1) CAR
2) notch filters
3) Hilbert transform on different bands
...
Saves to os.path.join(block_path, subject + '_B' + block + '_AA.h5')
Parameters
----------
block_path
rate
cfs: filter center frequencies. If None, use Chang lab defaults
sds: filer standard deviations. If None, use Chang lab defaults
takes about 20 minutes to run on 1 10-min block
"""
rng = None
if phase:
rng = np.random.RandomState(seed)
rate = 400.
cfs = bands.chang_lab['cfs']
sds = bands.chang_lab['sds']
subj_path, block_name = os.path.split(block_path)
start = time.time()
h5_ecog_path = os.path.join(block_path, 'ecog400', 'ecog.h5')
h5_ecog_tmp_path = os.path.join(block_path, 'ecog400', 'ecog_tmp.h5')
mat_ecog_path = os.path.join(block_path, 'ecog400', 'ecog.mat')
try:
raise IOError
# HDF5 format
with h5py.File(h5_ecog_path, 'r') as f:
X = f['ecogDS']['data'].value
fs = f['ecogDS']['sampFreq'].value
print('Load time for h5 {}: {} seconds'.format(block_name,
time.time() - start))
print('rates {}: {} {}'.format(block_name, rate, fs))
if not np.allclose(rate, fs):
assert rate < fs
X = resample(X, rate, fs)
except IOError:
try:
# Load raw HTK files
rd_path = os.path.join(block_path, 'RawHTK')
HTKoutR = HTK.read_HTKs(rd_path)
X = HTKoutR['data'] * 1e6
fs = HTKoutR['sampling_rate'] / srf
print('Load time for htk {}: {} seconds'.format(
block_name,
time.time() - start))
except IOError:
try:
# HDF5 .mat format
with h5py.File(mat_ecog_path, 'r') as f:
X = f['ecogDS']['data'][:].T
fs = f['ecogDS']['sampFreq'][0]
print('Load time for h5.mat {}:' +
' {} seconds'.format(block_name,
time.time() - start))
except IOError:
try:
# Old .mat format
X = None
fs = None
data = loadmat(mat_ecog_path)['ecogDS']
for ii, dtn in enumerate(data.dtype.names):
if dtn == 'data':
X = data.item()[ii]
elif dtn == 'sampFreq':
fs = data.item()[ii][0]
assert X is not None
assert fs is not None
print('Load time for mat {}: {} seconds'.format(
block_name,
time.time() - start))
except IOError:
# New Ben h5.mat
new_mat = os.path.join(block_path,
'{}_raw.mat'.format(block_name))
with h5py.File(new_mat, 'r') as f:
X = []
fs = None
for ii in range(1, 5):
if fs is None:
fs = f['data/streams/Wav{}/fs'.format(
ii)][0][0]
else:
assert fs == f['data/streams/Wav{}/fs'.format(
ii)][0][0]
X.append(f['data/streams/Wav{}/data'.format(
ii)].value.T)
X = np.concatenate(
X) * 1e6 # Values are too small otherwise
try:
os.mkdir(os.path.join(block_path, 'ecog400'))
except OSError:
pass
if not np.allclose(rate, fs):
assert rate < fs
start1 = time.time()
X = resample(X, rate, fs)
print('Downsample time for {}: {}, {}, {}'.format(
block_name,
time.time() - start1, rate, fs))
if not phase:
if np.allclose(rate, 400.):
start = time.time()
with h5py.File(h5_ecog_tmp_path, 'w') as f:
g = f.create_group('ecogDS')
g.create_dataset('data', data=X)
g.create_dataset('sampFreq', data=rate)
os.rename(h5_ecog_tmp_path, h5_ecog_path)
print('Save time for {}400: {} seconds'.format(
block_name,
time.time() - start))
if X.shape[0] != total_channels:
raise ValueError(block_name, X.shape, total_channels)
bad_elects = load_bad_electrodes(block_path)
if len(bad_elects) > 0:
X[bad_elects] = np.nan
# Subtract CAR
start = time.time()
X = subtract_CAR(X)
print('CAR subtract time for {}: {} seconds'.format(
block_name,
time.time() - start))
# Apply Notch filters
start = time.time()
X = linenoise_notch(X, rate)
print('Notch filter time for {}: {} seconds'.format(
block_name,
time.time() - start))
# Apply Hilbert transform and store
if suffix is None:
suffix_str = ''
else:
suffix_str = '_{}'.format(suffix)
if phase:
suffix_str = suffix_str + '_random_phase'
fname = '{}_AA{}.h5'.format(block_name, suffix_str)
AA_path = os.path.join(block_path, fname)
tmp_path = os.path.join(block_path, '{}_tmp.h5'.format(fname))
X = X.astype(float)
with h5py.File(tmp_path, 'w') as f:
note = 'applying Hilbert transform'
dset = f.create_dataset('X', (len(cfs), X.shape[0], X.shape[1]),
dtype=float)
theta = None
if phase:
theta = rng.rand(*X.shape) * 2. * np.pi
theta = np.sin(theta) + 1j * np.cos(theta)
X_fft_h = None
for ii, (cf, sd) in enumerate(zip(cfs, sds)):
kernel = gaussian(X, rate, cf, sd)
Xp, X_fft_h = hilbert_transform(X,
rate,
kernel,
phase=theta,
X_fft_h=X_fft_h)
dset[ii] = abs(Xp).astype(float)
dset.dims[0].label = 'filter'
dset.dims[1].label = 'channel'
dset.dims[2].label = 'time'
for val, name in ((cfs, 'filter_center'), (sds, 'filter_sigma')):
if name not in f.keys():
f[name] = val
dset.dims.create_scale(f[name], name)
dset.dims[0].attach_scale(f[name])
f.attrs['sampling_rate'] = rate
os.rename(tmp_path, AA_path)
print('{} finished'.format(block_name))
print('saved: {}'.format(AA_path))
def transform(block_path, suffix=None, phase=False, seed=20180928):
"""
Takes raw LFP data and does the standard hilb algorithm:
1) CAR
2) notch filters
3) Hilbert transform on different bands
...
Saves to os.path.join(block_path, subject + '_B' + block + '_AA.h5')
Parameters
----------
block_path
rate
cfs: filter center frequencies. If None, use Chang lab defaults
sds: filer standard deviations. If None, use Chang lab defaults
takes about 20 minutes to run on 1 10-min block
"""
rng = None
if phase:
rng = np.random.RandomState(seed)
rate = 400.
cfs = bands.chang_lab['cfs']
sds = bands.chang_lab['sds']
subj_path, block_name = os.path.split(block_path)
block_name = os.path.splitext(block_path)[0]
start = time.time()
with NWBHDF5IO(block_path, 'r') as io:
nwb = io.read()
# 1e6 scaling helps with numerical accuracy
X = nwb.acquisition['ECoG'].data[:].T * 1e6
fs = nwb.acquisition['ECoG'].rate
bad_elects = load_bad_electrodes(nwb)
print('Load time for h5 {}: {} seconds'.format(block_name,
time.time() - start))
print('rates {}: {} {}'.format(block_name, rate, fs))
if not np.allclose(rate, fs):
assert rate < fs
X = resample(X, rate, fs)
#if X.shape[0] != total_channels:
# raise ValueError(block_name, X.shape, total_channels)
if bad_elects.sum() > 0:
X[bad_elects] = np.nan
# Subtract CAR
start = time.time()
X = subtract_CAR(X)
print('CAR subtract time for {}: {} seconds'.format(block_name,
time.time()-start))
# Apply Notch filters
start = time.time()
X = linenoise_notch(X, rate)
print('Notch filter time for {}: {} seconds'.format(block_name,
time.time()-start))
# Apply Hilbert transform and store
if suffix is None:
suffix_str = ''
else:
suffix_str = '_{}'.format(suffix)
if phase:
suffix_str = suffix_str + '_random_phase'
fname = '{}_AA{}.h5'.format(block_name, suffix_str)
AA_path = os.path.join(block_path, fname)
tmp_path = os.path.join(block_path, '{}_tmp.h5'.format(fname))
X = X.astype('float32')
with h5py.File(tmp_path, 'w') as f:
note = 'applying Hilbert transform'
dset = f.create_dataset('X', (len(cfs),
X.shape[0], X.shape[1]),
dtype='float32')
theta = None
if phase:
theta = rng.rand(*X.shape) * 2. * np.pi
theta = np.sin(theta) + 1j * np.cos(theta)
X_fft_h = None
for ii, (cf, sd) in enumerate(zip(cfs, sds)):
kernel = gaussian(X, rate, cf, sd)
Xp, X_fft_h = hilbert_transform(X, rate, kernel, phase=theta, X_fft_h=X_fft_h)
dset[ii] = abs(Xp).astype('float32')
dset.dims[0].label = 'filter'
dset.dims[1].label = 'channel'
dset.dims[2].label = 'time'
for val, name in ((cfs, 'filter_center'),
(sds, 'filter_sigma')):
if name not in f.keys():
f[name] = val
dset.dims.create_scale(f[name], name)
dset.dims[0].attach_scale(f[name])
f.attrs['sampling_rate'] = rate
os.rename(tmp_path, AA_path)
print('{} finished'.format(block_name))
print('saved: {}'.format(AA_path))