def _hilbert_transform(X, rate, cfs, sds): Y = np.zeros(shape=(len(cfs), X.shape[0], X.shape[1]), dtype=np.complex) for i, (cf, sd) in enumerate( tqdm(zip(cfs, sds), 'Applying Hilbert transform', total=len(cfs))): kernel = gaussian(X, rate, cf, sd) Y[i] = hilbert_transform(X, rate, kernel) return Y
def test_hilbert_return(): """ Test the return shape and dtype. """ X = np.random.randn(32, 1000) rate = 200 filters = [gaussian(X, rate, 100, 5), hamming(X, rate, 60, 70)] Xh = hilbert_transform(X, rate, filters) assert Xh.shape == (len(filters), X.shape[0], X.shape[1]) assert Xh.dtype == np.complex Xh = hilbert_transform(X, rate) assert Xh.shape == X.shape assert Xh.dtype == np.complex
def transform(block_path, rate=400., cfs=None, sds=None, srf=1e4, neuro=False, suffix=None, total_channels=256): """ Takes raw LFP data and does the standard hilb algorithm: 1) CAR 2) notch filters 3) Hilbert transform on different bands ... Saves to os.path.join(block_path, subject + '_B' + block + '_Hilb.h5') Parameters ---------- block_path rate cfs: filter center frequencies. If None, use Chang lab defaults sds: filer standard deviations. If None, use Chang lab defaults srf: htk multiple takes about 20 minutes to run on 1 10-min block """ if neuro: band_names = bands.neuro['bands'] min_freqs = bands.neuro['min_freqs'] max_freqs = bands.neuro['max_freqs'] else: bands.neuro = bands.chang_lab if cfs is None: cfs = bands.neuro['cfs'] else: cfs = np.array(cfs) if sds is None: sds = bands.neuro['cfs'] else: sds = np.array(sds) subj_path, block_name = os.path.split(block_path) start = time.time() h5_ecog_path = os.path.join(block_path, 'ecog400', 'ecog.h5') h5_ecog_tmp_path = os.path.join(block_path, 'ecog400', 'ecog_tmp.h5') mat_ecog_path = os.path.join(block_path, 'ecog400', 'ecog.mat') try: # HDF5 format with h5py.File(h5_ecog_path, 'r') as f: X = f['ecogDS']['data'].value fs = f['ecogDS']['sampFreq'].value print('Load time for h5 {}: {} seconds'.format(block_name, time.time() - start)) print('rates {}: {} {}'.format(block_name, rate, fs)) if not np.allclose(rate, fs): assert rate < fs X = resample(X, rate, fs) except IOError: try: # HDF5 .mat format with h5py.File(mat_ecog_path, 'r') as f: X = f['ecogDS']['data'][:].T fs = f['ecogDS']['sampFreq'][0] print('Load time for h5.mat {}:' + ' {} seconds'.format(block_name, time.time() - start)) except IOError: try: # Old .mat format X = None fs = None data = loadmat(mat_ecog_path)['ecogDS'] for ii, dtn in enumerate(data.dtype.names): if dtn == 'data': X = data.item()[ii] elif dtn == 'sampFreq': fs = data.item()[ii][0] assert X is not None assert fs is not None print('Load time for mat {}:' + ' {} seconds'.format(block_name, time.time() - start)) except IOError: # Load raw HTK files rd_path = os.path.join(block_path, 'RawHTK') HTKoutR = HTK.read_HTKs(rd_path) X = HTKoutR['data'] fs = HTKoutR['sampling_rate'] / srf print('Load time for htk {}:' + ' {} seconds'.format(block_name, time.time() - start)) try: os.mkdir(os.path.join(block_path, 'ecog400')) except OSError: pass if not np.allclose(rate, fs): assert rate < fs X = resample(X, rate, fs) if np.allclose(rate, 400.): start = time.time() with h5py.File(h5_ecog_tmp_path, 'w') as f: g = f.create_group('ecogDS') g.create_dataset('data', data=X) g.create_dataset('sampFreq', data=rate) os.rename(h5_ecog_tmp_path, h5_ecog_path) print('Save time for {}: {} seconds'.format( block_name, time.time() - start)) assert X.shape[0] == total_channels, (block_name, X.shape) bad_elects = load_bad_electrodes(block_path) if len(bad_elects) > 0: X[bad_elects] = np.nan # Subtract CAR start = time.time() X = subtract_CAR(X) print('CAR subtract time for {}: {} seconds'.format( block_name, time.time() - start)) # Apply Notch filters start = time.time() X = linenoise_notch(X, rate) print('Notch filter time for {}: {} seconds'.format( block_name, time.time() - start)) # Apply Hilbert transform and store if suffix is None: suffix_str = '' else: suffix_str = '_{}'.format(suffix) if neuro: hilb_path = os.path.join( block_path, '{}_neuro_Hilb{}.h5'.format(block_name, suffix_str)) else: hilb_path = os.path.join(block_path, '{}_Hilb{}.h5'.format(block_name, suffix_str)) tmp_path = os.path.join(block_path, '{}_tmp.h5'.format(block_name)) with h5py.File(tmp_path, 'w') as f: note = 'applying Hilbert transform' if neuro: dset = f.create_dataset('X', (len(band_names), X.shape[0], X.shape[1]), np.complex, compression="gzip") kernels = [] for ii, (min_f, max_f) in enumerate( tqdm(zip(min_freqs, max_freqs), note, total=len(min_freqs))): kernels.append(hamming(X, rate, min_f, max_f)) dset[ii] = hilbert_transform(X, rate, kernels) dset.dims[0].label = 'band' dset.dims[1].label = 'channel' dset.dims[2].label = 'time' for val, name in ((min_freqs, 'min frequency'), (max_freqs, 'max frequency')): if name not in f.keys(): f[name] = val dset.dims.create_scale(f[name], name) dset.dims[0].attach_scale(f[name]) else: dset = f.create_dataset('X', (len(cfs), X.shape[0], X.shape[1]), np.complex, compression="gzip") for ii, (cf, sd) in enumerate(tqdm(zip(cfs, sds), note, total=len(cfs))): kernel = gaussian(X, rate, cf, sd) dset[ii] = hilbert_transform(X, rate, kernel) dset.dims[0].label = 'filter' dset.dims[1].label = 'channel' dset.dims[2].label = 'time' for val, name in ((cfs, 'filter_center'), (sds, 'filter_sigma')): if name not in f.keys(): f[name] = val dset.dims.create_scale(f[name], name) dset.dims[0].attach_scale(f[name]) f.attrs['sampling_rate'] = rate os.rename(tmp_path, hilb_path) print('{} finished'.format(block_name))
def transform(block_path, filter='default', bands_vals=None): """ Takes raw LFP data and does the standard Hilbert algorithm: 1) CAR 2) notch filters 3) Hilbert transform on different bands Takes about 20 minutes to run on 1 10-min block. Parameters ---------- block_path : str subject file path filter: str, optional Frequency bands to filter the signal. 'default' for Chang lab default values (Gaussian filters) 'custom' for user defined (Gaussian filters) bands_vals: 2D array, necessary only if filter='custom' [2,nBands] numpy array with Gaussian filter parameters, where: bands_vals[0,:] = filter centers [Hz] bands_vals[1,:] = filter sigmas [Hz] Returns ------- Saves preprocessed signals (LFP) and spectral power (DecompositionSeries) in the current NWB file. Only if containers for these data do not exist in the file. """ write_file = 1 rate = 400. # Define filter parameters if filter == 'default': band_param_0 = bands.chang_lab['cfs'] band_param_1 = bands.chang_lab['sds'] elif filter == 'high_gamma': band_param_0 = bands.chang_lab['cfs'][(bands.chang_lab['cfs'] > 70) & (bands.chang_lab['cfs'] < 150)] band_param_1 = bands.chang_lab['sds'][(bands.chang_lab['cfs'] > 70) & (bands.chang_lab['cfs'] < 150)] #band_param_0 = [ bands.neuro['min_freqs'][-1] ] #for hamming window filter #band_param_1 = [ bands.neuro['max_freqs'][-1] ] #band_param_0 = bands.chang_lab['cfs'][29:37] #for average of gaussian filters #band_param_1 = bands.chang_lab['sds'][29:37] elif filter == 'custom': band_param_0 = bands_vals[0, :] band_param_1 = bands_vals[1, :] block_name = os.path.splitext(block_path)[0] start = time.time() with NWBHDF5IO(block_path, 'a') as io: nwb = io.read() # Storage of processed signals on NWB file ----------------------------- if 'ecephys' not in nwb.modules: # Add module to NWB file nwb.create_processing_module( name='ecephys', description='Extracellular electrophysiology data.') ecephys_module = nwb.modules['ecephys'] # LFP: Downsampled and power line signal removed if 'LFP' in nwb.modules['ecephys'].data_interfaces: lfp_ts = nwb.modules['ecephys'].data_interfaces[ 'LFP'].electrical_series['preprocessed'] X = lfp_ts.data[:].T rate = lfp_ts.rate else: # 1e6 scaling helps with numerical accuracy X = nwb.acquisition['ECoG'].data[:].T * 1e6 fs = nwb.acquisition['ECoG'].rate bad_elects = load_bad_electrodes(nwb) print('Load time for h5 {}: {} seconds'.format( block_name, time.time() - start)) print('rates {}: {} {}'.format(block_name, rate, fs)) if not np.allclose(rate, fs): assert rate < fs start = time.time() X = resample(X, rate, fs) print('resample time for {}: {} seconds'.format( block_name, time.time() - start)) if bad_elects.sum() > 0: X[bad_elects] = np.nan # Subtract CAR start = time.time() X = subtract_CAR(X) print('CAR subtract time for {}: {} seconds'.format( block_name, time.time() - start)) # Apply Notch filters start = time.time() X = linenoise_notch(X, rate) print('Notch filter time for {}: {} seconds'.format( block_name, time.time() - start)) lfp = LFP() # Add preprocessed downsampled signals as an electrical_series lfp_ts = lfp.create_electrical_series( name='preprocessed', data=X.T, electrodes=nwb.acquisition['ECoG'].electrodes, rate=rate, description='') ecephys_module.add_data_interface(lfp) # Spectral band power if 'Bandpower_' + filter not in nwb.modules['ecephys'].data_interfaces: # Apply Hilbert transform X = X.astype('float32') # signal (nChannels,nSamples) nChannels = X.shape[0] nSamples = X.shape[1] nBands = len(band_param_0) Xp = np.zeros((nBands, nChannels, nSamples)) # power (nBands,nChannels,nSamples) X_fft_h = None for ii, (bp0, bp1) in enumerate(zip(band_param_0, band_param_1)): # if filter=='high_gamma': # kernel = hamming(X, rate, bp0, bp1) # else: kernel = gaussian(X, rate, bp0, bp1) X_analytic, X_fft_h = hilbert_transform(X, rate, kernel, phase=None, X_fft_h=X_fft_h) Xp[ii] = abs(X_analytic).astype('float32') # Scales signals back to Volt X /= 1e6 band_param_0V = VectorData( name='filter_param_0', description='frequencies for bandpass filters', data=band_param_0) band_param_1V = VectorData( name='filter_param_1', description='frequencies for bandpass filters', data=band_param_1) bandsTable = DynamicTable( name='bands', description='Series of filters used for Hilbert transform.', columns=[band_param_0V, band_param_1V], colnames=['filter_param_0', 'filter_param_1']) # data: (ndarray) dims: num_times * num_channels * num_bands Xp = np.swapaxes(Xp, 0, 2) decs = DecompositionSeries( name='Bandpower_' + filter, data=Xp, description='Band power estimated with Hilbert transform.', metric='power', unit='V**2/Hz', bands=bandsTable, rate=rate, source_timeseries=lfp_ts) ecephys_module.add_data_interface(decs) io.write(nwb) print('done', flush=True)
def transform(block_path, suffix=None, phase=False, total_channels=256, seed=20180928): """ Takes raw LFP data and does the standard hilb algorithm: 1) CAR 2) notch filters 3) Hilbert transform on different bands ... Saves to os.path.join(block_path, subject + '_B' + block + '_AA.h5') Parameters ---------- block_path rate cfs: filter center frequencies. If None, use Chang lab defaults sds: filer standard deviations. If None, use Chang lab defaults takes about 20 minutes to run on 1 10-min block """ rng = None if phase: rng = np.random.RandomState(seed) rate = 400. cfs = bands.chang_lab['cfs'] sds = bands.chang_lab['sds'] subj_path, block_name = os.path.split(block_path) start = time.time() h5_ecog_path = os.path.join(block_path, 'ecog400', 'ecog.h5') h5_ecog_tmp_path = os.path.join(block_path, 'ecog400', 'ecog_tmp.h5') mat_ecog_path = os.path.join(block_path, 'ecog400', 'ecog.mat') try: raise IOError # HDF5 format with h5py.File(h5_ecog_path, 'r') as f: X = f['ecogDS']['data'].value fs = f['ecogDS']['sampFreq'].value print('Load time for h5 {}: {} seconds'.format(block_name, time.time() - start)) print('rates {}: {} {}'.format(block_name, rate, fs)) if not np.allclose(rate, fs): assert rate < fs X = resample(X, rate, fs) except IOError: try: # Load raw HTK files rd_path = os.path.join(block_path, 'RawHTK') HTKoutR = HTK.read_HTKs(rd_path) X = HTKoutR['data'] * 1e6 fs = HTKoutR['sampling_rate'] / srf print('Load time for htk {}: {} seconds'.format( block_name, time.time() - start)) except IOError: try: # HDF5 .mat format with h5py.File(mat_ecog_path, 'r') as f: X = f['ecogDS']['data'][:].T fs = f['ecogDS']['sampFreq'][0] print('Load time for h5.mat {}:' + ' {} seconds'.format(block_name, time.time() - start)) except IOError: try: # Old .mat format X = None fs = None data = loadmat(mat_ecog_path)['ecogDS'] for ii, dtn in enumerate(data.dtype.names): if dtn == 'data': X = data.item()[ii] elif dtn == 'sampFreq': fs = data.item()[ii][0] assert X is not None assert fs is not None print('Load time for mat {}: {} seconds'.format( block_name, time.time() - start)) except IOError: # New Ben h5.mat new_mat = os.path.join(block_path, '{}_raw.mat'.format(block_name)) with h5py.File(new_mat, 'r') as f: X = [] fs = None for ii in range(1, 5): if fs is None: fs = f['data/streams/Wav{}/fs'.format( ii)][0][0] else: assert fs == f['data/streams/Wav{}/fs'.format( ii)][0][0] X.append(f['data/streams/Wav{}/data'.format( ii)].value.T) X = np.concatenate( X) * 1e6 # Values are too small otherwise try: os.mkdir(os.path.join(block_path, 'ecog400')) except OSError: pass if not np.allclose(rate, fs): assert rate < fs start1 = time.time() X = resample(X, rate, fs) print('Downsample time for {}: {}, {}, {}'.format( block_name, time.time() - start1, rate, fs)) if not phase: if np.allclose(rate, 400.): start = time.time() with h5py.File(h5_ecog_tmp_path, 'w') as f: g = f.create_group('ecogDS') g.create_dataset('data', data=X) g.create_dataset('sampFreq', data=rate) os.rename(h5_ecog_tmp_path, h5_ecog_path) print('Save time for {}400: {} seconds'.format( block_name, time.time() - start)) if X.shape[0] != total_channels: raise ValueError(block_name, X.shape, total_channels) bad_elects = load_bad_electrodes(block_path) if len(bad_elects) > 0: X[bad_elects] = np.nan # Subtract CAR start = time.time() X = subtract_CAR(X) print('CAR subtract time for {}: {} seconds'.format( block_name, time.time() - start)) # Apply Notch filters start = time.time() X = linenoise_notch(X, rate) print('Notch filter time for {}: {} seconds'.format( block_name, time.time() - start)) # Apply Hilbert transform and store if suffix is None: suffix_str = '' else: suffix_str = '_{}'.format(suffix) if phase: suffix_str = suffix_str + '_random_phase' fname = '{}_AA{}.h5'.format(block_name, suffix_str) AA_path = os.path.join(block_path, fname) tmp_path = os.path.join(block_path, '{}_tmp.h5'.format(fname)) X = X.astype(float) with h5py.File(tmp_path, 'w') as f: note = 'applying Hilbert transform' dset = f.create_dataset('X', (len(cfs), X.shape[0], X.shape[1]), dtype=float) theta = None if phase: theta = rng.rand(*X.shape) * 2. * np.pi theta = np.sin(theta) + 1j * np.cos(theta) X_fft_h = None for ii, (cf, sd) in enumerate(zip(cfs, sds)): kernel = gaussian(X, rate, cf, sd) Xp, X_fft_h = hilbert_transform(X, rate, kernel, phase=theta, X_fft_h=X_fft_h) dset[ii] = abs(Xp).astype(float) dset.dims[0].label = 'filter' dset.dims[1].label = 'channel' dset.dims[2].label = 'time' for val, name in ((cfs, 'filter_center'), (sds, 'filter_sigma')): if name not in f.keys(): f[name] = val dset.dims.create_scale(f[name], name) dset.dims[0].attach_scale(f[name]) f.attrs['sampling_rate'] = rate os.rename(tmp_path, AA_path) print('{} finished'.format(block_name)) print('saved: {}'.format(AA_path))
def transform(block_path, suffix=None, phase=False, seed=20180928): """ Takes raw LFP data and does the standard hilb algorithm: 1) CAR 2) notch filters 3) Hilbert transform on different bands ... Saves to os.path.join(block_path, subject + '_B' + block + '_AA.h5') Parameters ---------- block_path rate cfs: filter center frequencies. If None, use Chang lab defaults sds: filer standard deviations. If None, use Chang lab defaults takes about 20 minutes to run on 1 10-min block """ rng = None if phase: rng = np.random.RandomState(seed) rate = 400. cfs = bands.chang_lab['cfs'] sds = bands.chang_lab['sds'] subj_path, block_name = os.path.split(block_path) block_name = os.path.splitext(block_path)[0] start = time.time() with NWBHDF5IO(block_path, 'r') as io: nwb = io.read() # 1e6 scaling helps with numerical accuracy X = nwb.acquisition['ECoG'].data[:].T * 1e6 fs = nwb.acquisition['ECoG'].rate bad_elects = load_bad_electrodes(nwb) print('Load time for h5 {}: {} seconds'.format(block_name, time.time() - start)) print('rates {}: {} {}'.format(block_name, rate, fs)) if not np.allclose(rate, fs): assert rate < fs X = resample(X, rate, fs) #if X.shape[0] != total_channels: # raise ValueError(block_name, X.shape, total_channels) if bad_elects.sum() > 0: X[bad_elects] = np.nan # Subtract CAR start = time.time() X = subtract_CAR(X) print('CAR subtract time for {}: {} seconds'.format(block_name, time.time()-start)) # Apply Notch filters start = time.time() X = linenoise_notch(X, rate) print('Notch filter time for {}: {} seconds'.format(block_name, time.time()-start)) # Apply Hilbert transform and store if suffix is None: suffix_str = '' else: suffix_str = '_{}'.format(suffix) if phase: suffix_str = suffix_str + '_random_phase' fname = '{}_AA{}.h5'.format(block_name, suffix_str) AA_path = os.path.join(block_path, fname) tmp_path = os.path.join(block_path, '{}_tmp.h5'.format(fname)) X = X.astype('float32') with h5py.File(tmp_path, 'w') as f: note = 'applying Hilbert transform' dset = f.create_dataset('X', (len(cfs), X.shape[0], X.shape[1]), dtype='float32') theta = None if phase: theta = rng.rand(*X.shape) * 2. * np.pi theta = np.sin(theta) + 1j * np.cos(theta) X_fft_h = None for ii, (cf, sd) in enumerate(zip(cfs, sds)): kernel = gaussian(X, rate, cf, sd) Xp, X_fft_h = hilbert_transform(X, rate, kernel, phase=theta, X_fft_h=X_fft_h) dset[ii] = abs(Xp).astype('float32') dset.dims[0].label = 'filter' dset.dims[1].label = 'channel' dset.dims[2].label = 'time' for val, name in ((cfs, 'filter_center'), (sds, 'filter_sigma')): if name not in f.keys(): f[name] = val dset.dims.create_scale(f[name], name) dset.dims[0].attach_scale(f[name]) f.attrs['sampling_rate'] = rate os.rename(tmp_path, AA_path) print('{} finished'.format(block_name)) print('saved: {}'.format(AA_path))