def load_ddc(exp_path, test, electrode, drange, bandpass=(), notches=(), save=True, snip_transient=True, Fs=1000, units='nA', **extra): half = extra.get('half', False) avg = extra.get('avg', False) # returns data in coulombs, i.e. values are < 1e-12 (data, disconnected, trigs, pos_edge, chan_map) = _load_cooked(exp_path, test, half=half, avg=avg) if half or avg: Fs = Fs / 2.0 if 'a' in units.lower(): data *= Fs data = convert_scale(data, 'a', units) elif 'c' in units.lower(): data = convert_scale(data, 'c', units) if bandpass: (b, a) = ft.butter_bp(lo=bandpass[0], hi=bandpass[1], Fs=Fs) filtfilt(data, b, a) if notches: ft.notch_all(data, Fs, lines=notches, inplace=True, filtfilt=True) if snip_transient: snip_len = min(10000, pos_edge[0]) if len(pos_edge) else 10000 data = data[..., snip_len:].copy() disconnected = disconnected[..., snip_len:].copy() if len(pos_edge): trigs = trigs[..., snip_len:] pos_edge -= snip_len dset = Bunch() dset.data = data dset.pos_edge = pos_edge dset.trigs = trigs dset.ground_chans = disconnected dset.Fs = Fs dset.chan_map = chan_map dset.bandpass = bandpass dset.transient_snipped = snip_transient dset.units = units dset.notches = notches return dset
def _load_array_block(files, shared_array=False, antialias=True): Fs = 1 dtype = 'h' if quantized else 'd' # start on 1st index of 0th block n = 1 b_cnt = 0 b_idx = 1 ch_record = OE.loadContinuous(files[0], dtype=np.int16, verbose=False) d_len = ch_record['data'].shape[-1] sub_len = d_len // downsamp if sub_len * downsamp < d_len: sub_len += 1 proc_block = shm.shared_ndarray((page_size, d_len), typecode=dtype) proc_block[0] = ch_record['data'].astype('d') if shared_array: saved_array = shm.shared_ndarray((len(files), sub_len), typecode=dtype) else: saved_array = np.zeros((len(files), sub_len), dtype=dtype) for f in files[1:]: ch_record = OE.loadContinuous(f, dtype=np.int16, verbose=False) # load data Fs = float(ch_record['header']['sampleRate']) proc_block[b_idx] = ch_record['data'].astype(dtype) b_idx += 1 n += 1 if (b_idx == page_size) or (n == len(files)): # do dynamic range conversion and downsampling # on a block of data if not quantized: proc_block *= ch_record['header']['bitVolts'] if downsamp > 1 and antialias: filtfilt(proc_block, b_lp, a_lp) sl = slice(b_cnt * page_size, n) saved_array[sl] = proc_block[:b_idx, ::downsamp] # update / reset block counters b_idx = 0 b_cnt += 1 del proc_block while gc.collect(): pass return saved_array, Fs, ch_record['header']
def load_openephys_ddc(exp_path, test, electrode, drange, trigger_idx, rec_num='auto', bandpass=(), notches=(), save=False, snip_transient=True, units='nA', **extra): rawload = load_open_ephys_channels(exp_path, test, rec_num=rec_num) all_chans = rawload.chdata Fs = rawload.Fs d_chans = len(rows) ch_data = all_chans[:d_chans] if np.iterable(trigger_idx): trigger = all_chans[int(trigger_idx[0])] else: trigger = all_chans[int(trigger_idx)] electrode_chans = rows >= 0 chan_flat = mat_to_flat((8, 8), rows[electrode_chans], 7 - columns[electrode_chans], col_major=False) chan_map = ChannelMap(chan_flat, (8, 8), col_major=False, pitch=0.406) dr_lo, dr_hi = _dyn_range_lookup[drange] # drange 0 3 or 7 ch_data = convert_dyn_range(ch_data, (-2**15, 2**15), (dr_lo, dr_hi)) data = shm.shared_copy(ch_data[electrode_chans]) disconnected = ch_data[~electrode_chans] trigger -= trigger.mean() binary_trig = (trigger > 100).astype('i') if binary_trig.any(): pos_edge = np.where(np.diff(binary_trig) > 0)[0] + 1 else: pos_edge = () # change units if not nA if 'a' in units.lower(): # this puts it as picoamps data *= Fs data = convert_scale(data, 'pa', units) elif 'c' in units.lower(): data = convert_scale(data, 'pc', units) if bandpass: # how does this logic work? (b, a) = ft.butter_bp(lo=bandpass[0], hi=bandpass[1], Fs=Fs) filtfilt(data, b, a) if notches: ft.notch_all(data, Fs, lines=notches, inplace=True, filtfilt=True) if snip_transient: snip_len = min(10000, pos_edge[0]) if len(pos_edge) else 10000 data = data[..., snip_len:].copy() if len(disconnected): disconnected = disconnected[..., snip_len:].copy() if len(pos_edge): trigger = trigger[..., snip_len:] pos_edge -= snip_len dset = Bunch() dset.data = data dset.pos_edge = pos_edge dset.trigs = trigger dset.ground_chans = disconnected dset.Fs = Fs dset.chan_map = chan_map dset.bandpass = bandpass dset.transient_snipped = snip_transient dset.units = units dset.notches = notches return dset
def load_mux(exp_path, test, electrode, headstage, ni_daq_variant='', mux_connectors=(), bandpass=(), notches=(), trigger=0, bnc=(), mux_notches=(), save=False, snip_transient=True, units='uV'): """ Load data from the MUX style headstage acquisition. Data is expected to be organized along columns corresponding to the MUX units. The columns following sensor data columns are assumed to be a stimulus trigger followed by other BNC channels. The electrode information must be provided to determine the arrangement of recorded and grounded channels within the sensor data column. This preprocessing routine returns a Bunch container with the following items dset.data : nchan x ntime data array dset.ground_chans : m x ntime data array of grounded ADC channels dset.bnc : un-MUXed readout of the BNC channel(s) dset.chan_map : the channel-to-electrode mapping vector dset.Fs : sampling frequency dset.name : path + expID for the given data set dset.bandpass : bandpass filtering applied (if any) dset.trig : the logical value of the trigger channel (at MUX'd Fs) * If saving, then a table of the Bunch is written. * If snip_transient, then advance the timeseries past the bandpass filtering onset transient. """ try: dset = try_saved(exp_path, test, bandpass) return dset except DataPathError: pass # say no to shared memory since it's created later on in this method loaded = rawload_mux(exp_path, test, headstage, daq_variant=ni_daq_variant, shm=False) channels, Fs, dshape, info = loaded nrow, ncol_data = dshape if channels.shape[0] >= nrow * ncol_data: ncol = channels.shape[0] // nrow channels = channels.reshape(ncol, nrow, -1) else: ncol = channels.shape[0] channels.shape = (ncol, -1, nrow) channels = channels.transpose(0, 2, 1) ## Grab BNC data if bnc: bnc_chans = [ncol_data + int(b) for b in bnc] bnc = np.zeros((len(bnc), nrow * channels.shape[-1])) for bc, col in zip(bnc, bnc_chans): bc[:] = channels[col].transpose().ravel() bnc = bnc.squeeze() try: trig_chans = channels[ncol_data + trigger].copy() pos_edge, trig = process_trigger(trig_chans) except IndexError: pos_edge = () trig = () ## Realize channel mapping chan_map, disconnected, reference = epins.get_electrode_map( electrode, connectors=mux_connectors) ## Data channels # if any pre-processing of multiplexed channels, do it here first if mux_notches: mux_chans = shm.shared_ndarray((ncol_data, channels.shape[-1], nrow)) mux_chans[:] = channels[:ncol_data].transpose(0, 2, 1) mux_chans.shape = (ncol_data, -1) ft.notch_all(mux_chans, Fs, lines=mux_notches, filtfilt=True) mux_chans.shape = (ncol_data, channels.shape[-1], nrow) channels[:ncol_data] = mux_chans.transpose(0, 2, 1) del mux_chans rec_chans = channels[:ncol_data].reshape(nrow * ncol_data, -1) if units.lower() != 'v': convert_scale(rec_chans, 'v', units) g_chans = disconnected r_chans = reference d_chans = np.setdiff1d(np.arange(ncol_data * nrow), np.union1d(g_chans, r_chans)) data_chans = shm.shared_copy(rec_chans[d_chans]) if len(g_chans): gnd_data = rec_chans[g_chans] else: gnd_data = () if len(r_chans): ref_data = rec_chans[r_chans] else: ref_data = () del rec_chans del channels # do highpass filtering for stationarity if bandpass: # manually remove DC from channels before filtering if bandpass[0] > 0: data_chans -= data_chans.mean(1)[:, None] # do a high order highpass to really crush the crappy # low frequency noise b, a = ft.butter_bp(lo=bandpass[0], Fs=Fs, ord=5) # b, a = ft.cheby1_bp(0.5, lo=bandpass[0], Fs=Fs, ord=5) else: b = [1] a = [1] if bandpass[1] > 0: b_lp, a_lp = ft.butter_bp(hi=bandpass[1], Fs=Fs, ord=3) b = np.convolve(b, b_lp) a = np.convolve(a, a_lp) filtfilt(data_chans, b, a) if len(ref_data): with parallel_controller(False): ref_data = np.atleast_2d(ref_data) filtfilt(ref_data, b, a) ref_data = ref_data.squeeze() if notches: ft.notch_all(data_chans, Fs, lines=notches, inplace=True, filtfilt=True) if len(ref_data): with parallel_controller(False): ref_data = np.atleast_2d(ref_data) ft.notch_all(ref_data, Fs, lines=notches, inplace=True, filtfilt=True) ref_data = ref_data.squeeze() if snip_transient: if isinstance(snip_transient, bool): snip_len = int(Fs * 5) else: snip_len = int(Fs * snip_transient) if len(pos_edge): snip_len = max(0, min(snip_len, pos_edge[0] - int(Fs))) pos_edge = pos_edge - snip_len trig = trig[..., snip_len:].copy() data_chans = data_chans[..., snip_len:].copy() gnd_data = gnd_data[..., snip_len:].copy() if len(ref_data): ref_data = ref_data[..., snip_len:].copy() if len(bnc): bnc = bnc[..., snip_len * nrow:].copy() # do blockwise detrending for stationarity ## detrend_window = int(round(0.750*Fs)) ## ft.bdetrend(data_chans, bsize=detrend_window, type='linear', axis=-1) dset = ut.Bunch() dset.pos_edge = pos_edge dset.data = data_chans dset.ground_chans = gnd_data dset.ref_chans = ref_data dset.bnc = bnc dset.chan_map = chan_map dset.Fs = Fs #dset.name = os.path.join(exp_path, test) dset.bandpass = bandpass dset.trig = trig dset.transient_snipped = snip_transient dset.units = units dset.notches = notches dset.info = info if save: hf = os.path.join(exp_path, test + '_proc.h5') save_bunch(hf, '/', dset, mode='w') return dset
def load_blackrock( exp_path, test, electrode, connections=(), downsamp=15, page_size=10, bandpass=(), notches=(), save=True, snip_transient=True, lowpass_ord=12, units='uV', **extra ): """ Load raw data in an HDF5 table stripped from Blackrock NSx format. This data should be 16 bit signed integer sampled at 30 kHz. We take the approach of resampling to a lower rate (default 2 kHz) before subsequent bandpass filtering. This improves the numerical stability of the bandpass filter design. """ dsamp_path = p.join(exp_path, 'downsamp') nsx_path = p.join(exp_path, 'blackrock') ## Get array-to-channel pinouts chan_map, disconnected = epins.get_electrode_map(electrode, connectors=connections)[:2] # try preproc path first to see if this run has already been downsampled load_nsx = True if downsamp > 1: dsamp_Fs = 3e4 / downsamp try: test_file = p.join(dsamp_path, test) + '_Fs%d.h5'%dsamp_Fs print('searching for', test_file) h5f = tables.open_file(test_file) downsamp = 1 load_nsx = False except IOError: print('Resampled data not found: downsampling to %d Hz'%dsamp_Fs) if load_nsx: test_file = p.join(nsx_path, test+'.h5') h5f = tables.open_file(test_file) if downsamp > 1: (b, a) = cheby2_bp(60, hi=1.0/downsamp, Fs=2, ord=lowpass_ord) if not p.exists(dsamp_path): os.mkdir(dsamp_path) save_file = p.join(dsamp_path, test) + '_Fs%d.h5'%dsamp_Fs h5_save = tables.open_file(save_file, mode='w') h5_save.create_array(h5_save.root, 'Fs', dsamp_Fs) else: # in this case, either the preprocessed data has been found, # or downsampling was not requested, which will probably # *NEVER* happen if load_nsx: dlen, nchan = h5f.root.data.shape required_mem = dlen * nchan * np.dtype('d').itemsize if required_mem > 8e9: raise MemoryError( 'This dataset would eat %.2f GBytes RAM'%(required_mem/1e9,) ) dlen, nchan = h5f.root.data.shape if dlen < nchan: (dlen, nchan) = (nchan, dlen) tdim = 1 else: tdim = 0 sublen = dlen / downsamp if dlen - sublen*downsamp > 0: sublen += 1 # set up arrays for loaded data and ground chans subdata = array_split.shared_ndarray((len(chan_map), sublen)) if len(chan_map) < nchan: gndchan = np.empty((len(disconnected), sublen), 'd') else: gndchan = None # if saving downsampled results, set up H5 table (in c-major fashion) if downsamp > 1: atom = tables.Float64Atom() #filters = tables.Filters(complevel=5, complib='zlib') filters = None saved_array = h5_save.create_earray( h5_save.root, 'data', atom=atom, shape=(0, sublen), filters=filters, expectedrows=nchan ) if page_size < 0: page_size = nchan peel = array_split.shared_ndarray( (page_size, dlen) ) n = 0 dstop = 0 h5_data = h5f.root.data while n < nchan: start = n stop = min(nchan, n+page_size) print('processing BR channels %03d - %03d'%(start, stop-1)) if tdim == 0: peel[0:stop-n] = h5_data[:,start:stop].T.astype('d', order='C') else: peel[0:stop-n] = h5_data[start:stop,:].astype('d') if downsamp > 1: convert_dyn_range(peel, (-2**15, 2**15), (-8e-3, 8e-3), out=peel) print('parfilt', end=' ') sys.stdout.flush() filtfilt(peel[0:stop-n], b, a) print('done') sys.stdout.flush() print('saving chans', end=' ') sys.stdout.flush() saved_array.append(peel[0:stop-n,::downsamp]) print('done') sys.stdout.flush() if units.lower() != 'v': convert_scale(peel, 'v', units) data_chans = np.setdiff1d(np.arange(start,stop), disconnected) if len(data_chans): dstart = dstop dstop = dstart + len(data_chans) if len(data_chans) == (stop-start): # if all data channels, peel off in a straightforward way #print (dstart, dstop) subdata[dstart:dstop,:] = peel[0:stop-n,::downsamp] else: if len(data_chans): # get data channels first raw_data = peel[data_chans-n, :] #print (dstart, dstop), data_chans-n subdata[dstart:dstop, :] = raw_data[:, ::downsamp] # Now filter for ground channels within this set of channels: gnd_chans = [x for x in zip(disconnected, range(len(disconnected))) if x[0]>=start and x[0]<stop] for g in gnd_chans: gndchan[g[1], :] = peel[g[0]-n, ::downsamp] n += page_size del peel try: Fs = h5f.root.Fs.read()[0,0] / downsamp except TypeError: Fs = h5f.root.Fs.read() / downsamp trigs = h5f.root.trig_idx.read().squeeze() if not trigs.shape: trigs = () else: trigs = np.round( trigs / downsamp ).astype('i') h5f.close() if downsamp > 1: h5_save.create_array(h5_save.root, 'trig_idx', np.asarray(trigs)) h5_save.close() # seems to be more numerically stable to do highpass and # notch filtering after downsampling if bandpass: lo, hi = bandpass (b, a) = butter_bp(lo=lo, hi=hi, Fs=Fs, ord=4) filtfilt(subdata, b, a) if notches: notch_all(subdata, Fs, lines=notches, inplace=True, filtfilt=True) dset = ut.Bunch() dset.data = subdata dset.ground_chans = gndchan dset.chan_map = chan_map dset.Fs = Fs #dset.name = os.path.join(exp_path, test) dset.bandpass = bandpass dset.notches = notches dset.trig = trigs if len(trigs) == subdata.shape[-1]: dset.pos_edge = np.where( np.diff(trigs) > 0 )[0] + 1 else: dset.pos_edge = trigs dset.units = units gc.collect() return dset
def load_afe(exp_pth, test, electrode, n_data, range_code, cycle_rate, units='nA', bandpass=(), save=True, notches=(), snip_transient=True, **extra): h5 = tables.open_file(os.path.join(exp_pth, test + '.h5')) data = h5.root.data[:] Fs = h5.root.Fs[0, 0] if data.shape[1] > n_data: trig_chans = data[:, n_data:] trig = np.any(trig_chans > 1, axis=1).astype('i') pos_edge = np.where(np.diff(trig) > 0)[0] + 1 else: trig = None pos_edge = () data_chans = data[:, :n_data].T.copy(order='C') # convert dynamic range to charge or current if 'v' not in units.lower(): pico_coulombs = range_lookup[range_code] convert_dyn_range(data_chans, (-1.4, 1.4), pico_coulombs, out=data_chans) if 'a' in units.lower(): # To convert to amps, need to divide coulombs by the # integration period. This is found approximately by # finding out how many cycles in a scan period were spent # integrating. A scan period is now hard coded to be 500 # cycles. The cycling rate is given as an argument. # The integration period for channel i should be: # 500 - 2*(n_data - i) # That is, the 1st channel is clocked out over two cycles # immediately after the integration period. Meanwhile other # channels still acquire until they are clocked out. n_cycles = 500 #i_cycles = n_cycles - 2*(n_data - np.arange(n_data)) i_cycles = n_cycles - 2 * n_data i_period = i_cycles / cycle_rate data_chans /= i_period #[:,None] convert_scale(data_chans, 'pa', units) elif units.lower() != 'v': convert_scale(data, 'v', units) # only use this one electrode (for now) chan_map, disconnected = epins.get_electrode_map('psv_61_afe')[:2] connected = np.setdiff1d(np.arange(n_data), disconnected) disconnected = disconnected[disconnected < n_data] chan_map = chan_map.subset(list(range(len(connected)))) data = shm.shared_ndarray((len(connected), data_chans.shape[-1])) data[:, :] = data_chans[connected] ground_chans = data_chans[disconnected].copy() del data_chans if bandpass: # do a little extra to kill DC data -= data.mean(axis=1)[:, None] (b, a) = ft.butter_bp(lo=bandpass[0], hi=bandpass[1], Fs=Fs) filtfilt(data, b, a) if notches: for freq in notches: (b, a) = ft.notch(freq, Fs=Fs, ftype='cheby2') filtfilt(data, b, a) ## detrend_window = int(round(0.750*Fs)) ## ft.bdetrend(data, bsize=detrend_window, type='linear', axis=-1) else: data -= data.mean(axis=1)[:, None] if snip_transient: snip_len = min(10000, pos_edge[0]) if len(pos_edge) else 10000 data = data[..., snip_len:].copy() ground_chans = ground_chans[..., snip_len:].copy() if len(pos_edge): trig = trig[..., snip_len:] pos_edge -= snip_len dset = Bunch() dset.data = data dset.ground_chans = ground_chans dset.chan_map = chan_map dset.Fs = Fs dset.pos_edge = pos_edge dset.bandpass = bandpass dset.trig = trig dset.transient_snipped = snip_transient dset.units = units dset.notches = notches return dset
def load_afe_aug21(exp_pth, test, electrode, n_data, range_code, cycle_rate, units='nA', bandpass=(), save=False, notches=(), snip_transient=True, **extra): h5 = tables.open_file(os.path.join(exp_pth, test + '.h5')) Fs = h5.root.Fs.read() n_row = h5.root.numRow.read() n_data_col = h5.root.numCol.read() n_col = h5.root.numChan.read() # data rows are 4:67 -- acquiring AFE chans 31:0 and 63:32 on two columns data_rows = slice(4, 67, 2) full_data = h5.root.data[:].reshape(n_col, n_row, -1) #data_chans = shm.shared_ndarray( (32*n_data_col, full_data.shape[-1]) ) data_chans = full_data[:n_data_col, data_rows].reshape(-1, full_data.shape[-1]) trig_chans = full_data[-10:, -1] del full_data trig = np.any(trig_chans > 1, axis=0).astype('i') pos_edge = np.where(np.diff(trig) > 0)[0] + 1 # convert dynamic range to charge or current if 'v' not in units.lower(): pico_coulombs = range_lookup[range_code] convert_dyn_range(data_chans, (-1.4, 1.4), pico_coulombs, out=data_chans) if 'a' in units.lower(): i_period = 563e-6 data_chans /= i_period convert_scale(data_chans, 'pa', units) elif units.lower() != 'v': convert_scale(data_chans, 'v', units) ## # only use this one electrode (for now) chan_map, disconnected = epins.get_electrode_map('psv_61_afe')[:2] connected = np.setdiff1d(np.arange(n_data), disconnected) disconnected = disconnected[disconnected < n_data] data = shm.shared_ndarray((len(connected), data_chans.shape[-1])) data[:, :] = data_chans[connected] ground_chans = data_chans[disconnected] del data_chans # do a little extra to kill DC data -= data.mean(axis=1)[:, None] if bandpass: (b, a) = ft.butter_bp(lo=bandpass[0], hi=bandpass[1], Fs=Fs) filtfilt(data, b, a) if notches: ft.notch_all(data, Fs, lines=notches, inplace=True, filtfilt=True) if snip_transient: snip_len = min(10000, pos_edge[0]) if len(pos_edge) else 10000 data = data[..., snip_len:].copy() if ground_chans is not None: ground_chans = ground_chans[..., snip_len:].copy() if len(pos_edge): trig = trig[..., snip_len:] pos_edge -= snip_len dset = ut.Bunch() dset.data = data dset.ground_chans = ground_chans dset.chan_map = chan_map dset.Fs = Fs dset.pos_edge = pos_edge dset.bandpass = bandpass dset.trig = trig dset.transient_snipped = snip_transient dset.units = units dset.notches = notches return dset