def rmsmap(fbin, spectra=True): """ Computes RMS map in time domain and spectra for each channel of Neuropixel probe :param fbin: binary file in spike glx format (will look for attached metatdata) :type fbin: str or pathlib.Path :param spectra: whether to compute the power spectrum (only need for lfp data) :type: bool :return: a dictionary with amplitudes in channeltime space, channelfrequency space, time and frequency scales """ if not isinstance(fbin, spikeglx.Reader): sglx = spikeglx.Reader(fbin) rms_win_length_samples = 2**np.ceil(np.log2(sglx.fs * RMS_WIN_LENGTH_SECS)) # the window generator will generates window indices wingen = dsp.WindowGenerator(ns=sglx.ns, nswin=rms_win_length_samples, overlap=0) # pre-allocate output dictionary of numpy arrays win = { 'TRMS': np.zeros((wingen.nwin, sglx.nc)), 'nsamples': np.zeros((wingen.nwin, )), 'fscale': dsp.fscale(WELCH_WIN_LENGTH_SAMPLES, 1 / sglx.fs, one_sided=True), 'tscale': wingen.tscale(fs=sglx.fs) } win['spectral_density'] = np.zeros((len(win['fscale']), sglx.nc)) # loop through the whole session for first, last in wingen.firstlast: D = sglx.read_samples(first_sample=first, last_sample=last)[0].transpose() # remove low frequency noise below 1 Hz D = dsp.hp(D, 1 / sglx.fs, [0, 1]) iw = wingen.iw win['TRMS'][iw, :] = dsp.rms(D) win['nsamples'][iw] = D.shape[1] if spectra: # the last window may be smaller than what is needed for welch if last - first < WELCH_WIN_LENGTH_SAMPLES: continue # compute a smoothed spectrum using welch method _, w = signal.welch(D, fs=sglx.fs, window='hanning', nperseg=WELCH_WIN_LENGTH_SAMPLES, detrend='constant', return_onesided=True, scaling='density', axis=-1) win['spectral_density'] += w.T # print at least every 20 windows if (iw % min(20, max(int(np.floor(wingen.nwin / 75)), 1))) == 0: print_progress(iw, wingen.nwin) return win
def _sync_to_alf(raw_ephys_apfile, output_path=None, save=False, parts=''): """ Extracts sync.times, sync.channels and sync.polarities from binary ephys dataset :param raw_ephys_apfile: bin file containing ephys data or spike :param output_path: output directory :param save: bool write to disk only if True :param parts: string or list of strings that will be appended to the filename before extension :return: """ # handles input argument: support ibllib.io.spikeglx.Reader, str and pathlib.Path if isinstance(raw_ephys_apfile, spikeglx.Reader): sr = raw_ephys_apfile else: raw_ephys_apfile = Path(raw_ephys_apfile) sr = spikeglx.Reader(raw_ephys_apfile) # if no output, need a temp folder to swap for big files if not output_path: output_path = raw_ephys_apfile.parent file_ftcp = Path(output_path).joinpath( f'fronts_times_channel_polarity{str(uuid.uuid4())}.bin') # loop over chunks of the raw ephys file wg = dsp.WindowGenerator(sr.ns, int(SYNC_BATCH_SIZE_SECS * sr.fs), overlap=1) fid_ftcp = open(file_ftcp, 'wb') for sl in wg.slice: ss = sr.read_sync(sl) ind, fronts = dsp.fronts(ss, axis=0) # a = sr.read_sync_analog(sl) sav = np.c_[(ind[0, :] + sl.start) / sr.fs, ind[1, :], fronts.astype(np.double)] sav.tofile(fid_ftcp) # print progress wg.print_progress() # close temp file, read from it and delete fid_ftcp.close() tim_chan_pol = np.fromfile(str(file_ftcp)) tim_chan_pol = tim_chan_pol.reshape((int(tim_chan_pol.size / 3), 3)) file_ftcp.unlink() sync = { 'times': tim_chan_pol[:, 0], 'channels': tim_chan_pol[:, 1], 'polarities': tim_chan_pol[:, 2] } if save: out_files = alf.io.save_object_npy(output_path, sync, '_spikeglx_sync', parts=parts) return Bunch(sync), out_files else: return Bunch(sync)
def welchogram(fs, wav, nswin=NS_WIN, overlap=OVERLAP, nperseg=NS_WELCH): """ Computes a spectrogram on a very large audio file. :param fs: sampling frequency (Hz) :param wav: wav signal (vector or memmap) :param nswin: n samples of the sliding window :param overlap: n samples of the overlap between windows :param nperseg: n samples for the computation of the spectrogram :return: tscale, fscale, downsampled_spectrogram """ ns = wav.shape[0] window_generator = dsp.WindowGenerator(ns=ns, nswin=nswin, overlap=overlap) nwin = window_generator.nwin fscale = dsp.fscale(nperseg, 1 / fs, one_sided=True) W = np.zeros((nwin, len(fscale))) tscale = window_generator.tscale(fs=fs) detect = [] for first, last in window_generator.firstlast: # load the current window into memory w = np.float64(wav[first:last]) * _get_conversion_factor() # detection of ready tones a = [d + first for d in _detect_ready_tone(w, fs)] if len(a): detect += a # the last window may not allow a pwelch if (last - first) < nperseg: continue # compute PSD estimate for the current window iw = window_generator.iw _, W[iw, :] = signal.welch(w, fs=fs, window='hanning', nperseg=nperseg, axis=-1, detrend='constant', return_onesided=True, scaling='density') if (iw % 50) == 0: window_generator.print_progress() window_generator.print_progress() # the onset detection may have duplicates with sliding window, average them and remove detect = np.sort(np.array(detect)) / fs ind = np.where(np.diff(detect) < 0.1)[0] detect[ind] = (detect[ind] + detect[ind + 1]) / 2 detect = np.delete(detect, ind + 1) return tscale, fscale, W, detect
def _sync_to_alf(raw_ephys_apfile, output_path=None, save=False): """ Extracts sync.times, sync.channels and sync.polarities from binary ephys dataset :param raw_ephys_apfile: bin file containing ephys data or spike :param out_dir: output directory :return: """ if not output_path: file_ftcp = tempfile.TemporaryFile() else: file_ftcp = Path(output_path) / 'fronts_times_channel_polarity.bin' if isinstance(raw_ephys_apfile, ibllib.io.spikeglx.Reader): sr = raw_ephys_apfile else: sr = ibllib.io.spikeglx.Reader(raw_ephys_apfile) # loop over chunks of the raw ephys file wg = dsp.WindowGenerator(sr.ns, SYNC_BATCH_SIZE_SAMPLES, overlap=1) fid_ftcp = open(file_ftcp, 'wb') for sl in wg.slice: ss = sr.read_sync(sl) ind, fronts = dsp.fronts(ss, axis=0) sav = np.c_[(ind[0, :] + sl.start) / sr.fs, ind[1, :], fronts.astype(np.double)] sav.tofile(fid_ftcp) # print progress wg.print_progress() # close temp file, read from it and delete fid_ftcp.close() tim_chan_pol = np.fromfile(str(file_ftcp)) tim_chan_pol = tim_chan_pol.reshape((int(tim_chan_pol.size / 3), 3)) file_ftcp.unlink() sync = { 'times': tim_chan_pol[:, 0], 'channels': tim_chan_pol[:, 1], 'polarities': tim_chan_pol[:, 2] } if save: alf.io.save_object_npy(output_path, sync, '_spikeglx_sync') return sync
import ibllib.dsp as dsp import ibllib.io.spikeglx import ibllib.io.extractors.ephys_fpga BATCH_SIZE_SAMPLES = 50000 # full path to the raw ephys raw_ephys_apfile = ( '/datadisk/Data/Subjects/ZM_1150/2019-05-07/001/raw_ephys_data/probe_right/' 'ephysData_g0_t0.imec.ap.bin') output_path = '/home/olivier/scratch' # load reader object, and extract sync traces sr = ibllib.io.spikeglx.Reader(raw_ephys_apfile) sync = ibllib.io.extractors.ephys_fpga._sync_to_alf(sr, output_path, save=False) # if the data is needed as well, loop over the file # raw data contains raw ephys traces, while raw_sync contains the 16 sync traces wg = dsp.WindowGenerator(sr.ns, BATCH_SIZE_SAMPLES, overlap=1) for first, last in wg.firstlast: rawdata, rawsync = sr.read_samples(first, last) wg.print_progress()
def event_extraction_and_comparison(sr): # it took 8 min to run that for 6 min of data, all 300 ish channels # silent channels for Guido's set: # [36,75,112,151,188,227,264,303,317,340,379,384] # sr,sync=get_ephys_data(sync_test_folder) """ this function first finds the times of square signal fronts in ephys and compares them to corresponding ones in the sync signal. Iteratively for small data chunks """ _logger.info('starting event_extraction_and_comparison') period_duration = 30000 # in observations, 30 kHz BATCH_SIZE_SAMPLES = period_duration # in observations, 30 kHz # only used to find first pulse wg = dsp.WindowGenerator(sr.ns, BATCH_SIZE_SAMPLES, overlap=1) # if the data is needed as well, loop over the file # raw data contains raw ephys traces, while raw_sync contains the 16 sync # traces rawdata, _ = sr.read_samples(0, BATCH_SIZE_SAMPLES) _, chans = rawdata.shape chan_fronts = {} sync_fronts = {} sync_fronts['fpga up fronts'] = [] for j in range(chans): chan_fronts[j] = {} chan_fronts[j]['ephys up fronts'] = [] for first, last in list(wg.firstlast): _, rawsync = sr.read_samples(first, last) diffs = np.diff(rawsync.T[0]) sync_up_fronts = np.where(diffs == 1)[0] + first if len(sync_up_fronts) != 0: break k = 0 # assure there is exactly one pulse per cut segment for pulse in range(500): # there are 500 square pulses if pulse == 0: first = int(sync_up_fronts[0] - period_duration / 4) last = int(first + period_duration) else: first = int(sync_up_fronts[0] - period_duration / 4 + period_duration) last = int(first + period_duration) if k % 100 == 0: print('segment %s of %s' % (k, 500)) k += 1 rawdata, rawsync = sr.read_samples(first, last) # get fronts for sync signal diffs = np.diff(rawsync.T[0]) sync_up_fronts = np.where(diffs == 1)[0] + first sync_fronts['fpga up fronts'].append(sync_up_fronts) # get fronts for only one valid ephys channel obs, chans = rawdata.shape i = 0 # assume channel 0 is valid (to be generalized maybe) Mean = np.median(rawdata.T[i]) Std = np.std(rawdata.T[i]) ups = np.invert(rawdata.T[i] > Mean + 6 * Std) up_fronts = [] # Activity front at least 10 samples long (empirical) up_fronts.append(first_occ_index(ups, 3) + first) chan_fronts[i]['ephys up fronts'].append(up_fronts) return chan_fronts, sync_fronts
def event_extraction_and_comparison(sr): # it took 8 min to run that for 6 min of data, all 300 ish channels # silent channels for Guido's set: # [36,75,112,151,188,227,264,303,317,340,379,384] # sr,sync=get_ephys_data(sync_test_folder) """ this function first finds the times of square signal fronts in ephys and compares them to corresponding ones in the sync signal. Iteratively for small data chunks """ _logger.info('starting event_extraction_and_comparison') period_duration = 30000 # in observations, 30 kHz BATCH_SIZE_SAMPLES = period_duration # in observations, 30 kHz # only used to find first pulse wg = dsp.WindowGenerator(sr.ns, BATCH_SIZE_SAMPLES, overlap=1) # if the data is needed as well, loop over the file # raw data contains raw ephys traces, while raw_sync contains the 16 sync # traces rawdata, _ = sr.read_samples(0, BATCH_SIZE_SAMPLES) _, chans = rawdata.shape chan_fronts = {} sync_fronts = {} sync_fronts['fpga up fronts'] = [] sync_fronts['fpga down fronts'] = [] for j in range(chans): chan_fronts[j] = {} chan_fronts[j]['ephys up fronts'] = [] chan_fronts[j]['ephys down fronts'] = [] # find time of first pulse (take first channel with square signal) for i in range(chans): try: # assuming a signal in the first minute for first, last in list(wg.firstlast)[:120]: rawdata, rawsync = sr.read_samples(first, last) diffs = np.diff(rawsync.T[0]) sync_up_fronts = np.where(diffs == 1)[0] + first if len(sync_up_fronts) != 0: break assert len(sync_up_fronts) != 0 Channel = i break except BaseException: print('channel %s shows no pulse signal, checking next' % i) assert i < 10, \ "something wrong, \ the first 10 channels don't show a square signal" continue start_of_chopping = sync_up_fronts[0] - period_duration / 4 k = 0 # assure there is exactly one pulse per cut segment for pulse in range(500): # there are 500 square pulses first = int(start_of_chopping + period_duration * pulse) last = int(first + period_duration) if k % 100 == 0: print('segment %s of %s' % (k, 500)) k += 1 rawdata, rawsync = sr.read_samples(first, last) # get fronts for sync signal diffs = np.diff(rawsync.T[0]) # can that thing be a global variable? sync_up_fronts = np.where(diffs == 1)[0] + first sync_down_fronts = np.where(diffs == -1)[0] + first sync_fronts['fpga up fronts'].append(sync_up_fronts) sync_fronts['fpga down fronts'].append(sync_down_fronts) # get fronts for only one valid ephys channel obs, chans = rawdata.shape i = Channel Mean = np.median(rawdata.T[i]) Std = np.std(rawdata.T[i]) ups = np.invert(rawdata.T[i] > Mean + 6 * Std) downs = np.invert(rawdata.T[i] < Mean - 6 * Std) up_fronts = [] down_fronts = [] # Activity front at least 10 samples long (empirical) up_fronts.append(first_occ_index(ups, 20) + first) down_fronts.append(first_occ_index(downs, 20) + first) chan_fronts[i]['ephys up fronts'].append(up_fronts) chan_fronts[i]['ephys down fronts'].append(down_fronts) return chan_fronts, sync_fronts # all differences