def rmsmap(fbin, spectra=True):
"""
Computes RMS map in time domain and spectra for each channel of Neuropixel probe
:param fbin: binary file in spike glx format (will look for attached metatdata)
:type fbin: str or pathlib.Path
:param spectra: whether to compute the power spectrum (only need for lfp data)
:type: bool
:return: a dictionary with amplitudes in channeltime space, channelfrequency space, time
and frequency scales
"""
if not isinstance(fbin, spikeglx.Reader):
sglx = spikeglx.Reader(fbin)
rms_win_length_samples = 2**np.ceil(np.log2(sglx.fs * RMS_WIN_LENGTH_SECS))
# the window generator will generates window indices
wingen = dsp.WindowGenerator(ns=sglx.ns,
nswin=rms_win_length_samples,
overlap=0)
# pre-allocate output dictionary of numpy arrays
win = {
'TRMS': np.zeros((wingen.nwin, sglx.nc)),
'nsamples': np.zeros((wingen.nwin, )),
'fscale': dsp.fscale(WELCH_WIN_LENGTH_SAMPLES,
1 / sglx.fs,
one_sided=True),
'tscale': wingen.tscale(fs=sglx.fs)
}
win['spectral_density'] = np.zeros((len(win['fscale']), sglx.nc))
# loop through the whole session
for first, last in wingen.firstlast:
D = sglx.read_samples(first_sample=first,
last_sample=last)[0].transpose()
# remove low frequency noise below 1 Hz
D = dsp.hp(D, 1 / sglx.fs, [0, 1])
iw = wingen.iw
win['TRMS'][iw, :] = dsp.rms(D)
win['nsamples'][iw] = D.shape[1]
if spectra:
# the last window may be smaller than what is needed for welch
if last - first < WELCH_WIN_LENGTH_SAMPLES:
continue
# compute a smoothed spectrum using welch method
_, w = signal.welch(D,
fs=sglx.fs,
window='hanning',
nperseg=WELCH_WIN_LENGTH_SAMPLES,
detrend='constant',
return_onesided=True,
scaling='density',
axis=-1)
win['spectral_density'] += w.T
# print at least every 20 windows
if (iw % min(20, max(int(np.floor(wingen.nwin / 75)), 1))) == 0:
print_progress(iw, wingen.nwin)
return win
def _sync_to_alf(raw_ephys_apfile, output_path=None, save=False, parts=''):
"""
Extracts sync.times, sync.channels and sync.polarities from binary ephys dataset
:param raw_ephys_apfile: bin file containing ephys data or spike
:param output_path: output directory
:param save: bool write to disk only if True
:param parts: string or list of strings that will be appended to the filename before extension
:return:
"""
# handles input argument: support ibllib.io.spikeglx.Reader, str and pathlib.Path
if isinstance(raw_ephys_apfile, spikeglx.Reader):
sr = raw_ephys_apfile
else:
raw_ephys_apfile = Path(raw_ephys_apfile)
sr = spikeglx.Reader(raw_ephys_apfile)
# if no output, need a temp folder to swap for big files
if not output_path:
output_path = raw_ephys_apfile.parent
file_ftcp = Path(output_path).joinpath(
f'fronts_times_channel_polarity{str(uuid.uuid4())}.bin')
# loop over chunks of the raw ephys file
wg = dsp.WindowGenerator(sr.ns,
int(SYNC_BATCH_SIZE_SECS * sr.fs),
overlap=1)
fid_ftcp = open(file_ftcp, 'wb')
for sl in wg.slice:
ss = sr.read_sync(sl)
ind, fronts = dsp.fronts(ss, axis=0)
# a = sr.read_sync_analog(sl)
sav = np.c_[(ind[0, :] + sl.start) / sr.fs, ind[1, :],
fronts.astype(np.double)]
sav.tofile(fid_ftcp)
# print progress
wg.print_progress()
# close temp file, read from it and delete
fid_ftcp.close()
tim_chan_pol = np.fromfile(str(file_ftcp))
tim_chan_pol = tim_chan_pol.reshape((int(tim_chan_pol.size / 3), 3))
file_ftcp.unlink()
sync = {
'times': tim_chan_pol[:, 0],
'channels': tim_chan_pol[:, 1],
'polarities': tim_chan_pol[:, 2]
}
if save:
out_files = alf.io.save_object_npy(output_path,
sync,
'_spikeglx_sync',
parts=parts)
return Bunch(sync), out_files
else:
return Bunch(sync)
def welchogram(fs, wav, nswin=NS_WIN, overlap=OVERLAP, nperseg=NS_WELCH):
"""
Computes a spectrogram on a very large audio file.
:param fs: sampling frequency (Hz)
:param wav: wav signal (vector or memmap)
:param nswin: n samples of the sliding window
:param overlap: n samples of the overlap between windows
:param nperseg: n samples for the computation of the spectrogram
:return: tscale, fscale, downsampled_spectrogram
"""
ns = wav.shape[0]
window_generator = dsp.WindowGenerator(ns=ns, nswin=nswin, overlap=overlap)
nwin = window_generator.nwin
fscale = dsp.fscale(nperseg, 1 / fs, one_sided=True)
W = np.zeros((nwin, len(fscale)))
tscale = window_generator.tscale(fs=fs)
detect = []
for first, last in window_generator.firstlast:
# load the current window into memory
w = np.float64(wav[first:last]) * _get_conversion_factor()
# detection of ready tones
a = [d + first for d in _detect_ready_tone(w, fs)]
if len(a):
detect += a
# the last window may not allow a pwelch
if (last - first) < nperseg:
continue
# compute PSD estimate for the current window
iw = window_generator.iw
_, W[iw, :] = signal.welch(w,
fs=fs,
window='hanning',
nperseg=nperseg,
axis=-1,
detrend='constant',
return_onesided=True,
scaling='density')
if (iw % 50) == 0:
window_generator.print_progress()
window_generator.print_progress()
# the onset detection may have duplicates with sliding window, average them and remove
detect = np.sort(np.array(detect)) / fs
ind = np.where(np.diff(detect) < 0.1)[0]
detect[ind] = (detect[ind] + detect[ind + 1]) / 2
detect = np.delete(detect, ind + 1)
return tscale, fscale, W, detect
def _sync_to_alf(raw_ephys_apfile, output_path=None, save=False):
"""
Extracts sync.times, sync.channels and sync.polarities from binary ephys dataset
:param raw_ephys_apfile: bin file containing ephys data or spike
:param out_dir: output directory
:return:
"""
if not output_path:
file_ftcp = tempfile.TemporaryFile()
else:
file_ftcp = Path(output_path) / 'fronts_times_channel_polarity.bin'
if isinstance(raw_ephys_apfile, ibllib.io.spikeglx.Reader):
sr = raw_ephys_apfile
else:
sr = ibllib.io.spikeglx.Reader(raw_ephys_apfile)
# loop over chunks of the raw ephys file
wg = dsp.WindowGenerator(sr.ns, SYNC_BATCH_SIZE_SAMPLES, overlap=1)
fid_ftcp = open(file_ftcp, 'wb')
for sl in wg.slice:
ss = sr.read_sync(sl)
ind, fronts = dsp.fronts(ss, axis=0)
sav = np.c_[(ind[0, :] + sl.start) / sr.fs, ind[1, :],
fronts.astype(np.double)]
sav.tofile(fid_ftcp)
# print progress
wg.print_progress()
# close temp file, read from it and delete
fid_ftcp.close()
tim_chan_pol = np.fromfile(str(file_ftcp))
tim_chan_pol = tim_chan_pol.reshape((int(tim_chan_pol.size / 3), 3))
file_ftcp.unlink()
sync = {
'times': tim_chan_pol[:, 0],
'channels': tim_chan_pol[:, 1],
'polarities': tim_chan_pol[:, 2]
}
if save:
alf.io.save_object_npy(output_path, sync, '_spikeglx_sync')
return sync
import ibllib.dsp as dsp
import ibllib.io.spikeglx
import ibllib.io.extractors.ephys_fpga
BATCH_SIZE_SAMPLES = 50000
# full path to the raw ephys
raw_ephys_apfile = (
'/datadisk/Data/Subjects/ZM_1150/2019-05-07/001/raw_ephys_data/probe_right/'
'ephysData_g0_t0.imec.ap.bin')
output_path = '/home/olivier/scratch'
# load reader object, and extract sync traces
sr = ibllib.io.spikeglx.Reader(raw_ephys_apfile)
sync = ibllib.io.extractors.ephys_fpga._sync_to_alf(sr,
output_path,
save=False)
# if the data is needed as well, loop over the file
# raw data contains raw ephys traces, while raw_sync contains the 16 sync traces
wg = dsp.WindowGenerator(sr.ns, BATCH_SIZE_SAMPLES, overlap=1)
for first, last in wg.firstlast:
rawdata, rawsync = sr.read_samples(first, last)
wg.print_progress()
def event_extraction_and_comparison(sr):
# it took 8 min to run that for 6 min of data, all 300 ish channels
# silent channels for Guido's set:
# [36,75,112,151,188,227,264,303,317,340,379,384]
# sr,sync=get_ephys_data(sync_test_folder)
"""
this function first finds the times of square signal fronts in ephys and
compares them to corresponding ones in the sync signal.
Iteratively for small data chunks
"""
_logger.info('starting event_extraction_and_comparison')
period_duration = 30000 # in observations, 30 kHz
BATCH_SIZE_SAMPLES = period_duration # in observations, 30 kHz
# only used to find first pulse
wg = dsp.WindowGenerator(sr.ns, BATCH_SIZE_SAMPLES, overlap=1)
# if the data is needed as well, loop over the file
# raw data contains raw ephys traces, while raw_sync contains the 16 sync
# traces
rawdata, _ = sr.read_samples(0, BATCH_SIZE_SAMPLES)
_, chans = rawdata.shape
chan_fronts = {}
sync_fronts = {}
sync_fronts['fpga up fronts'] = []
for j in range(chans):
chan_fronts[j] = {}
chan_fronts[j]['ephys up fronts'] = []
for first, last in list(wg.firstlast):
_, rawsync = sr.read_samples(first, last)
diffs = np.diff(rawsync.T[0])
sync_up_fronts = np.where(diffs == 1)[0] + first
if len(sync_up_fronts) != 0:
break
k = 0
# assure there is exactly one pulse per cut segment
for pulse in range(500): # there are 500 square pulses
if pulse == 0:
first = int(sync_up_fronts[0] - period_duration / 4)
last = int(first + period_duration)
else:
first = int(sync_up_fronts[0] - period_duration / 4 + period_duration)
last = int(first + period_duration)
if k % 100 == 0:
print('segment %s of %s' % (k, 500))
k += 1
rawdata, rawsync = sr.read_samples(first, last)
# get fronts for sync signal
diffs = np.diff(rawsync.T[0])
sync_up_fronts = np.where(diffs == 1)[0] + first
sync_fronts['fpga up fronts'].append(sync_up_fronts)
# get fronts for only one valid ephys channel
obs, chans = rawdata.shape
i = 0 # assume channel 0 is valid (to be generalized maybe)
Mean = np.median(rawdata.T[i])
Std = np.std(rawdata.T[i])
ups = np.invert(rawdata.T[i] > Mean + 6 * Std)
up_fronts = []
# Activity front at least 10 samples long (empirical)
up_fronts.append(first_occ_index(ups, 3) + first)
chan_fronts[i]['ephys up fronts'].append(up_fronts)
return chan_fronts, sync_fronts
def event_extraction_and_comparison(sr):
# it took 8 min to run that for 6 min of data, all 300 ish channels
# silent channels for Guido's set:
# [36,75,112,151,188,227,264,303,317,340,379,384]
# sr,sync=get_ephys_data(sync_test_folder)
"""
this function first finds the times of square signal fronts in ephys and
compares them to corresponding ones in the sync signal.
Iteratively for small data chunks
"""
_logger.info('starting event_extraction_and_comparison')
period_duration = 30000 # in observations, 30 kHz
BATCH_SIZE_SAMPLES = period_duration # in observations, 30 kHz
# only used to find first pulse
wg = dsp.WindowGenerator(sr.ns, BATCH_SIZE_SAMPLES, overlap=1)
# if the data is needed as well, loop over the file
# raw data contains raw ephys traces, while raw_sync contains the 16 sync
# traces
rawdata, _ = sr.read_samples(0, BATCH_SIZE_SAMPLES)
_, chans = rawdata.shape
chan_fronts = {}
sync_fronts = {}
sync_fronts['fpga up fronts'] = []
sync_fronts['fpga down fronts'] = []
for j in range(chans):
chan_fronts[j] = {}
chan_fronts[j]['ephys up fronts'] = []
chan_fronts[j]['ephys down fronts'] = []
# find time of first pulse (take first channel with square signal)
for i in range(chans):
try:
# assuming a signal in the first minute
for first, last in list(wg.firstlast)[:120]:
rawdata, rawsync = sr.read_samples(first, last)
diffs = np.diff(rawsync.T[0])
sync_up_fronts = np.where(diffs == 1)[0] + first
if len(sync_up_fronts) != 0:
break
assert len(sync_up_fronts) != 0
Channel = i
break
except BaseException:
print('channel %s shows no pulse signal, checking next' % i)
assert i < 10, \
"something wrong, \
the first 10 channels don't show a square signal"
continue
start_of_chopping = sync_up_fronts[0] - period_duration / 4
k = 0
# assure there is exactly one pulse per cut segment
for pulse in range(500): # there are 500 square pulses
first = int(start_of_chopping + period_duration * pulse)
last = int(first + period_duration)
if k % 100 == 0:
print('segment %s of %s' % (k, 500))
k += 1
rawdata, rawsync = sr.read_samples(first, last)
# get fronts for sync signal
diffs = np.diff(rawsync.T[0]) # can that thing be a global variable?
sync_up_fronts = np.where(diffs == 1)[0] + first
sync_down_fronts = np.where(diffs == -1)[0] + first
sync_fronts['fpga up fronts'].append(sync_up_fronts)
sync_fronts['fpga down fronts'].append(sync_down_fronts)
# get fronts for only one valid ephys channel
obs, chans = rawdata.shape
i = Channel
Mean = np.median(rawdata.T[i])
Std = np.std(rawdata.T[i])
ups = np.invert(rawdata.T[i] > Mean + 6 * Std)
downs = np.invert(rawdata.T[i] < Mean - 6 * Std)
up_fronts = []
down_fronts = []
# Activity front at least 10 samples long (empirical)
up_fronts.append(first_occ_index(ups, 20) + first)
down_fronts.append(first_occ_index(downs, 20) + first)
chan_fronts[i]['ephys up fronts'].append(up_fronts)
chan_fronts[i]['ephys down fronts'].append(down_fronts)
return chan_fronts, sync_fronts # all differences
