def _merge_level(s_, d_, level='/'):
if 'chan_mask' in s_:
print('end-node')
src_mask = s_.chan_mask
dst_mask = d_.chan_mask
print('Source mask: {0} channels'.format(src_mask.sum()))
print('Dest mask: {0} channels'.format(dst_mask.sum()))
choice = 'Overwrite destination? ([y]/n) '
if choice.lower() in ('y', ''):
# print 'would write', d_, 'at path:', level
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# remove trailing slash
k = level[:-1]
s_load = load_bunch(source, k)
save_bunch(dest, k, s_load, overwrite_paths=True)
return
src_keys = set(s_.keys())
dst_keys = set(d_.keys())
new_keys = src_keys.difference(dst_keys)
for nk in new_keys:
# print 'would write', s_[nk], 'at path:', '/'+nk
with warnings.catch_warnings():
warnings.simplefilter("ignore")
s_load = load_bunch(source, level + nk)
save_bunch(dest, level + nk, s_load)
sim_keys = src_keys.intersection(dst_keys)
for sk in sim_keys:
_merge_level(s_[sk], d_[sk], level + sk + '/')
def rawload_mux(exp_path,
test,
version,
daq_variant='',
data_only=False,
shm=True):
"""
Find and load data recorded from the MUX style headstage. Return all
recording columns by default, otherwise only return the electrode
data.
"""
raw_data = None
shm_arr = ('/data', ) if shm else ()
try:
raw_data = load_bunch(os.path.join(exp_path, test + '.h5'),
'/',
shared_arrays=shm_arr)
except IOError:
raw_data = load_bunch(os.path.join(exp_path, test + '.mat'),
'/',
shared_arrays=shm_arr)
try:
Fs = raw_data.Fs
except:
Fs = raw_data.fs
shape = raw_data.data.shape
if shape[1] < shape[0]:
raw_data.data = raw_data.data.transpose().copy()
nrow, ncol_data = list(map(int, (raw_data.numRow, raw_data.numCol)))
# scale data channels
raw_data.data[:ncol_data * nrow] /= mux_gain[version]
# correct for permuted digital out sampling
if not daq_variant:
# if daq info (new style) is not given, try to look up sampling order
# based on the mux version (old style)
daq_variant = version
raw_data.data = _permute_mux(raw_data.data, nrow, daq_variant)
if data_only:
raw_data.data = raw_data.data[:ncol_data * nrow]
try:
# stim-mux converted h5 files (Virginia's conversion)
# do not have info
info = tdms_info(raw_data.info)
except AttributeError:
info = None
return raw_data.data, Fs, (nrow, ncol_data), info
def lookup(self, dset_name):
path = self._dset_to_path(dset_name)
try:
node = load_bunch(self.dbfile, path)
except IOError:
return Bunch()
except NoSuchNodeError:
return Bunch()
return node
def try_saved(exp_path, test, bandpass):
try:
dset = load_bunch(os.path.join(exp_path, test + '_proc.h5'), '/')
#h5 = tables.open_file(os.path.join(exp_path, test+'_proc.h5'))
except IOError as exc:
raise DataPathError
if dset.bandpass != bandpass:
del dset
raise DataPathError(
'saved data bandpass does not match requested bandpass')
return dset
def list_paths(self):
b = load_bunch(self.dbfile, '/', load=False)
_walk_paths(b)
def load_open_ephys_channels(exp_path,
test,
rec_num='auto',
shared_array=False,
downsamp=1,
target_Fs=-1,
lowpass_ord=12,
page_size=8,
save_downsamp=True,
use_stored=True,
store_path='',
quantized=False):
# first off, check if there is a stored file at target_Fs (if valid)
if use_stored and target_Fs > 0:
# Look for a previously downsampled data stash
fname_part = '*{0}*_Fs{1}.h5'.format(test, int(target_Fs))
# try store_path (if given) and also exp_path
for p_ in (store_path, exp_path):
fname = glob(osp.join(p_, fname_part))
if len(fname) and osp.exists(fname[0]):
print('Loading from', fname[0])
channel_data = load_bunch(fname[0], '/')
return channel_data
rec_path, rec_num = prepare_paths(exp_path, test, rec_num)
trueFs = get_robust_samplingrate(rec_path)
if downsamp == 1 and target_Fs > 0:
if trueFs is None:
# do nothing
print('Sampling frequency not robustly determined, '
'downsample not calculated for {0:.1f} Hz'.format(target_Fs))
raise ValueError
else:
# find the correct (integer) downsample rate
# to get (approx) target Fs
# target_fs * downsamp <= Fs
# downsamp <= Fs / target_fs
downsamp = int(trueFs // target_Fs)
print(('downsample rate:', downsamp))
if downsamp > 1 and quantized:
print('Cannot return quantized data when downsampling')
quantized = False
downsamp = int(downsamp)
all_files = list()
for pre in rec_num:
all_files.extend(glob(osp.join(rec_path, pre + '*.continuous')))
if not len(all_files):
raise IOError('No files found')
c_nums = list()
chan_files = list()
aux_files = list()
aux_nums = list()
adc_files = list()
adc_nums = list()
for f in all_files:
f_part = osp.splitext(osp.split(f)[1])[0]
# File names can be: Proc#_{ADC/CH/AUX}[_N].continuous
# (the last _N part is not always present!! disgard for now)
f_parts = f_part.split('_')
if len(f_parts[-1]) == 1 and f_parts[-1] in '0123456789':
f_parts = f_parts[:-1]
ch = f_parts[-1] # last file part is CHx or AUXx
if ch[0:2] == 'CH':
chan_files.append(f)
c_nums.append(int(ch[2:]))
elif ch[0:3] == 'AUX': # separate chan and AUX files
aux_files.append(f)
aux_nums.append(int(ch[3:]))
elif ch[0:3] == 'ADC':
adc_files.append(f)
adc_nums.append(int(ch[3:]))
if downsamp > 1:
(b_lp, a_lp) = cheby2_bp(60, hi=1.0 / downsamp, Fs=2, ord=lowpass_ord)
def _load_array_block(files, shared_array=False, antialias=True):
Fs = 1
dtype = 'h' if quantized else 'd'
# start on 1st index of 0th block
n = 1
b_cnt = 0
b_idx = 1
ch_record = OE.loadContinuous(files[0], dtype=np.int16, verbose=False)
d_len = ch_record['data'].shape[-1]
sub_len = d_len // downsamp
if sub_len * downsamp < d_len:
sub_len += 1
proc_block = shm.shared_ndarray((page_size, d_len), typecode=dtype)
proc_block[0] = ch_record['data'].astype('d')
if shared_array:
saved_array = shm.shared_ndarray((len(files), sub_len),
typecode=dtype)
else:
saved_array = np.zeros((len(files), sub_len), dtype=dtype)
for f in files[1:]:
ch_record = OE.loadContinuous(f, dtype=np.int16,
verbose=False) # load data
Fs = float(ch_record['header']['sampleRate'])
proc_block[b_idx] = ch_record['data'].astype(dtype)
b_idx += 1
n += 1
if (b_idx == page_size) or (n == len(files)):
# do dynamic range conversion and downsampling
# on a block of data
if not quantized:
proc_block *= ch_record['header']['bitVolts']
if downsamp > 1 and antialias:
filtfilt(proc_block, b_lp, a_lp)
sl = slice(b_cnt * page_size, n)
saved_array[sl] = proc_block[:b_idx, ::downsamp]
# update / reset block counters
b_idx = 0
b_cnt += 1
del proc_block
while gc.collect():
pass
return saved_array, Fs, ch_record['header']
# sort CH, AUX, and ADC by the channel number
sorted_chans = np.argsort(c_nums)
# sorts list ed on sorted_chans
chan_files = [chan_files[n] for n in sorted_chans]
chdata, Fs, header = _load_array_block(chan_files,
shared_array=shared_array)
aux_data = list()
if len(aux_files) > 0:
sorted_aux = np.argsort(aux_nums)
aux_files = [aux_files[n] for n in sorted_aux]
aux_data, _, _ = _load_array_block(aux_files, antialias=False)
adc_data = list()
if len(adc_files) > 0:
sorted_adc = np.argsort(adc_nums)
adc_files = [adc_files[n] for n in sorted_adc]
adc_data, _, _ = _load_array_block(adc_files, antialias=False)
if not trueFs:
print('settings.xml not found, relying on sampling rate from '
'recording header files')
trueFs = Fs
if downsamp > 1:
trueFs /= downsamp
dset = Bunch(chdata=chdata,
aux=aux_data,
adc=adc_data,
Fs=trueFs,
header=header)
if save_downsamp and downsamp > 1:
fname = '{0}_Fs{1}.h5'.format(osp.split(rec_path)[-1], int(dset.Fs))
if not len(store_path):
store_path = exp_path
mkdir_p(store_path)
fname = osp.join(store_path, fname)
print('saving', fname)
save_bunch(fname, '/', dset, mode='w')
return dset