def _load_array_block(files, shared_array=False, antialias=True):
Fs = 1
dtype = 'h' if quantized else 'd'
# start on 1st index of 0th block
n = 1
b_cnt = 0
b_idx = 1
ch_record = OE.loadContinuous(files[0], dtype=np.int16, verbose=False)
d_len = ch_record['data'].shape[-1]
sub_len = d_len // downsamp
if sub_len * downsamp < d_len:
sub_len += 1
proc_block = shm.shared_ndarray((page_size, d_len), typecode=dtype)
proc_block[0] = ch_record['data'].astype('d')
if shared_array:
saved_array = shm.shared_ndarray((len(files), sub_len),
typecode=dtype)
else:
saved_array = np.zeros((len(files), sub_len), dtype=dtype)
for f in files[1:]:
ch_record = OE.loadContinuous(f, dtype=np.int16,
verbose=False) # load data
Fs = float(ch_record['header']['sampleRate'])
proc_block[b_idx] = ch_record['data'].astype(dtype)
b_idx += 1
n += 1
if (b_idx == page_size) or (n == len(files)):
# do dynamic range conversion and downsampling
# on a block of data
if not quantized:
proc_block *= ch_record['header']['bitVolts']
if downsamp > 1 and antialias:
filtfilt(proc_block, b_lp, a_lp)
sl = slice(b_cnt * page_size, n)
saved_array[sl] = proc_block[:b_idx, ::downsamp]
# update / reset block counters
b_idx = 0
b_cnt += 1
del proc_block
while gc.collect():
pass
return saved_array, Fs, ch_record['header']
def upsample(x, fs, appx_fs=None, r=None, interp='sinc'):
if appx_fs is None and r is None:
return x
if appx_fs is not None and r is not None:
raise ValueError('only specify new fs or resample factor, not both')
if appx_fs is not None:
# new sampling interval must be a multiple of old sample interval,
# so find the closest match that is <= appx_fs
r = int(np.floor(appx_fs / fs))
x_up = shm.shared_ndarray(x.shape[:-1] + (r * x.shape[-1],), x.dtype.char)
x_up[..., ::r] = x
new_fs = fs * r
from ecogdata.filt.blocks import BlockedSignal
from scipy.fftpack import fft, ifft, fftfreq
from scipy.interpolate import interp1d
if interp == 'sinc':
x_up *= r
op_size = 2**16
xb = BlockedSignal(x_up, op_size, partial_block=True)
for block in xb.fwd():
Xf = fft(block, axis=-1)
freq = fftfreq(Xf.shape[-1]) * new_fs
Xf[..., np.abs(freq) > fs/2.0] = 0
block[:] = ifft(Xf).real
return x_up, new_fs
elif interp == 'linear':
# always pick up the first and last sample when skipping by r
op_size = r * 10000 + 1
t = np.arange(op_size)
xb = BlockedSignal(x_up, op_size, overlap=1, partial_block=True)
for block in xb.fwd():
N = block.shape[-1]
fn = interp1d(t[:N][::r], block[..., ::r], axis=-1,
bounds_error=False, fill_value=0,
assume_sorted=True)
block[:] = fn(t[:N])
return x_up, new_fs
# design a cheby-1 lowpass filter
# wp: 0.4 * new_fs
# ws: 0.5 * new_fs
# design specs with halved numbers, since filtfilt will be used
wp = 2 * 0.4 * fs / new_fs
ws = 2 * 0.5 * fs / new_fs
ord, wc = signal.cheb1ord(wp, ws, 0.25, 10)
fdesign = dict(ripple=0.25, hi=0.5 * wc * new_fs, Fs=new_fs, ord=ord)
filter_array(x_up, ftype='cheby1', inplace=True, design_kwargs=fdesign)
x_up *= r
return x_up, new_fs
def test_parfilt2():
from ecogdata.parallel.split_methods import bfilter as bfilter_p
import ecogdata.parallel.sharedmem as shm
r = np.random.randn(20, 2000)
b, a = butter_bp(lo=30, hi=100, Fs=1000)
zi = lfilter_zi(b, a)
f1, _ = lfilter(b, a, r, axis=1, zi=zi * r[:, :1])
f2 = shm.shared_copy(r)
f3 = shm.shared_ndarray(f2.shape, f2.dtype.char)
# test w/o blocking
bfilter_p(b, a, f2, out=f3, axis=1)
assert (f1 == f3).all()
# test w/ blocking
f2 = shm.shared_copy(r)
bfilter_p(b, a, f2, out=f3, bsize=234, axis=1)
assert (f1 == f3).all()
def get_output_array(self, slicer, only_shape=False):
"""
Allocate an array for read outs (or just the output shape).
Parameters
----------
slicer: slice
Read slice
only_shape: bool
If True, just return the output shape
Returns
-------
output_array: ndarray
Allocated array for output (unless only_shape is given).
"""
# Currently this is used to
# 1) check a slice size
# 2) create an output array for slice caching
# **NOTE** that the output needs to respect `transpose_state`
slicer = _abs_slicer(slicer, self.shape)
# patch inter-operability from 2.10 to 3.x
if h5py.version.version_tuple.major >= 3:
_null_dataset = None
shape_attr = 'array_shape'
else:
_null_dataset = 0
shape_attr = 'mshape'
selection = select(self.shape, slicer, _null_dataset)
out_shape = getattr(selection, shape_attr)
# if the output will be transposed, then pre-create the array in the right order
if self._transpose_state:
out_shape = out_shape[::-1]
if only_shape:
return out_shape
typecode = self.dtype.char
if self._read_output is None:
if self._shared_mem_read:
return shm.shared_ndarray(out_shape, typecode)
else:
return np.empty(out_shape, typecode)
else:
return self._read_output
def _convert(rec,
from_unit,
to_unit,
inverted=False,
tfs=(),
Rct=None,
Cdl=None,
Rs=None,
prewarp=False,
**sm_kwargs):
from_type = from_unit[-1]
to_type = to_unit[-1]
def _override_tf(b, a):
b, a = signal.normalize(b, a)
Rs_, Rct_, Cdl_ = tf_to_circuit(b, a)
b, a = circuit_to_tf((Rs if Rs else Rs_), (Rct if Rct else Rct_),
(Cdl if Cdl else Cdl_))
return b, a
if not len(tfs):
from ecogdata.expconfig import params
session = rec.name.split('.')[0]
# backwards compatibility
session = session.split('/')[-1]
if session in _transform_lookup:
# XXX: ideally should find the expected unit scale of the
# transforms in the transform Bunch -- for now assume pico-scale
to_scale = convert_scale(1, from_unit, 'p' + from_type)
from_scale = convert_scale(1, 'p' + to_type, to_unit)
stash_path = pt.sep.join(
[params.stash_path, 'devices', 'impedance_transfer'])
transforms = pt.join(stash_path, _transform_lookup[session])
tfs = load_bunch(transforms, '/')
else:
# do an analog integrator --
# will be converted to 1 + z / (1 - z) later
# do this on pico scale also (?)
to_scale = convert_scale(1, from_unit, 'p' + from_type)
from_scale = convert_scale(1, 'p' + to_type, to_unit)
tfs = Bunch(aa=np.array([1, 0]), bb=np.array([0, 1]))
else:
to_scale = convert_scale(1, from_unit, 'p' + from_type)
from_scale = convert_scale(1, 'p' + to_type, to_unit)
if tfs.aa.ndim > 1:
from ecogdata.filt.time import bfilter
conv = rec.deepcopy()
cmap = conv.chan_map
bb, aa = smooth_transfer_functions(tfs.bb.T, tfs.aa.T, **sm_kwargs)
for n, ij in enumerate(zip(*cmap.to_mat())):
b = bb[ij]
a = aa[ij]
b, a = _override_tf(b, a)
if prewarp:
T = conv.Fs**-1
z, p, k = signal.tf2zpk(b, a)
z = 2 / T * np.tan(z * T / 2)
p = 2 / T * np.tan(p * T / 2)
b, a = signal.zpk2tf(z, p, k)
if inverted:
zb, za = signal.bilinear(a, b, fs=conv.Fs)
else:
zb, za = signal.bilinear(b, a, fs=conv.Fs)
bfilter(zb, za, conv.data[n], bsize=10000)
else:
from ecogdata.parallel.split_methods import bfilter
# avoid needless copy of data array
rec_data = rec.pop('data')
conv = rec.deepcopy()
rec.data = rec_data
conv_data = shm.shared_ndarray(rec_data.shape, rec_data.dtype.char)
conv_data[:] = rec_data
b = tfs.bb
a = tfs.aa
if prewarp:
T = conv.Fs**-1
z, p, k = signal.tf2zpk(b, a)
z = 2 / T * np.tan(z * T / 2)
p = 2 / T * np.tan(p * T / 2)
b, a = signal.zpk2tf(z, p, k)
if inverted:
zb, za = signal.bilinear(a, b, fs=conv.Fs)
else:
zb, za = signal.bilinear(b, a, fs=conv.Fs)
bfilter(zb, za, conv_data, bsize=10000)
conv.data = conv_data
conv.data *= (from_scale * to_scale)
conv.units = to_unit
return conv
def filter_array(
arr, ftype='butterworth', inplace=True, out=None, block_filter='parallel',
design_kwargs=dict(), filt_kwargs=dict()
):
"""
Filter an ND array timeseries on the last dimension. For computational
efficiency, the timeseries are blocked into partitions (10000 points
by default) and split over multiple threads (not supported on Windoze).
Parameters
----------
arr: ndarray
Timeseries in the last dimension (can be 1D).
ftype: str
Filter type to design.
inplace: bool
If True, then arr must be a shared memory array. Otherwise a
shared copy will be made from the input. This is a shortcut for using "out=arr".
out: ndarray
If not None, place filter output here (if inplace is specified, any output array is ignored).
block_filter: str or callable
Specify the run-time block filter to apply. Can be "parallel" or
"serial", or can be a callable that follows the basic signature of
`ecogdata.filt.time.block_filter.bfilter`.
design_kwargs: dict
Design parameters for the filter (e.g. lo, hi, Fs, ord)
filt_kwargs: dict
Processing parameters (e.g. filtfilt, bsize)
Returns
-------
arr_f: ndarray
Filtered timeseries, same shape as input.
"""
b, a = _get_poles_zeros(ftype, **design_kwargs)
check_shm = False
if isinstance(block_filter, str):
if block_filter.lower() == 'parallel':
from ecogdata.parallel.split_methods import bfilter
block_filter = bfilter
check_shm = True
elif block_filter.lower() == 'serial':
from .blocked_filter import bfilter
block_filter = bfilter
else:
raise ValueError('Block filter type not known: {}'.format(block_filter))
if not callable(block_filter):
raise ValueError('Provided block filter is not callable: {}'.format(block_filter))
def_args = get_default_args(block_filter)
# Set some defaults and then update with filt_kwargs
def_args['bsize'] = 10000
def_args['filtfilt'] = True
def_args.update(filt_kwargs)
def_args['out'] = out
if inplace:
# enforce that def_args['out'] is arr?
def_args['out'] = arr
block_filter(b, a, arr, **def_args)
return arr
else:
# still use bfilter for memory efficiency
# Work in progress to use this syntax
# use_shm = hasattr(block_filter, 'uses_parallel') and block_filter.uses_parallel(b, a, arr)
if check_shm:
# signal shared memory usage if check_shm remains true
try:
check_shm = block_filter(b, a, arr, check_parallel=True, **def_args)
except TypeError:
check_shm = False
if def_args['out'] is None:
if check_shm:
arr_f = shm.shared_ndarray(arr.shape, arr.dtype.char)
else:
arr_f = np.empty_like(arr)
def_args['out'] = arr_f
block_filter(b, a, arr, **def_args)
return def_args['out']
def load_mux(exp_path,
test,
electrode,
headstage,
ni_daq_variant='',
mux_connectors=(),
bandpass=(),
notches=(),
trigger=0,
bnc=(),
mux_notches=(),
save=False,
snip_transient=True,
units='uV'):
"""
Load data from the MUX style headstage acquisition. Data is expected
to be organized along columns corresponding to the MUX units. The
columns following sensor data columns are assumed to be a stimulus
trigger followed by other BNC channels.
The electrode information must be provided to determine the
arrangement of recorded and grounded channels within the sensor
data column.
This preprocessing routine returns a Bunch container with the
following items
dset.data : nchan x ntime data array
dset.ground_chans : m x ntime data array of grounded ADC channels
dset.bnc : un-MUXed readout of the BNC channel(s)
dset.chan_map : the channel-to-electrode mapping vector
dset.Fs : sampling frequency
dset.name : path + expID for the given data set
dset.bandpass : bandpass filtering applied (if any)
dset.trig : the logical value of the trigger channel (at MUX'd Fs)
* If saving, then a table of the Bunch is written.
* If snip_transient, then advance the timeseries past the bandpass
filtering onset transient.
"""
try:
dset = try_saved(exp_path, test, bandpass)
return dset
except DataPathError:
pass
# say no to shared memory since it's created later on in this method
loaded = rawload_mux(exp_path,
test,
headstage,
daq_variant=ni_daq_variant,
shm=False)
channels, Fs, dshape, info = loaded
nrow, ncol_data = dshape
if channels.shape[0] >= nrow * ncol_data:
ncol = channels.shape[0] // nrow
channels = channels.reshape(ncol, nrow, -1)
else:
ncol = channels.shape[0]
channels.shape = (ncol, -1, nrow)
channels = channels.transpose(0, 2, 1)
## Grab BNC data
if bnc:
bnc_chans = [ncol_data + int(b) for b in bnc]
bnc = np.zeros((len(bnc), nrow * channels.shape[-1]))
for bc, col in zip(bnc, bnc_chans):
bc[:] = channels[col].transpose().ravel()
bnc = bnc.squeeze()
try:
trig_chans = channels[ncol_data + trigger].copy()
pos_edge, trig = process_trigger(trig_chans)
except IndexError:
pos_edge = ()
trig = ()
## Realize channel mapping
chan_map, disconnected, reference = epins.get_electrode_map(
electrode, connectors=mux_connectors)
## Data channels
# if any pre-processing of multiplexed channels, do it here first
if mux_notches:
mux_chans = shm.shared_ndarray((ncol_data, channels.shape[-1], nrow))
mux_chans[:] = channels[:ncol_data].transpose(0, 2, 1)
mux_chans.shape = (ncol_data, -1)
ft.notch_all(mux_chans, Fs, lines=mux_notches, filtfilt=True)
mux_chans.shape = (ncol_data, channels.shape[-1], nrow)
channels[:ncol_data] = mux_chans.transpose(0, 2, 1)
del mux_chans
rec_chans = channels[:ncol_data].reshape(nrow * ncol_data, -1)
if units.lower() != 'v':
convert_scale(rec_chans, 'v', units)
g_chans = disconnected
r_chans = reference
d_chans = np.setdiff1d(np.arange(ncol_data * nrow),
np.union1d(g_chans, r_chans))
data_chans = shm.shared_copy(rec_chans[d_chans])
if len(g_chans):
gnd_data = rec_chans[g_chans]
else:
gnd_data = ()
if len(r_chans):
ref_data = rec_chans[r_chans]
else:
ref_data = ()
del rec_chans
del channels
# do highpass filtering for stationarity
if bandpass:
# manually remove DC from channels before filtering
if bandpass[0] > 0:
data_chans -= data_chans.mean(1)[:, None]
# do a high order highpass to really crush the crappy
# low frequency noise
b, a = ft.butter_bp(lo=bandpass[0], Fs=Fs, ord=5)
# b, a = ft.cheby1_bp(0.5, lo=bandpass[0], Fs=Fs, ord=5)
else:
b = [1]
a = [1]
if bandpass[1] > 0:
b_lp, a_lp = ft.butter_bp(hi=bandpass[1], Fs=Fs, ord=3)
b = np.convolve(b, b_lp)
a = np.convolve(a, a_lp)
filtfilt(data_chans, b, a)
if len(ref_data):
with parallel_controller(False):
ref_data = np.atleast_2d(ref_data)
filtfilt(ref_data, b, a)
ref_data = ref_data.squeeze()
if notches:
ft.notch_all(data_chans,
Fs,
lines=notches,
inplace=True,
filtfilt=True)
if len(ref_data):
with parallel_controller(False):
ref_data = np.atleast_2d(ref_data)
ft.notch_all(ref_data,
Fs,
lines=notches,
inplace=True,
filtfilt=True)
ref_data = ref_data.squeeze()
if snip_transient:
if isinstance(snip_transient, bool):
snip_len = int(Fs * 5)
else:
snip_len = int(Fs * snip_transient)
if len(pos_edge):
snip_len = max(0, min(snip_len, pos_edge[0] - int(Fs)))
pos_edge = pos_edge - snip_len
trig = trig[..., snip_len:].copy()
data_chans = data_chans[..., snip_len:].copy()
gnd_data = gnd_data[..., snip_len:].copy()
if len(ref_data):
ref_data = ref_data[..., snip_len:].copy()
if len(bnc):
bnc = bnc[..., snip_len * nrow:].copy()
# do blockwise detrending for stationarity
## detrend_window = int(round(0.750*Fs))
## ft.bdetrend(data_chans, bsize=detrend_window, type='linear', axis=-1)
dset = ut.Bunch()
dset.pos_edge = pos_edge
dset.data = data_chans
dset.ground_chans = gnd_data
dset.ref_chans = ref_data
dset.bnc = bnc
dset.chan_map = chan_map
dset.Fs = Fs
#dset.name = os.path.join(exp_path, test)
dset.bandpass = bandpass
dset.trig = trig
dset.transient_snipped = snip_transient
dset.units = units
dset.notches = notches
dset.info = info
if save:
hf = os.path.join(exp_path, test + '_proc.h5')
save_bunch(hf, '/', dset, mode='w')
return dset
def map_raw_data(self, data_file, open_mode, electrode_chans, ground_chans,
ref_chans, downsample_ratio):
"""
Unlike the parent method, this method will directly load and downsample .continuous files to in-memory arrays
without creating an intermediary MappedSource.
Parameters
----------
data_file: str
Open Ephys data path or HDF5 file
open_mode:
File mode (for HDF5 only)
electrode_chans: sequence
ground_chans: sequence
ref_chans: sequence
downsample_ratio: int
Returns
-------
datasource: ElectrodeArraySource
Datasource for electrodes.
ground_chans: ElectrodeArraySource
Datasource for ground channels. May be an empty list
ref_chans: ElectrodeArraySource:
Datasource for reference channels. May be an empty list
"""
# downsample_ratio is 1 if a pre-computed file exists or if a full resolution HDF5 needs to be created here.
# In these cases, revert to parent method
if downsample_ratio == 1:
if os.path.isdir(data_file):
# mapped_file = data_file + '.h5'
with TempFilePool(mode='ab') as mf:
mapped_file = mf
print(
'Take note!! Creating full resolution map file {}'.format(
mapped_file))
# Get a little tricky and see if this source should be writeable. If no, then leave it quantized with
# a scaling factor. If yes, then convert the source to microvolts.
quantized = not (self.ensure_writeable or self.bandpass
or self.notches)
hdf5_open_ephys_channels(self.experiment_path,
self.recording,
str(mapped_file),
data_chans='all',
quantized=quantized)
if quantized:
open_mode = 'r'
else:
open_mode = 'r+'
self.units_scale = convert_scale(1, 'uv', self.units)
# Note: pass file "name" directly as a TempFilePool object so that file-closing can be registered
# downstream!!
data_file = mapped_file
else:
# the data_file is possibly an HDF file that should be treated as a read-only source
with h5py.File(data_file, 'r') as fw:
if fw['chdata'].dtype in np.sctypes['float']:
self.units_scale = convert_scale(1, 'uv', self.units)
open_mode = 'r'
print('Calling super to map/load {} in mode {} scaling units {}'.
format(data_file, open_mode, self.units_scale))
return super(OpenEphysLoader,
self).map_raw_data(data_file, open_mode,
electrode_chans, ground_chans,
ref_chans, downsample_ratio)
# If downsample_ratio > 1 then we are downsampling straight to memory. Invoke a custom loader that handles
# continuous files
loaded = load_open_ephys_channels(self.experiment_path,
self.recording,
shared_array=False,
downsamp=downsample_ratio,
save_downsamp=False,
use_stored=False,
quantized=False)
chdata = loaded.chdata
electrode_data = shm.shared_ndarray(
(len(electrode_chans), chdata.shape[1]), chdata.dtype.char)
np.take(chdata, electrode_chans, axis=0, out=electrode_data)
datasource = PlainArraySource(electrode_data,
use_shared_mem=False,
adc=loaded.adc,
aux=loaded.aux)
if ground_chans:
ground_chans = PlainArraySource(chdata[ground_chans],
use_shared_mem=False)
if ref_chans:
ref_chans = PlainArraySource(chdata[ref_chans],
use_shared_mem=False)
return datasource, ground_chans, ref_chans
def mirror(self, new_rate_ratio=None, writeable=True, mapped=True, channel_compatible=False, filename='',
copy='', new_sources=dict(), **map_args):
# TODO: channel order permutation in mirrored source
"""
Create an empty ElectrodeDataSource based on the current source, possibly with a new sampling rate and new
access permissions.
Parameters
----------
new_rate_ratio: int or None
Ratio of old to new sample rate for the mirrored array (> 1).
writeable: bool
Make any new MappedSource arrays writeable. This implies 1) datatype casting to floats, and 2) there is
no more units conversion on the primary array.
mapped: bool
If False, mirror to a PlainArraySource (in memory). Else mirror into a new MappedSource.
channel_compatible: bool
If True, preserve the same number of raw data channels in a MappedSource. Otherwise, reduce the channels
to just the set of active channels. Currently not supported for non-mapped mirrors.
filename: str
Name of the new MappedSource. If empty, use a self-destroying temporary file.
copy: str
Code whether to copy any arrays, which is only valid when new_rate_ratio is None or 1. 'aligned' copies
aligned arrays. 'electrode' copies electrode data: only valid if channel_compatible is False.
'all' copies all arrays. By default, nothing is copied.
new_sources: dict
If mapped=False, then pre-allocated arrays can be provided for each source (i.e. 'data_buffer' and any
aligned arrays).
map_args: dict
Any other MappedSource arguments
Returns
-------
datasource: ElectrodeDataSource subtype
"""
T = self.shape[1]
if new_rate_ratio:
T = calc_new_samples(T, new_rate_ratio)
# if casting to floating point, check for preferred precision
fp_precision = load_params().floating_point.lower()
fp_dtype = 'f' if fp_precision == 'single' else 'd'
fp_dtype = np.dtype(fp_dtype)
# unpack copy mode
copy_electrodes_coded = copy.lower() in ('all', 'electrode')
copy_aligned_coded = copy.lower() in ('all', 'aligned')
diff_rate = T != self.shape[1]
if copy_electrodes_coded:
if diff_rate or channel_compatible:
copy_electrodes = False
print('Not copying electrode channels. Diff rate '
'({}) or indirect channel map ({})'.format(diff_rate, channel_compatible))
else:
copy_electrodes = True
else:
copy_electrodes = False
if copy_aligned_coded:
if diff_rate:
copy_aligned = False
print('Not copying aligned arrays: different sample rate')
else:
copy_aligned = True
else:
copy_aligned = False
if mapped:
if channel_compatible:
electrode_channels = self._electrode_channels
C = self.data_buffer.shape[1] if self._transpose else self.data_buffer.shape[0]
channel_mask = self.binary_channel_mask
else:
C = self.shape[0]
electrode_channels = None
channel_mask = None
if writeable:
new_dtype = fp_dtype
reopen_mode = 'r+'
units_scale = None
else:
new_dtype = self.data_buffer.map_dtype
reopen_mode = 'r'
units_scale = self.data_buffer.units_scale
tempfile = not filename
if tempfile:
with TempFilePool(mode='ab') as f:
# punt on the unlink-on-close issue for now with "delete=False"
# f.file.close()
filename = f
# open as string just in case its a TempFilePool
# TODO: (need to figure out how to make it work seamless as a string)
with h5py.File(str(filename), 'w', libver='latest') as fw:
# Create all new datasets as non-transposed
fw.create_dataset(self._electrode_field, shape=(C, T), dtype=new_dtype, chunks=True)
if copy_electrodes:
for block, sl in self.iter_blocks(return_slice=True, sharedmem=False):
fw[self._electrode_field][sl] = block
for name in self.aligned_arrays:
arr = getattr(self, name)
if len(arr.shape) > 1:
dims = (arr.shape[1], T) if self._transpose else (arr.shape[0], T)
# is this correct ???
dtype = fp_dtype if writeable else arr.dtype
fw.create_dataset(name, shape=dims, dtype=dtype, chunks=True)
if copy_aligned:
aligned = getattr(self, name)[:]
fw[name][:] = aligned.T if self._transpose else aligned
# set single write, multiple read mode AFTER datasets are created
# Skip for now -- SWMR issue pending
# fw.swmr_mode = True
print(str(filename), 'reopen mode', reopen_mode)
f_mapped = h5py.File(str(filename), reopen_mode, libver='latest')
# Skip for now -- SWMR issue pending
# if writeable:
# f_mapped.swmr_mode = True
if isinstance(filename, TempFilePool):
filename.register_to_close(f_mapped)
# If the original buffer's unit scaling is 1.0 then we should keep real slices on
real_slices = units_scale == 1.0
return MappedSource.from_hdf_sources(f_mapped, self._electrode_field, units_scale=units_scale,
real_slices=real_slices, aligned_arrays=self.aligned_arrays,
transpose=False, electrode_channels=electrode_channels,
channel_mask=channel_mask, **map_args)
# return MappedSource(f_mapped, self._electrode_field, electrode_channels=electrode_channels,
# channel_mask=channel_mask, aligned_arrays=self.aligned_arrays,
# transpose=False, **map_args)
else:
if channel_compatible:
print('RAM mirrors are not channel compatible--use mapped=True')
self._check_slice_size(np.s_[:, :T])
C = self.shape[0]
if 'data_buffer' in new_sources:
new_source = new_sources['data_buffer']
else:
new_source = shm.shared_ndarray((C, T), fp_dtype.char)
if copy_electrodes:
for block, sl in self.iter_blocks(return_slice=True, sharedmem=False):
new_source[sl] = block
# Kind of tricky with aligned fields -- assume that transpose means the same thing for them?
# But also un-transpose them on this mirroring step
aligned_arrays = dict()
for name in self.aligned_arrays:
arr = getattr(self, name)
if len(arr.shape) > 1:
dims = (arr.shape[1], T) if self._transpose else (arr.shape[0], T)
else:
dims = (T,)
if name in new_sources:
aligned_arrays[name] = new_sources[name]
else:
aligned_arrays[name] = shm.shared_ndarray(dims, fp_dtype.char)
if copy_aligned:
aligned = getattr(self, name)[:]
aligned_arrays[name][:] = aligned.T if self._transpose else aligned
return PlainArraySource(new_source, **aligned_arrays)
def load_afe(exp_pth,
test,
electrode,
n_data,
range_code,
cycle_rate,
units='nA',
bandpass=(),
save=True,
notches=(),
snip_transient=True,
**extra):
h5 = tables.open_file(os.path.join(exp_pth, test + '.h5'))
data = h5.root.data[:]
Fs = h5.root.Fs[0, 0]
if data.shape[1] > n_data:
trig_chans = data[:, n_data:]
trig = np.any(trig_chans > 1, axis=1).astype('i')
pos_edge = np.where(np.diff(trig) > 0)[0] + 1
else:
trig = None
pos_edge = ()
data_chans = data[:, :n_data].T.copy(order='C')
# convert dynamic range to charge or current
if 'v' not in units.lower():
pico_coulombs = range_lookup[range_code]
convert_dyn_range(data_chans, (-1.4, 1.4),
pico_coulombs,
out=data_chans)
if 'a' in units.lower():
# To convert to amps, need to divide coulombs by the
# integration period. This is found approximately by
# finding out how many cycles in a scan period were spent
# integrating. A scan period is now hard coded to be 500
# cycles. The cycling rate is given as an argument.
# The integration period for channel i should be:
# 500 - 2*(n_data - i)
# That is, the 1st channel is clocked out over two cycles
# immediately after the integration period. Meanwhile other
# channels still acquire until they are clocked out.
n_cycles = 500
#i_cycles = n_cycles - 2*(n_data - np.arange(n_data))
i_cycles = n_cycles - 2 * n_data
i_period = i_cycles / cycle_rate
data_chans /= i_period #[:,None]
convert_scale(data_chans, 'pa', units)
elif units.lower() != 'v':
convert_scale(data, 'v', units)
# only use this one electrode (for now)
chan_map, disconnected = epins.get_electrode_map('psv_61_afe')[:2]
connected = np.setdiff1d(np.arange(n_data), disconnected)
disconnected = disconnected[disconnected < n_data]
chan_map = chan_map.subset(list(range(len(connected))))
data = shm.shared_ndarray((len(connected), data_chans.shape[-1]))
data[:, :] = data_chans[connected]
ground_chans = data_chans[disconnected].copy()
del data_chans
if bandpass:
# do a little extra to kill DC
data -= data.mean(axis=1)[:, None]
(b, a) = ft.butter_bp(lo=bandpass[0], hi=bandpass[1], Fs=Fs)
filtfilt(data, b, a)
if notches:
for freq in notches:
(b, a) = ft.notch(freq, Fs=Fs, ftype='cheby2')
filtfilt(data, b, a)
## detrend_window = int(round(0.750*Fs))
## ft.bdetrend(data, bsize=detrend_window, type='linear', axis=-1)
else:
data -= data.mean(axis=1)[:, None]
if snip_transient:
snip_len = min(10000, pos_edge[0]) if len(pos_edge) else 10000
data = data[..., snip_len:].copy()
ground_chans = ground_chans[..., snip_len:].copy()
if len(pos_edge):
trig = trig[..., snip_len:]
pos_edge -= snip_len
dset = Bunch()
dset.data = data
dset.ground_chans = ground_chans
dset.chan_map = chan_map
dset.Fs = Fs
dset.pos_edge = pos_edge
dset.bandpass = bandpass
dset.trig = trig
dset.transient_snipped = snip_transient
dset.units = units
dset.notches = notches
return dset
def load_afe_aug21(exp_pth,
test,
electrode,
n_data,
range_code,
cycle_rate,
units='nA',
bandpass=(),
save=False,
notches=(),
snip_transient=True,
**extra):
h5 = tables.open_file(os.path.join(exp_pth, test + '.h5'))
Fs = h5.root.Fs.read()
n_row = h5.root.numRow.read()
n_data_col = h5.root.numCol.read()
n_col = h5.root.numChan.read()
# data rows are 4:67 -- acquiring AFE chans 31:0 and 63:32 on two columns
data_rows = slice(4, 67, 2)
full_data = h5.root.data[:].reshape(n_col, n_row, -1)
#data_chans = shm.shared_ndarray( (32*n_data_col, full_data.shape[-1]) )
data_chans = full_data[:n_data_col,
data_rows].reshape(-1, full_data.shape[-1])
trig_chans = full_data[-10:, -1]
del full_data
trig = np.any(trig_chans > 1, axis=0).astype('i')
pos_edge = np.where(np.diff(trig) > 0)[0] + 1
# convert dynamic range to charge or current
if 'v' not in units.lower():
pico_coulombs = range_lookup[range_code]
convert_dyn_range(data_chans, (-1.4, 1.4),
pico_coulombs,
out=data_chans)
if 'a' in units.lower():
i_period = 563e-6
data_chans /= i_period
convert_scale(data_chans, 'pa', units)
elif units.lower() != 'v':
convert_scale(data_chans, 'v', units)
## # only use this one electrode (for now)
chan_map, disconnected = epins.get_electrode_map('psv_61_afe')[:2]
connected = np.setdiff1d(np.arange(n_data), disconnected)
disconnected = disconnected[disconnected < n_data]
data = shm.shared_ndarray((len(connected), data_chans.shape[-1]))
data[:, :] = data_chans[connected]
ground_chans = data_chans[disconnected]
del data_chans
# do a little extra to kill DC
data -= data.mean(axis=1)[:, None]
if bandpass:
(b, a) = ft.butter_bp(lo=bandpass[0], hi=bandpass[1], Fs=Fs)
filtfilt(data, b, a)
if notches:
ft.notch_all(data, Fs, lines=notches, inplace=True, filtfilt=True)
if snip_transient:
snip_len = min(10000, pos_edge[0]) if len(pos_edge) else 10000
data = data[..., snip_len:].copy()
if ground_chans is not None:
ground_chans = ground_chans[..., snip_len:].copy()
if len(pos_edge):
trig = trig[..., snip_len:]
pos_edge -= snip_len
dset = ut.Bunch()
dset.data = data
dset.ground_chans = ground_chans
dset.chan_map = chan_map
dset.Fs = Fs
dset.pos_edge = pos_edge
dset.bandpass = bandpass
dset.trig = trig
dset.transient_snipped = snip_transient
dset.units = units
dset.notches = notches
return dset
def traverse_table(f,
path='/',
load=True,
scan=False,
shared_paths=(),
skip_stale_pickles=True,
attempt_reload=False):
# Walk nodes and stuff arrays into the bunch.
# If we encounter a group, then loop back into this method
from ecogdata.devices.load.file2data import FileLoader
if not isinstance(f, tables.file.File):
if load or scan:
# If scan is True, load should be forced False here
load = not scan
with closing(tables.open_file(f, mode='r')) as f:
return traverse_table(f,
path=path,
load=load,
scan=scan,
shared_paths=shared_paths,
skip_stale_pickles=skip_stale_pickles,
attempt_reload=attempt_reload)
else:
f = tables.open_file(f, mode='r')
try:
return traverse_table(f,
path=path,
load=load,
scan=scan,
shared_paths=shared_paths,
skip_stale_pickles=skip_stale_pickles,
attempt_reload=attempt_reload)
except:
f.close()
raise
if load or scan:
gbunch = Bunch()
else:
gbunch = HDF5Bunch(f)
(p, g) = os.path.split(path)
if g == '':
g = p
nlist = f.list_nodes(path)
#for n in f.walk_nodes(where=path):
for n in nlist:
if isinstance(n, tables.Array):
if load:
if n.dtype.char == 'O':
arr = 'Not loaded: ' + n.name
elif '/'.join([path, n.name]) in shared_paths:
arr = shm.shared_ndarray(n.shape)
arr[:] = n.read()
else:
arr = n.read()
if isinstance(arr, np.ndarray) and n.shape:
if arr.shape == (1, 1):
arr = arr[0, 0]
if arr == 0:
arr = None
else:
arr = arr.squeeze()
else:
arr = n
gbunch[n.name] = arr
elif isinstance(n, tables.VLArray):
if load:
try:
obj = n.read()[0]
except (ModuleNotFoundError, PickleError, PicklingError):
if not skip_stale_pickles:
raise
gbunch[n.name] = 'unloadable pickle'
continue
# if it's a generic Bunch Pickle, then update the bunch
if n.name == 'b_pickle':
gbunch.update(obj)
else:
gbunch[n.name] = obj
else:
# ignore the empty pickle
if n.name == 'b_pickle' and n.size_in_memory > 32:
gbunch[n.name] = 'unloaded pickle'
elif isinstance(n, tables.Group):
gname = n._v_name
# walk_nodes() includes the current group:
# don't try to descend into this node!
if gname == g:
continue
if gname == '#refs#':
continue
subbunch = traverse_table(f,
path='/'.join([path, gname]),
load=load,
scan=scan,
shared_paths=shared_paths,
skip_stale_pickles=skip_stale_pickles,
attempt_reload=attempt_reload)
gbunch[gname] = subbunch
else:
gbunch[n.name] = 'Not Loaded!'
this_node = f.get_node(path)
for attr in this_node._v_attrs._f_list():
gbunch[attr] = this_node._v_attrs[attr]
loaders = [v for v in gbunch.values() if isinstance(v, FileLoader)]
if attempt_reload and loaders:
for loader in loaders:
print('Attempting load from {}'.format(loader.primary_data_file))
dataset = loader.create_dataset()
new_keys = set(dataset.keys()) - set(gbunch.keys())
for k in new_keys:
gbunch[k] = dataset.pop(k)
return gbunch