def test_empty(self):
dummy = AnalogData()
assert len(dummy.cfg) == 0
assert dummy.dimord == None
for attr in ["channel", "data", "hdr", "sampleinfo", "trialinfo"]:
assert getattr(dummy, attr) is None
with pytest.raises(SPYTypeError):
AnalogData({})
def test_filename(self):
# ensure we're salting sufficiently to create at least `numf`
# distinct pseudo-random filenames in `__storage__`
numf = 1000
dummy = AnalogData()
fnames = []
for k in range(numf):
fnames.append(dummy._gen_filename())
assert np.unique(fnames).size == numf
def test_dataselection(self):
dummy = AnalogData(data=self.data,
trialdefinition=self.trl,
samplerate=self.samplerate)
trialSelections = [
"all", # enforce below selections in all trials of `dummy`
[3, 1] # minimally unordered
]
chanSelections = [
["channel03", "channel01", "channel01",
"channel02"], # string selection w/repetition + unordered
[4, 2, 2, 5, 5], # repetition + unorderd
range(5, 8), # narrow range
slice(-2, None) # negative-start slice
]
toiSelections = [
"all", # non-type-conform string
[0.6], # single inexact match
[-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45,
3.] # unordered, inexact, repetions
]
toilimSelections = [
[0.5, 1.5], # regular range
[1.5, 2.0], # minimal range (just two-time points)
[1.0, np.inf] # unbounded from above
]
timeSelections = list(zip(["toi"] * len(toiSelections), toiSelections)) \
+ list(zip(["toilim"] * len(toilimSelections), toilimSelections))
idx = [slice(None)] * len(dummy.dimord)
timeIdx = dummy.dimord.index("time")
chanIdx = dummy.dimord.index("channel")
for trialSel in trialSelections:
for chanSel in chanSelections:
for timeSel in timeSelections:
kwdict = {}
kwdict["trials"] = trialSel
kwdict["channels"] = chanSel
kwdict[timeSel[0]] = timeSel[1]
cfg = StructDict(kwdict)
# data selection via class-method + `Selector` instance for indexing
selected = dummy.selectdata(**kwdict)
selector = Selector(dummy, kwdict)
idx[chanIdx] = selector.channel
for tk, trialno in enumerate(selector.trials):
idx[timeIdx] = selector.time[tk]
assert np.array_equal(
selected.trials[tk].squeeze(),
dummy.trials[trialno][idx[0], :][:,
idx[1]].squeeze())
cfg.data = dummy
cfg.out = AnalogData(dimord=AnalogData._defaultDimord)
# data selection via package function and `cfg`: ensure equality
selectdata(cfg)
assert np.array_equal(cfg.out.channel, selected.channel)
assert np.array_equal(cfg.out.data, selected.data)
def test_nparray(self):
dummy = AnalogData(data=self.data)
assert dummy.dimord == AnalogData._defaultDimord
assert dummy.channel.size == self.nc
assert (dummy.sampleinfo == [0, self.ns]).min()
assert dummy.trialinfo.shape == (1, 0)
assert np.array_equal(dummy.data, self.data)
# wrong shape for data-type
with pytest.raises(SPYValueError):
AnalogData(np.ones((3, )))
def filter_manager(data,
b=None,
a=None,
out=None,
select=None,
chan_per_worker=None,
keeptrials=True,
parallel=False,
parallel_store=None,
log_dict=None):
newOut = False
if out is None:
newOut = True
out = AnalogData(dimord=AnalogData._defaultDimord)
myfilter = LowPassFilter(b, a=a)
myfilter.initialize(data,
out._stackingDim,
chan_per_worker=chan_per_worker,
keeptrials=keeptrials)
myfilter.compute(data,
out,
parallel=parallel,
parallel_store=parallel_store,
log_dict=log_dict)
return out if newOut else None
def test_virtualdata(self):
with tempfile.TemporaryDirectory() as tdir:
fname = os.path.join(tdir, "dummy.npy")
np.save(fname, self.data)
dmap = open_memmap(fname, mode="r")
vdata = VirtualData([dmap, dmap])
dummy = AnalogData(vdata)
assert dummy.channel.size == 2 * self.nc
assert len(dummy._filename) == 2
assert isinstance(dummy.filename, str)
del dmap, dummy, vdata
def test_trialretrieval(self):
# test ``_get_trial`` with NumPy array: regular order
dummy = AnalogData(data=self.data, trialdefinition=self.trl)
for trlno, start in enumerate(range(0, self.ns, 5)):
trl_ref = self.data[start:start + 5, :]
assert np.array_equal(dummy._get_trial(trlno), trl_ref)
# test ``_get_trial`` with NumPy array: swapped dimensions
dummy = AnalogData(self.data.T,
trialdefinition=self.trl,
dimord=["channel", "time"])
for trlno, start in enumerate(range(0, self.ns, 5)):
trl_ref = self.data.T[:, start:start + 5]
assert np.array_equal(dummy._get_trial(trlno), trl_ref)
# # test ``_copy_trial`` with memmap'ed data
# with tempfile.TemporaryDirectory() as tdir:
# fname = os.path.join(tdir, "dummy.npy")
# np.save(fname, self.data)
# mm = open_memmap(fname, mode="r")
# dummy = AnalogData(mm, trialdefinition=self.trl)
# for trlno, start in enumerate(range(0, self.ns, 5)):
# trl_ref = self.data[start:start + 5, :]
# trl_tmp = dummy._copy_trial(trlno,
# dummy.filename,
# dummy.dimord,
# dummy.sampleinfo,
# dummy.hdr)
# assert np.array_equal(trl_tmp, trl_ref)
#
# # Delete all open references to file objects b4 closing tmp dir
# del mm, dummy
del dummy
def test_save_mmap(self):
with tempfile.TemporaryDirectory() as tdir:
fname = os.path.join(tdir, "vdat.npy")
dname = os.path.join(tdir, "dummy")
vdata = np.ones((1000, 5000)) # ca. 38.2 MB
np.save(fname, vdata)
del vdata
dmap = open_memmap(fname)
adata = AnalogData(dmap, samplerate=10)
# Ensure memory consumption stays within provided bounds
mem = memory_usage()[0]
save(adata, filename=dname, memuse=60)
assert (mem - memory_usage()[0]) < 70
# Delete all open references to file objects b4 closing tmp dir
del dmap, adata
def test_clear(self):
with tempfile.TemporaryDirectory() as tdir:
fname = os.path.join(tdir, "dummy.npy")
data = np.ones((5000, 1000)) # ca. 38.2 MB
np.save(fname, data)
del data
dmap = open_memmap(fname)
# test consistency and efficacy of clear method
dummy = AnalogData(dmap)
data = np.array(dummy.data)
dummy.clear()
assert np.array_equal(data, dummy.data)
mem = memory_usage()[0]
dummy.clear()
time.sleep(1)
assert np.abs(mem - memory_usage()[0]) > 30
# Delete all open references to file objects b4 closing tmp dir
del dmap, dummy
def test_trialsetting(self):
# Create sampleinfo w/ EventData vs. AnalogData samplerate
sr_e = 2
sr_a = 1
pre = 2
post = 1
msk = self.data[:, 1] == 1
sinfo = np.vstack(
[self.data[msk, 0] / sr_e - pre,
self.data[msk, 0] / sr_e + post]).T
sinfo_e = np.round(sinfo * sr_e).astype(int)
sinfo_a = np.round(sinfo * sr_a).astype(int)
# Compute sampleinfo w/pre, post and trigger
evt_dummy = EventData(self.data, samplerate=sr_e)
evt_dummy.definetrial(pre=pre, post=post, trigger=1)
assert np.array_equal(evt_dummy.sampleinfo, sinfo_e)
# Compute sampleinfo w/ start/stop combination
evt_dummy = EventData(self.data, samplerate=sr_e)
evt_dummy.definetrial(start=0, stop=1)
sinfo2 = np.vstack([
self.data[np.where(self.data[:, 1] == 0)[0], 0],
self.data[np.where(self.data[:, 1] == 1)[0], 0]
]).T
assert np.array_equal(sinfo2, evt_dummy.sampleinfo)
# Same w/ more complicated data array
samples = np.arange(0, int(self.ns / 3), 3)[1:]
dappend = np.vstack([samples, np.full(samples.shape, 2)]).T
data3 = np.vstack([self.data, dappend])
idx = np.argsort(data3[:, 0])
data3 = data3[idx, :]
evt_dummy = EventData(data3, samplerate=sr_e)
evt_dummy.definetrial(start=0, stop=1)
assert np.array_equal(sinfo2, evt_dummy.sampleinfo)
# Compute sampleinfo w/start/stop arrays instead of scalars
starts = [2, 2, 1]
stops = [1, 2, 0]
sinfo3 = np.empty((3, 2))
dsamps = list(data3[:, 0])
dcodes = list(data3[:, 1])
for sk, (start, stop) in enumerate(zip(starts, stops)):
idx = dcodes.index(start)
start = dsamps[idx]
dcodes = dcodes[idx + 1:]
dsamps = dsamps[idx + 1:]
idx = dcodes.index(stop)
stop = dsamps[idx]
dcodes = dcodes[idx + 1:]
dsamps = dsamps[idx + 1:]
sinfo3[sk, :] = [start, stop]
evt_dummy = EventData(data3, samplerate=sr_e)
evt_dummy.definetrial(start=[2, 2, 1], stop=[1, 2, 0])
assert np.array_equal(evt_dummy.sampleinfo, sinfo3)
# Attach computed sampleinfo to AnalogData (data and data3 must yield identical resutls)
evt_dummy = EventData(data=self.data, samplerate=sr_e)
evt_dummy.definetrial(pre=pre, post=post, trigger=1)
ang_dummy = AnalogData(self.adata, samplerate=sr_a)
ang_dummy.definetrial(evt_dummy)
assert np.array_equal(ang_dummy.sampleinfo, sinfo_a)
evt_dummy = EventData(data=data3, samplerate=sr_e)
evt_dummy.definetrial(pre=pre, post=post, trigger=1)
ang_dummy.definetrial(evt_dummy)
assert np.array_equal(ang_dummy.sampleinfo, sinfo_a)
# Compute and attach sampleinfo on the fly
evt_dummy = EventData(data=self.data, samplerate=sr_e)
ang_dummy = AnalogData(self.adata, samplerate=sr_a)
ang_dummy.definetrial(evt_dummy, pre=pre, post=post, trigger=1)
assert np.array_equal(ang_dummy.sampleinfo, sinfo_a)
evt_dummy = EventData(data=data3, samplerate=sr_e)
ang_dummy = AnalogData(self.adata, samplerate=sr_a)
ang_dummy.definetrial(evt_dummy, pre=pre, post=post, trigger=1)
assert np.array_equal(ang_dummy.sampleinfo, sinfo_a)
# Extend data and provoke an exception due to out of bounds erro
smp = np.vstack([
np.arange(self.ns, int(2.5 * self.ns), 5),
np.zeros((int((1.5 * self.ns) / 5), ))
]).T
smp[1::2, 1] = 1
data4 = np.vstack([data3, smp])
evt_dummy = EventData(data=data4, samplerate=sr_e)
evt_dummy.definetrial(pre=pre, post=post, trigger=1)
# with pytest.raises(SPYValueError):
# ang_dummy.definetrial(evt_dummy)
# Trimming edges produces zero-length trial
with pytest.raises(SPYValueError):
ang_dummy.definetrial(evt_dummy, clip_edges=True)
# We need `clip_edges` to make trial-definition work
data4 = data4[:-2, :]
data4[-2, 0] = data4[-1, 0]
evt_dummy = EventData(data=data4, samplerate=sr_e)
evt_dummy.definetrial(pre=pre, post=post, trigger=1)
# with pytest.raises(SPYValueError):
# ang_dummy.definetrial(evt_dummy)
ang_dummy.definetrial(evt_dummy, clip_edges=True)
assert ang_dummy.sampleinfo[-1, 1] == self.ns
# Check both pre/start and/or post/stop being None
evt_dummy = EventData(data=self.data, samplerate=sr_e)
with pytest.raises(SPYValueError):
evt_dummy.definetrial(trigger=1, post=post)
with pytest.raises(SPYValueError):
evt_dummy.definetrial(pre=pre, trigger=1)
with pytest.raises(SPYValueError):
evt_dummy.definetrial(start=0)
with pytest.raises(SPYValueError):
evt_dummy.definetrial(stop=1)
with pytest.raises(SPYValueError):
evt_dummy.definetrial(trigger=1)
with pytest.raises(SPYValueError):
evt_dummy.definetrial(pre=pre, post=post)
# Try to define trials w/o samplerate set
evt_dummy = EventData(data=self.data)
with pytest.raises(SPYValueError):
evt_dummy.definetrial(pre=pre, post=post, trigger=1)
evt_dummy = EventData(data=self.data, samplerate=sr_e)
ang_dummy = AnalogData(self.adata)
with pytest.raises(SPYValueError):
ang_dummy.definetrial(evt_dummy, pre=pre, post=post, trigger=1)
# Try to define trials w/o data
evt_dummy = EventData(samplerate=sr_e)
with pytest.raises(SPYValueError):
evt_dummy.definetrial(pre=pre, post=post, trigger=1)
ang_dummy = AnalogData(self.adata, samplerate=sr_a)
with pytest.raises(SPYValueError):
ang_dummy.definetrial(evt_dummy, pre=pre, post=post, trigger=1)
def generate_artificial_data(nTrials=2,
nChannels=2,
equidistant=True,
seed=None,
overlapping=False,
inmemory=True,
dimord="default"):
"""
Create :class:`~syncopy.AnalogData` object with synthetic trigonometric signal(s)
Parameters
----------
nTrials : int
Number of trials to populate synthetic data object with
nChannels : int
Number of channels to populate synthetic object with
equidistant : bool
If `True`, trials of equal length are defined
seed : None or int
If `None`, imposed noise is completely random. If `seed` is an integer,
it is used to fix the (initial) state of NumPy's random number generator
:func:`numpy.random.default_rng`, i.e., objects created wtih same `seed`
will be populated with identical artificial signals.
overlapping : bool
If `True`, constructed trials overlap
inmemory : bool
If `True`, the full `data` array (all channels across all trials) is allocated
in memory (fast but dangerous for large arrays), otherwise the output data
object's corresponding backing HDF5 file in `__storage__` is filled with
synthetic data in a trial-by-trial manner (slow but safe even for very
large datasets).
dimord : str or list
If `dimord` is "default", the constructed output object uses the default
dimensional layout of a standard :class:`~syncopy.AnalogData` object.
If `dimord` is a list (i.e., ``["channel", "time"]``) the provided sequence
of dimensions is used.
Returns
-------
out : :class:`~syncopy.AnalogData` object
Syncopy :class:`~syncopy.AnalogData` object with specified properties
populated with a synthetic multivariate trigonometric signal.
Notes
-----
This is an auxiliary method that is intended purely for internal use. Thus,
no error checking is performed.
Examples
--------
Generate small artificial :class:`~syncopy.AnalogData` object in memory
.. code-block:: python
>>> iAmSmall = generate_artificial_data(nTrials=5, nChannels=10, inmemory=True)
>>> iAmSmall
Syncopy AnalogData object with fields
cfg : dictionary with keys ''
channel : [10] element <class 'numpy.ndarray'>
container : None
data : 5 trials of length 3000 defined on [15000 x 10] float32 Dataset of size 0.57 MB
dimord : 2 element list
filename : /Users/pantaray/.spy/spy_158f_4d4153e3.analog
mode : r+
sampleinfo : [5 x 2] element <class 'numpy.ndarray'>
samplerate : 1000.0
tag : None
time : 5 element list
trialinfo : [5 x 0] element <class 'numpy.ndarray'>
trials : 5 element iterable
Use `.log` to see object history
Generate artificial :class:`~syncopy.AnalogData` object of more substantial
size on disk
.. code-block:: python
>>> iAmBig = generate_artificial_data(nTrials=50, nChannels=1024, inmemory=False)
>>> iAmBig
Syncopy AnalogData object with fields
cfg : dictionary with keys ''
channel : [1024] element <class 'numpy.ndarray'>
container : None
data : 200 trials of length 3000 defined on [600000 x 1024] float32 Dataset of size 2.29 GB
dimord : 2 element list
filename : /Users/pantaray/.spy/spy_158f_b80715fe.analog
mode : r+
sampleinfo : [200 x 2] element <class 'numpy.ndarray'>
samplerate : 1000.0
tag : None
time : 200 element list
trialinfo : [200 x 0] element <class 'numpy.ndarray'>
trials : 200 element iterable
Use `.log` to see object history
"""
# Create dummy 1d signal that will be blown up to fill channels later
dt = 0.001
t = np.arange(0, 3, dt, dtype="float32") - 1.0
sig = np.cos(2 * np.pi * (7 * (np.heaviside(t, 1) * t - 1) + 10) * t)
# Depending on chosen `dimord` either get default position of time-axis
# in `AnalogData` objects or use provided `dimord` and reshape signal accordingly
if dimord == "default":
dimord = AnalogData._defaultDimord
timeAxis = dimord.index("time")
idx = [1, 1]
idx[timeAxis] = -1
sig = np.repeat(sig.reshape(*idx), axis=idx.index(1), repeats=nChannels)
# Initialize random number generator (with possibly user-provided seed-value)
rng = np.random.default_rng(seed)
# Either construct the full data array in memory using tiling or create
# an HDF5 container in `__storage__` and fill it trial-by-trial
# NOTE: use `swapaxes` here to ensure two objects created w/same seed really
# are affected w/identical additive noise patterns, no matter their respective
# `dimord`.
out = AnalogData(samplerate=1 / dt, dimord=dimord)
if inmemory:
idx[timeAxis] = nTrials
sig = np.tile(sig, idx)
shp = [slice(None), slice(None)]
for iTrial in range(nTrials):
shp[timeAxis] = slice(iTrial * t.size, (iTrial + 1) * t.size)
noise = rng.standard_normal(
(t.size, nChannels)).astype(sig.dtype) * 0.5
sig[tuple(shp)] += np.swapaxes(noise, timeAxis, 0)
out.data = sig
else:
with h5py.File(out.filename, "w") as h5f:
shp = list(sig.shape)
shp[timeAxis] *= nTrials
dset = h5f.create_dataset("data",
shape=tuple(shp),
dtype=sig.dtype)
shp = [slice(None), slice(None)]
for iTrial in range(nTrials):
shp[timeAxis] = slice(iTrial * t.size, (iTrial + 1) * t.size)
noise = rng.standard_normal(
(t.size, nChannels)).astype(sig.dtype) * 0.5
dset[tuple(shp)] = sig + np.swapaxes(noise, timeAxis, 0)
dset.flush()
out.data = h5py.File(out.filename, "r+")["data"]
# Define by-trial offsets to generate (non-)equidistant/(non-)overlapping trials
trialdefinition = np.zeros((nTrials, 3), dtype='int')
if equidistant:
equiOffset = 0
if overlapping:
equiOffset = 100
offsets = np.full((nTrials, ), equiOffset, dtype=sig.dtype)
else:
offsets = rng.integers(low=int(0.1 * t.size),
high=int(0.2 * t.size),
size=(nTrials, ))
# Using generated offsets, construct trialdef array and make sure initial
# and end-samples are within data bounds (only relevant if overlapping
# trials are built)
shift = (-1)**(not overlapping)
for iTrial in range(nTrials):
trialdefinition[iTrial, :] = np.array([
iTrial * t.size - shift * offsets[iTrial],
(iTrial + 1) * t.size + shift * offsets[iTrial], -1000
])
if equidistant:
trialdefinition[0, :2] += equiOffset
trialdefinition[-1, :2] -= equiOffset
else:
trialdefinition[0, 0] = 0
trialdefinition[-1, 1] = nTrials * t.size
out.definetrial(trialdefinition)
return out
def test_continuous_toitoilim(self):
# this only works w/the equidistant trials constructed above!!!
selDict = {
"toi": (
None, # trivial "selection" of entire contents
"all", # trivial "selection" of entire contents
[0.5], # single entry lists
[0.6], # inexact match
[1.0, 2.5], # two disjoint time-points
[1.2, 2.7], # inexact from above
[1.9, 2.4], # inexact from below
[0.4, 2.1], # inexact from below, inexact from above
[1.6, 1.9], # inexact from above, inexact from below
[-0.2, 0.6, 0.9, 1.1, 1.3, 1.6, 1.8, 2.2, 2.45,
3.], # alternating madness
[2.0, 0.5, 2.5], # unsorted list
[1.0, 0.5, 0.5, 1.5], # repetition
[0.5, 0.5, 1.0,
1.5], # preserve repetition, don't convert to slice
[0.5, 1.0,
1.5]), # sorted list (should be converted to slice-selection)
"toilim": (
None, # trivial "selection" of entire contents
"all", # trivial "selection" of entire contents
[0.5, 1.5], # regular range
[1.5, 2.0], # minimal range (just two-time points)
[1.0, np.inf], # unbounded from above
[-np.inf, 1.0])
} # unbounded from below
# all trials have same time-scale: take 1st one as reference
trlTime = (np.arange(
0, self.trl["AnalogData"][0, 1] - self.trl["AnalogData"][0, 0]) +
self.trl["AnalogData"][0, 2]) / self.samplerate
ang = AnalogData(data=self.data["AnalogData"],
trialdefinition=self.trl["AnalogData"],
samplerate=self.samplerate)
angIdx = [slice(None)] * len(ang.dimord)
timeIdx = ang.dimord.index("time")
# the below check only works for equidistant trials!
for tselect in ["toi", "toilim"]:
for timeSel in selDict[tselect]:
sel = Selector(ang, {tselect: timeSel}).time
if timeSel is None or timeSel == "all":
idx = slice(None)
else:
if tselect == "toi":
idx = []
for tp in timeSel:
idx.append(np.abs(trlTime - tp).argmin())
else:
idx = np.intersect1d(
np.where(trlTime >= timeSel[0])[0],
np.where(trlTime <= timeSel[1])[0])
# check that correct data was selected (all trials identical, just take 1st one)
assert np.array_equal(ang.trials[0][idx, :],
ang.trials[0][sel[0], :])
if not isinstance(idx, slice) and len(idx) > 1:
timeSteps = np.diff(idx)
if timeSteps.min() == timeSteps.max() == 1:
idx = slice(idx[0], idx[-1] + 1, 1)
result = [idx] * len(ang.trials)
# check correct format of selector (list -> slice etc.)
assert np.array_equal(result, sel)
# perform actual data-selection and ensure identity of results
selected = selectdata(ang, {tselect: timeSel})
for trialno in range(len(ang.trials)):
angIdx[timeIdx] = result[trialno]
assert np.array_equal(selected.trials[trialno],
ang.trials[trialno][tuple(angIdx)])
def test_general(self):
# construct expected results for `DiscreteData` objects defined above
mapDict = {"unit": "SpikeData", "eventid": "EventData"}
for prop, dclass in mapDict.items():
discrete = getattr(spd, dclass)(data=self.data[dclass],
trialdefinition=self.trl[dclass],
samplerate=self.samplerate)
propIdx = discrete.dimord.index(prop)
# convert selection from `selectDict` to a usable integer-list
allResults = []
for selection in self.selectDict[prop]["valid"]:
if isinstance(selection, slice):
if selection.start is selection.stop is None:
selects = [None]
else:
selects = list(range(getattr(discrete,
prop).size))[selection]
elif isinstance(selection, range):
selects = list(selection)
elif isinstance(selection, str) or selection is None:
selects = [None]
else: # selection is list/ndarray
if isinstance(selection[0], str):
avail = getattr(discrete, prop)
else:
avail = np.arange(getattr(discrete, prop).size)
selects = []
for sel in selection:
selects += list(np.where(avail == sel)[0])
# alternate (expensive) way to get by-trial selection indices
result = []
for trial in discrete.trials:
if selects[0] is None:
res = slice(0, trial.shape[0], 1)
else:
res = []
for sel in selects:
res += list(np.where(trial[:, propIdx] == sel)[0])
if len(res) > 1:
steps = np.diff(res)
if steps.min() == steps.max() == 1:
res = slice(res[0], res[-1] + 1, 1)
result.append(res)
allResults.append(result)
self.selectDict[prop]["result"] = tuple(allResults)
# wrong type of data and/or selector
with pytest.raises(SPYTypeError):
Selector(np.empty((3, )), {})
with pytest.raises(SPYValueError):
Selector(spd.AnalogData(), {})
ang = AnalogData(data=self.data["AnalogData"],
trialdefinition=self.trl["AnalogData"],
samplerate=self.samplerate)
with pytest.raises(SPYTypeError):
Selector(ang, ())
with pytest.raises(SPYValueError):
Selector(ang, {"wrongkey": [1]})
# go through all data-classes defined above
for dclass in self.classes:
dummy = getattr(spd, dclass)(data=self.data[dclass],
trialdefinition=self.trl[dclass],
samplerate=self.samplerate)
# test trial selection
selection = Selector(dummy, {"trials": [3, 1]})
assert selection.trials == [3, 1]
selected = selectdata(dummy, trials=[3, 1])
assert np.array_equal(selected.trials[0], dummy.trials[3])
assert np.array_equal(selected.trials[1], dummy.trials[1])
assert selected.trialdefinition.shape == (2, 4)
assert np.array_equal(selected.trialdefinition[:, -1],
dummy.trialdefinition[[3, 1], -1])
for trlSec in [None, "all"]:
selection = Selector(dummy, {"trials": trlSec})
assert selection.trials == list(range(len(dummy.trials)))
selected = selectdata(dummy, trials=trlSec)
for tk, trl in enumerate(selected.trials):
assert np.array_equal(trl, dummy.trials[tk])
assert np.array_equal(selected.trialdefinition,
dummy.trialdefinition)
with pytest.raises(SPYValueError):
Selector(dummy, {"trials": [-1, 9]})
# test "simple" property setters handled by `_selection_setter`
# for prop in ["eventid"]:
for prop in ["channel", "taper", "unit", "eventid"]:
if hasattr(dummy, prop):
expected = self.selectDict[prop]["result"]
for sk, sel in enumerate(self.selectDict[prop]["valid"]):
solution = expected[sk]
if dclass == "SpikeData" and prop == "channel":
if isinstance(solution, slice):
start, stop, step = solution.start, solution.stop, solution.step
if start is None:
start = 0
elif start < 0:
start = len(dummy.channel) + start
if stop is None:
stop = len(dummy.channel)
elif stop < 0:
stop = len(dummy.channel) + stop
if step not in [None, 1]:
solution = list(range(start,
stop))[solution]
else:
solution = slice(start, stop, step)
# once we're sure `Selector` works, actually select data
selection = Selector(dummy, {prop + "s": sel})
assert getattr(selection, prop) == solution
selected = selectdata(dummy, {prop + "s": sel})
# process `unit` and `enventid`
if prop in selection._byTrialProps:
propIdx = selected.dimord.index(prop)
propArr = np.unique(
selected.data[:, propIdx]).astype(np.intp)
assert set(getattr(selected, prop)) == set(
getattr(dummy, prop)[propArr])
tk = 0
for trialno in range(len(dummy.trials)):
if solution[
trialno]: # do not try to compare empty selections
assert np.array_equal(
selected.trials[tk],
dummy.trials[trialno][
solution[trialno], :])
tk += 1
# `channel` is a special case for `SpikeData` objects
elif dclass == "SpikeData" and prop == "channel":
chanIdx = selected.dimord.index("channel")
chanArr = np.arange(dummy.channel.size)
assert set(selected.data[:, chanIdx]).issubset(
chanArr[solution])
assert set(selected.channel) == set(
dummy.channel[solution])
# everything else (that is not a `DiscreteData` child)
else:
idx = [slice(None)] * len(dummy.dimord)
idx[dummy.dimord.index(prop)] = solution
assert np.array_equal(
np.array(dummy.data)[tuple(idx)],
selected.data)
assert np.array_equal(
getattr(selected, prop),
getattr(dummy, prop)[solution])
# ensure invalid selection trigger expected errors
for ik, isel in enumerate(
self.selectDict[prop]["invalid"]):
with pytest.raises(
self.selectDict[prop]["errors"][ik]):
Selector(dummy, {prop + "s": isel})
else:
# ensure objects that don't have a `prop` attribute complain
with pytest.raises(SPYValueError):
Selector(dummy, {prop + "s": [0]})
# ensure invalid `toi` + `toilim` specifications trigger expected errors
if hasattr(dummy, "time") or hasattr(dummy, "trialtime"):
for selection in ["toi", "toilim"]:
for ik, isel in enumerate(
self.selectDict[selection]["invalid"]):
with pytest.raises(
self.selectDict[selection]["errors"][ik]):
Selector(dummy, {selection: isel})
# provide both `toi` and `toilim`
with pytest.raises(SPYValueError):
Selector(dummy, {"toi": [0], "toilim": [0, 1]})
else:
# ensure objects that don't have `time` props complain properly
with pytest.raises(SPYValueError):
Selector(dummy, {"toi": [0]})
with pytest.raises(SPYValueError):
Selector(dummy, {"toilim": [0]})
# ensure invalid `foi` + `foilim` specifications trigger expected errors
if hasattr(dummy, "freq"):
for selection in ["foi", "foilim"]:
for ik, isel in enumerate(
self.selectDict[selection]["invalid"]):
with pytest.raises(
self.selectDict[selection]["errors"][ik]):
Selector(dummy, {selection: isel})
# provide both `foi` and `foilim`
with pytest.raises(SPYValueError):
Selector(dummy, {"foi": [0], "foilim": [0, 1]})
else:
# ensure objects without `freq` property complain properly
with pytest.raises(SPYValueError):
Selector(dummy, {"foi": [0]})
with pytest.raises(SPYValueError):
Selector(dummy, {"foilim": [0]})
def test_saveload(self):
with tempfile.TemporaryDirectory() as tdir:
fname = os.path.join(tdir, "dummy")
# basic but most important: ensure object integrity is preserved
checkAttr = [
"channel", "data", "dimord", "sampleinfo", "samplerate",
"trialinfo"
]
dummy = AnalogData(data=self.data, samplerate=1000)
dummy.save(fname)
filename = construct_spy_filename(fname, dummy)
# NOTE: We removed support for loading data via the constructor
# dummy2 = AnalogData(filename)
# for attr in checkAttr:
# assert np.array_equal(getattr(dummy, attr), getattr(dummy2, attr))
dummy3 = load(fname)
for attr in checkAttr:
assert np.array_equal(getattr(dummy3, attr),
getattr(dummy, attr))
save(dummy3, container=os.path.join(tdir, "ymmud"))
dummy4 = load(os.path.join(tdir, "ymmud"))
for attr in checkAttr:
assert np.array_equal(getattr(dummy4, attr),
getattr(dummy, attr))
del dummy, dummy3, dummy4 # avoid PermissionError in Windows
# # FIXME: either remove or repair this
# # save object hosting VirtualData
# np.save(fname + ".npy", self.data)
# dmap = open_memmap(fname + ".npy", mode="r")
# vdata = VirtualData([dmap, dmap])
# dummy = AnalogData(vdata, samplerate=1000)
# dummy.save(fname, overwrite=True)
# dummy2 = AnalogData(filename)
# assert dummy2.mode == "r+"
# assert np.array_equal(dummy2.data, vdata[:, :])
# del dummy, dummy2 # avoid PermissionError in Windows
# ensure trialdefinition is saved and loaded correctly
dummy = AnalogData(data=self.data,
trialdefinition=self.trl,
samplerate=1000)
dummy.save(fname + "_trl")
filename = construct_spy_filename(fname + "_trl", dummy)
dummy2 = load(filename)
assert np.array_equal(dummy.trialdefinition,
dummy2.trialdefinition)
# test getters
assert np.array_equal(dummy.sampleinfo, dummy2.sampleinfo)
assert np.array_equal(dummy._t0, dummy2._t0)
assert np.array_equal(dummy.trialinfo, dummy2.trialinfo)
del dummy, dummy2 # avoid PermissionError in Windows
# swap dimensions and ensure `dimord` is preserved
dummy = AnalogData(data=self.data,
dimord=["channel", "time"],
samplerate=1000)
dummy.save(fname + "_dimswap")
filename = construct_spy_filename(fname + "_dimswap", dummy)
dummy2 = load(filename)
assert dummy2.dimord == dummy.dimord
assert dummy2.channel.size == self.ns # swapped
assert dummy2.data.shape == dummy.data.shape
# Delete all open references to file objects and wait 0.1s for changes
# to take effect (thanks, Windows!)
del dummy, dummy2
time.sleep(0.1)
class TestComputationalRoutine():
# Construct linear combination of low- and high-frequency sine waves
# and use an IIR filter to reconstruct the low-frequency component
nChannels = 32
nTrials = 8
fData = 2
fNoise = 64
fs = 1000
t = np.linspace(-1, 1, fs)
orig = np.sin(2 * np.pi * fData * t)
sig = orig + np.sin(2 * np.pi * fNoise * t)
cutoff = 50
b, a = signal.butter(8, 2 * cutoff / fs)
# Blow up the signal to have "channels" and "trials" and inflate the low-
# frequency component accordingly for ad-hoc comparisons later
sig = np.repeat(sig.reshape(-1, 1), axis=1, repeats=nChannels)
sig = np.tile(sig, (nTrials, 1))
orig = np.repeat(orig.reshape(-1, 1), axis=1, repeats=nChannels)
orig = np.tile(orig, (nTrials, 1))
# Construct artificial equidistant trial-definition array
trl = np.zeros((nTrials, 3), dtype="int")
for ntrial in range(nTrials):
trl[ntrial, :] = np.array([ntrial * fs, (ntrial + 1) * fs, -500])
# Create reference AnalogData objects with equidistant trial spacing
sigdata = AnalogData(data=sig,
samplerate=fs,
trialdefinition=trl,
dimord=["time", "channel"])
origdata = AnalogData(data=orig,
samplerate=fs,
trialdefinition=trl,
dimord=["time", "channel"])
# Set by-worker channel-count for channel-parallelization
chanPerWrkr = 7
# Data selections to be tested w/`sigdata`
sigdataSelections = [
None, {
"trials": [3, 1, 0],
"channels": ["channel" + str(i) for i in range(12, 28)][::-1]
}, {
"trials": [0, 1, 2],
"channels": range(0, int(nChannels / 2)),
"toilim": [-0.25, 0.25]
}
]
# Data selections to be tested w/`artdata` generated below (use fixed but arbitrary
# random number seed to randomly select time-points for `toi` (with repetitions)
seed = np.random.RandomState(13)
artdataSelections = [
None, {
"trials": [3, 1, 0],
"channels": ["channel" + str(i) for i in range(12, 28)][::-1],
"toi": None
}, {
"trials": [0, 1, 2],
"channels": range(0, int(nChannels / 2)),
"toilim": [-0.5, 0.6]
}
]
# Error tolerances and respective quality metrics (depend on data selection!)
tols = [1e-6, 1e-6, 1e-2]
metrix = [np.max, np.max, np.mean]
def test_sequential_equidistant(self):
for sk, select in enumerate(self.sigdataSelections):
sel = Selector(self.sigdata, select)
out = filter_manager(self.sigdata, self.b, self.a, select=select)
# check correct signal filtering (especially wrt data-selection)
if select is None:
reference = self.orig
else:
ref = []
for tk, trlno in enumerate(sel.trials):
ref.append(self.origdata.trials[trlno][sel.time[tk],
sel.channel])
# check for correct time selection
assert np.array_equal(
out.time[tk], self.sigdata.time[trlno][sel.time[tk]])
reference = np.vstack(ref)
assert self.metrix[sk](
np.abs(out.data - reference)) < self.tols[sk]
assert np.array_equal(out.channel,
self.sigdata.channel[sel.channel])
# ensure pre-selection is equivalent to in-place selection
if select is None:
selected = self.sigdata.selectdata()
else:
selected = self.sigdata.selectdata(**select)
out_sel = filter_manager(selected, self.b, self.a)
assert np.array_equal(out.data, out_sel.data)
assert np.array_equal(out.channel, out_sel.channel)
assert np.array_equal(out.time, out_sel.time)
out = filter_manager(self.sigdata,
self.b,
self.a,
select=select,
keeptrials=False)
# check correct signal filtering (especially wrt data-selection)
if select is None:
reference = self.orig[:self.t.size, :]
else:
ref = np.zeros(out.trials[0].shape)
for tk, trlno in enumerate(sel.trials):
ref += self.origdata.trials[trlno][sel.time[tk],
sel.channel]
# check for correct time selection (accounting for trial-averaging)
assert np.array_equal(out.time[0],
self.sigdata.time[0][sel.time[0]])
reference = ref / len(sel.trials)
assert self.metrix[sk](
np.abs(out.data - reference)) < self.tols[sk]
assert np.array_equal(out.channel,
self.sigdata.channel[sel.channel])
# ensure pre-selection is equivalent to in-place selection
if select is None:
selected = self.sigdata.selectdata()
else:
selected = self.sigdata.selectdata(**select)
out_sel = filter_manager(selected,
self.b,
self.a,
keeptrials=False)
assert np.array_equal(out.data, out_sel.data)
assert np.array_equal(out.channel, out_sel.channel)
assert np.array_equal(out.time, out_sel.time)
def test_sequential_nonequidistant(self):
for overlapping in [False, True]:
nonequidata = generate_artificial_data(nTrials=self.nTrials,
nChannels=self.nChannels,
equidistant=False,
overlapping=overlapping,
inmemory=False)
# unsorted, w/repetitions
toi = self.seed.choice(nonequidata.time[0],
int(nonequidata.time[0].size))
self.artdataSelections[1]["toi"] = toi
for select in self.artdataSelections:
sel = Selector(nonequidata, select)
out = filter_manager(nonequidata,
self.b,
self.a,
select=select)
# compare expected w/actual shape of computed data
reference = 0
for tk, trlno in enumerate(sel.trials):
reference += nonequidata.trials[trlno][
sel.time[tk]].shape[0]
# check for correct time selection
# FIXME: remove `if` below as soon as `time` prop for lists is fixed
if not isinstance(sel.time[0], list):
assert np.array_equal(
out.time[tk],
nonequidata.time[trlno][sel.time[tk]])
assert out.data.shape[0] == reference
assert np.array_equal(out.channel,
nonequidata.channel[sel.channel])
# ensure pre-selection is equivalent to in-place selection
if select is None:
selected = nonequidata.selectdata()
else:
selected = nonequidata.selectdata(**select)
out_sel = filter_manager(selected, self.b, self.a)
assert np.array_equal(out.data, out_sel.data)
assert np.array_equal(out.channel, out_sel.channel)
for tk in range(len(out.trials)):
assert np.array_equal(out.time[tk], out_sel.time[tk])
def test_sequential_saveload(self):
for sk, select in enumerate(self.sigdataSelections):
sel = Selector(self.sigdata, select)
out = filter_manager(self.sigdata,
self.b,
self.a,
select=select,
log_dict={
"a": "this is a",
"b": "this is b"
})
# only keyword args (`a` in this case here) are stored in `cfg`
assert set(["a"]) == set(out.cfg.keys())
assert np.array_equal(out.cfg["a"], self.a)
assert len(out.trials) == len(sel.trials)
# ensure our `log_dict` specification was respected
assert "lowpass" in out._log
assert "a = this is a" in out._log
assert "b = this is b" in out._log
# ensure pre-selection is equivalent to in-place selection
if select is None:
selected = self.sigdata.selectdata()
else:
selected = self.sigdata.selectdata(**select)
out_sel = filter_manager(selected,
self.b,
self.a,
log_dict={
"a": "this is a",
"b": "this is b"
})
assert set(["a"]) == set(out.cfg.keys())
assert np.array_equal(out.cfg["a"], self.a)
assert len(out.trials) == len(out_sel.trials)
assert "lowpass" in out._log
assert "a = this is a" in out._log
assert "b = this is b" in out._log
# save and re-load result, ensure nothing funky happens
with tempfile.TemporaryDirectory() as tdir:
fname = os.path.join(tdir, "dummy")
out.save(fname)
dummy = load(fname)
assert "a" in dummy.cfg.keys()
assert np.array_equal(dummy.cfg["a"], self.a)
assert out.filename == dummy.filename
if select is None:
reference = self.orig
else:
ref = []
for tk, trlno in enumerate(sel.trials):
ref.append(self.origdata.trials[trlno][sel.time[tk],
sel.channel])
assert np.array_equal(
dummy.time[tk],
self.sigdata.time[trlno][sel.time[tk]])
reference = np.vstack(ref)
assert self.metrix[sk](
np.abs(dummy.data - reference)) < self.tols[sk]
assert np.array_equal(dummy.channel,
self.sigdata.channel[sel.channel])
time.sleep(0.01)
del out
# ensure out_sel is written/read correctly
fname2 = os.path.join(tdir, "dummy2")
out_sel.save(fname2)
dummy2 = load(fname2)
assert "a" in dummy2.cfg.keys()
assert np.array_equal(dummy2.cfg["a"], dummy.cfg["a"])
assert np.array_equal(dummy.data, dummy2.data)
assert np.array_equal(dummy.channel, dummy2.channel)
assert np.array_equal(dummy.time, dummy2.time)
del dummy, dummy2, out_sel
@skip_without_acme
def test_parallel_equidistant(self, testcluster):
client = dd.Client(testcluster)
for parallel_store in [True, False]:
for chan_per_worker in [None, self.chanPerWrkr]:
for sk, select in enumerate(self.sigdataSelections):
# FIXME: remove as soon as channel-parallelization works w/channel selectors
if chan_per_worker is not None:
select = None
sel = Selector(self.sigdata, select)
out = filter_manager(self.sigdata,
self.b,
self.a,
select=select,
chan_per_worker=chan_per_worker,
parallel=True,
parallel_store=parallel_store)
assert out.data.is_virtual == parallel_store
# check correct signal filtering (especially wrt data-selection)
if select is None:
reference = self.orig
else:
ref = []
for tk, trlno in enumerate(sel.trials):
ref.append(
self.origdata.trials[trlno][sel.time[tk],
sel.channel])
# check for correct time selection
assert np.array_equal(
out.time[tk],
self.sigdata.time[trlno][sel.time[tk]])
reference = np.vstack(ref)
assert self.metrix[sk](
np.abs(out.data - reference)) < self.tols[sk]
assert np.array_equal(out.channel,
self.sigdata.channel[sel.channel])
# ensure correct no. HDF5 files were generated for virtual data-set
if parallel_store:
nfiles = len(
glob(
os.path.join(
os.path.splitext(out.filename)[0],
"*.h5")))
if chan_per_worker is None:
assert nfiles == len(sel.trials)
else:
assert nfiles == len(sel.trials) * (
int(out.channel.size / chan_per_worker) +
int(out.channel.size % chan_per_worker > 0))
# ensure pre-selection is equivalent to in-place selection
if select is None:
selected = self.sigdata.selectdata()
else:
selected = self.sigdata.selectdata(**select)
out_sel = filter_manager(selected,
self.b,
self.a,
chan_per_worker=chan_per_worker,
parallel=True,
parallel_store=parallel_store)
assert np.allclose(out.data, out_sel.data)
assert np.array_equal(out.channel, out_sel.channel)
assert np.array_equal(out.time, out_sel.time)
out = filter_manager(self.sigdata,
self.b,
self.a,
select=select,
parallel=True,
parallel_store=parallel_store,
keeptrials=False)
# check correct signal filtering (especially wrt data-selection)
if select is None:
reference = self.orig[:self.t.size, :]
else:
ref = np.zeros(out.trials[0].shape)
for tk, trlno in enumerate(sel.trials):
ref += self.origdata.trials[trlno][sel.time[tk],
sel.channel]
# check for correct time selection (accounting for trial-averaging)
assert np.array_equal(
out.time[0], self.sigdata.time[0][sel.time[0]])
reference = ref / len(sel.trials)
assert self.metrix[sk](
np.abs(out.data - reference)) < self.tols[sk]
assert np.array_equal(out.channel,
self.sigdata.channel[sel.channel])
assert out.data.is_virtual == False
# ensure pre-selection is equivalent to in-place selection
out_sel = filter_manager(selected,
self.b,
self.a,
parallel=True,
parallel_store=parallel_store,
keeptrials=False)
assert np.allclose(out.data, out_sel.data)
assert np.array_equal(out.channel, out_sel.channel)
assert np.array_equal(out.time, out_sel.time)
client.close()
@skip_without_acme
def test_parallel_nonequidistant(self, testcluster):
client = dd.Client(testcluster)
for overlapping in [False, True]:
nonequidata = generate_artificial_data(nTrials=self.nTrials,
nChannels=self.nChannels,
equidistant=False,
overlapping=overlapping,
inmemory=False)
# unsorted, w/repetitions
toi = self.seed.choice(nonequidata.time[0],
int(nonequidata.time[0].size))
self.artdataSelections[1]["toi"] = toi
for parallel_store in [True, False]:
for chan_per_worker in [None, self.chanPerWrkr]:
for select in self.artdataSelections:
# FIXME: remove as soon as channel-parallelization works w/channel selectors
if chan_per_worker is not None:
select = None
sel = Selector(nonequidata, select)
out = filter_manager(nonequidata,
self.b,
self.a,
select=select,
chan_per_worker=chan_per_worker,
parallel=True,
parallel_store=parallel_store)
# compare expected w/actual shape of computed data
reference = 0
for tk, trlno in enumerate(sel.trials):
reference += nonequidata.trials[trlno][
sel.time[tk]].shape[0]
# check for correct time selection
# FIXME: remove `if` below as soon as `time` prop for lists is fixed
if not isinstance(sel.time[0], list):
assert np.array_equal(
out.time[tk],
nonequidata.time[trlno][sel.time[tk]])
assert out.data.shape[0] == reference
assert np.array_equal(out.channel,
nonequidata.channel[sel.channel])
assert out.data.is_virtual == parallel_store
if parallel_store:
nfiles = len(
glob(
os.path.join(
os.path.splitext(out.filename)[0],
"*.h5")))
if chan_per_worker is None:
assert nfiles == len(sel.trials)
else:
assert nfiles == len(sel.trials) * (
int(out.channel.size / chan_per_worker) +
int(out.channel.size % chan_per_worker > 0)
)
# ensure pre-selection is equivalent to in-place selection
if select is None:
selected = nonequidata.selectdata()
else:
selected = nonequidata.selectdata(**select)
out_sel = filter_manager(
selected,
self.b,
self.a,
chan_per_worker=chan_per_worker,
parallel=True,
parallel_store=parallel_store)
assert np.allclose(out.data, out_sel.data)
assert np.array_equal(out.channel, out_sel.channel)
for tk in range(len(out.trials)):
assert np.array_equal(out.time[tk],
out_sel.time[tk])
client.close()
@skip_without_acme
def test_parallel_saveload(self, testcluster):
client = dd.Client(testcluster)
for parallel_store in [True, False]:
for sk, select in enumerate(self.sigdataSelections):
sel = Selector(self.sigdata, select)
out = filter_manager(self.sigdata,
self.b,
self.a,
select=select,
log_dict={
"a": "this is a",
"b": "this is b"
},
parallel=True,
parallel_store=parallel_store)
# only keyword args (`a` in this case here) are stored in `cfg`
assert set(["a"]) == set(out.cfg.keys())
assert np.array_equal(out.cfg["a"], self.a)
assert len(out.trials) == len(sel.trials)
# ensure our `log_dict` specification was respected
assert "lowpass" in out._log
assert "a = this is a" in out._log
assert "b = this is b" in out._log
# ensure pre-selection is equivalent to in-place selection
if select is None:
selected = self.sigdata.selectdata()
else:
selected = self.sigdata.selectdata(**select)
out_sel = filter_manager(selected,
self.b,
self.a,
log_dict={
"a": "this is a",
"b": "this is b"
},
parallel=True,
parallel_store=parallel_store)
# only keyword args (`a` in this case here) are stored in `cfg`
assert set(["a"]) == set(out.cfg.keys())
assert np.array_equal(out.cfg["a"], self.a)
assert len(out.trials) == len(sel.trials)
# ensure our `log_dict` specification was respected
assert "lowpass" in out._log
assert "a = this is a" in out._log
assert "b = this is b" in out._log
# save and re-load result, ensure nothing funky happens
with tempfile.TemporaryDirectory() as tdir:
fname = os.path.join(tdir, "dummy")
out.save(fname)
dummy = load(fname)
assert "a" in dummy.cfg.keys()
assert np.array_equal(dummy.cfg["a"], self.a)
assert out.filename == dummy.filename
assert not out.data.is_virtual
if select is None:
reference = self.orig
else:
ref = []
for tk, trlno in enumerate(sel.trials):
ref.append(
self.origdata.trials[trlno][sel.time[tk],
sel.channel])
assert np.array_equal(
dummy.time[tk],
self.sigdata.time[trlno][sel.time[tk]])
reference = np.vstack(ref)
assert self.metrix[sk](
np.abs(dummy.data - reference)) < self.tols[sk]
assert np.array_equal(dummy.channel,
self.sigdata.channel[sel.channel])
# del dummy, out
# ensure out_sel is written/read correctly
fname2 = os.path.join(tdir, "dummy2")
out_sel.save(fname2)
dummy2 = load(fname2)
assert "a" in dummy2.cfg.keys()
assert np.array_equal(dummy2.cfg["a"], dummy.cfg["a"])
assert np.array_equal(dummy.data, dummy2.data)
assert np.array_equal(dummy.channel, dummy2.channel)
assert np.array_equal(dummy.time, dummy2.time)
assert not dummy2.data.is_virtual
del dummy, dummy2, out, out_sel
client.close()