def test_checksum(self):
with tempfile.TemporaryDirectory() as tdir:
for dclass in self.classes:
dname = os.path.join(tdir, "dummy")
dummy = getattr(swd, dclass)(self.data[dclass],
samplerate=1000)
dummy.trialdefintion = self.trl[dclass]
save(dummy, dname)
# perform checksum-matching - this must work
dummy2 = load(dname, checksum=True)
# manipulate data file
hname = dummy._filename
del dummy, dummy2
time.sleep(0.1) # wait to kick-off garbage collection
h5f = h5py.File(hname, "r+")
dset = h5f["data"]
# provoke checksum error by adding 1 to all datasets
dset[()] += 1
h5f.close()
with pytest.raises(SPYValueError):
load(dname, checksum=True)
shutil.rmtree(dname + ".spy")
def test_saveload(self):
with tempfile.TemporaryDirectory() as tdir:
fname = os.path.join(tdir, "dummy")
# basic but most important: ensure object integrity is preserved
checkAttr = [
"channel", "data", "dimord", "freq", "sampleinfo",
"samplerate", "taper", "trialinfo"
]
dummy = SpectralData(self.data, samplerate=1000)
dummy.save(fname)
filename = construct_spy_filename(fname, dummy)
# dummy2 = SpectralData(filename)
# for attr in checkAttr:
# assert np.array_equal(getattr(dummy, attr), getattr(dummy2, attr))
dummy3 = load(fname)
for attr in checkAttr:
assert np.array_equal(getattr(dummy3, attr),
getattr(dummy, attr))
save(dummy3, container=os.path.join(tdir, "ymmud"))
dummy4 = load(os.path.join(tdir, "ymmud"))
for attr in checkAttr:
assert np.array_equal(getattr(dummy4, attr),
getattr(dummy, attr))
del dummy, dummy3, dummy4 # avoid PermissionError in Windows
# ensure trialdefinition is saved and loaded correctly
dummy = SpectralData(self.data,
trialdefinition=self.trl,
samplerate=1000)
dummy.save(fname, overwrite=True)
dummy2 = load(filename)
assert np.array_equal(dummy.trialdefinition,
dummy2.trialdefinition)
# test getters
assert np.array_equal(dummy.sampleinfo, dummy2.sampleinfo)
assert np.array_equal(dummy._t0, dummy2._t0)
assert np.array_equal(dummy.trialinfo, dummy2.trialinfo)
# swap dimensions and ensure `dimord` is preserved
dummy = SpectralData(self.data,
dimord=["time", "channel", "taper", "freq"],
samplerate=1000)
dummy.save(fname + "_dimswap")
filename = construct_spy_filename(fname + "_dimswap", dummy)
dummy2 = load(filename)
assert dummy2.dimord == dummy.dimord
assert dummy2.channel.size == self.nt # swapped
assert dummy2.taper.size == self.nf # swapped
assert dummy2.data.shape == dummy.data.shape
# Delete all open references to file objects b4 closing tmp dir
del dummy, dummy2
def test_logging(self):
with tempfile.TemporaryDirectory() as tdir:
fname = os.path.join(tdir, "dummy")
dummy = generate_artificial_data(inmemory=True)
ldum = len(dummy._log)
save(dummy, filename=fname)
# ensure saving is logged correctly
assert len(dummy._log) > ldum
assert dummy.filename in dummy._log
assert dummy.filename + FILE_EXT["info"] in dummy._log
assert dummy.cfg["method"] == "save"
assert dummy.filename in dummy.cfg["files"]
assert dummy.filename + FILE_EXT["info"] in dummy.cfg["files"]
# ensure loading is logged correctly
dummy2 = load(filename=fname + ".analog")
assert len(dummy2._log) > len(dummy._log)
assert dummy2.filename in dummy2._log
assert dummy2.filename + FILE_EXT["info"] in dummy._log
assert dummy2.cfg.cfg["method"] == "load"
assert dummy2.filename in dummy2.cfg.cfg["files"]
assert dummy2.filename + FILE_EXT["info"] in dummy2.cfg.cfg[
"files"]
# Delete all open references to file objects b4 closing tmp dir
del dummy, dummy2
def test_container_saveload(self):
with tempfile.TemporaryDirectory() as tdir:
for dclass in self.classes:
dummy = getattr(swd, dclass)(self.data[dclass],
samplerate=1000)
dummy.trialdefintion = self.trl[dclass]
# load single file from container
container = "single_container_" + dummy._classname_to_extension(
)[1:]
dummy.save(container=os.path.join(tdir, container))
dummy2 = load(os.path.join(tdir, container))
for attr in ["data", "sampleinfo", "trialinfo"]:
assert np.array_equal(getattr(dummy, attr),
getattr(dummy2, attr))
del dummy2
# load single file via dataclass
dummy2 = load(os.path.join(tdir, container),
dataclass=dummy._classname_to_extension())
# save and load single file via tag
container2 = "another_single_container_" + dummy._classname_to_extension(
)[1:]
dummy2.save(container=os.path.join(tdir, container2),
tag="sometag")
dummy3 = load(os.path.join(tdir, container2), tag="sometag")
for attr in ["data", "sampleinfo", "trialinfo"]:
assert np.array_equal(getattr(dummy2, attr),
getattr(dummy3, attr))
del dummy, dummy2, dummy3
# tag mismatch in single-file container
with pytest.raises(SPYIOError):
load(os.path.join(tdir, container2), tag="invalid")
# dataclass mismatch in single-file container
wrong_ext = getattr(
swd,
list(set(self.classes).difference(
[dclass]))[0])()._classname_to_extension()
with pytest.raises(SPYIOError):
load(os.path.join(tdir, container), dataclass=wrong_ext)
# invalid dataclass specification
with pytest.raises(SPYValueError):
load(os.path.join(tdir, container), dataclass='.invalid')
def test_saveload(self):
with tempfile.TemporaryDirectory() as tdir:
fname = os.path.join(tdir, "dummy")
# basic but most important: ensure object integrity is preserved
checkAttr = [
"data", "dimord", "sampleinfo", "samplerate", "trialinfo"
]
dummy = EventData(self.data, samplerate=10)
dummy.save(fname)
filename = construct_spy_filename(fname, dummy)
dummy2 = load(filename)
for attr in checkAttr:
assert np.array_equal(getattr(dummy, attr),
getattr(dummy2, attr))
dummy3 = load(fname)
for attr in checkAttr:
assert np.array_equal(getattr(dummy3, attr),
getattr(dummy, attr))
save(dummy3, container=os.path.join(tdir, "ymmud"))
dummy4 = load(os.path.join(tdir, "ymmud"))
for attr in checkAttr:
assert np.array_equal(getattr(dummy4, attr),
getattr(dummy, attr))
del dummy2, dummy3, dummy4 # avoid PermissionError in Windows
# overwrite existing file w/new data
dummy.samplerate = 20
dummy.save()
dummy2 = load(filename=filename)
assert dummy2.samplerate == 20
del dummy, dummy2
time.sleep(0.1) # wait to kick-off garbage collection
# ensure trialdefinition is saved and loaded correctly
dummy = EventData(self.data,
trialdefinition=self.trl,
samplerate=10)
dummy.save(fname, overwrite=True)
dummy2 = load(filename)
assert np.array_equal(dummy.sampleinfo, dummy2.sampleinfo)
assert np.array_equal(dummy._t0, dummy2._t0)
assert np.array_equal(dummy.trialinfo, dummy2.trialinfo)
del dummy, dummy2
# swap dimensions and ensure `dimord` is preserved
dummy = EventData(self.data,
dimord=["eventid", "sample"],
samplerate=10)
dummy.save(fname + "_dimswap")
filename = construct_spy_filename(fname + "_dimswap", dummy)
dummy2 = load(filename)
assert dummy2.dimord == dummy.dimord
assert dummy2.eventid.size == self.num_smp # swapped
assert dummy2.data.shape == dummy.data.shape
# Delete all open references to file objects b4 closing tmp dir
del dummy, dummy2
time.sleep(0.1)
def test_multi_saveload(self):
with tempfile.TemporaryDirectory() as tdir:
for dk, dclass in enumerate(self.classes):
dummy = getattr(swd, dclass)(self.data[dclass],
samplerate=1000)
dummy.trialdefintion = self.trl[dclass]
# save to joint container
container = "multi_container"
dummy.save(container=os.path.join(tdir, container))
# try to load last class from joint container before it's there
if dk == len(self.classes) - 2:
not_yet = getattr(
swd, self.classes[dk + 1])()._classname_to_extension()
with pytest.raises(SPYIOError):
load(os.path.join(tdir, container), dataclass=not_yet)
# try to load everything but the first class at the beginning
if dk == 0:
ext_list = []
for attr in self.classes[1:]:
ext_list.append(
getattr(swd, attr)()._classname_to_extension())
fname = os.path.join(
os.path.join(tdir, container + FILE_EXT["dir"]),
container + dummy._classname_to_extension())
with pytest.raises(SPYValueError):
load(fname, dataclass=ext_list)
del dummy
# load all files created above
container = "multi_container"
objDict = load(os.path.join(tdir, container))
assert len(objDict.keys()) == len(self.classes)
all_ext = []
for attr in self.classes:
all_ext.append(getattr(swd, attr)()._classname_to_extension())
fnameList = []
for ext in all_ext:
fnameList.append(
os.path.join(
os.path.join(tdir, container + FILE_EXT["dir"]),
container + ext))
for name, obj in objDict.items():
assert obj.filename in fnameList
fname = fnameList.pop(fnameList.index(obj.filename))
assert name in fname
assert len(fnameList) == 0
del objDict
# load single file from joint container via dataclass
dummy = load(os.path.join(tdir, container), dataclass="analog")
assert dummy.filename == os.path.join(
os.path.join(tdir, container + FILE_EXT["dir"]),
container + ".analog")
dummy.save(tag="2ndanalog")
del dummy
# load single file from joint container using tag
dummy = load(os.path.join(tdir, container), tag="2ndanalog")
dummy.save(tag="3rdanalog")
del dummy
# load single file from joint container using dataclass and tag
dummy = load(os.path.join(tdir, container),
dataclass="analog",
tag="3rdanalog")
# load single file from joint container using multiple dataclasses and single tag
dummy2 = load(os.path.join(tdir, container),
dataclass=["analog", "spectral"],
tag="3rdanalog")
assert dummy2.filename == dummy.filename
# load single file from joint container using single dataclass and multiple tags
dummy3 = load(os.path.join(tdir, container),
dataclass="analog",
tag=["3rdanalog", "invalid"])
assert dummy3.filename == dummy.filename
# load single file from joint container using multiple dataclasses and tags
dummy4 = load(os.path.join(tdir, container),
dataclass=["analog", "spectral"],
tag=["3rdanalog", "invalid"])
assert dummy4.filename == dummy.filename
del dummy, dummy2, dummy3, dummy4
# load multiple files from joint container using single tag
objDict = load(os.path.join(tdir, container), tag="analog")
assert len(objDict.keys()) == 2
wanted = ["2nd", "3rd"]
for name in objDict.keys():
inWanted = [tag in name for tag in wanted]
assert any(inWanted)
inWanted.pop(inWanted.index(True))
del objDict
# load multiple files from joint container using multiple tags
objDict = load(os.path.join(tdir, container), tag=["2nd", "3rd"])
assert len(objDict.keys()) == 2
wanted = ["2nd", "3rd"]
for name in objDict.keys():
inWanted = [tag in name for tag in wanted]
assert any(inWanted)
inWanted.pop(inWanted.index(True))
del objDict
# load all AnalogData files from joint container via single dataclass
objDict = load(os.path.join(tdir, container), dataclass="analog")
assert len(objDict.keys()) == 3
wanted = ["3rdanalog", "2ndanalog", "analog"]
for name in objDict.keys():
inWanted = [tag in name for tag in wanted]
assert any(inWanted)
inWanted.pop(inWanted.index(True))
del objDict
# load multiple files from joint container via multiple dataclasses
wanted = ["spectral", "spike"]
objDict = load(os.path.join(tdir, container), dataclass=wanted)
assert len(wanted) == len(objDict.keys())
for ext in wanted:
basename = container + "." + ext
assert basename in objDict.keys()
fname = objDict[basename].filename
assert fname == os.path.join(
os.path.join(tdir, container + FILE_EXT["dir"]), basename)
del objDict
# load multiple files from joint container via multiple dataclasses and single tag
objDict = load(os.path.join(tdir, container),
tag="analog",
dataclass=["analog", "analog"])
wanted = ["2nd", "3rd"]
for name, obj in objDict.items():
assert isinstance(obj, AnalogData)
inWanted = [tag in name for tag in wanted]
assert any(inWanted)
inWanted.pop(inWanted.index(True))
del objDict
# load multiple files from joint container via single dataclass and multiple tags
objDict = load(os.path.join(tdir, container),
tag=["2nd", "3rd"],
dataclass="analog")
wanted = ["2nd", "3rd"]
for name, obj in objDict.items():
assert isinstance(obj, AnalogData)
inWanted = [tag in name for tag in wanted]
assert any(inWanted)
inWanted.pop(inWanted.index(True))
del objDict
# load multiple files from joint container via multiple dataclasses and tags
objDict = load(os.path.join(tdir, container),
tag=["2nd", "3rd"],
dataclass=["analog", "analog"])
wanted = ["2nd", "3rd"]
for name, obj in objDict.items():
assert isinstance(obj, AnalogData)
inWanted = [tag in name for tag in wanted]
assert any(inWanted)
inWanted.pop(inWanted.index(True))
del obj, objDict
# invalid combinations of tag/dataclass
with pytest.raises(SPYIOError):
load(os.path.join(tdir, container),
dataclass="analog",
tag="invalid")
with pytest.raises(SPYIOError):
load(os.path.join(tdir, container),
dataclass="spike",
tag="2nd")
with pytest.raises(SPYIOError):
load(os.path.join(tdir, container),
dataclass=["analog", "spike"],
tag="invalid")
with pytest.raises(SPYIOError):
load(os.path.join(tdir, container),
dataclass=["spike", "invalid"],
tag="2nd")
with pytest.raises(SPYIOError):
load(os.path.join(tdir, container),
dataclass="analog",
tag=["invalid", "stillinvalid"])
with pytest.raises(SPYIOError):
load(os.path.join(tdir, container),
dataclass="spike",
tag=["2nd", "3rd"])
with pytest.raises(SPYIOError):
load(os.path.join(tdir, container),
dataclass=["spike", "event"],
tag=["2nd", "3rd"])
with pytest.raises(SPYIOError):
load(os.path.join(tdir, container),
dataclass=["analog", "analog"],
tag=["invalid", "stillinvalid"])
with pytest.raises(SPYIOError):
load(os.path.join(tdir, container),
dataclass=["spike", "event"],
tag=["invalid", "stillinvalid"])
with pytest.raises(SPYValueError):
load(os.path.join(tdir, container),
dataclass="invalid",
tag="2nd")
with pytest.raises(SPYValueError):
load(os.path.join(tdir, container),
dataclass=["invalid", "stillinvalid"],
tag="2nd")
def test_saveload(self):
with tempfile.TemporaryDirectory() as tdir:
fname = os.path.join(tdir, "dummy")
# basic but most important: ensure object integrity is preserved
checkAttr = [
"channel", "data", "dimord", "sampleinfo", "samplerate",
"trialinfo"
]
dummy = AnalogData(data=self.data, samplerate=1000)
dummy.save(fname)
filename = construct_spy_filename(fname, dummy)
# NOTE: We removed support for loading data via the constructor
# dummy2 = AnalogData(filename)
# for attr in checkAttr:
# assert np.array_equal(getattr(dummy, attr), getattr(dummy2, attr))
dummy3 = load(fname)
for attr in checkAttr:
assert np.array_equal(getattr(dummy3, attr),
getattr(dummy, attr))
save(dummy3, container=os.path.join(tdir, "ymmud"))
dummy4 = load(os.path.join(tdir, "ymmud"))
for attr in checkAttr:
assert np.array_equal(getattr(dummy4, attr),
getattr(dummy, attr))
del dummy, dummy3, dummy4 # avoid PermissionError in Windows
# # FIXME: either remove or repair this
# # save object hosting VirtualData
# np.save(fname + ".npy", self.data)
# dmap = open_memmap(fname + ".npy", mode="r")
# vdata = VirtualData([dmap, dmap])
# dummy = AnalogData(vdata, samplerate=1000)
# dummy.save(fname, overwrite=True)
# dummy2 = AnalogData(filename)
# assert dummy2.mode == "r+"
# assert np.array_equal(dummy2.data, vdata[:, :])
# del dummy, dummy2 # avoid PermissionError in Windows
# ensure trialdefinition is saved and loaded correctly
dummy = AnalogData(data=self.data,
trialdefinition=self.trl,
samplerate=1000)
dummy.save(fname + "_trl")
filename = construct_spy_filename(fname + "_trl", dummy)
dummy2 = load(filename)
assert np.array_equal(dummy.trialdefinition,
dummy2.trialdefinition)
# test getters
assert np.array_equal(dummy.sampleinfo, dummy2.sampleinfo)
assert np.array_equal(dummy._t0, dummy2._t0)
assert np.array_equal(dummy.trialinfo, dummy2.trialinfo)
del dummy, dummy2 # avoid PermissionError in Windows
# swap dimensions and ensure `dimord` is preserved
dummy = AnalogData(data=self.data,
dimord=["channel", "time"],
samplerate=1000)
dummy.save(fname + "_dimswap")
filename = construct_spy_filename(fname + "_dimswap", dummy)
dummy2 = load(filename)
assert dummy2.dimord == dummy.dimord
assert dummy2.channel.size == self.ns # swapped
assert dummy2.data.shape == dummy.data.shape
# Delete all open references to file objects and wait 0.1s for changes
# to take effect (thanks, Windows!)
del dummy, dummy2
time.sleep(0.1)
def test_parallel_saveload(self, testcluster):
client = dd.Client(testcluster)
for parallel_store in [True, False]:
for sk, select in enumerate(self.sigdataSelections):
sel = Selector(self.sigdata, select)
out = filter_manager(self.sigdata,
self.b,
self.a,
select=select,
log_dict={
"a": "this is a",
"b": "this is b"
},
parallel=True,
parallel_store=parallel_store)
# only keyword args (`a` in this case here) are stored in `cfg`
assert set(["a"]) == set(out.cfg.keys())
assert np.array_equal(out.cfg["a"], self.a)
assert len(out.trials) == len(sel.trials)
# ensure our `log_dict` specification was respected
assert "lowpass" in out._log
assert "a = this is a" in out._log
assert "b = this is b" in out._log
# ensure pre-selection is equivalent to in-place selection
if select is None:
selected = self.sigdata.selectdata()
else:
selected = self.sigdata.selectdata(**select)
out_sel = filter_manager(selected,
self.b,
self.a,
log_dict={
"a": "this is a",
"b": "this is b"
},
parallel=True,
parallel_store=parallel_store)
# only keyword args (`a` in this case here) are stored in `cfg`
assert set(["a"]) == set(out.cfg.keys())
assert np.array_equal(out.cfg["a"], self.a)
assert len(out.trials) == len(sel.trials)
# ensure our `log_dict` specification was respected
assert "lowpass" in out._log
assert "a = this is a" in out._log
assert "b = this is b" in out._log
# save and re-load result, ensure nothing funky happens
with tempfile.TemporaryDirectory() as tdir:
fname = os.path.join(tdir, "dummy")
out.save(fname)
dummy = load(fname)
assert "a" in dummy.cfg.keys()
assert np.array_equal(dummy.cfg["a"], self.a)
assert out.filename == dummy.filename
assert not out.data.is_virtual
if select is None:
reference = self.orig
else:
ref = []
for tk, trlno in enumerate(sel.trials):
ref.append(
self.origdata.trials[trlno][sel.time[tk],
sel.channel])
assert np.array_equal(
dummy.time[tk],
self.sigdata.time[trlno][sel.time[tk]])
reference = np.vstack(ref)
assert self.metrix[sk](
np.abs(dummy.data - reference)) < self.tols[sk]
assert np.array_equal(dummy.channel,
self.sigdata.channel[sel.channel])
# del dummy, out
# ensure out_sel is written/read correctly
fname2 = os.path.join(tdir, "dummy2")
out_sel.save(fname2)
dummy2 = load(fname2)
assert "a" in dummy2.cfg.keys()
assert np.array_equal(dummy2.cfg["a"], dummy.cfg["a"])
assert np.array_equal(dummy.data, dummy2.data)
assert np.array_equal(dummy.channel, dummy2.channel)
assert np.array_equal(dummy.time, dummy2.time)
assert not dummy2.data.is_virtual
del dummy, dummy2, out, out_sel
client.close()
