def test_checksum(self): with tempfile.TemporaryDirectory() as tdir: for dclass in self.classes: dname = os.path.join(tdir, "dummy") dummy = getattr(swd, dclass)(self.data[dclass], samplerate=1000) dummy.trialdefintion = self.trl[dclass] save(dummy, dname) # perform checksum-matching - this must work dummy2 = load(dname, checksum=True) # manipulate data file hname = dummy._filename del dummy, dummy2 time.sleep(0.1) # wait to kick-off garbage collection h5f = h5py.File(hname, "r+") dset = h5f["data"] # provoke checksum error by adding 1 to all datasets dset[()] += 1 h5f.close() with pytest.raises(SPYValueError): load(dname, checksum=True) shutil.rmtree(dname + ".spy")
def test_logging(self): with tempfile.TemporaryDirectory() as tdir: fname = os.path.join(tdir, "dummy") dummy = generate_artificial_data(inmemory=True) ldum = len(dummy._log) save(dummy, filename=fname) # ensure saving is logged correctly assert len(dummy._log) > ldum assert dummy.filename in dummy._log assert dummy.filename + FILE_EXT["info"] in dummy._log assert dummy.cfg["method"] == "save" assert dummy.filename in dummy.cfg["files"] assert dummy.filename + FILE_EXT["info"] in dummy.cfg["files"] # ensure loading is logged correctly dummy2 = load(filename=fname + ".analog") assert len(dummy2._log) > len(dummy._log) assert dummy2.filename in dummy2._log assert dummy2.filename + FILE_EXT["info"] in dummy._log assert dummy2.cfg.cfg["method"] == "load" assert dummy2.filename in dummy2.cfg.cfg["files"] assert dummy2.filename + FILE_EXT["info"] in dummy2.cfg.cfg[ "files"] # Delete all open references to file objects b4 closing tmp dir del dummy, dummy2
def test_saveload(self): with tempfile.TemporaryDirectory() as tdir: fname = os.path.join(tdir, "dummy") # basic but most important: ensure object integrity is preserved checkAttr = [ "data", "dimord", "sampleinfo", "samplerate", "trialinfo" ] dummy = EventData(self.data, samplerate=10) dummy.save(fname) filename = construct_spy_filename(fname, dummy) dummy2 = load(filename) for attr in checkAttr: assert np.array_equal(getattr(dummy, attr), getattr(dummy2, attr)) dummy3 = load(fname) for attr in checkAttr: assert np.array_equal(getattr(dummy3, attr), getattr(dummy, attr)) save(dummy3, container=os.path.join(tdir, "ymmud")) dummy4 = load(os.path.join(tdir, "ymmud")) for attr in checkAttr: assert np.array_equal(getattr(dummy4, attr), getattr(dummy, attr)) del dummy2, dummy3, dummy4 # avoid PermissionError in Windows # overwrite existing file w/new data dummy.samplerate = 20 dummy.save() dummy2 = load(filename=filename) assert dummy2.samplerate == 20 del dummy, dummy2 time.sleep(0.1) # wait to kick-off garbage collection # ensure trialdefinition is saved and loaded correctly dummy = EventData(self.data, trialdefinition=self.trl, samplerate=10) dummy.save(fname, overwrite=True) dummy2 = load(filename) assert np.array_equal(dummy.sampleinfo, dummy2.sampleinfo) assert np.array_equal(dummy._t0, dummy2._t0) assert np.array_equal(dummy.trialinfo, dummy2.trialinfo) del dummy, dummy2 # swap dimensions and ensure `dimord` is preserved dummy = EventData(self.data, dimord=["eventid", "sample"], samplerate=10) dummy.save(fname + "_dimswap") filename = construct_spy_filename(fname + "_dimswap", dummy) dummy2 = load(filename) assert dummy2.dimord == dummy.dimord assert dummy2.eventid.size == self.num_smp # swapped assert dummy2.data.shape == dummy.data.shape # Delete all open references to file objects b4 closing tmp dir del dummy, dummy2 time.sleep(0.1)
def test_saveload(self): with tempfile.TemporaryDirectory() as tdir: fname = os.path.join(tdir, "dummy") # basic but most important: ensure object integrity is preserved checkAttr = [ "channel", "data", "dimord", "freq", "sampleinfo", "samplerate", "taper", "trialinfo" ] dummy = SpectralData(self.data, samplerate=1000) dummy.save(fname) filename = construct_spy_filename(fname, dummy) # dummy2 = SpectralData(filename) # for attr in checkAttr: # assert np.array_equal(getattr(dummy, attr), getattr(dummy2, attr)) dummy3 = load(fname) for attr in checkAttr: assert np.array_equal(getattr(dummy3, attr), getattr(dummy, attr)) save(dummy3, container=os.path.join(tdir, "ymmud")) dummy4 = load(os.path.join(tdir, "ymmud")) for attr in checkAttr: assert np.array_equal(getattr(dummy4, attr), getattr(dummy, attr)) del dummy, dummy3, dummy4 # avoid PermissionError in Windows # ensure trialdefinition is saved and loaded correctly dummy = SpectralData(self.data, trialdefinition=self.trl, samplerate=1000) dummy.save(fname, overwrite=True) dummy2 = load(filename) assert np.array_equal(dummy.trialdefinition, dummy2.trialdefinition) # test getters assert np.array_equal(dummy.sampleinfo, dummy2.sampleinfo) assert np.array_equal(dummy._t0, dummy2._t0) assert np.array_equal(dummy.trialinfo, dummy2.trialinfo) # swap dimensions and ensure `dimord` is preserved dummy = SpectralData(self.data, dimord=["time", "channel", "taper", "freq"], samplerate=1000) dummy.save(fname + "_dimswap") filename = construct_spy_filename(fname + "_dimswap", dummy) dummy2 = load(filename) assert dummy2.dimord == dummy.dimord assert dummy2.channel.size == self.nt # swapped assert dummy2.taper.size == self.nf # swapped assert dummy2.data.shape == dummy.data.shape # Delete all open references to file objects b4 closing tmp dir del dummy, dummy2
def test_save_mmap(self): with tempfile.TemporaryDirectory() as tdir: fname = os.path.join(tdir, "vdat.npy") dname = os.path.join(tdir, "dummy") vdata = np.ones((1000, 5000)) # ca. 38.2 MB np.save(fname, vdata) del vdata dmap = open_memmap(fname) adata = AnalogData(dmap, samplerate=10) # Ensure memory consumption stays within provided bounds mem = memory_usage()[0] save(adata, filename=dname, memuse=60) assert (mem - memory_usage()[0]) < 70 # Delete all open references to file objects b4 closing tmp dir del dmap, adata
def test_save_fname(self): for dclass in self.classes: with tempfile.TemporaryDirectory() as tdir: dummy = getattr(swd, dclass)(self.data[dclass], samplerate=1000) dummy.trialdefintion = self.trl[dclass] # object w/o container association with pytest.raises(SPYError): dummy.save() # filename without extension filename = "some_filename" save(dummy, filename=os.path.join(tdir, filename)) assert len(glob(os.path.join(tdir, filename + "*"))) == 2 # filename with extension filename = "some_filename_w_ext" + dummy._classname_to_extension( ) save(dummy, filename=os.path.join(tdir, filename)) assert len(glob(os.path.join(tdir, filename + "*"))) == 2 # filename with invalid extension filename = "some_filename_w_ext.invalid" with pytest.raises(SPYError): save(dummy, filename=os.path.join(tdir, filename)) # filename with multiple extensions filename = "some_filename.w.ext" + dummy._classname_to_extension( ) with pytest.raises(SPYError): save(dummy, filename=os.path.join(tdir, filename)) # container with extension container = "test_container.spy" save(dummy, container=os.path.join(tdir, container)) assert len(glob(os.path.join(tdir, container, "*"))) == 2 # container with invalid extension container = "test_container.invalid" with pytest.raises(SPYError): save(dummy, container=os.path.join(tdir, container)) # container with multiple extensions container = "test_container.invalid.too" with pytest.raises(SPYValueError): save(dummy, container=os.path.join(tdir, container)) # container w/o extension container = "test_container2" save(dummy, container=os.path.join(tdir, container)) assert len(glob(os.path.join(tdir, container + ".spy", "*"))) == 2 # container with extension and tag container = "test_container.spy" tag = "sometag" save(dummy, container=os.path.join(tdir, container), tag=tag) assert len( glob( os.path.join(tdir, container, "test_container_sometag*"))) == 2 # explicit overwrite save(dummy, container=os.path.join(tdir, container), tag=tag, overwrite=True) assert len( glob( os.path.join(tdir, container, "test_container_sometag*"))) == 2 # implicit overwrite dummy.save() assert len( glob( os.path.join(tdir, container, "test_container_sometag*"))) == 2 # attempted overwrite w/o keyword with pytest.raises(SPYIOError): save(dummy, container=os.path.join(tdir, container), tag=tag) # shortcut with new tag dummy.save(tag="newtag") assert len( glob( os.path.join(tdir, container, "test_container_newtag*"))) == 2 # overwrite new tag dummy.save(tag="newtag", overwrite=True) assert len( glob( os.path.join(tdir, container, "test_container_newtag*"))) == 2 # attempted overwrite w/o keyword with pytest.raises(SPYIOError): dummy.save(tag="newtag") # both container and filename with pytest.raises(SPYError): save(dummy, container="container", filename="someFile") # neither container nor filename with pytest.raises(SPYError): save(dummy) del dummy
def test_saveload(self): with tempfile.TemporaryDirectory() as tdir: fname = os.path.join(tdir, "dummy") # basic but most important: ensure object integrity is preserved checkAttr = [ "channel", "data", "dimord", "sampleinfo", "samplerate", "trialinfo" ] dummy = AnalogData(data=self.data, samplerate=1000) dummy.save(fname) filename = construct_spy_filename(fname, dummy) # NOTE: We removed support for loading data via the constructor # dummy2 = AnalogData(filename) # for attr in checkAttr: # assert np.array_equal(getattr(dummy, attr), getattr(dummy2, attr)) dummy3 = load(fname) for attr in checkAttr: assert np.array_equal(getattr(dummy3, attr), getattr(dummy, attr)) save(dummy3, container=os.path.join(tdir, "ymmud")) dummy4 = load(os.path.join(tdir, "ymmud")) for attr in checkAttr: assert np.array_equal(getattr(dummy4, attr), getattr(dummy, attr)) del dummy, dummy3, dummy4 # avoid PermissionError in Windows # # FIXME: either remove or repair this # # save object hosting VirtualData # np.save(fname + ".npy", self.data) # dmap = open_memmap(fname + ".npy", mode="r") # vdata = VirtualData([dmap, dmap]) # dummy = AnalogData(vdata, samplerate=1000) # dummy.save(fname, overwrite=True) # dummy2 = AnalogData(filename) # assert dummy2.mode == "r+" # assert np.array_equal(dummy2.data, vdata[:, :]) # del dummy, dummy2 # avoid PermissionError in Windows # ensure trialdefinition is saved and loaded correctly dummy = AnalogData(data=self.data, trialdefinition=self.trl, samplerate=1000) dummy.save(fname + "_trl") filename = construct_spy_filename(fname + "_trl", dummy) dummy2 = load(filename) assert np.array_equal(dummy.trialdefinition, dummy2.trialdefinition) # test getters assert np.array_equal(dummy.sampleinfo, dummy2.sampleinfo) assert np.array_equal(dummy._t0, dummy2._t0) assert np.array_equal(dummy.trialinfo, dummy2.trialinfo) del dummy, dummy2 # avoid PermissionError in Windows # swap dimensions and ensure `dimord` is preserved dummy = AnalogData(data=self.data, dimord=["channel", "time"], samplerate=1000) dummy.save(fname + "_dimswap") filename = construct_spy_filename(fname + "_dimswap", dummy) dummy2 = load(filename) assert dummy2.dimord == dummy.dimord assert dummy2.channel.size == self.ns # swapped assert dummy2.data.shape == dummy.data.shape # Delete all open references to file objects and wait 0.1s for changes # to take effect (thanks, Windows!) del dummy, dummy2 time.sleep(0.1)