class NixIOReadTest(NixIOTest):
filename = "testfile_readtest.h5"
nixfile = None
nix_blocks = None
original_methods = dict()
@classmethod
def setUpClass(cls):
if HAVE_NIX:
cls.nixfile = cls.create_full_nix_file(cls.filename)
def setUp(self):
self.io = NixIO(self.filename, "ro")
self.original_methods["_read_cascade"] = self.io._read_cascade
self.original_methods["_update_maps"] = self.io._update_maps
@classmethod
def tearDownClass(cls):
if HAVE_NIX:
cls.nixfile.close()
os.remove(cls.filename)
def tearDown(self):
self.io.close()
def test_all_read(self):
neo_blocks = self.io.read_all_blocks(lazy=False)
nix_blocks = self.io.nix_file.blocks
self.compare_blocks(neo_blocks, nix_blocks)
def setUp(self):
self.filename = "nixio_testfile_write.h5"
self.writer = NixIO(self.filename, "ow")
self.io = self.writer
self.reader = nix.File.open(self.filename,
nix.FileMode.ReadOnly,
backend="h5py")
class NixIOReadTest(NixIOTest):
nixfile = None
nix_blocks = None
@classmethod
def setUpClass(cls):
cls.tempdir = mkdtemp(prefix="nixiotest")
cls.filename = os.path.join(cls.tempdir, "testnixio.nix")
if HAVE_NIX:
cls.nixfile = cls.create_full_nix_file(cls.filename)
def setUp(self):
self.io = NixIO(self.filename, "ro")
@classmethod
def tearDownClass(cls):
if HAVE_NIX:
cls.nixfile.close()
shutil.rmtree(cls.tempdir)
def tearDown(self):
self.io.close()
def test_all_read(self):
neo_blocks = self.io.read_all_blocks()
nix_blocks = self.io.nix_file.blocks
self.compare_blocks(neo_blocks, nix_blocks)
def test_iter_read(self):
blocknames = [blk.name for blk in self.nixfile.blocks]
for blk, nixname in zip(self.io.iter_blocks(), blocknames):
self.assertEqual(blk.annotations["nix_name"], nixname)
def test_nix_name_read(self):
for nixblock in self.nixfile.blocks:
nixname = nixblock.name
neoblock = self.io.read_block(nixname=nixname)
self.assertEqual(neoblock.annotations["nix_name"], nixname)
def test_index_read(self):
for idx, nixblock in enumerate(self.nixfile.blocks):
neoblock = self.io.read_block(index=idx)
self.assertEqual(neoblock.annotations["nix_name"], nixblock.name)
def test_auto_index_read(self):
for nixblock in self.nixfile.blocks:
neoblock = self.io.read_block() # don't specify index
self.assertEqual(neoblock.annotations["nix_name"], nixblock.name)
# No more blocks - should return None
self.assertIs(self.io.read_block(), None)
self.assertIs(self.io.read_block(), None)
self.assertIs(self.io.read_block(), None)
def test_neo_name_read(self):
for nixblock in self.nixfile.blocks:
neoname = nixblock.metadata["neo_name"]
neoblock = self.io.read_block(neoname=neoname)
self.assertEqual(neoblock.annotations["nix_name"], nixblock.name)
def setUp(self):
self.tempdir = mkdtemp(prefix="nixiotest")
self.filename = os.path.join(self.tempdir, "testnixio.nix")
self.writer = NixIO(self.filename, "ow")
self.io = self.writer
self.reader = nix.File.open(self.filename,
nix.FileMode.ReadOnly,
backend="h5py")
def setUp(self):
super().setUp()
self.testfilename = self.get_local_path('nix/nixio_fr.nix')
self.reader_fr = NixIOfr(filename=self.testfilename)
self.reader_norm = NixIO(filename=self.testfilename, mode='ro')
self.blk = self.reader_fr.read_block(block_index=1, load_waveforms=True)
# read block with NixIOfr
self.blk1 = self.reader_norm.read_block(index=1) # read same block with NixIO
def test_file_with_ver(self):
someversion = '0.100.10'
nixfile = nix.File.open(self.filename, nix.FileMode.Overwrite)
filemd = nixfile.create_section("neo", "neo.metadata")
filemd["version"] = someversion
nixfile.close()
with NixIO(self.filename, "ro") as iofile:
self.assertEqual(iofile._file_version, someversion)
with NixIO(self.filename, "rw") as iofile:
self.assertEqual(iofile._file_version, someversion)
with NixIO(self.filename, "ow") as iofile:
self.assertEqual(iofile._file_version, neover)
def test_annotations(self):
self.testfilename = self.get_filename_path('nixio_fr_ann.nix')
with NixIO(filename=self.testfilename, mode='ow') as io:
annotations = {'my_custom_annotation': 'hello block'}
bl = Block(**annotations)
annotations = {'something': 'hello hello000'}
seg = Segment(**annotations)
an =AnalogSignal([[1, 2, 3], [4, 5, 6]], units='V',
sampling_rate=1*pq.Hz)
an.annotations['ansigrandom'] = 'hello chars'
sp = SpikeTrain([3, 4, 5]* s, t_stop=10.0)
sp.annotations['railway'] = 'hello train'
ev = Event(np.arange(0, 30, 10)*pq.Hz,
labels=np.array(['trig0', 'trig1', 'trig2'], dtype='S'))
ev.annotations['venue'] = 'hello event'
ev2 = Event(np.arange(0, 30, 10) * pq.Hz,
labels=np.array(['trig0', 'trig1', 'trig2'], dtype='S'))
ev2.annotations['evven'] = 'hello ev'
seg.spiketrains.append(sp)
seg.events.append(ev)
seg.events.append(ev2)
seg.analogsignals.append(an)
bl.segments.append(seg)
io.write_block(bl)
io.close()
with NixIOfr(filename=self.testfilename) as frio:
frbl = frio.read_block()
assert 'my_custom_annotation' in frbl.annotations
assert 'something' in frbl.segments[0].annotations
# assert 'ansigrandom' in frbl.segments[0].analogsignals[0].annotations
assert 'railway' in frbl.segments[0].spiketrains[0].annotations
assert 'venue' in frbl.segments[0].events[0].annotations
assert 'evven' in frbl.segments[0].events[1].annotations
os.remove(self.testfilename)
def setUp(self):
self.filename = "nixio_testfile_write.h5"
self.writer = NixIO(self.filename, "ow")
self.io = self.writer
self.reader = nix.File.open(self.filename,
nix.FileMode.ReadOnly,
backend="h5py")
def test_new_file(self):
with NixIO(self.filename, "ow") as iofile:
self.assertEqual(iofile._file_version, neover)
nixfile = nix.File.open(self.filename, nix.FileMode.ReadOnly)
filever = nixfile.sections["neo"]["version"]
self.assertEqual(filever, neover)
nixfile.close()
def setUp(self):
super(TestNixfr, self).setUp()
self.testfilename = self.get_filename_path('nixio_fr.nix')
self.reader_fr = NixIOfr(filename=self.testfilename)
self.reader_norm = NixIO(filename=self.testfilename, mode='ro')
self.blk = self.reader_fr.read_block(block_index=1, load_waveforms=True)
# read block with NixIOfr
self.blk1 = self.reader_norm.read_block(index=1) # read same block with NixIO
def test_oldfile_nover(self):
nixfile = nix.File.open(self.filename, nix.FileMode.Overwrite)
nixfile.close()
with NixIO(self.filename, "ro") as iofile:
self.assertEqual(iofile._file_version, '0.5.2') # compat version
nixfile = nix.File.open(self.filename, nix.FileMode.ReadOnly)
self.assertNotIn("neo", nixfile.sections)
nixfile.close()
with NixIO(self.filename, "rw") as iofile:
self.assertEqual(iofile._file_version, '0.5.2') # compat version
# section should have been created now
nixfile = nix.File.open(self.filename, nix.FileMode.ReadOnly)
self.assertIn("neo", nixfile.sections)
self.assertEqual(nixfile.sections["neo"]["version"], '0.5.2')
nixfile.close()
def test_context_write(self):
neoblock = Block(name=self.rword(), description=self.rsentence())
with NixIO(self.filename, "ow") as iofile:
iofile.write_block(neoblock)
nixfile = nix.File.open(self.filename, nix.FileMode.ReadOnly,
backend="h5py")
self.compare_blocks([neoblock], nixfile.blocks)
nixfile.close()
neoblock.annotate(**self.rdict(5))
with NixIO(self.filename, "rw") as iofile:
iofile.write_block(neoblock)
nixfile = nix.File.open(self.filename, nix.FileMode.ReadOnly,
backend="h5py")
self.compare_blocks([neoblock], nixfile.blocks)
nixfile.close()
os.remove(self.filename)
def test_no_segment_write(self):
# Tests storing AnalogSignal, IrregularlySampledSignal, and SpikeTrain
# objects in the secondary (ChannelIndex) substructure without them
# being attached to a Segment.
blk = Block("segmentless block")
signal = AnalogSignal(name="sig1",
signal=[0, 1, 2],
units="mV",
sampling_period=pq.Quantity(1, "ms"))
othersignal = IrregularlySampledSignal(name="i1",
signal=[0, 0, 0],
units="mV",
times=[1, 2, 3],
time_units="ms")
sta = SpikeTrain(name="the train of spikes",
times=[0.1, 0.2, 10.3],
t_stop=11,
units="us")
stb = SpikeTrain(name="the train of spikes b",
times=[1.1, 2.2, 10.1],
t_stop=100,
units="ms")
chidx = ChannelIndex([8, 13, 21])
blk.channel_indexes.append(chidx)
chidx.analogsignals.append(signal)
chidx.irregularlysampledsignals.append(othersignal)
unit = Unit()
chidx.units.append(unit)
unit.spiketrains.extend([sta, stb])
self.writer.write_block(blk)
self.compare_blocks([blk], self.reader.blocks)
self.writer.close()
reader = NixIO(self.filename, "ro")
blk = reader.read_block(neoname="segmentless block")
chx = blk.channel_indexes[0]
self.assertEqual(len(chx.analogsignals), 1)
self.assertEqual(len(chx.irregularlysampledsignals), 1)
self.assertEqual(len(chx.units[0].spiketrains), 2)
def dump_electrode_data_circus(in_filename, out_filename, chunks=1e9):
in_file = NixIO(in_filename)
block = in_file.read_all_blocks()[0]
signals = block.segments[2].analogsignals
names = [signal.annotations['nix_name'] for signal in signals]
indices = sorted(range(len(signals)), key=lambda x: names[x])
signals = [signals[i] for i in indices]
names = [names[i] for i in indices]
itemsize = np.array([0.0], dtype=np.float32).nbytes
n = len(signals[0]) # num samples per channel
n_items = int(chunks // itemsize) # num chunked samples per chan
total_n = sum(len(value) for value in signals) # num samples total
pbar = tqdm(total=total_n * itemsize,
file=sys.stdout,
unit_scale=1,
unit='bytes')
mmap_array = open_memmap(out_filename,
mode='w+',
dtype=np.float32,
shape=(n, len(signals)))
for k, signal in enumerate(signals):
i = 0
n = len(signal)
while i * n_items < n:
items = np.array(signal[i * n_items:min((i + 1) * n_items, n)],
dtype=np.float32)[:, 0]
mmap_array[i * n_items:i * n_items + len(items), k] = items
pbar.update(len(items) * itemsize)
i += 1
pbar.close()
print('Channel order is: {}'.format(names))
def test_context_read(self):
nixfile = nix.File.open(self.filename,
nix.FileMode.Overwrite,
backend="h5py")
name_one = self.rword()
name_two = self.rword()
nixfile.create_block(name_one, "neo.block")
nixfile.create_block(name_two, "neo.block")
nixfile.close()
with NixIO(self.filename, "ro") as iofile:
blocks = iofile.read_all_blocks()
self.assertEqual(blocks[0].name, name_one)
self.assertEqual(blocks[1].name, name_two)
class NixIOReadTest(NixIOTest):
filename = "testfile_readtest.h5"
nixfile = None
nix_blocks = None
original_methods = dict()
@classmethod
def setUpClass(cls):
if HAVE_NIX:
cls.nixfile = cls.create_full_nix_file(cls.filename)
def setUp(self):
self.io = NixIO(self.filename, "ro")
self.original_methods["_read_cascade"] = self.io._read_cascade
self.original_methods["_update_maps"] = self.io._update_maps
@classmethod
def tearDownClass(cls):
if HAVE_NIX:
cls.nixfile.close()
def tearDown(self):
self.io.close()
def test_all_read(self):
neo_blocks = self.io.read_all_blocks(cascade=True, lazy=False)
nix_blocks = self.io.nix_file.blocks
self.compare_blocks(neo_blocks, nix_blocks)
def test_lazyload_fullcascade_read(self):
neo_blocks = self.io.read_all_blocks(cascade=True, lazy=True)
nix_blocks = self.io.nix_file.blocks
# data objects should be empty
for block in neo_blocks:
for seg in block.segments:
for asig in seg.analogsignals:
self.assertEqual(len(asig), 0)
for isig in seg.irregularlysampledsignals:
self.assertEqual(len(isig), 0)
for epoch in seg.epochs:
self.assertEqual(len(epoch), 0)
for event in seg.events:
self.assertEqual(len(event), 0)
for st in seg.spiketrains:
self.assertEqual(len(st), 0)
self.compare_blocks(neo_blocks, nix_blocks)
def test_lazyload_lazycascade_read(self):
neo_blocks = self.io.read_all_blocks(cascade="lazy", lazy=True)
nix_blocks = self.io.nix_file.blocks
self.compare_blocks(neo_blocks, nix_blocks)
def test_lazycascade_read(self):
def getitem(self, index):
return self._data.__getitem__(index)
from neo.io.nixio import LazyList
getitem_original = LazyList.__getitem__
LazyList.__getitem__ = getitem
neo_blocks = self.io.read_all_blocks(cascade="lazy", lazy=False)
for block in neo_blocks:
self.assertIsInstance(block.segments, LazyList)
self.assertIsInstance(block.channel_indexes, LazyList)
for seg in block.segments:
self.assertIsInstance(seg, string_types)
for chx in block.channel_indexes:
self.assertIsInstance(chx, string_types)
LazyList.__getitem__ = getitem_original
def test_load_lazy_cascade(self):
from neo.io.nixio import LazyList
neo_blocks = self.io.read_all_blocks(cascade="lazy", lazy=False)
for block in neo_blocks:
self.assertIsInstance(block.segments, LazyList)
self.assertIsInstance(block.channel_indexes, LazyList)
name = block.name
block = self.io.load_lazy_cascade("/" + name, lazy=False)
self.assertIsInstance(block.segments, list)
self.assertIsInstance(block.channel_indexes, list)
for seg in block.segments:
self.assertIsInstance(seg.analogsignals, list)
self.assertIsInstance(seg.irregularlysampledsignals, list)
self.assertIsInstance(seg.epochs, list)
self.assertIsInstance(seg.events, list)
self.assertIsInstance(seg.spiketrains, list)
def test_nocascade_read(self):
self.io._read_cascade = mock.Mock()
neo_blocks = self.io.read_all_blocks(cascade=False)
self.io._read_cascade.assert_not_called()
for block in neo_blocks:
self.assertEqual(len(block.segments), 0)
nix_block = self.io.nix_file.blocks[block.name]
self.compare_attr(block, nix_block)
def test_lazy_load_subschema(self):
blk = self.io.nix_file.blocks[0]
segpath = "/" + blk.name + "/segments/" + blk.groups[0].name
segment = self.io.load_lazy_cascade(segpath, lazy=True)
self.assertIsInstance(segment, Segment)
self.assertEqual(segment.name, blk.groups[0].name)
self.assertIs(segment.block, None)
self.assertEqual(len(segment.analogsignals[0]), 0)
segment = self.io.load_lazy_cascade(segpath, lazy=False)
self.assertEqual(np.shape(segment.analogsignals[0]), (100, 3))
# random sampling times for data_b
data_b_t = np.cumsum(np.random.random(1200))
# Create one AnalogSignal and one IrregularlySampledSignal
asig_a = AnalogSignal(name="Data A",
signal=data_a,
units="mV",
sampling_rate=pq.Quantity(10, "Hz"))
isig_a = IrregularlySampledSignal(name="Sampled data",
signal=data_b,
units="nA",
times=data_b_t,
time_units="ms")
# Attach the signals to the first segment
seg_a.analogsignals.append(asig_a)
seg_a.irregularlysampledsignals.append(isig_a)
# Create a second AnalogSignal and attach it to the second segment
st_a = AnalogSignal(name="Data A",
signal=data_c * pq.mV,
sampling_rate=20 * pq.Hz)
seg_b.analogsignals.append(st_a)
# Write the Block to file using the NixIO
# Replace NixIO with any other IO to change storage format
io = NixIO("example-file.nix", mode='ow')
io.write_block(block)
io.close()
class NixIOWriteTest(NixIOTest):
def setUp(self):
self.filename = "nixio_testfile_write.h5"
self.writer = NixIO(self.filename, "ow")
self.io = self.writer
self.reader = nix.File.open(self.filename,
nix.FileMode.ReadOnly,
backend="h5py")
def tearDown(self):
self.writer.close()
self.reader.close()
os.remove(self.filename)
def write_and_compare(self, blocks):
self.writer.write_all_blocks(blocks)
self.compare_blocks(self.writer.read_all_blocks(), self.reader.blocks)
def test_block_write(self):
block = Block(name=self.rword(), description=self.rsentence())
self.write_and_compare([block])
block.annotate(**self.rdict(5))
self.write_and_compare([block])
def test_segment_write(self):
block = Block(name=self.rword())
segment = Segment(name=self.rword(), description=self.rword())
block.segments.append(segment)
self.write_and_compare([block])
segment.annotate(**self.rdict(2))
self.write_and_compare([block])
def test_channel_index_write(self):
block = Block(name=self.rword())
chx = ChannelIndex(name=self.rword(),
description=self.rsentence(),
index=[1, 2, 3, 5, 8, 13])
block.channel_indexes.append(chx)
self.write_and_compare([block])
chx.annotate(**self.rdict(3))
self.write_and_compare([block])
def test_signals_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
asig = AnalogSignal(signal=self.rquant((10, 3), pq.mV),
sampling_rate=pq.Quantity(10, "Hz"))
seg.analogsignals.append(asig)
self.write_and_compare([block])
anotherblock = Block("ir signal block")
seg = Segment("ir signal seg")
anotherblock.segments.append(seg)
irsig = IrregularlySampledSignal(signal=np.random.random((20, 3)),
times=self.rquant(20, pq.ms, True),
units=pq.A)
seg.irregularlysampledsignals.append(irsig)
self.write_and_compare([anotherblock])
block.segments[0].analogsignals.append(
AnalogSignal(signal=[10.0, 1.0, 3.0],
units=pq.S,
sampling_period=pq.Quantity(3, "s"),
dtype=np.double,
name="signal42",
description="this is an analogsignal",
t_start=45 * pq.ms), )
self.write_and_compare([block, anotherblock])
block.segments[0].irregularlysampledsignals.append(
IrregularlySampledSignal(times=np.random.random(10),
signal=np.random.random((10, 3)),
units="mV",
time_units="s",
dtype=np.float,
name="some sort of signal",
description="the signal is described"))
self.write_and_compare([block, anotherblock])
def test_epoch_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
epoch = Epoch(times=[1, 1, 10, 3] * pq.ms,
durations=[3, 3, 3, 1] * pq.ms,
labels=np.array(["one", "two", "three", "four"]),
name="test epoch",
description="an epoch for testing")
seg.epochs.append(epoch)
self.write_and_compare([block])
def test_event_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
event = Event(times=np.arange(0, 30, 10) * pq.s,
labels=np.array(["0", "1", "2"]),
name="event name",
description="event description")
seg.events.append(event)
self.write_and_compare([block])
def test_spiketrain_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
spiketrain = SpikeTrain(times=[3, 4, 5] * pq.s,
t_stop=10.0,
name="spikes!",
description="sssssspikes")
seg.spiketrains.append(spiketrain)
self.write_and_compare([block])
waveforms = self.rquant((3, 5, 10), pq.mV)
spiketrain = SpikeTrain(times=[1, 1.1, 1.2] * pq.ms,
t_stop=1.5 * pq.s,
name="spikes with wf",
description="spikes for waveform test",
waveforms=waveforms)
seg.spiketrains.append(spiketrain)
self.write_and_compare([block])
spiketrain.left_sweep = np.random.random(10) * pq.ms
self.write_and_compare([block])
def test_metadata_structure_write(self):
neoblk = self.create_all_annotated()
self.io.write_block(neoblk)
blk = self.io.nix_file.blocks[0]
blkmd = blk.metadata
self.assertEqual(blk.name, blkmd.name)
grp = blk.groups[0] # segment
self.assertIn(grp.name, blkmd.sections)
grpmd = blkmd.sections[grp.name]
for da in grp.data_arrays: # signals
name = ".".join(da.name.split(".")[:-1])
self.assertIn(name, grpmd.sections)
for mtag in grp.multi_tags: # spiketrains, events, and epochs
self.assertIn(mtag.name, grpmd.sections)
srcchx = blk.sources[0] # chx
self.assertIn(srcchx.name, blkmd.sections)
for srcunit in blk.sources: # units
self.assertIn(srcunit.name, blkmd.sections)
self.write_and_compare([neoblk])
def test_anonymous_objects_write(self):
nblocks = 2
nsegs = 2
nanasig = 4
nirrseg = 2
nepochs = 3
nevents = 4
nspiketrains = 3
nchx = 5
nunits = 10
times = self.rquant(1, pq.s)
signal = self.rquant(1, pq.V)
blocks = []
for blkidx in range(nblocks):
blk = Block()
blocks.append(blk)
for segidx in range(nsegs):
seg = Segment()
blk.segments.append(seg)
for anaidx in range(nanasig):
seg.analogsignals.append(
AnalogSignal(signal=signal, sampling_rate=pq.Hz))
for irridx in range(nirrseg):
seg.irregularlysampledsignals.append(
IrregularlySampledSignal(times=times,
signal=signal,
time_units=pq.s))
for epidx in range(nepochs):
seg.epochs.append(Epoch(times=times, durations=times))
for evidx in range(nevents):
seg.events.append(Event(times=times))
for stidx in range(nspiketrains):
seg.spiketrains.append(
SpikeTrain(times=times, t_stop=pq.s, units=pq.s))
for chidx in range(nchx):
chx = ChannelIndex(name="chx{}".format(chidx), index=[1, 2])
blk.channel_indexes.append(chx)
for unidx in range(nunits):
unit = Unit()
chx.units.append(unit)
self.writer.write_all_blocks(blocks)
self.compare_blocks(blocks, self.reader.blocks)
def test_to_value(self):
section = self.io.nix_file.create_section("Metadata value test",
"Test")
writeprop = self.io._write_property
# quantity
qvalue = pq.Quantity(10, "mV")
writeprop(section, "qvalue", qvalue)
self.assertEqual(section["qvalue"], 10)
self.assertEqual(section.props["qvalue"].unit, "mV")
# datetime
dt = self.rdate()
writeprop(section, "dt", dt)
self.assertEqual(datetime.fromtimestamp(section["dt"]), dt)
# string
randstr = self.rsentence()
writeprop(section, "randstr", randstr)
self.assertEqual(section["randstr"], randstr)
# bytes
bytestring = b"bytestring"
writeprop(section, "randbytes", bytestring)
self.assertEqual(section["randbytes"], bytestring.decode())
# iterables
randlist = np.random.random(10).tolist()
writeprop(section, "randlist", randlist)
self.assertEqual(randlist, section["randlist"])
randarray = np.random.random(10)
writeprop(section, "randarray", randarray)
np.testing.assert_almost_equal(randarray, section["randarray"])
# numpy item
npval = np.float64(2398)
writeprop(section, "npval", npval)
self.assertEqual(npval, section["npval"])
# number
val = 42
writeprop(section, "val", val)
self.assertEqual(val, section["val"])
def setUp(self):
self.io = NixIO(self.filename, "rw")
self.neo_blocks = self.io.read_all_blocks()
self.original_methods["_write_attr_annotations"] =\
self.io._write_attr_annotations
class TestNixfr(BaseTestIO, unittest.TestCase, ):
ioclass = NixIOfr
entities_to_download = [
'nix/nixio_fr.nix'
]
entities_to_test = [
'nix/nixio_fr.nix'
]
def setUp(self):
super().setUp()
self.testfilename = self.get_local_path('nix/nixio_fr.nix')
self.reader_fr = NixIOfr(filename=self.testfilename)
self.reader_norm = NixIO(filename=self.testfilename, mode='ro')
self.blk = self.reader_fr.read_block(block_index=1, load_waveforms=True)
# read block with NixIOfr
self.blk1 = self.reader_norm.read_block(index=1) # read same block with NixIO
def tearDown(self):
self.reader_fr.file.close()
self.reader_norm.close()
def test_check_same_neo_structure(self):
self.assertEqual(len(self.blk.segments), len(self.blk1.segments))
for seg1, seg2 in zip(self.blk.segments, self.blk1.segments):
self.assertEqual(len(seg1.analogsignals), len(seg2.analogsignals))
self.assertEqual(len(seg1.spiketrains), len(seg2.spiketrains))
self.assertEqual(len(seg1.events), len(seg2.events))
self.assertEqual(len(seg1.epochs), len(seg2.epochs))
def test_check_same_data_content(self):
for seg1, seg2 in zip(self.blk.segments, self.blk1.segments):
for asig1, asig2 in zip(seg1.analogsignals, seg2.analogsignals):
np.testing.assert_almost_equal(asig1.magnitude, asig2.magnitude)
# not completely equal
for st1, st2 in zip(seg1.spiketrains, seg2.spiketrains):
np.testing.assert_array_equal(st1.magnitude, st2.times)
for wf1, wf2 in zip(st1.waveforms, st2.waveforms):
np.testing.assert_array_equal(wf1.shape, wf2.shape)
np.testing.assert_almost_equal(wf1.magnitude, wf2.magnitude)
for ev1, ev2 in zip(seg1.events, seg2.events):
np.testing.assert_almost_equal(ev1.times, ev2.times)
assert np.all(ev1.labels == ev2.labels)
for ep1, ep2 in zip(seg1.epochs, seg2.epochs):
assert len(ep1.durations) == len(ep2.times)
np.testing.assert_almost_equal(ep1.times, ep2.times)
np.testing.assert_array_equal(ep1.durations, ep2.durations)
np.testing.assert_array_equal(ep1.labels, ep2.labels)
def test_analog_signal(self):
seg1 = self.blk.segments[0]
an_sig1 = seg1.analogsignals[0]
assert len(an_sig1) == 30
an_sig2 = seg1.analogsignals[1]
assert an_sig2.shape == (50, 3)
def test_spike_train(self):
st1 = self.blk.segments[0].spiketrains[0]
assert np.all(st1.times == np.cumsum(np.arange(0, 1, 0.1)).tolist() * pq.s + 10 * pq.s)
def test_event(self):
seg1 = self.blk.segments[0]
event1 = seg1.events[0]
raw_time = 10 + np.cumsum(np.array([0, 1, 2, 3, 4]))
assert np.all(event1.times == np.array(raw_time * pq.s / 1000))
assert np.all(event1.labels == np.array(['A', 'B', 'C', 'D', 'E'], dtype='U'))
assert len(seg1.events) == 1
def test_epoch(self):
seg1 = self.blk.segments[1]
seg2 = self.blk1.segments[1]
epoch1 = seg1.epochs[0]
epoch2 = seg2.epochs[0]
assert len(epoch1.durations) == len(epoch1.times)
assert np.all(epoch1.durations == epoch2.durations)
assert np.all(epoch1.labels == epoch2.labels)
def test_annotations(self):
self.testfilename = self.get_local_path('nix/nixio_fr_ann.nix')
with NixIO(filename=self.testfilename, mode='ow') as io:
annotations = {'my_custom_annotation': 'hello block'}
bl = Block(**annotations)
annotations = {'something': 'hello hello000'}
seg = Segment(**annotations)
an =AnalogSignal([[1, 2, 3], [4, 5, 6]], units='V',
sampling_rate=1 * pq.Hz)
an.annotate(ansigrandom='hello chars')
an.array_annotate(custom_id=[1, 2, 3])
sp = SpikeTrain([3, 4, 5]* s, t_stop=10.0)
sp.annotations['railway'] = 'hello train'
ev = Event(np.arange(0, 30, 10)*pq.Hz,
labels=np.array(['trig0', 'trig1', 'trig2'], dtype='S'))
ev.annotations['venue'] = 'hello event'
ev2 = Event(np.arange(0, 30, 10) * pq.Hz,
labels=np.array(['trig0', 'trig1', 'trig2'], dtype='S'))
ev2.annotations['evven'] = 'hello ev'
seg.spiketrains.append(sp)
seg.events.append(ev)
seg.events.append(ev2)
seg.analogsignals.append(an)
bl.segments.append(seg)
io.write_block(bl)
io.close()
with NixIOfr(filename=self.testfilename) as frio:
frbl = frio.read_block()
assert 'my_custom_annotation' in frbl.annotations
assert 'something' in frbl.segments[0].annotations
assert 'ansigrandom' in frbl.segments[0].analogsignals[0].annotations
assert 'railway' in frbl.segments[0].spiketrains[0].annotations
assert 'venue' in frbl.segments[0].events[0].annotations
assert 'evven' in frbl.segments[0].events[1].annotations
assert 'custom_id' in frbl.segments[0].analogsignals[0].array_annotations
for anasig in frbl.segments[0].analogsignals:
for value in anasig.array_annotations.values():
assert anasig.shape[-1] == len(value)
os.remove(self.testfilename)
class NixIOPartialWriteTest(NixIOTest):
filename = "testfile_partialwrite.h5"
nixfile = None
neo_blocks = None
original_methods = dict()
@classmethod
def setUpClass(cls):
if HAVE_NIX:
cls.nixfile = cls.create_full_nix_file(cls.filename)
def setUp(self):
self.io = NixIO(self.filename, "rw")
self.neo_blocks = self.io.read_all_blocks()
self.original_methods["_write_attr_annotations"] =\
self.io._write_attr_annotations
@classmethod
def tearDownClass(cls):
if HAVE_NIX:
cls.nixfile.close()
def tearDown(self):
self.restore_methods()
self.io.close()
def restore_methods(self):
for name, method in self.original_methods.items():
setattr(self.io, name, self.original_methods[name])
def _mock_write_attr(self, objclass):
typestr = str(objclass.__name__).lower()
self.io._write_attr_annotations = mock.Mock(
wraps=self.io._write_attr_annotations,
side_effect=self.check_obj_type("neo.{}".format(typestr)))
neo_blocks = self.neo_blocks
self.modify_objects(neo_blocks, excludes=[objclass])
self.io.write_all_blocks(neo_blocks)
self.restore_methods()
def check_obj_type(self, typestring):
neq = self.assertNotEqual
def side_effect_func(*args, **kwargs):
obj = kwargs.get("nixobj", args[0])
if isinstance(obj, list):
for sig in obj:
neq(sig.type, typestring)
else:
neq(obj.type, typestring)
return side_effect_func
@classmethod
def modify_objects(cls, objs, excludes=()):
excludes = tuple(excludes)
for obj in objs:
if not (excludes and isinstance(obj, excludes)):
obj.description = cls.rsentence()
for container in getattr(obj, "_child_containers", []):
children = getattr(obj, container)
cls.modify_objects(children, excludes)
def test_partial(self):
for objclass in NixIO.supported_objects:
self._mock_write_attr(objclass)
self.compare_blocks(self.neo_blocks, self.io.nix_file.blocks)
def test_no_modifications(self):
self.io._write_attr_annotations = mock.Mock()
self.io.write_all_blocks(self.neo_blocks)
self.io._write_attr_annotations.assert_not_called()
self.compare_blocks(self.neo_blocks, self.io.nix_file.blocks)
# clearing hashes and checking again
for k in self.io._object_hashes.keys():
self.io._object_hashes[k] = None
self.io.write_all_blocks(self.neo_blocks)
self.io._write_attr_annotations.assert_not_called()
# changing hashes to force rewrite
for k in self.io._object_hashes.keys():
self.io._object_hashes[k] = "_"
self.io.write_all_blocks(self.neo_blocks)
callcount = self.io._write_attr_annotations.call_count
self.assertEqual(callcount, len(self.io._object_hashes))
self.compare_blocks(self.neo_blocks, self.io.nix_file.blocks)
class TestNixfr(BaseTestIO, unittest.TestCase, ):
ioclass = NixIOfr
files_to_test = ['nixio_fr.nix']
files_to_download = ['nixio_fr.nix']
def setUp(self):
super(TestNixfr, self).setUp()
self.testfilename = self.get_filename_path('nixio_fr.nix')
self.reader_fr = NixIOfr(filename=self.testfilename)
self.reader_norm = NixIO(filename=self.testfilename, mode='ro')
self.blk = self.reader_fr.read_block(block_index=1, load_waveforms=True)
# read block with NixIOfr
self.blk1 = self.reader_norm.read_block(index=1) # read same block with NixIO
def tearDown(self):
self.reader_fr.file.close()
self.reader_norm.close()
def test_check_same_neo_structure(self):
self.assertEqual(len(self.blk.segments), len(self.blk1.segments))
for seg1, seg2 in zip(self.blk.segments, self.blk1.segments):
self.assertEqual(len(seg1.analogsignals), len(seg2.analogsignals))
self.assertEqual(len(seg1.spiketrains), len(seg2.spiketrains))
self.assertEqual(len(seg1.events), len(seg2.events))
self.assertEqual(len(seg1.epochs), len(seg2.epochs))
def test_check_same_data_content(self):
for seg1, seg2 in zip(self.blk.segments, self.blk1.segments):
for asig1, asig2 in zip(seg1.analogsignals, seg2.analogsignals):
np.testing.assert_almost_equal(asig1.magnitude, asig2.magnitude)
# not completely equal
for st1, st2 in zip(seg1.spiketrains, seg2.spiketrains):
np.testing.assert_array_equal(st1.magnitude, st2.times)
for wf1, wf2 in zip(st1.waveforms, st2.waveforms):
np.testing.assert_array_equal(wf1.shape, wf2.shape)
np.testing.assert_almost_equal(wf1.magnitude, wf2.magnitude)
for ev1, ev2 in zip(seg1.events, seg2.events):
np.testing.assert_almost_equal(ev1.times, ev2.times)
assert np.all(ev1.labels == ev2.labels)
for ep1, ep2 in zip(seg1.epochs, seg2.epochs):
assert len(ep1.durations) == len(ep2.times)
np.testing.assert_almost_equal(ep1.times, ep2.times)
np.testing.assert_array_equal(ep1.durations, ep2.durations)
np.testing.assert_array_equal(ep1.labels, ep2.labels)
# Not testing for channel_index as rawio always read from seg
for chid1, chid2 in zip(self.blk.channel_indexes, self.blk1.channel_indexes):
for asig1, asig2 in zip(chid1.analogsignals, chid2.analogsignals):
np.testing.assert_almost_equal(asig1.magnitude, asig2.magnitude)
def test_analog_signal(self):
seg1 = self.blk.segments[0]
an_sig1 = seg1.analogsignals[0]
assert len(an_sig1) == 30
an_sig2 = seg1.analogsignals[1]
assert an_sig2.shape == (50, 3)
def test_spike_train(self):
st1 = self.blk.segments[0].spiketrains[0]
assert np.all(st1.times == np.cumsum(np.arange(0, 1, 0.1)).tolist() * pq.s + 10 * pq.s)
def test_event(self):
seg1 = self.blk.segments[0]
event1 = seg1.events[0]
raw_time = 10 + np.cumsum(np.array([0, 1, 2, 3, 4]))
assert np.all(event1.times == np.array(raw_time * pq.s / 1000))
assert np.all(event1.labels == np.array([b'A', b'B', b'C', b'D', b'E']))
assert len(seg1.events) == 1
def test_epoch(self):
seg1 = self.blk.segments[1]
seg2 = self.blk1.segments[1]
epoch1 = seg1.epochs[0]
epoch2 = seg2.epochs[0]
assert len(epoch1.durations) == len(epoch1.times)
assert np.all(epoch1.durations == epoch2.durations)
assert np.all(epoch1.labels == epoch2.labels)
class NixIOReadTest(NixIOTest):
filename = "testfile_readtest.h5"
nixfile = None
nix_blocks = None
original_methods = dict()
@classmethod
def setUpClass(cls):
if HAVE_NIX:
cls.nixfile = cls.create_full_nix_file(cls.filename)
def setUp(self):
self.io = NixIO(self.filename, "ro")
self.original_methods["_read_cascade"] = self.io._read_cascade
self.original_methods["_update_maps"] = self.io._update_maps
@classmethod
def tearDownClass(cls):
if HAVE_NIX:
cls.nixfile.close()
os.remove(cls.filename)
def tearDown(self):
self.io.close()
def test_all_read(self):
neo_blocks = self.io.read_all_blocks(cascade=True, lazy=False)
nix_blocks = self.io.nix_file.blocks
self.compare_blocks(neo_blocks, nix_blocks)
def test_lazyload_fullcascade_read(self):
neo_blocks = self.io.read_all_blocks(cascade=True, lazy=True)
nix_blocks = self.io.nix_file.blocks
# data objects should be empty
for block in neo_blocks:
for seg in block.segments:
for asig in seg.analogsignals:
self.assertEqual(len(asig), 0)
for isig in seg.irregularlysampledsignals:
self.assertEqual(len(isig), 0)
for epoch in seg.epochs:
self.assertEqual(len(epoch), 0)
for event in seg.events:
self.assertEqual(len(event), 0)
for st in seg.spiketrains:
self.assertEqual(len(st), 0)
self.compare_blocks(neo_blocks, nix_blocks)
def test_lazyload_lazycascade_read(self):
neo_blocks = self.io.read_all_blocks(cascade="lazy", lazy=True)
nix_blocks = self.io.nix_file.blocks
self.compare_blocks(neo_blocks, nix_blocks)
def test_lazycascade_read(self):
def getitem(self, index):
return self._data.__getitem__(index)
from neo.io.nixio import LazyList
getitem_original = LazyList.__getitem__
LazyList.__getitem__ = getitem
neo_blocks = self.io.read_all_blocks(cascade="lazy", lazy=False)
for block in neo_blocks:
self.assertIsInstance(block.segments, LazyList)
self.assertIsInstance(block.channel_indexes, LazyList)
for seg in block.segments:
self.assertIsInstance(seg, string_types)
for chx in block.channel_indexes:
self.assertIsInstance(chx, string_types)
LazyList.__getitem__ = getitem_original
def test_load_lazy_cascade(self):
from neo.io.nixio import LazyList
neo_blocks = self.io.read_all_blocks(cascade="lazy", lazy=False)
for block in neo_blocks:
self.assertIsInstance(block.segments, LazyList)
self.assertIsInstance(block.channel_indexes, LazyList)
name = block.annotations["nix_name"]
block = self.io.load_lazy_cascade("/" + name, lazy=False)
self.assertIsInstance(block.segments, list)
self.assertIsInstance(block.channel_indexes, list)
for seg in block.segments:
self.assertIsInstance(seg.analogsignals, list)
self.assertIsInstance(seg.irregularlysampledsignals, list)
self.assertIsInstance(seg.epochs, list)
self.assertIsInstance(seg.events, list)
self.assertIsInstance(seg.spiketrains, list)
def test_nocascade_read(self):
self.io._read_cascade = mock.Mock()
neo_blocks = self.io.read_all_blocks(cascade=False)
self.io._read_cascade.assert_not_called()
for block in neo_blocks:
self.assertEqual(len(block.segments), 0)
nix_block = self.io.nix_file.blocks[block.annotations["nix_name"]]
self.compare_attr(block, nix_block)
def test_lazy_load_subschema(self):
blk = self.io.nix_file.blocks[0]
segpath = "/" + blk.name + "/segments/" + blk.groups[0].name
segment = self.io.load_lazy_cascade(segpath, lazy=True)
self.assertIsInstance(segment, Segment)
self.assertEqual(segment.annotations["nix_name"], blk.groups[0].name)
self.assertIs(segment.block, None)
self.assertEqual(len(segment.analogsignals[0]), 0)
segment = self.io.load_lazy_cascade(segpath, lazy=False)
self.assertEqual(np.shape(segment.analogsignals[0]), (100, 3))
def setUp(self):
self.io = NixIO(self.filename, "rw")
self.neo_blocks = self.io.read_all_blocks()
self.original_methods["_write_attr_annotations"] =\
self.io._write_attr_annotations
class NixIOWriteTest(NixIOTest):
def setUp(self):
self.filename = "nixio_testfile_write.h5"
self.writer = NixIO(self.filename, "ow")
self.io = self.writer
self.reader = nix.File.open(self.filename,
nix.FileMode.ReadOnly,
backend="h5py")
def tearDown(self):
self.writer.close()
self.reader.close()
os.remove(self.filename)
def write_and_compare(self, blocks):
self.writer.write_all_blocks(blocks)
self.compare_blocks(self.writer.read_all_blocks(), self.reader.blocks)
self.compare_blocks(blocks, self.reader.blocks)
def test_block_write(self):
block = Block(name=self.rword(),
description=self.rsentence())
self.write_and_compare([block])
block.annotate(**self.rdict(5))
self.write_and_compare([block])
def test_segment_write(self):
block = Block(name=self.rword())
segment = Segment(name=self.rword(), description=self.rword())
block.segments.append(segment)
self.write_and_compare([block])
segment.annotate(**self.rdict(2))
self.write_and_compare([block])
def test_channel_index_write(self):
block = Block(name=self.rword())
chx = ChannelIndex(name=self.rword(),
description=self.rsentence(),
channel_ids=[10, 20, 30, 50, 80, 130],
index=[1, 2, 3, 5, 8, 13])
block.channel_indexes.append(chx)
self.write_and_compare([block])
chx.annotate(**self.rdict(3))
self.write_and_compare([block])
chx.channel_names = ["one", "two", "three", "five",
"eight", "xiii"]
self.write_and_compare([block])
def test_signals_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
asig = AnalogSignal(signal=self.rquant((10, 3), pq.mV),
sampling_rate=pq.Quantity(10, "Hz"))
seg.analogsignals.append(asig)
self.write_and_compare([block])
anotherblock = Block("ir signal block")
seg = Segment("ir signal seg")
anotherblock.segments.append(seg)
irsig = IrregularlySampledSignal(
signal=np.random.random((20, 3)),
times=self.rquant(20, pq.ms, True),
units=pq.A
)
seg.irregularlysampledsignals.append(irsig)
self.write_and_compare([block, anotherblock])
block.segments[0].analogsignals.append(
AnalogSignal(signal=[10.0, 1.0, 3.0], units=pq.S,
sampling_period=pq.Quantity(3, "s"),
dtype=np.double, name="signal42",
description="this is an analogsignal",
t_start=45 * pq.ms),
)
self.write_and_compare([block, anotherblock])
block.segments[0].irregularlysampledsignals.append(
IrregularlySampledSignal(times=np.random.random(10),
signal=np.random.random((10, 3)),
units="mV", time_units="s",
dtype=np.float,
name="some sort of signal",
description="the signal is described")
)
self.write_and_compare([block, anotherblock])
def test_epoch_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
epoch = Epoch(times=[1, 1, 10, 3]*pq.ms, durations=[3, 3, 3, 1]*pq.ms,
labels=np.array(["one", "two", "three", "four"]),
name="test epoch", description="an epoch for testing")
seg.epochs.append(epoch)
self.write_and_compare([block])
def test_event_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
event = Event(times=np.arange(0, 30, 10)*pq.s,
labels=np.array(["0", "1", "2"]),
name="event name",
description="event description")
seg.events.append(event)
self.write_and_compare([block])
def test_spiketrain_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
spiketrain = SpikeTrain(times=[3, 4, 5]*pq.s, t_stop=10.0,
name="spikes!", description="sssssspikes")
seg.spiketrains.append(spiketrain)
self.write_and_compare([block])
waveforms = self.rquant((3, 5, 10), pq.mV)
spiketrain = SpikeTrain(times=[1, 1.1, 1.2]*pq.ms, t_stop=1.5*pq.s,
name="spikes with wf",
description="spikes for waveform test",
waveforms=waveforms)
seg.spiketrains.append(spiketrain)
self.write_and_compare([block])
spiketrain.left_sweep = np.random.random(10)*pq.ms
self.write_and_compare([block])
def test_metadata_structure_write(self):
neoblk = self.create_all_annotated()
self.io.write_block(neoblk)
blk = self.io.nix_file.blocks[0]
blkmd = blk.metadata
self.assertEqual(blk.name, blkmd.name)
grp = blk.groups[0] # segment
self.assertIn(grp.name, blkmd.sections)
grpmd = blkmd.sections[grp.name]
for da in grp.data_arrays: # signals
name = ".".join(da.name.split(".")[:-1])
self.assertIn(name, grpmd.sections)
for mtag in grp.multi_tags: # spiketrains, events, and epochs
self.assertIn(mtag.name, grpmd.sections)
srcchx = blk.sources[0] # chx
self.assertIn(srcchx.name, blkmd.sections)
for srcunit in blk.sources: # units
self.assertIn(srcunit.name, blkmd.sections)
self.write_and_compare([neoblk])
def test_anonymous_objects_write(self):
nblocks = 2
nsegs = 2
nanasig = 4
nirrseg = 2
nepochs = 3
nevents = 4
nspiketrains = 3
nchx = 5
nunits = 10
times = self.rquant(1, pq.s)
signal = self.rquant(1, pq.V)
blocks = []
for blkidx in range(nblocks):
blk = Block()
blocks.append(blk)
for segidx in range(nsegs):
seg = Segment()
blk.segments.append(seg)
for anaidx in range(nanasig):
seg.analogsignals.append(AnalogSignal(signal=signal,
sampling_rate=pq.Hz))
for irridx in range(nirrseg):
seg.irregularlysampledsignals.append(
IrregularlySampledSignal(times=times,
signal=signal,
time_units=pq.s)
)
for epidx in range(nepochs):
seg.epochs.append(Epoch(times=times, durations=times))
for evidx in range(nevents):
seg.events.append(Event(times=times))
for stidx in range(nspiketrains):
seg.spiketrains.append(SpikeTrain(times=times,
t_stop=times[-1]+pq.s,
units=pq.s))
for chidx in range(nchx):
chx = ChannelIndex(name="chx{}".format(chidx),
index=[1, 2],
channel_ids=[11, 22])
blk.channel_indexes.append(chx)
for unidx in range(nunits):
unit = Unit()
chx.units.append(unit)
self.writer.write_all_blocks(blocks)
self.compare_blocks(blocks, self.reader.blocks)
def test_multiref_write(self):
blk = Block("blk1")
signal = AnalogSignal(name="sig1", signal=[0, 1, 2], units="mV",
sampling_period=pq.Quantity(1, "ms"))
for idx in range(3):
segname = "seg" + str(idx)
seg = Segment(segname)
blk.segments.append(seg)
seg.analogsignals.append(signal)
chidx = ChannelIndex([10, 20, 29])
seg = blk.segments[0]
st = SpikeTrain(name="choochoo", times=[10, 11, 80], t_stop=1000,
units="s")
seg.spiketrains.append(st)
blk.channel_indexes.append(chidx)
for idx in range(6):
unit = Unit("unit" + str(idx))
chidx.units.append(unit)
unit.spiketrains.append(st)
self.writer.write_block(blk)
self.compare_blocks([blk], self.reader.blocks)
def test_to_value(self):
section = self.io.nix_file.create_section("Metadata value test",
"Test")
writeprop = self.io._write_property
# quantity
qvalue = pq.Quantity(10, "mV")
writeprop(section, "qvalue", qvalue)
self.assertEqual(section["qvalue"], 10)
self.assertEqual(section.props["qvalue"].unit, "mV")
# datetime
dt = self.rdate()
writeprop(section, "dt", dt)
self.assertEqual(datetime.fromtimestamp(section["dt"]), dt)
# string
randstr = self.rsentence()
writeprop(section, "randstr", randstr)
self.assertEqual(section["randstr"], randstr)
# bytes
bytestring = b"bytestring"
writeprop(section, "randbytes", bytestring)
self.assertEqual(section["randbytes"], bytestring.decode())
# iterables
randlist = np.random.random(10).tolist()
writeprop(section, "randlist", randlist)
self.assertEqual(randlist, section["randlist"])
randarray = np.random.random(10)
writeprop(section, "randarray", randarray)
np.testing.assert_almost_equal(randarray, section["randarray"])
# numpy item
npval = np.float64(2398)
writeprop(section, "npval", npval)
self.assertEqual(npval, section["npval"])
# number
val = 42
writeprop(section, "val", val)
self.assertEqual(val, section["val"])
class NixIOPartialWriteTest(NixIOTest):
filename = "testfile_partialwrite.h5"
nixfile = None
neo_blocks = None
original_methods = dict()
@classmethod
def setUpClass(cls):
if HAVE_NIX:
cls.nixfile = cls.create_full_nix_file(cls.filename)
def setUp(self):
self.io = NixIO(self.filename, "rw")
self.neo_blocks = self.io.read_all_blocks()
self.original_methods["_write_attr_annotations"] =\
self.io._write_attr_annotations
@classmethod
def tearDownClass(cls):
if HAVE_NIX:
cls.nixfile.close()
os.remove(cls.filename)
def tearDown(self):
self.restore_methods()
self.io.close()
def restore_methods(self):
for name, method in self.original_methods.items():
setattr(self.io, name, self.original_methods[name])
def _mock_write_attr(self, objclass):
typestr = str(objclass.__name__).lower()
self.io._write_attr_annotations = mock.Mock(
wraps=self.io._write_attr_annotations,
side_effect=self.check_obj_type("neo.{}".format(typestr))
)
neo_blocks = self.neo_blocks
self.modify_objects(neo_blocks, excludes=[objclass])
self.io.write_all_blocks(neo_blocks)
self.restore_methods()
def check_obj_type(self, typestring):
neq = self.assertNotEqual
def side_effect_func(*args, **kwargs):
obj = kwargs.get("nixobj", args[0])
if isinstance(obj, list):
for sig in obj:
neq(sig.type, typestring)
else:
neq(obj.type, typestring)
return side_effect_func
@classmethod
def modify_objects(cls, objs, excludes=()):
excludes = tuple(excludes)
for obj in objs:
if not (excludes and isinstance(obj, excludes)):
obj.description = cls.rsentence()
for container in getattr(obj, "_child_containers", []):
children = getattr(obj, container)
cls.modify_objects(children, excludes)
def test_partial(self):
for objclass in NixIO.supported_objects:
self._mock_write_attr(objclass)
self.compare_blocks(self.neo_blocks, self.io.nix_file.blocks)
def test_no_modifications(self):
self.io._write_attr_annotations = mock.Mock()
self.io.write_all_blocks(self.neo_blocks)
self.io._write_attr_annotations.assert_not_called()
self.compare_blocks(self.neo_blocks, self.io.nix_file.blocks)
# clearing hashes and checking again
for k in self.io._object_hashes.keys():
self.io._object_hashes[k] = None
self.io.write_all_blocks(self.neo_blocks)
self.io._write_attr_annotations.assert_not_called()
# changing hashes to force rewrite
for k in self.io._object_hashes.keys():
self.io._object_hashes[k] = "_"
self.io.write_all_blocks(self.neo_blocks)
callcount = self.io._write_attr_annotations.call_count
self.assertEqual(callcount, len(self.io._object_hashes))
self.compare_blocks(self.neo_blocks, self.io.nix_file.blocks)
def setUp(self):
self.io = NixIO(self.filename, "ro")
def create_nix_file(file_pat,
output=None,
signed=False,
rate=1,
electrode_scale=0.0104,
analog_scale=12.5122,
channels=channels,
chunks=256 * 1024 * 1024):
'''The default resolution (i.e. voltage per bit) in uV '''
filenames = sorted(glob(file_pat))
dtype = np.int16 if signed else np.uint16
N = len(channels)
slice_size = N * 2
chunks = chunks - chunks % slice_size # round to equal blocks
reader = read_files(filenames, dtype, chunks, slice_size)
data = next(reader)
if signed:
af = lambda x: x.astype(np.float32) * analog_scale
ef = lambda x: x.astype(np.float32) * electrode_scale
df = lambda x: np.array(x, dtype=np.bool_)
else:
af = lambda x: (x.astype(np.float32) - 2**15) * analog_scale
ef = lambda x: (x.astype(np.float32) - 2**15) * electrode_scale
df = lambda x: np.array(x.astype(np.int32) - 2**15, dtype=np.bool_)
analogs = [(i, ch) for i, ch in enumerate(channels) if ch.startswith('A')]
digitals = [(i, ch) for i, ch in enumerate(channels) if ch.startswith('D')]
electrodes = [(i, ch) for i, ch in enumerate(channels)
if not ch.startswith('D') and not ch.startswith('A')]
groups = (('Analog', analogs, af, np.float32),
('Digital', digitals, df, np.bool_), ('Electrodes', electrodes,
ef, np.float32))
if output is None:
output = '{}.h5'.format(splitext(filenames[0])[0])
ofile = NixIO(output, mode='ow')
blk = Block()
for group_name, channels, f, dtype in groups:
seg = Segment(name=group_name)
for slice_idx, chan_name in channels:
seg.analogsignals.append(
AnalogSignal(f(data[slice_idx::N]),
dtype=dtype,
units=uV,
sampling_rate=rate * Hz,
name=chan_name))
blk.segments.append(seg)
ofile.write_block(blk)
nix_file = ofile.nix_file
nix_groups = nix_file.blocks[0].groups
for data in reader:
for k, (group_name, channels, f, _) in enumerate(groups):
data_arrays = nix_groups[k].data_arrays
for i, (slice_idx, _) in enumerate(channels):
# nix_file._h5file.flush()
data_arrays[i].append(f(data[slice_idx::N]))
nix_file.close()
return output
def setUp(self):
self.io = NixIO(self.filename, "ro")
self.original_methods["_read_cascade"] = self.io._read_cascade
self.original_methods["_update_maps"] = self.io._update_maps
def setUp(self):
self.io = NixIO(self.filename, "ro")
self.original_methods["_read_cascade"] = self.io._read_cascade
self.original_methods["_update_maps"] = self.io._update_maps
epoch_times = tstart + np.cumsum(np.array([0,1,2])) * pq.ms
epoch = Epoch(name="Seg {} :: Epoch".format(idx),
times=epoch_times,
durations=np.array([0,1,2])*pq.ms,
labels=["A+", "B+", "C+"])
seg.epochs.append(epoch)
tstart = 10 *pq.s
st_times = tstart + np.cumsum(np.arange(0,1,0.1)) * pq.s
tstop = max(event_times[-1], epoch_times[-1], st_times[-1]) + 1 * pq.s
st = SpikeTrain(name="Seg {} :: SpikeTrain".format(idx),
times=st_times, t_start=tstart, t_stop=tstop)
wf = np.random.random((len(st_times), nchannels, 30)) * pq.mV
st.waveforms = wf
st.sampling_rate = sampling_rate
seg.spiketrains.append(st)
unit = Unit(name="unit-{}".format(idx))
print(unit)
unit.spiketrains.append(st)
chx.units.append(unit)
# Write the Block to file using the NixIO
# Any existing file will be overwritten
fname = "test_case.nix"
io = NixIO(fname, "ow")
io.write_block(block1)
io.write_block(block2)
io.close()
class NixIOWriteTest(NixIOTest):
def setUp(self):
self.tempdir = mkdtemp(prefix="nixiotest")
self.filename = os.path.join(self.tempdir, "testnixio.nix")
self.writer = NixIO(self.filename, "ow")
self.io = self.writer
self.reader = nix.File.open(self.filename,
nix.FileMode.ReadOnly,
backend="h5py")
def tearDown(self):
self.writer.close()
self.reader.close()
shutil.rmtree(self.tempdir)
def write_and_compare(self, blocks):
self.writer.write_all_blocks(blocks)
self.compare_blocks(blocks, self.reader.blocks)
self.compare_blocks(self.writer.read_all_blocks(), self.reader.blocks)
self.compare_blocks(blocks, self.reader.blocks)
def test_block_write(self):
block = Block(name=self.rword(), description=self.rsentence())
self.write_and_compare([block])
block.annotate(**self.rdict(5))
self.write_and_compare([block])
def test_segment_write(self):
block = Block(name=self.rword())
segment = Segment(name=self.rword(), description=self.rword())
block.segments.append(segment)
self.write_and_compare([block])
segment.annotate(**self.rdict(2))
self.write_and_compare([block])
def test_channel_index_write(self):
block = Block(name=self.rword())
chx = ChannelIndex(name=self.rword(),
description=self.rsentence(),
channel_ids=[10, 20, 30, 50, 80, 130],
index=[1, 2, 3, 5, 8, 13])
block.channel_indexes.append(chx)
self.write_and_compare([block])
chx.annotate(**self.rdict(3))
self.write_and_compare([block])
chx.channel_names = ["one", "two", "three", "five", "eight", "xiii"]
chx.coordinates = self.rquant((6, 3), pq.um)
self.write_and_compare([block])
# add an empty channel index and check again
newchx = ChannelIndex(np.array([]))
block.channel_indexes.append(newchx)
self.write_and_compare([block])
def test_signals_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
asig = AnalogSignal(signal=self.rquant((10, 3), pq.mV),
sampling_rate=pq.Quantity(10, "Hz"))
seg.analogsignals.append(asig)
self.write_and_compare([block])
anotherblock = Block("ir signal block")
seg = Segment("ir signal seg")
anotherblock.segments.append(seg)
irsig = IrregularlySampledSignal(signal=np.random.random((20, 3)),
times=self.rquant(20, pq.ms, True),
units=pq.A)
seg.irregularlysampledsignals.append(irsig)
self.write_and_compare([block, anotherblock])
block.segments[0].analogsignals.append(
AnalogSignal(signal=[10.0, 1.0, 3.0],
units=pq.S,
sampling_period=pq.Quantity(3, "s"),
dtype=np.double,
name="signal42",
description="this is an analogsignal",
t_start=45 * pq.ms), )
self.write_and_compare([block, anotherblock])
block.segments[0].irregularlysampledsignals.append(
IrregularlySampledSignal(times=np.random.random(10),
signal=np.random.random((10, 3)),
units="mV",
time_units="s",
dtype=np.float,
name="some sort of signal",
description="the signal is described"))
self.write_and_compare([block, anotherblock])
def test_signals_compound_units(self):
block = Block()
seg = Segment()
block.segments.append(seg)
units = pq.CompoundUnit("1/30000*V")
srate = pq.Quantity(10, pq.CompoundUnit("1.0/10 * Hz"))
asig = AnalogSignal(signal=self.rquant((10, 3), units),
sampling_rate=srate)
seg.analogsignals.append(asig)
self.write_and_compare([block])
anotherblock = Block("ir signal block")
seg = Segment("ir signal seg")
anotherblock.segments.append(seg)
irsig = IrregularlySampledSignal(signal=np.random.random((20, 3)),
times=self.rquant(
20, pq.CompoundUnit("0.1 * ms"),
True),
units=pq.CompoundUnit("10 * V / s"))
seg.irregularlysampledsignals.append(irsig)
self.write_and_compare([block, anotherblock])
block.segments[0].analogsignals.append(
AnalogSignal(signal=[10.0, 1.0, 3.0],
units=pq.S,
sampling_period=pq.Quantity(3, "s"),
dtype=np.double,
name="signal42",
description="this is an analogsignal",
t_start=45 * pq.CompoundUnit("3.14 * s")), )
self.write_and_compare([block, anotherblock])
times = self.rquant(10, pq.CompoundUnit("3 * year"), True)
block.segments[0].irregularlysampledsignals.append(
IrregularlySampledSignal(times=times,
signal=np.random.random((10, 3)),
units="mV",
dtype=np.float,
name="some sort of signal",
description="the signal is described"))
self.write_and_compare([block, anotherblock])
def test_epoch_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
epoch = Epoch(times=[1, 1, 10, 3] * pq.ms,
durations=[3, 3, 3, 1] * pq.ms,
labels=np.array(["one", "two", "three", "four"]),
name="test epoch",
description="an epoch for testing")
seg.epochs.append(epoch)
self.write_and_compare([block])
def test_event_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
event = Event(times=np.arange(0, 30, 10) * pq.s,
labels=np.array(["0", "1", "2"]),
name="event name",
description="event description")
seg.events.append(event)
self.write_and_compare([block])
def test_spiketrain_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
spiketrain = SpikeTrain(times=[3, 4, 5] * pq.s,
t_stop=10.0,
name="spikes!",
description="sssssspikes")
seg.spiketrains.append(spiketrain)
self.write_and_compare([block])
waveforms = self.rquant((3, 5, 10), pq.mV)
spiketrain = SpikeTrain(times=[1, 1.1, 1.2] * pq.ms,
t_stop=1.5 * pq.s,
name="spikes with wf",
description="spikes for waveform test",
waveforms=waveforms)
seg.spiketrains.append(spiketrain)
self.write_and_compare([block])
spiketrain.left_sweep = np.random.random(10) * pq.ms
self.write_and_compare([block])
spiketrain.left_sweep = pq.Quantity(-10, "ms")
self.write_and_compare([block])
def test_metadata_structure_write(self):
neoblk = self.create_all_annotated()
self.io.write_block(neoblk)
blk = self.io.nix_file.blocks[0]
blkmd = blk.metadata
self.assertEqual(blk.name, blkmd.name)
grp = blk.groups[0] # segment
self.assertIn(grp.name, blkmd.sections)
grpmd = blkmd.sections[grp.name]
for da in grp.data_arrays: # signals
name = ".".join(da.name.split(".")[:-1])
self.assertIn(name, grpmd.sections)
for mtag in grp.multi_tags: # spiketrains, events, and epochs
self.assertIn(mtag.name, grpmd.sections)
srcchx = blk.sources[0] # chx
self.assertIn(srcchx.name, blkmd.sections)
for srcunit in blk.sources: # units
self.assertIn(srcunit.name, blkmd.sections)
self.write_and_compare([neoblk])
def test_anonymous_objects_write(self):
nblocks = 2
nsegs = 2
nanasig = 4
nirrseg = 2
nepochs = 3
nevents = 4
nspiketrains = 3
nchx = 5
nunits = 10
times = self.rquant(1, pq.s)
signal = self.rquant(1, pq.V)
blocks = []
for blkidx in range(nblocks):
blk = Block()
blocks.append(blk)
for segidx in range(nsegs):
seg = Segment()
blk.segments.append(seg)
for anaidx in range(nanasig):
seg.analogsignals.append(
AnalogSignal(signal=signal, sampling_rate=pq.Hz))
for irridx in range(nirrseg):
seg.irregularlysampledsignals.append(
IrregularlySampledSignal(times=times,
signal=signal,
time_units=pq.s))
for epidx in range(nepochs):
seg.epochs.append(Epoch(times=times, durations=times))
for evidx in range(nevents):
seg.events.append(Event(times=times))
for stidx in range(nspiketrains):
seg.spiketrains.append(
SpikeTrain(times=times,
t_stop=times[-1] + pq.s,
units=pq.s))
for chidx in range(nchx):
chx = ChannelIndex(name="chx{}".format(chidx),
index=[1, 2],
channel_ids=[11, 22])
blk.channel_indexes.append(chx)
for unidx in range(nunits):
unit = Unit()
chx.units.append(unit)
self.writer.write_all_blocks(blocks)
self.compare_blocks(blocks, self.reader.blocks)
def test_multiref_write(self):
blk = Block("blk1")
signal = AnalogSignal(name="sig1",
signal=[0, 1, 2],
units="mV",
sampling_period=pq.Quantity(1, "ms"))
othersignal = IrregularlySampledSignal(name="i1",
signal=[0, 0, 0],
units="mV",
times=[1, 2, 3],
time_units="ms")
event = Event(name="Evee", times=[0.3, 0.42], units="year")
epoch = Epoch(name="epoche",
times=[0.1, 0.2] * pq.min,
durations=[0.5, 0.5] * pq.min)
st = SpikeTrain(name="the train of spikes",
times=[0.1, 0.2, 10.3],
t_stop=11,
units="us")
for idx in range(3):
segname = "seg" + str(idx)
seg = Segment(segname)
blk.segments.append(seg)
seg.analogsignals.append(signal)
seg.irregularlysampledsignals.append(othersignal)
seg.events.append(event)
seg.epochs.append(epoch)
seg.spiketrains.append(st)
chidx = ChannelIndex([10, 20, 29])
seg = blk.segments[0]
st = SpikeTrain(name="choochoo",
times=[10, 11, 80],
t_stop=1000,
units="s")
seg.spiketrains.append(st)
blk.channel_indexes.append(chidx)
for idx in range(6):
unit = Unit("unit" + str(idx))
chidx.units.append(unit)
unit.spiketrains.append(st)
self.writer.write_block(blk)
self.compare_blocks([blk], self.reader.blocks)
def test_no_segment_write(self):
# Tests storing AnalogSignal, IrregularlySampledSignal, and SpikeTrain
# objects in the secondary (ChannelIndex) substructure without them
# being attached to a Segment.
blk = Block("segmentless block")
signal = AnalogSignal(name="sig1",
signal=[0, 1, 2],
units="mV",
sampling_period=pq.Quantity(1, "ms"))
othersignal = IrregularlySampledSignal(name="i1",
signal=[0, 0, 0],
units="mV",
times=[1, 2, 3],
time_units="ms")
sta = SpikeTrain(name="the train of spikes",
times=[0.1, 0.2, 10.3],
t_stop=11,
units="us")
stb = SpikeTrain(name="the train of spikes b",
times=[1.1, 2.2, 10.1],
t_stop=100,
units="ms")
chidx = ChannelIndex([8, 13, 21])
blk.channel_indexes.append(chidx)
chidx.analogsignals.append(signal)
chidx.irregularlysampledsignals.append(othersignal)
unit = Unit()
chidx.units.append(unit)
unit.spiketrains.extend([sta, stb])
self.writer.write_block(blk)
self.compare_blocks([blk], self.reader.blocks)
self.writer.close()
reader = NixIO(self.filename, "ro")
blk = reader.read_block(neoname="segmentless block")
chx = blk.channel_indexes[0]
self.assertEqual(len(chx.analogsignals), 1)
self.assertEqual(len(chx.irregularlysampledsignals), 1)
self.assertEqual(len(chx.units[0].spiketrains), 2)
def test_to_value(self):
section = self.io.nix_file.create_section("Metadata value test",
"Test")
writeprop = self.io._write_property
# quantity
qvalue = pq.Quantity(10, "mV")
writeprop(section, "qvalue", qvalue)
self.assertEqual(section["qvalue"], 10)
self.assertEqual(section.props["qvalue"].unit, "mV")
# datetime
dt = self.rdate()
writeprop(section, "dt", dt)
self.assertEqual(datetime.fromtimestamp(section["dt"]), dt)
# string
randstr = self.rsentence()
writeprop(section, "randstr", randstr)
self.assertEqual(section["randstr"], randstr)
# bytes
bytestring = b"bytestring"
writeprop(section, "randbytes", bytestring)
self.assertEqual(section["randbytes"], bytestring.decode())
# iterables
randlist = np.random.random(10).tolist()
writeprop(section, "randlist", randlist)
self.assertEqual(randlist, section["randlist"])
randarray = np.random.random(10)
writeprop(section, "randarray", randarray)
np.testing.assert_almost_equal(randarray, section["randarray"])
# numpy item
npval = np.float64(2398)
writeprop(section, "npval", npval)
self.assertEqual(npval, section["npval"])
# number
val = 42
writeprop(section, "val", val)
self.assertEqual(val, section["val"])
# empty string
writeprop(section, "emptystring", "")
self.assertEqual("", section["emptystring"])
def test_annotations_special_cases(self):
# Special cases for annotations: empty list, list of strings,
# multidimensional lists/arrays
# These are handled differently on read, so we test them on a block
# instead of just checking the property writer method
# empty value
# empty list
wblock = Block("block with empty list", an_empty_list=list())
self.writer.write_block(wblock)
rblock = self.writer.read_block(neoname="block with empty list")
self.assertEqual(rblock.annotations["an_empty_list"], list())
# empty tuple (gets read out as list)
wblock = Block("block with empty tuple", an_empty_tuple=tuple())
self.writer.write_block(wblock)
rblock = self.writer.read_block(neoname="block with empty tuple")
self.assertEqual(rblock.annotations["an_empty_tuple"], list())
# list of strings
losval = ["one", "two", "one million"]
wblock = Block("block with list of strings", los=losval)
self.writer.write_block(wblock)
rblock = self.writer.read_block(neoname="block with list of strings")
self.assertEqual(rblock.annotations["los"], losval)