def setUp(self):
self.io = NixIO(self.filename, "ro")
self.original_methods["_read_cascade"] = self.io._read_cascade
self.original_methods["_update_maps"] = self.io._update_maps
def main():
startmsg = "Starting conversion task at {}".format(
datetime.now().isoformat())
printerr("-"*len(startmsg))
printerr(startmsg)
printerr("-"*len(startmsg))
if "-v" in sys.argv:
verbose = True
else:
verbose = False
for datafilename in [f for f in os.listdir(".") if os.path.isfile(f)]:
print("Processing {}".format(datafilename))
try:
reader = neo.io.get_io(datafilename)
print("File type: {}".format(reader.name))
data = reader.read()
except OSError:
printerr("NOTICE: file {} does not have an extension "
"known to Neo.".format(datafilename))
continue
except ImportError as ie:
printerr("ERROR importing reader for file {}.".format(datafilename))
printerr(" {}".format(ie))
continue
except Exception as exc:
printerr("ERROR reading file {}.".format(datafilename))
printerr(" {}".format(exc))
continue
blocks = []
try:
blkiter = iter(data)
except TypeError:
blkiter = iter([data])
for item in blkiter:
if isinstance(item, neo.core.Block):
# filter out non-blocks
blocks.append(item)
if blocks:
if verbose:
print_neo(blocks)
nixfilename = datafilename.replace(".", "_")+"_nix.h5"
nixio = None
try:
print("Writing data to {}".format(nixfilename))
nixio = NixIO(nixfilename, mode="ow")
nixio.write_all_blocks(blocks)
print("DONE: file {} converted and saved to {}".
format(datafilename, nixfilename))
except RuntimeError as re:
printerr("ERROR creating file {}".format(nixfilename))
printerr(" {}".format(re))
except Exception as exc:
printerr("ERROR: The following unexpected error occurred during"
" conversion of file {}.".format(datafilename))
printerr(" {}".format(exc))
finally:
if nixio:
del nixio
else:
print("File does not contain Blocks. Skipping.")
print()
class NixIOReadTest(NixIOTest):
filename = "testfile_readtest.h5"
nixfile = None
nix_blocks = None
original_methods = dict()
@classmethod
def setUpClass(cls):
cls.nixfile = cls.create_full_nix_file(cls.filename)
def setUp(self):
self.io = NixIO(self.filename, "ro")
self.original_methods["_read_cascade"] = self.io._read_cascade
self.original_methods["_update_maps"] = self.io._update_maps
@classmethod
def tearDownClass(cls):
cls.nixfile.close()
def tearDown(self):
del self.io
def test_all_read(self):
neo_blocks = self.io.read_all_blocks(cascade=True, lazy=False)
nix_blocks = self.io.nix_file.blocks
self.compare_blocks(neo_blocks, nix_blocks)
def test_lazyload_fullcascade_read(self):
neo_blocks = self.io.read_all_blocks(cascade=True, lazy=True)
nix_blocks = self.io.nix_file.blocks
# data objects should be empty
for block in neo_blocks:
for seg in block.segments:
for asig in seg.analogsignals:
self.assertEqual(len(asig), 0)
for isig in seg.irregularlysampledsignals:
self.assertEqual(len(isig), 0)
for epoch in seg.epochs:
self.assertEqual(len(epoch), 0)
for event in seg.events:
self.assertEqual(len(event), 0)
for st in seg.spiketrains:
self.assertEqual(len(st), 0)
self.compare_blocks(neo_blocks, nix_blocks)
def test_lazyload_lazycascade_read(self):
neo_blocks = self.io.read_all_blocks(cascade="lazy", lazy=True)
nix_blocks = self.io.nix_file.blocks
self.compare_blocks(neo_blocks, nix_blocks)
def test_lazycascade_read(self):
def getitem(self, index):
return self._data.__getitem__(index)
from neonix.io.nixio import LazyList
getitem_original = LazyList.__getitem__
LazyList.__getitem__ = getitem
neo_blocks = self.io.read_all_blocks(cascade="lazy", lazy=False)
for block in neo_blocks:
self.assertIsInstance(block.segments, LazyList)
self.assertIsInstance(block.channel_indexes, LazyList)
for seg in block.segments:
self.assertIsInstance(seg, string_types)
for chx in block.channel_indexes:
self.assertIsInstance(chx, string_types)
LazyList.__getitem__ = getitem_original
def test_load_lazy_cascade(self):
from neonix.io.nixio import LazyList
neo_blocks = self.io.read_all_blocks(cascade="lazy", lazy=False)
for block in neo_blocks:
self.assertIsInstance(block.segments, LazyList)
self.assertIsInstance(block.channel_indexes, LazyList)
name = block.name
block = self.io.load_lazy_cascade("/" + name, lazy=False)
self.assertIsInstance(block.segments, list)
self.assertIsInstance(block.channel_indexes, list)
for seg in block.segments:
self.assertIsInstance(seg.analogsignals, list)
self.assertIsInstance(seg.irregularlysampledsignals, list)
self.assertIsInstance(seg.epochs, list)
self.assertIsInstance(seg.events, list)
self.assertIsInstance(seg.spiketrains, list)
def test_nocascade_read(self):
self.io._read_cascade = mock.Mock()
neo_blocks = self.io.read_all_blocks(cascade=False)
self.io._read_cascade.assert_not_called()
for block in neo_blocks:
self.assertEqual(len(block.segments), 0)
nix_block = self.io.nix_file.blocks[block.name]
self.compare_attr(block, nix_block)
def test_lazy_load_subschema(self):
blk = self.io.nix_file.blocks[0]
segpath = "/" + blk.name + "/segments/" + blk.groups[0].name
segment = self.io.load_lazy_cascade(segpath, lazy=True)
self.assertIsInstance(segment, Segment)
self.assertEqual(segment.name, blk.groups[0].name)
self.assertIs(segment.block, None)
self.assertEqual(len(segment.analogsignals[0]), 0)
segment = self.io.load_lazy_cascade(segpath, lazy=False)
self.assertEqual(np.shape(segment.analogsignals[0]), (100, 3))
def setUp(self):
self.filename = "nixio_testfile_write.h5"
self.writer = NixIO(self.filename, "ow")
self.io = self.writer
self.reader = nixio.File.open(self.filename,
nixio.FileMode.ReadOnly)
class NixIOWriteTest(NixIOTest):
def setUp(self):
self.filename = "nixio_testfile_write.h5"
self.writer = NixIO(self.filename, "ow")
self.io = self.writer
self.reader = nixio.File.open(self.filename,
nixio.FileMode.ReadOnly)
def tearDown(self):
del self.writer
self.reader.close()
os.remove(self.filename)
def write_and_compare(self, blocks):
self.writer.write_all_blocks(blocks)
self.compare_blocks(self.writer.read_all_blocks(), self.reader.blocks)
def test_block_write(self):
block = Block(name=self.rword(),
description=self.rsentence())
self.write_and_compare([block])
block.annotate(**self.rdict(5))
self.write_and_compare([block])
def test_segment_write(self):
block = Block(name=self.rword())
segment = Segment(name=self.rword(), description=self.rword())
block.segments.append(segment)
self.write_and_compare([block])
segment.annotate(**self.rdict(2))
self.write_and_compare([block])
def test_channel_index_write(self):
block = Block(name=self.rword())
chx = ChannelIndex(name=self.rword(),
description=self.rsentence(),
index=[1, 2, 3, 5, 8, 13])
block.channel_indexes.append(chx)
self.write_and_compare([block])
chx.annotate(**self.rdict(3))
self.write_and_compare([block])
def test_signals_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
asig = AnalogSignal(signal=self.rquant((10, 3), pq.mV),
sampling_rate=pq.Quantity(10, "Hz"))
seg.analogsignals.append(asig)
self.write_and_compare([block])
anotherblock = Block("ir signal block")
seg = Segment("ir signal seg")
anotherblock.segments.append(seg)
irsig = IrregularlySampledSignal(
signal=np.random.random((20, 3)),
times=self.rquant(20, pq.ms, True),
units=pq.A
)
seg.irregularlysampledsignals.append(irsig)
self.write_and_compare([anotherblock])
block.segments[0].analogsignals.append(
AnalogSignal(signal=[10.0, 1.0, 3.0], units=pq.S,
sampling_period=pq.Quantity(3, "s"),
dtype=np.double, name="signal42",
description="this is an analogsignal",
t_start=45 * pq.ms),
)
self.write_and_compare([block, anotherblock])
block.segments[0].irregularlysampledsignals.append(
IrregularlySampledSignal(times=np.random.random(10),
signal=np.random.random((10, 3)),
units="mV", time_units="s",
dtype=np.float,
name="some sort of signal",
description="the signal is described")
)
self.write_and_compare([block, anotherblock])
def test_epoch_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
epoch = Epoch(times=[1, 1, 10, 3]*pq.ms, durations=[3, 3, 3, 1]*pq.ms,
labels=np.array(["one", "two", "three", "four"]),
name="test epoch", description="an epoch for testing")
seg.epochs.append(epoch)
self.write_and_compare([block])
def test_event_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
event = Event(times=np.arange(0, 30, 10)*pq.s,
labels=np.array(["0", "1", "2"]),
name="event name",
description="event description")
seg.events.append(event)
self.write_and_compare([block])
def test_spiketrain_write(self):
block = Block()
seg = Segment()
block.segments.append(seg)
spiketrain = SpikeTrain(times=[3, 4, 5]*pq.s, t_stop=10.0,
name="spikes!", description="sssssspikes")
seg.spiketrains.append(spiketrain)
self.write_and_compare([block])
waveforms = self.rquant((20, 5, 10), pq.mV)
spiketrain = SpikeTrain(times=[1, 1.1, 1.2]*pq.ms, t_stop=1.5*pq.s,
name="spikes with wf",
description="spikes for waveform test",
waveforms=waveforms)
seg.spiketrains.append(spiketrain)
self.write_and_compare([block])
spiketrain.left_sweep = np.random.random(10)*pq.ms
self.write_and_compare([block])
def test_metadata_structure_write(self):
neoblk = self.create_all_annotated()
self.io.write_block(neoblk)
blk = self.io.nix_file.blocks[0]
blkmd = blk.metadata
self.assertEqual(blk.name, blkmd.name)
grp = blk.groups[0] # segment
self.assertIn(grp.name, blkmd.sections)
grpmd = blkmd.sections[grp.name]
for da in grp.data_arrays: # signals
name = ".".join(da.name.split(".")[:-1])
self.assertIn(name, grpmd.sections)
for mtag in grp.multi_tags: # spiketrains, events, and epochs
self.assertIn(mtag.name, grpmd.sections)
srcchx = blk.sources[0] # chx
self.assertIn(srcchx.name, blkmd.sections)
for srcunit in blk.sources: # units
self.assertIn(srcunit.name, blkmd.sections)
def test_anonymous_objects_write(self):
nblocks = 2
nsegs = 2
nanasig = 4
nirrseg = 2
nepochs = 3
nevents = 4
nspiketrains = 3
nchx = 5
nunits = 10
times = self.rquant(1, pq.s)
signal = self.rquant(1, pq.V)
blocks = []
for blkidx in range(nblocks):
blk = Block()
blocks.append(blk)
for segidx in range(nsegs):
seg = Segment()
blk.segments.append(seg)
for anaidx in range(nanasig):
seg.analogsignals.append(AnalogSignal(signal=signal,
sampling_rate=pq.Hz))
for irridx in range(nirrseg):
seg.irregularlysampledsignals.append(
IrregularlySampledSignal(times=times,
signal=signal,
time_units=pq.s)
)
for epidx in range(nepochs):
seg.epochs.append(Epoch(times=times, durations=times))
for evidx in range(nevents):
seg.events.append(Event(times=times))
for stidx in range(nspiketrains):
seg.spiketrains.append(SpikeTrain(times=times, t_stop=pq.s,
units=pq.s))
for chidx in range(nchx):
chx = ChannelIndex(name="chx{}".format(chidx),
index=[1, 2])
blk.channel_indexes.append(chx)
for unidx in range(nunits):
unit = Unit()
chx.units.append(unit)
self.writer.write_all_blocks(blocks)
self.compare_blocks(blocks, self.reader.blocks)
def test_to_value(self):
section = self.io.nix_file.create_section("Metadata value test", "Test")
tovalue = self.io._to_value
# quantity
# qvalue = pq.Quantity(10, "mV")
# section["qvalue"] = tovalue(qvalue)
# self.assertEqual(section["qvalue"], 10)
# datetime
dt = self.rdate()
section["dt"] = tovalue(dt)
self.assertEqual(datetime.fromtimestamp(section["dt"]), dt)
# string
randstr = self.rsentence()
section["randstr"] = tovalue(randstr)
self.assertEqual(section["randstr"], randstr)
# bytes
bytestring = b"bytestring"
section["randbytes"] = tovalue(bytestring)
self.assertEqual(section["randbytes"], bytestring.decode())
# iterables
# mdlist = [[1, 2, 3], [4, 5, 6]]
# self.assertIs(tovalue(mdlist), None)
#
# mdarray = np.random.random((10, 3))
# self.assertIs(tovalue(mdarray), None)
randlist = np.random.random(10).tolist()
section["randlist"] = tovalue(randlist)
self.assertEqual(randlist, section["randlist"])
randarray = np.random.random(10)
section["randarray"] = tovalue(randarray)
np.testing.assert_almost_equal(randarray, section["randarray"])
empty = []
self.assertIs(tovalue(empty), None)
# numpy item
npval = np.float64(2398)
section["npval"] = tovalue(npval)
self.assertEqual(npval, section["npval"])
# number
val = 42
section["val"] = tovalue(val)
self.assertEqual(val, section["val"])
def setUp(self):
self.io = NixIO(self.filename, "rw")
self.neo_blocks = self.io.read_all_blocks()
self.original_methods["_write_attr_annotations"] =\
self.io._write_attr_annotations
class NixIOPartialWriteTest(NixIOTest):
filename = "testfile_partialwrite.h5"
nixfile = None
neo_blocks = None
original_methods = dict()
@classmethod
def setUpClass(cls):
cls.nixfile = cls.create_full_nix_file(cls.filename)
def setUp(self):
self.io = NixIO(self.filename, "rw")
self.neo_blocks = self.io.read_all_blocks()
self.original_methods["_write_attr_annotations"] =\
self.io._write_attr_annotations
@classmethod
def tearDownClass(cls):
cls.nixfile.close()
def tearDown(self):
self.restore_methods()
del self.io
def restore_methods(self):
for name, method in self.original_methods.items():
setattr(self.io, name, self.original_methods[name])
def _mock_write_attr(self, objclass):
typestr = str(objclass.__name__).lower()
self.io._write_attr_annotations = mock.Mock(
wraps=self.io._write_attr_annotations,
side_effect=self.check_obj_type("neo.{}".format(typestr))
)
neo_blocks = self.neo_blocks
self.modify_objects(neo_blocks, excludes=[objclass])
self.io.write_all_blocks(neo_blocks)
self.restore_methods()
def check_obj_type(self, typestring):
neq = self.assertNotEqual
def side_effect_func(*args, **kwargs):
obj = kwargs.get("nixobj", args[0])
if isinstance(obj, list):
for sig in obj:
neq(sig.type, typestring)
else:
neq(obj.type, typestring)
return side_effect_func
@classmethod
def modify_objects(cls, objs, excludes=()):
excludes = tuple(excludes)
for obj in objs:
if not (excludes and isinstance(obj, excludes)):
obj.description = cls.rsentence()
for container in getattr(obj, "_child_containers", []):
children = getattr(obj, container)
cls.modify_objects(children, excludes)
def test_partial(self):
for objclass in NixIO.supported_objects:
self._mock_write_attr(objclass)
self.compare_blocks(self.neo_blocks, self.io.nix_file.blocks)
def test_no_modifications(self):
self.io._write_attr_annotations = mock.Mock()
self.io.write_all_blocks(self.neo_blocks)
self.io._write_attr_annotations.assert_not_called()
self.compare_blocks(self.neo_blocks, self.io.nix_file.blocks)
# clearing hashes and checking again
for k in self.io._object_hashes.keys():
self.io._object_hashes[k] = None
self.io.write_all_blocks(self.neo_blocks)
self.io._write_attr_annotations.assert_not_called()
# changing hashes to force rewrite
for k in self.io._object_hashes.keys():
self.io._object_hashes[k] = "_"
self.io.write_all_blocks(self.neo_blocks)
callcount = self.io._write_attr_annotations.call_count
self.assertEqual(callcount, len(self.io._object_hashes))
self.compare_blocks(self.neo_blocks, self.io.nix_file.blocks)
