def setUp(self): self.io = NixIO(self.filename, "ro") self.original_methods["_read_cascade"] = self.io._read_cascade self.original_methods["_update_maps"] = self.io._update_maps
def main(): startmsg = "Starting conversion task at {}".format( datetime.now().isoformat()) printerr("-"*len(startmsg)) printerr(startmsg) printerr("-"*len(startmsg)) if "-v" in sys.argv: verbose = True else: verbose = False for datafilename in [f for f in os.listdir(".") if os.path.isfile(f)]: print("Processing {}".format(datafilename)) try: reader = neo.io.get_io(datafilename) print("File type: {}".format(reader.name)) data = reader.read() except OSError: printerr("NOTICE: file {} does not have an extension " "known to Neo.".format(datafilename)) continue except ImportError as ie: printerr("ERROR importing reader for file {}.".format(datafilename)) printerr(" {}".format(ie)) continue except Exception as exc: printerr("ERROR reading file {}.".format(datafilename)) printerr(" {}".format(exc)) continue blocks = [] try: blkiter = iter(data) except TypeError: blkiter = iter([data]) for item in blkiter: if isinstance(item, neo.core.Block): # filter out non-blocks blocks.append(item) if blocks: if verbose: print_neo(blocks) nixfilename = datafilename.replace(".", "_")+"_nix.h5" nixio = None try: print("Writing data to {}".format(nixfilename)) nixio = NixIO(nixfilename, mode="ow") nixio.write_all_blocks(blocks) print("DONE: file {} converted and saved to {}". format(datafilename, nixfilename)) except RuntimeError as re: printerr("ERROR creating file {}".format(nixfilename)) printerr(" {}".format(re)) except Exception as exc: printerr("ERROR: The following unexpected error occurred during" " conversion of file {}.".format(datafilename)) printerr(" {}".format(exc)) finally: if nixio: del nixio else: print("File does not contain Blocks. Skipping.") print()
class NixIOReadTest(NixIOTest): filename = "testfile_readtest.h5" nixfile = None nix_blocks = None original_methods = dict() @classmethod def setUpClass(cls): cls.nixfile = cls.create_full_nix_file(cls.filename) def setUp(self): self.io = NixIO(self.filename, "ro") self.original_methods["_read_cascade"] = self.io._read_cascade self.original_methods["_update_maps"] = self.io._update_maps @classmethod def tearDownClass(cls): cls.nixfile.close() def tearDown(self): del self.io def test_all_read(self): neo_blocks = self.io.read_all_blocks(cascade=True, lazy=False) nix_blocks = self.io.nix_file.blocks self.compare_blocks(neo_blocks, nix_blocks) def test_lazyload_fullcascade_read(self): neo_blocks = self.io.read_all_blocks(cascade=True, lazy=True) nix_blocks = self.io.nix_file.blocks # data objects should be empty for block in neo_blocks: for seg in block.segments: for asig in seg.analogsignals: self.assertEqual(len(asig), 0) for isig in seg.irregularlysampledsignals: self.assertEqual(len(isig), 0) for epoch in seg.epochs: self.assertEqual(len(epoch), 0) for event in seg.events: self.assertEqual(len(event), 0) for st in seg.spiketrains: self.assertEqual(len(st), 0) self.compare_blocks(neo_blocks, nix_blocks) def test_lazyload_lazycascade_read(self): neo_blocks = self.io.read_all_blocks(cascade="lazy", lazy=True) nix_blocks = self.io.nix_file.blocks self.compare_blocks(neo_blocks, nix_blocks) def test_lazycascade_read(self): def getitem(self, index): return self._data.__getitem__(index) from neonix.io.nixio import LazyList getitem_original = LazyList.__getitem__ LazyList.__getitem__ = getitem neo_blocks = self.io.read_all_blocks(cascade="lazy", lazy=False) for block in neo_blocks: self.assertIsInstance(block.segments, LazyList) self.assertIsInstance(block.channel_indexes, LazyList) for seg in block.segments: self.assertIsInstance(seg, string_types) for chx in block.channel_indexes: self.assertIsInstance(chx, string_types) LazyList.__getitem__ = getitem_original def test_load_lazy_cascade(self): from neonix.io.nixio import LazyList neo_blocks = self.io.read_all_blocks(cascade="lazy", lazy=False) for block in neo_blocks: self.assertIsInstance(block.segments, LazyList) self.assertIsInstance(block.channel_indexes, LazyList) name = block.name block = self.io.load_lazy_cascade("/" + name, lazy=False) self.assertIsInstance(block.segments, list) self.assertIsInstance(block.channel_indexes, list) for seg in block.segments: self.assertIsInstance(seg.analogsignals, list) self.assertIsInstance(seg.irregularlysampledsignals, list) self.assertIsInstance(seg.epochs, list) self.assertIsInstance(seg.events, list) self.assertIsInstance(seg.spiketrains, list) def test_nocascade_read(self): self.io._read_cascade = mock.Mock() neo_blocks = self.io.read_all_blocks(cascade=False) self.io._read_cascade.assert_not_called() for block in neo_blocks: self.assertEqual(len(block.segments), 0) nix_block = self.io.nix_file.blocks[block.name] self.compare_attr(block, nix_block) def test_lazy_load_subschema(self): blk = self.io.nix_file.blocks[0] segpath = "/" + blk.name + "/segments/" + blk.groups[0].name segment = self.io.load_lazy_cascade(segpath, lazy=True) self.assertIsInstance(segment, Segment) self.assertEqual(segment.name, blk.groups[0].name) self.assertIs(segment.block, None) self.assertEqual(len(segment.analogsignals[0]), 0) segment = self.io.load_lazy_cascade(segpath, lazy=False) self.assertEqual(np.shape(segment.analogsignals[0]), (100, 3))
def setUp(self): self.filename = "nixio_testfile_write.h5" self.writer = NixIO(self.filename, "ow") self.io = self.writer self.reader = nixio.File.open(self.filename, nixio.FileMode.ReadOnly)
class NixIOWriteTest(NixIOTest): def setUp(self): self.filename = "nixio_testfile_write.h5" self.writer = NixIO(self.filename, "ow") self.io = self.writer self.reader = nixio.File.open(self.filename, nixio.FileMode.ReadOnly) def tearDown(self): del self.writer self.reader.close() os.remove(self.filename) def write_and_compare(self, blocks): self.writer.write_all_blocks(blocks) self.compare_blocks(self.writer.read_all_blocks(), self.reader.blocks) def test_block_write(self): block = Block(name=self.rword(), description=self.rsentence()) self.write_and_compare([block]) block.annotate(**self.rdict(5)) self.write_and_compare([block]) def test_segment_write(self): block = Block(name=self.rword()) segment = Segment(name=self.rword(), description=self.rword()) block.segments.append(segment) self.write_and_compare([block]) segment.annotate(**self.rdict(2)) self.write_and_compare([block]) def test_channel_index_write(self): block = Block(name=self.rword()) chx = ChannelIndex(name=self.rword(), description=self.rsentence(), index=[1, 2, 3, 5, 8, 13]) block.channel_indexes.append(chx) self.write_and_compare([block]) chx.annotate(**self.rdict(3)) self.write_and_compare([block]) def test_signals_write(self): block = Block() seg = Segment() block.segments.append(seg) asig = AnalogSignal(signal=self.rquant((10, 3), pq.mV), sampling_rate=pq.Quantity(10, "Hz")) seg.analogsignals.append(asig) self.write_and_compare([block]) anotherblock = Block("ir signal block") seg = Segment("ir signal seg") anotherblock.segments.append(seg) irsig = IrregularlySampledSignal( signal=np.random.random((20, 3)), times=self.rquant(20, pq.ms, True), units=pq.A ) seg.irregularlysampledsignals.append(irsig) self.write_and_compare([anotherblock]) block.segments[0].analogsignals.append( AnalogSignal(signal=[10.0, 1.0, 3.0], units=pq.S, sampling_period=pq.Quantity(3, "s"), dtype=np.double, name="signal42", description="this is an analogsignal", t_start=45 * pq.ms), ) self.write_and_compare([block, anotherblock]) block.segments[0].irregularlysampledsignals.append( IrregularlySampledSignal(times=np.random.random(10), signal=np.random.random((10, 3)), units="mV", time_units="s", dtype=np.float, name="some sort of signal", description="the signal is described") ) self.write_and_compare([block, anotherblock]) def test_epoch_write(self): block = Block() seg = Segment() block.segments.append(seg) epoch = Epoch(times=[1, 1, 10, 3]*pq.ms, durations=[3, 3, 3, 1]*pq.ms, labels=np.array(["one", "two", "three", "four"]), name="test epoch", description="an epoch for testing") seg.epochs.append(epoch) self.write_and_compare([block]) def test_event_write(self): block = Block() seg = Segment() block.segments.append(seg) event = Event(times=np.arange(0, 30, 10)*pq.s, labels=np.array(["0", "1", "2"]), name="event name", description="event description") seg.events.append(event) self.write_and_compare([block]) def test_spiketrain_write(self): block = Block() seg = Segment() block.segments.append(seg) spiketrain = SpikeTrain(times=[3, 4, 5]*pq.s, t_stop=10.0, name="spikes!", description="sssssspikes") seg.spiketrains.append(spiketrain) self.write_and_compare([block]) waveforms = self.rquant((20, 5, 10), pq.mV) spiketrain = SpikeTrain(times=[1, 1.1, 1.2]*pq.ms, t_stop=1.5*pq.s, name="spikes with wf", description="spikes for waveform test", waveforms=waveforms) seg.spiketrains.append(spiketrain) self.write_and_compare([block]) spiketrain.left_sweep = np.random.random(10)*pq.ms self.write_and_compare([block]) def test_metadata_structure_write(self): neoblk = self.create_all_annotated() self.io.write_block(neoblk) blk = self.io.nix_file.blocks[0] blkmd = blk.metadata self.assertEqual(blk.name, blkmd.name) grp = blk.groups[0] # segment self.assertIn(grp.name, blkmd.sections) grpmd = blkmd.sections[grp.name] for da in grp.data_arrays: # signals name = ".".join(da.name.split(".")[:-1]) self.assertIn(name, grpmd.sections) for mtag in grp.multi_tags: # spiketrains, events, and epochs self.assertIn(mtag.name, grpmd.sections) srcchx = blk.sources[0] # chx self.assertIn(srcchx.name, blkmd.sections) for srcunit in blk.sources: # units self.assertIn(srcunit.name, blkmd.sections) def test_anonymous_objects_write(self): nblocks = 2 nsegs = 2 nanasig = 4 nirrseg = 2 nepochs = 3 nevents = 4 nspiketrains = 3 nchx = 5 nunits = 10 times = self.rquant(1, pq.s) signal = self.rquant(1, pq.V) blocks = [] for blkidx in range(nblocks): blk = Block() blocks.append(blk) for segidx in range(nsegs): seg = Segment() blk.segments.append(seg) for anaidx in range(nanasig): seg.analogsignals.append(AnalogSignal(signal=signal, sampling_rate=pq.Hz)) for irridx in range(nirrseg): seg.irregularlysampledsignals.append( IrregularlySampledSignal(times=times, signal=signal, time_units=pq.s) ) for epidx in range(nepochs): seg.epochs.append(Epoch(times=times, durations=times)) for evidx in range(nevents): seg.events.append(Event(times=times)) for stidx in range(nspiketrains): seg.spiketrains.append(SpikeTrain(times=times, t_stop=pq.s, units=pq.s)) for chidx in range(nchx): chx = ChannelIndex(name="chx{}".format(chidx), index=[1, 2]) blk.channel_indexes.append(chx) for unidx in range(nunits): unit = Unit() chx.units.append(unit) self.writer.write_all_blocks(blocks) self.compare_blocks(blocks, self.reader.blocks) def test_to_value(self): section = self.io.nix_file.create_section("Metadata value test", "Test") tovalue = self.io._to_value # quantity # qvalue = pq.Quantity(10, "mV") # section["qvalue"] = tovalue(qvalue) # self.assertEqual(section["qvalue"], 10) # datetime dt = self.rdate() section["dt"] = tovalue(dt) self.assertEqual(datetime.fromtimestamp(section["dt"]), dt) # string randstr = self.rsentence() section["randstr"] = tovalue(randstr) self.assertEqual(section["randstr"], randstr) # bytes bytestring = b"bytestring" section["randbytes"] = tovalue(bytestring) self.assertEqual(section["randbytes"], bytestring.decode()) # iterables # mdlist = [[1, 2, 3], [4, 5, 6]] # self.assertIs(tovalue(mdlist), None) # # mdarray = np.random.random((10, 3)) # self.assertIs(tovalue(mdarray), None) randlist = np.random.random(10).tolist() section["randlist"] = tovalue(randlist) self.assertEqual(randlist, section["randlist"]) randarray = np.random.random(10) section["randarray"] = tovalue(randarray) np.testing.assert_almost_equal(randarray, section["randarray"]) empty = [] self.assertIs(tovalue(empty), None) # numpy item npval = np.float64(2398) section["npval"] = tovalue(npval) self.assertEqual(npval, section["npval"]) # number val = 42 section["val"] = tovalue(val) self.assertEqual(val, section["val"])
def setUp(self): self.io = NixIO(self.filename, "rw") self.neo_blocks = self.io.read_all_blocks() self.original_methods["_write_attr_annotations"] =\ self.io._write_attr_annotations
class NixIOPartialWriteTest(NixIOTest): filename = "testfile_partialwrite.h5" nixfile = None neo_blocks = None original_methods = dict() @classmethod def setUpClass(cls): cls.nixfile = cls.create_full_nix_file(cls.filename) def setUp(self): self.io = NixIO(self.filename, "rw") self.neo_blocks = self.io.read_all_blocks() self.original_methods["_write_attr_annotations"] =\ self.io._write_attr_annotations @classmethod def tearDownClass(cls): cls.nixfile.close() def tearDown(self): self.restore_methods() del self.io def restore_methods(self): for name, method in self.original_methods.items(): setattr(self.io, name, self.original_methods[name]) def _mock_write_attr(self, objclass): typestr = str(objclass.__name__).lower() self.io._write_attr_annotations = mock.Mock( wraps=self.io._write_attr_annotations, side_effect=self.check_obj_type("neo.{}".format(typestr)) ) neo_blocks = self.neo_blocks self.modify_objects(neo_blocks, excludes=[objclass]) self.io.write_all_blocks(neo_blocks) self.restore_methods() def check_obj_type(self, typestring): neq = self.assertNotEqual def side_effect_func(*args, **kwargs): obj = kwargs.get("nixobj", args[0]) if isinstance(obj, list): for sig in obj: neq(sig.type, typestring) else: neq(obj.type, typestring) return side_effect_func @classmethod def modify_objects(cls, objs, excludes=()): excludes = tuple(excludes) for obj in objs: if not (excludes and isinstance(obj, excludes)): obj.description = cls.rsentence() for container in getattr(obj, "_child_containers", []): children = getattr(obj, container) cls.modify_objects(children, excludes) def test_partial(self): for objclass in NixIO.supported_objects: self._mock_write_attr(objclass) self.compare_blocks(self.neo_blocks, self.io.nix_file.blocks) def test_no_modifications(self): self.io._write_attr_annotations = mock.Mock() self.io.write_all_blocks(self.neo_blocks) self.io._write_attr_annotations.assert_not_called() self.compare_blocks(self.neo_blocks, self.io.nix_file.blocks) # clearing hashes and checking again for k in self.io._object_hashes.keys(): self.io._object_hashes[k] = None self.io.write_all_blocks(self.neo_blocks) self.io._write_attr_annotations.assert_not_called() # changing hashes to force rewrite for k in self.io._object_hashes.keys(): self.io._object_hashes[k] = "_" self.io.write_all_blocks(self.neo_blocks) callcount = self.io._write_attr_annotations.call_count self.assertEqual(callcount, len(self.io._object_hashes)) self.compare_blocks(self.neo_blocks, self.io.nix_file.blocks)