def setUpBuilder(self):
ids_builder = DatasetBuilder('id', [0, 1],
attributes={'neurodata_type': 'ElementIdentifiers',
'namespace': 'core'})
st_builder = DatasetBuilder('spike_times', [0, 1, 2, 3, 4, 5],
attributes={'neurodata_type': 'VectorData',
'namespace': 'core',
'description': 'the spike times for each unit'})
sti_builder = DatasetBuilder('spike_times_index',
[3, 6],
attributes={'neurodata_type': 'VectorIndex',
'namespace': 'core',
'target': ReferenceBuilder(st_builder)})
obs_builder = DatasetBuilder('obs_intervals', [[0, 1], [2, 3], [2, 5], [6, 7]],
attributes={'neurodata_type': 'VectorData',
'namespace': 'core',
'description': 'the observation intervals for each unit'})
obsi_builder = DatasetBuilder('obs_intervals_index',
[2, 4],
attributes={'neurodata_type': 'VectorIndex',
'namespace': 'core',
'target': ReferenceBuilder(obs_builder)})
return GroupBuilder('UnitsTest',
attributes={'neurodata_type': 'Units',
'namespace': 'core',
'description': 'a simple table for testing Units',
'colnames': (b'spike_times', b'obs_intervals',)},
datasets={'id': ids_builder,
'spike_times': st_builder,
'spike_times_index': sti_builder,
'obs_intervals': obs_builder,
'obs_intervals_index': obsi_builder})
def __read_dataset(self, h5obj, name=None):
h5obj_maxshape = h5obj.shape
kwargs = {
"attributes": self.__read_attrs(h5obj),
"dtype": h5obj.dtype,
"maxshape": h5obj_maxshape
}
if name is None:
name = str(os.path.basename(h5obj.name))
kwargs['source'] = h5obj.file.filename
ndims = len(h5obj.shape)
if ndims == 0: # read scalar
scalar = h5obj[()]
if isinstance(scalar, bytes):
scalar = scalar.decode('UTF-8')
# TO DO Reference #
deref_obj = None
if isinstance(scalar, str) and scalar != '':
try:
deref_obj = h5obj.file[scalar]
except:
pass
if deref_obj is not None:
# TODO (AJTRITT): This should call __read_ref to support Group references
target = deref_obj
target = self.__set_rgroup(target)
target_builder = self.__read_dataset(target)
self.__set_built(target.file.filename, target.id, target_builder)
# TO DO Region Reference #
# TO DO #
kwargs['data'] = ReferenceBuilder(target_builder)
else:
kwargs["data"] = scalar
elif ndims == 1:
d = None
if h5obj.dtype.kind == 'O' and len(h5obj) > 0:
elem1 = h5obj[0]
if isinstance(elem1, (str, bytes)):
d = h5obj
# TO DO #
elif h5obj.dtype == 'uint64' and len(h5obj) > 0:
d = BuilderH5ReferenceDataset(HDMFArray(h5obj), self) # read list of references
# TO DO Region Reference #
elif h5obj.dtype.kind == 'V': # table
cpd_dt = h5obj.dtype
# TO DO check_dtype #
ref_cols = [cpd_dt[i] == 'uint64' for i in range(len(cpd_dt))]
d = BuilderH5TableDataset(HDMFArray(h5obj), self, ref_cols)
else:
d = h5obj
kwargs["data"] = d
else:
kwargs["data"] = h5obj
ret = DatasetBuilder(name, **kwargs)
ret.written = True
return ret
def setUpBuilder(self):
ps_builder = TestPlaneSegmentation.get_plane_segmentation_builder(self)
return GroupBuilder(
'test_roi_response_series',
attributes={
'namespace': base.CORE_NAMESPACE,
'comments': 'no comments',
'description': 'no description',
'neurodata_type': 'RoiResponseSeries',
'help': ('ROI responses over an imaging plane. Each element on the second dimension of data[] '
'should correspond to the signal from one ROI')},
datasets={
'data': DatasetBuilder(
'data', [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
attributes={
'unit': 'lumens',
'conversion': 1.0,
'resolution': 0.0}
),
'timestamps': DatasetBuilder('timestamps', [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
attributes={'unit': 'Seconds', 'interval': 1}),
'rois': DatasetBuilder('rois', data=[0],
attributes={'help': 'a subset (i.e. slice or region) of a DynamicTable',
'description': 'the first of two ROIs',
'table': ReferenceBuilder(ps_builder),
'namespace': 'core',
'neurodata_type': 'DynamicTableRegion'}),
})
def test_build(self):
''' Test default mapping functionality when no attributes are nested '''
foo = Foo('my_foo1', [1, 2, 3], 'string', 10)
baz = Baz('MyBaz', [foo], 'abcdefghijklmnopqrstuvwxyz')
foo_builder = self.manager.build(foo)
baz_builder = self.manager.build(baz, root=True)
expected = DatasetBuilder('MyBaz', [ReferenceBuilder(foo_builder)],
attributes={
'baz_attr': 'abcdefghijklmnopqrstuvwxyz',
'data_type': 'Baz',
'namespace': CORE_NAMESPACE,
'object_id': baz.object_id
})
self.assertBuilderEqual(baz_builder, expected)
def setUpBuilder(self):
table_builder = self.get_table_builder(self)
data = list(zip(range(10), range(10, 20)))
timestamps = list(map(lambda x: x / 10, range(10)))
return GroupBuilder(
'test_eS',
attributes={
'namespace':
base.CORE_NAMESPACE,
'comments':
'no comments',
'description':
'no description',
'neurodata_type':
'ElectricalSeries',
'help':
'Stores acquired voltage data from extracellular recordings'
},
datasets={
'data':
DatasetBuilder('data',
data,
attributes={
'unit': 'volt',
'conversion': 1.0,
'resolution': 0.0
}),
'timestamps':
DatasetBuilder('timestamps',
timestamps,
attributes={
'unit': 'Seconds',
'interval': 1
}),
'electrodes':
DatasetBuilder(
'electrodes',
data=[0, 2],
attributes={
'neurodata_type': 'DynamicTableRegion',
'namespace': 'core',
'table': ReferenceBuilder(table_builder),
'description': 'the first and third electrodes',
'help':
'a subset (i.e. slice or region) of a DynamicTable'
})
}) # noqa: E501
def setUpBuilder(self):
table_builder = self.get_table_builder(self)
data = list(zip(range(10), range(10, 20)))
timestamps = list(map(lambda x: x / 10, range(10)))
data_builder = DatasetBuilder('data',
data,
attributes={
'unit': 'volts',
'conversion': 1.0,
'resolution': 0.0
})
timestamps_builder = DatasetBuilder('timestamps',
timestamps,
attributes={
'unit': 'seconds',
'interval': 1
})
elec_builder = DatasetBuilder('electrodes',
data=[0, 2],
attributes={
'neurodata_type':
'DynamicTableRegion',
'namespace':
'core',
'table':
ReferenceBuilder(table_builder),
'description':
'the first and third electrodes'
})
return GroupBuilder('test_eS',
attributes={
'namespace': base.CORE_NAMESPACE,
'comments': 'no comments',
'description': 'no description',
'neurodata_type': 'ElectricalSeries'
},
datasets={
'data': data_builder,
'timestamps': timestamps_builder,
'electrodes': elec_builder
})
def test_build_attr_ref(self):
''' Test default mapping functionality when one container contains an attribute reference to another container.
'''
foo_inst = Foo('my_foo')
bar_inst1 = Bar('my_bar1', list(range(10)), 'value1', 10, foo=foo_inst)
bar_inst2 = Bar('my_bar2', list(range(10)), 'value1', 10)
foo_builder = self.manager.build(foo_inst, root=True)
bar1_builder = self.manager.build(bar_inst1, root=True) # adds refs
bar2_builder = self.manager.build(bar_inst2, root=True)
foo_expected = GroupBuilder('my_foo',
attributes={
'data_type': 'Foo',
'namespace': CORE_NAMESPACE,
'object_id': foo_inst.object_id
})
bar1_expected = GroupBuilder(
'n/a', # name doesn't matter
datasets={'data': DatasetBuilder('data', list(range(10)))},
attributes={
'attr1': 'value1',
'attr2': 10,
'foo': ReferenceBuilder(foo_expected),
'data_type': 'Bar',
'namespace': CORE_NAMESPACE,
'object_id': bar_inst1.object_id
})
bar2_expected = GroupBuilder(
'n/a', # name doesn't matter
datasets={'data': DatasetBuilder('data', list(range(10)))},
attributes={
'attr1': 'value1',
'attr2': 10,
'data_type': 'Bar',
'namespace': CORE_NAMESPACE,
'object_id': bar_inst2.object_id
})
self.assertDictEqual(foo_builder, foo_expected)
self.assertDictEqual(bar1_builder, bar1_expected)
self.assertDictEqual(bar2_builder, bar2_expected)
def setUpBuilder(self):
ps_builder = TestPlaneSegmentation.get_plane_segmentation_builder(self)
return GroupBuilder(
'test_roi_response_series',
attributes={
'namespace': base.CORE_NAMESPACE,
'comments': 'no comments',
'description': 'no description',
'neurodata_type': 'RoiResponseSeries'
},
datasets={
'data':
DatasetBuilder('data', [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],
attributes={
'unit': 'lumens',
'conversion': 1.0,
'resolution': 0.0
}),
'timestamps':
DatasetBuilder(
'timestamps',
[0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
attributes={
'unit': 'seconds',
'interval': 1
}),
'rois':
DatasetBuilder('rois',
data=[0],
attributes={
'description': 'the first of two ROIs',
'table': ReferenceBuilder(ps_builder),
'namespace': 'core',
'neurodata_type': 'DynamicTableRegion'
}),
})
def setUpBuilder(self):
device = GroupBuilder('device_name',
attributes={
'neurodata_type': 'Device',
'help': 'A recording device e.g. amplifier',
'namespace': 'core',
})
datasets = [
DatasetBuilder('slice', data=u'tissue slice'),
DatasetBuilder('resistance', data=u'something measured in ohms'),
DatasetBuilder('seal', data=u'sealing method'),
DatasetBuilder('description', data=u'a fake electrode object'),
DatasetBuilder('location', data=u'Springfield Elementary School'),
DatasetBuilder('filtering',
data=u'a meaningless free-form text field'),
DatasetBuilder('initial_access_resistance',
data=u'I guess this changes'),
]
elec = GroupBuilder('elec0',
attributes={
'help':
'Metadata about an intracellular electrode',
'namespace': 'core',
'neurodata_type': 'IntracellularElectrode',
},
datasets={d.name: d
for d in datasets},
links={'device': LinkBuilder(device, 'device')})
datasets = [
DatasetBuilder(
'gain',
data=0.126,
attributes={},
),
DatasetBuilder('data',
data=[1, 2, 3, 4, 5],
attributes={
'conversion': 1.0,
'resolution': 0.0,
'unit': u'A',
}),
DatasetBuilder('starting_time',
data=123.6,
attributes={
'rate': 10000.0,
'unit': 'Seconds',
}),
]
attributes = {
'neurodata_type': 'PatchClampSeries',
'namespace': 'core',
'comments': u'no comments',
'help': 'Superclass definition for patch-clamp data',
'description': u'no description',
'stimulus_description': u'gotcha ya!',
}
attributes['sweep_number'] = 4711
pcs1 = GroupBuilder(
'pcs1',
attributes=attributes,
links={'electrode': LinkBuilder(elec, 'electrode')},
datasets={d.name: d
for d in datasets},
)
attributes['sweep_number'] = 4712
pcs2a = GroupBuilder(
'pcs2a',
attributes=attributes,
links={'electrode': LinkBuilder(elec, 'electrode')},
datasets={d.name: d
for d in datasets},
)
pcs2b = GroupBuilder(
'pcs2b',
attributes=attributes,
links={'electrode': LinkBuilder(elec, 'electrode')},
datasets={d.name: d
for d in datasets},
)
column_id = DatasetBuilder(
'id', [0, 1, 2],
attributes={
'neurodata_type': 'ElementIdentifiers',
'namespace': 'core',
'help': 'unique identifiers for a list of elements',
})
column_series = DatasetBuilder(
'series',
attributes={
'neurodata_type': 'VectorData',
'namespace': 'core',
'help': 'Values for a list of elements',
'description': u'PatchClampSeries with the same sweep number',
},
data=[LinkBuilder(pcs) for pcs in (pcs1, pcs2a, pcs2b)])
column_index = DatasetBuilder(
'series_index',
[1, 2, 3],
attributes={
'neurodata_type': 'VectorIndex',
'namespace': 'core',
'help': 'indexes into a list of values for a list of elements',
'target': ReferenceBuilder(column_series),
},
)
column_sweep_number = DatasetBuilder(
'sweep_number',
data=[4711, 4712, 4712],
attributes={
'neurodata_type': 'VectorData',
'namespace': 'core',
'help': 'Values for a list of elements',
'description': u'Sweep number of the entries in that row',
})
columns = [column_id, column_series, column_index, column_sweep_number]
sweep_table = GroupBuilder(
'sweep_table',
datasets={c.name: c
for c in columns},
attributes={
'neurodata_type':
'SweepTable',
'namespace':
'core',
'colnames': (b'series', b'sweep_number'),
'help':
'The table which groups different PatchClampSeries together',
'description':
u'A sweep table groups different PatchClampSeries together.',
},
)
return sweep_table
def get_plane_segmentation_builder(self):
self.optchan_builder = GroupBuilder(
'test_optical_channel',
attributes={
'neurodata_type': 'OpticalChannel',
'namespace': 'core'
},
datasets={
'description':
DatasetBuilder('description', 'optical channel description'),
'emission_lambda':
DatasetBuilder('emission_lambda', 500.)
},
)
device_builder = GroupBuilder('dev1',
attributes={
'neurodata_type': 'Device',
'namespace': 'core'
})
self.imgpln_builder = GroupBuilder(
'imgpln1',
attributes={
'neurodata_type': 'ImagingPlane',
'namespace': 'core'
},
datasets={
'description':
DatasetBuilder('description', 'imaging plane description'),
'excitation_lambda':
DatasetBuilder('excitation_lambda', 600.),
'imaging_rate':
DatasetBuilder('imaging_rate', 300.),
'indicator':
DatasetBuilder('indicator', 'GFP'),
'manifold':
DatasetBuilder('manifold', (((1., 2., 3.), (4., 5., 6.)), ),
attributes={
'conversion': 4.0,
'unit': 'manifold unit'
}),
'reference_frame':
DatasetBuilder('reference_frame', 'A frame to refer to'),
'location':
DatasetBuilder('location', 'somewhere in the brain')
},
groups={'optchan1': self.optchan_builder},
links={'device': LinkBuilder(device_builder, 'device')})
ts = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
self.is_builder = GroupBuilder(
'test_iS',
attributes={
'namespace': 'core',
'neurodata_type': 'ImageSeries',
'description': 'no description',
'comments': 'no comments'
},
datasets={
'timestamps':
DatasetBuilder('timestamps',
ts,
attributes={
'unit': 'seconds',
'interval': 1
}),
'external_file':
DatasetBuilder('external_file', ['images.tiff'],
attributes={'starting_frame': [1, 2, 3]}),
'format':
DatasetBuilder('format', 'tiff'),
'dimension':
DatasetBuilder('dimension', [2]),
})
self.pixel_masks_builder = DatasetBuilder(
'pixel_mask',
self.pix_mask,
attributes={
'namespace': 'core',
'neurodata_type': 'VectorData',
'description': 'Pixel masks for each ROI'
})
self.pxmsk_index_builder = DatasetBuilder(
'pixel_mask_index',
self.pxmsk_index,
attributes={
'namespace': 'core',
'neurodata_type': 'VectorIndex',
'target': ReferenceBuilder(self.pixel_masks_builder)
})
self.image_masks_builder = DatasetBuilder(
'image_mask',
self.img_mask,
attributes={
'namespace': 'core',
'neurodata_type': 'VectorData',
'description': 'Image masks for each ROI'
})
ps_builder = GroupBuilder(
'test_plane_seg_name',
attributes={
'neurodata_type': 'PlaneSegmentation',
'namespace': 'core',
'description': 'plane segmentation description',
'colnames': (b'image_mask', b'pixel_mask')
},
datasets={
'id':
DatasetBuilder('id',
data=[0, 1],
attributes={
'namespace': 'core',
'neurodata_type': 'ElementIdentifiers'
}),
'pixel_mask':
self.pixel_masks_builder,
'pixel_mask_index':
self.pxmsk_index_builder,
'image_mask':
self.image_masks_builder,
},
groups={
'reference_images':
GroupBuilder('reference_images',
groups={'test_iS': self.is_builder}),
},
links={
'imaging_plane': LinkBuilder(self.imgpln_builder,
'imaging_plane')
})
return ps_builder
def get_table_builder(self):
self.device_builder = GroupBuilder(
'dev1',
attributes={
'neurodata_type': 'Device',
'namespace': 'core',
'help': 'A recording device e.g. amplifier'
})
self.eg_builder = GroupBuilder(
'tetrode1',
attributes={
'neurodata_type': 'ElectrodeGroup',
'namespace': 'core',
'help': 'A physical grouping of channels',
'description': 'tetrode description',
'location': 'tetrode location'
},
links={'device': LinkBuilder(self.device_builder, 'device')})
datasets = [
DatasetBuilder('id',
data=[1, 2, 3, 4],
attributes={
'help':
'unique identifiers for a list of elements',
'neurodata_type': 'ElementIdentifiers',
'namespace': 'core'
}),
DatasetBuilder('x',
data=[1.0, 1.0, 1.0, 1.0],
attributes={
'help': 'Values for a list of elements',
'description':
'the x coordinate of the channel location',
'neurodata_type': 'VectorData',
'namespace': 'core'
}),
DatasetBuilder('y',
data=[2.0, 2.0, 2.0, 2.0],
attributes={
'help': 'Values for a list of elements',
'description':
'the y coordinate of the channel location',
'neurodata_type': 'VectorData',
'namespace': 'core'
}),
DatasetBuilder('z',
data=[3.0, 3.0, 3.0, 3.0],
attributes={
'help': 'Values for a list of elements',
'description':
'the z coordinate of the channel location',
'neurodata_type': 'VectorData',
'namespace': 'core'
}),
DatasetBuilder('imp',
data=[-1.0, -2.0, -3.0, -4.0],
attributes={
'help': 'Values for a list of elements',
'description': 'the impedance of the channel',
'neurodata_type': 'VectorData',
'namespace': 'core'
}),
DatasetBuilder(
'location',
data=['CA1', 'CA1', 'CA1', 'CA1'],
attributes={
'help': 'Values for a list of elements',
'description':
'the location of channel within the subject e.g. brain region',
'neurodata_type': 'VectorData',
'namespace': 'core'
}),
DatasetBuilder('filtering',
data=['none', 'none', 'none', 'none'],
attributes={
'help': 'Values for a list of elements',
'description':
'description of hardware filtering',
'neurodata_type': 'VectorData',
'namespace': 'core'
}),
DatasetBuilder(
'group',
data=[
ReferenceBuilder(self.eg_builder),
ReferenceBuilder(self.eg_builder),
ReferenceBuilder(self.eg_builder),
ReferenceBuilder(self.eg_builder)
],
attributes={
'help': 'Values for a list of elements',
'description':
'a reference to the ElectrodeGroup this electrode is a part of',
'neurodata_type': 'VectorData',
'namespace': 'core'
}),
DatasetBuilder(
'group_name',
data=['tetrode1', 'tetrode1', 'tetrode1', 'tetrode1'],
attributes={
'help': 'Values for a list of elements',
'description':
'the name of the ElectrodeGroup this electrode is a part of',
'neurodata_type': 'VectorData',
'namespace': 'core'
}),
]
return GroupBuilder('electrodes',
datasets={d.name: d
for d in datasets},
attributes={
'colnames':
(b'x', b'y', b'z', b'imp', b'location',
b'filtering', b'group', b'group_name'),
'description':
'metadata about extracellular electrodes',
'neurodata_type':
'DynamicTable',
'namespace':
'core',
'help':
'A column-centric table'
})
def setUpBuilder(self):
device = GroupBuilder('device_name',
attributes={
'neurodata_type': 'Device',
'namespace': 'core',
})
datasets = [
DatasetBuilder('slice', data=u'tissue slice'),
DatasetBuilder('resistance', data=u'something measured in ohms'),
DatasetBuilder('seal', data=u'sealing method'),
DatasetBuilder('description', data=u'a fake electrode object'),
DatasetBuilder('location', data=u'Springfield Elementary School'),
DatasetBuilder('filtering',
data=u'a meaningless free-form text field'),
DatasetBuilder('initial_access_resistance',
data=u'I guess this changes'),
]
elec = GroupBuilder('elec0',
attributes={
'namespace': 'core',
'neurodata_type': 'IntracellularElectrode',
},
datasets={d.name: d
for d in datasets},
links={'device': LinkBuilder(device, 'device')})
datasets = [
DatasetBuilder(
'gain',
data=0.126,
attributes={},
),
DatasetBuilder('data',
data=[1, 2, 3, 4, 5],
attributes={
'conversion': 1.0,
'resolution': 0.0,
'unit': u'A',
}),
DatasetBuilder('starting_time',
data=123.6,
attributes={
'rate': 10000.0,
'unit': 'Seconds',
}),
]
pcs = GroupBuilder(
'pcs',
attributes={
'neurodata_type': 'PatchClampSeries',
'namespace': 'core',
'comments': u'no comments',
'description': u'no description',
'stimulus_description': u'gotcha ya!',
'sweep_number': 4711
},
links={'electrode': LinkBuilder(elec, 'electrode')},
datasets={d.name: d
for d in datasets},
)
column_id = DatasetBuilder('id', [0],
attributes={
'neurodata_type': 'ElementIdentifiers',
'namespace': 'core'
})
column_series = DatasetBuilder(
'series',
attributes={
'neurodata_type': 'VectorData',
'namespace': 'core',
'description': u'PatchClampSeries with the same sweep number',
},
data=[LinkBuilder(pcs)])
column_index = DatasetBuilder(
'series_index',
[1],
attributes={
'neurodata_type': 'VectorIndex',
'namespace': 'core',
'target': ReferenceBuilder(column_series),
},
)
column_sweep_number = DatasetBuilder(
'sweep_number',
data=[4711],
attributes={
'neurodata_type': 'VectorData',
'namespace': 'core',
'description': u'Sweep number of the entries in that row',
})
columns = [column_id, column_series, column_index, column_sweep_number]
sweep_table = GroupBuilder(
'sweep_table',
datasets={c.name: c
for c in columns},
attributes={
'neurodata_type':
'SweepTable',
'namespace':
'core',
'colnames': (b'series', b'sweep_number'),
'description':
u'A sweep table groups different PatchClampSeries together.',
},
)
return sweep_table
def test_constructor(self):
db = DatasetBuilder('db1', [1, 2, 3])
rb = ReferenceBuilder(db)
self.assertIs(rb.builder, db)
