def test_register_generator(self):
"""Test TypeMap.register_generator and ClassGenerator.register_generator."""
class MyClassGenerator(CustomClassGenerator):
@classmethod
def apply_generator_to_field(cls, field_spec, bases, type_map):
return True
@classmethod
def process_field_spec(cls, classdict, docval_args, parent_cls,
attr_name, not_inherited_fields, type_map,
spec):
# append attr_name to classdict['__custom_fields__'] list
classdict.setdefault('process_field_spec',
list()).append(attr_name)
@classmethod
def post_process(cls, classdict, bases, docval_args, spec):
classdict['post_process'] = True
spec = GroupSpec(doc='A test group specification with a data type',
data_type_def='Baz',
attributes=[
AttributeSpec(name='attr1',
doc='a string attribute',
dtype='text')
])
spec_catalog = SpecCatalog()
spec_catalog.register_spec(spec, 'test.yaml')
namespace = SpecNamespace(doc='a test namespace',
name=CORE_NAMESPACE,
schema=[{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
type_map = TypeMap(namespace_catalog)
type_map.register_generator(MyClassGenerator)
cls = type_map.get_dt_container_cls('Baz', CORE_NAMESPACE)
self.assertEqual(cls.process_field_spec, ['attr1'])
self.assertTrue(cls.post_process)
class TestDynamicDynamicTable(TestCase):
def setUp(self):
self.dt_spec = GroupSpec(
'A test extension that contains a dynamic table',
data_type_def='TestTable',
data_type_inc='DynamicTable',
datasets=[
DatasetSpec(
data_type_inc='VectorData',
name='my_col',
doc='a test column',
dtype='float'
),
DatasetSpec(
data_type_inc='VectorData',
name='indexed_col',
doc='a test column',
dtype='float'
),
DatasetSpec(
data_type_inc='VectorIndex',
name='indexed_col_index',
doc='a test column',
),
DatasetSpec(
data_type_inc='VectorData',
name='optional_col1',
doc='a test column',
dtype='float',
quantity='?',
),
DatasetSpec(
data_type_inc='VectorData',
name='optional_col2',
doc='a test column',
dtype='float',
quantity='?',
)
]
)
self.dt_spec2 = GroupSpec(
'A test extension that contains a dynamic table',
data_type_def='TestDTRTable',
data_type_inc='DynamicTable',
datasets=[
DatasetSpec(
data_type_inc='DynamicTableRegion',
name='ref_col',
doc='a test column',
),
DatasetSpec(
data_type_inc='DynamicTableRegion',
name='indexed_ref_col',
doc='a test column',
),
DatasetSpec(
data_type_inc='VectorIndex',
name='indexed_ref_col_index',
doc='a test column',
),
DatasetSpec(
data_type_inc='DynamicTableRegion',
name='optional_ref_col',
doc='a test column',
quantity='?'
),
DatasetSpec(
data_type_inc='DynamicTableRegion',
name='optional_indexed_ref_col',
doc='a test column',
quantity='?'
),
DatasetSpec(
data_type_inc='VectorIndex',
name='optional_indexed_ref_col_index',
doc='a test column',
quantity='?'
),
DatasetSpec(
data_type_inc='VectorData',
name='optional_col3',
doc='a test column',
dtype='float',
quantity='?',
)
]
)
from hdmf.spec.write import YAMLSpecWriter
writer = YAMLSpecWriter(outdir='.')
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.dt_spec, 'test.yaml')
self.spec_catalog.register_spec(self.dt_spec2, 'test.yaml')
self.namespace = SpecNamespace(
'a test namespace', CORE_NAMESPACE,
[
dict(
namespace='hdmf-common',
),
dict(source='test.yaml'),
],
version='0.1.0',
catalog=self.spec_catalog
)
self.test_dir = tempfile.mkdtemp()
spec_fpath = os.path.join(self.test_dir, 'test.yaml')
namespace_fpath = os.path.join(self.test_dir, 'test-namespace.yaml')
writer.write_spec(dict(groups=[self.dt_spec, self.dt_spec2]), spec_fpath)
writer.write_namespace(self.namespace, namespace_fpath)
self.namespace_catalog = NamespaceCatalog()
hdmf_typemap = get_type_map()
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.merge(hdmf_typemap, ns_catalog=True)
self.type_map.load_namespaces(namespace_fpath)
self.manager = BuildManager(self.type_map)
self.TestTable = self.type_map.get_dt_container_cls('TestTable', CORE_NAMESPACE)
self.TestDTRTable = self.type_map.get_dt_container_cls('TestDTRTable', CORE_NAMESPACE)
def tearDown(self) -> None:
shutil.rmtree(self.test_dir)
def test_dynamic_table(self):
assert issubclass(self.TestTable, DynamicTable)
assert self.TestTable.__columns__[0] == dict(
name='my_col',
description='a test column',
required=True
)
def test_forbids_incorrect_col(self):
test_table = self.TestTable(name='test_table', description='my test table')
with self.assertRaises(ValueError):
test_table.add_row(my_col=3.0, indexed_col=[1.0, 3.0], incorrect_col=5)
def test_dynamic_column(self):
test_table = self.TestTable(name='test_table', description='my test table')
test_table.add_column('dynamic_column', 'this is a dynamic column')
test_table.add_row(
my_col=3.0, indexed_col=[1.0, 3.0], dynamic_column=4, optional_col2=.5,
)
test_table.add_row(
my_col=4.0, indexed_col=[2.0, 4.0], dynamic_column=4, optional_col2=.5,
)
np.testing.assert_array_equal(test_table['indexed_col'].target.data, [1., 3., 2., 4.])
np.testing.assert_array_equal(test_table['dynamic_column'].data, [4, 4])
def test_optional_col(self):
test_table = self.TestTable(name='test_table', description='my test table')
test_table.add_row(my_col=3.0, indexed_col=[1.0, 3.0], optional_col2=.5)
test_table.add_row(my_col=4.0, indexed_col=[2.0, 4.0], optional_col2=.5)
def test_dynamic_table_region(self):
test_table = self.TestTable(name='test_table', description='my test table')
test_table.add_row(my_col=3.0, indexed_col=[1.0, 3.0], optional_col2=.5)
test_table.add_row(my_col=4.0, indexed_col=[2.0, 4.0], optional_col2=.5)
test_dtr_table = self.TestDTRTable(name='test_dtr_table', description='my table',
target_tables={'ref_col': test_table,
'indexed_ref_col': test_table})
self.assertIs(test_dtr_table['ref_col'].table, test_table)
self.assertIs(test_dtr_table['indexed_ref_col'].target.table, test_table)
test_dtr_table.add_row(ref_col=0, indexed_ref_col=[0, 1])
test_dtr_table.add_row(ref_col=0, indexed_ref_col=[0, 1])
np.testing.assert_array_equal(test_dtr_table['indexed_ref_col'].target.data, [0, 1, 0, 1])
np.testing.assert_array_equal(test_dtr_table['ref_col'].data, [0, 0])
def test_dynamic_table_region_optional(self):
test_table = self.TestTable(name='test_table', description='my test table')
test_table.add_row(my_col=3.0, indexed_col=[1.0, 3.0], optional_col2=.5)
test_table.add_row(my_col=4.0, indexed_col=[2.0, 4.0], optional_col2=.5)
test_dtr_table = self.TestDTRTable(name='test_dtr_table', description='my table',
target_tables={'optional_ref_col': test_table,
'optional_indexed_ref_col': test_table})
self.assertIs(test_dtr_table['optional_ref_col'].table, test_table)
self.assertIs(test_dtr_table['optional_indexed_ref_col'].target.table, test_table)
test_dtr_table.add_row(ref_col=0, indexed_ref_col=[0, 1],
optional_ref_col=0, optional_indexed_ref_col=[0, 1])
test_dtr_table.add_row(ref_col=0, indexed_ref_col=[0, 1],
optional_ref_col=0, optional_indexed_ref_col=[0, 1])
np.testing.assert_array_equal(test_dtr_table['optional_indexed_ref_col'].target.data, [0, 1, 0, 1])
np.testing.assert_array_equal(test_dtr_table['optional_ref_col'].data, [0, 0])
def test_dynamic_table_region_bad_target_col(self):
test_table = self.TestTable(name='test_table', description='my test table')
test_table.add_row(my_col=3.0, indexed_col=[1.0, 3.0], optional_col2=.5)
test_table.add_row(my_col=4.0, indexed_col=[2.0, 4.0], optional_col2=.5)
msg = r"^'bad' is not the name of a predefined column of table .*"
with self.assertRaisesRegex(ValueError, msg):
self.TestDTRTable(name='test_dtr_table', description='my table', target_tables={'bad': test_table})
def test_dynamic_table_region_non_dtr_target(self):
test_table = self.TestTable(name='test_table', description='my test table')
test_table.add_row(my_col=3.0, indexed_col=[1.0, 3.0], optional_col2=.5)
test_table.add_row(my_col=4.0, indexed_col=[2.0, 4.0], optional_col2=.5)
msg = "Column 'optional_col3' must be a DynamicTableRegion to have a target table."
with self.assertRaisesWith(ValueError, msg):
self.TestDTRTable(name='test_dtr_table', description='my table',
target_tables={'optional_col3': test_table})
def test_roundtrip(self):
# NOTE this does not use H5RoundTripMixin because this requires custom validation
test_table = self.TestTable(name='test_table', description='my test table')
test_table.add_column('dynamic_column', 'this is a dynamic column')
test_table.add_row(
my_col=3.0, indexed_col=[1.0, 3.0], dynamic_column=4, optional_col2=.5,
)
self.filename = os.path.join(self.test_dir, 'test_TestTable.h5')
with HDF5IO(self.filename, manager=self.manager, mode='w') as write_io:
write_io.write(test_table, cache_spec=True)
self.reader = HDF5IO(self.filename, manager=self.manager, mode='r')
read_container = self.reader.read()
self.assertIsNotNone(str(test_table)) # added as a test to make sure printing works
self.assertIsNotNone(str(read_container))
# make sure we get a completely new object
self.assertNotEqual(id(test_table), id(read_container))
# the name of the root container of a file is always 'root' (see h5tools.py ROOT_NAME)
# thus, ignore the name of the container when comparing original container vs read container
self.assertContainerEqual(read_container, test_table, ignore_name=True)
builder = self.reader.read_builder()
# TODO fix ValueError: No specification for 'Container' in namespace 'test_core'
validator = ValidatorMap(self.manager.namespace_catalog.get_namespace(name=CORE_NAMESPACE))
errors = validator.validate(builder)
if errors:
for err in errors:
raise Exception(err)
self.reader.close()
class TestGetClassSeparateNamespace(TestCase):
def setUp(self):
self.test_dir = tempfile.mkdtemp()
if os.path.exists(self.test_dir): # start clean
self.tearDown()
os.mkdir(self.test_dir)
self.bar_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='Bar',
datasets=[DatasetSpec(name='data', doc='a dataset', dtype='int')],
attributes=[
AttributeSpec(name='attr1',
doc='a string attribute',
dtype='text'),
AttributeSpec(name='attr2',
doc='an integer attribute',
dtype='int')
])
self.type_map = TypeMap()
create_load_namespace_yaml(namespace_name=CORE_NAMESPACE,
specs=[self.bar_spec],
output_dir=self.test_dir,
incl_types=dict(),
type_map=self.type_map)
def tearDown(self):
shutil.rmtree(self.test_dir)
def test_get_class_separate_ns(self):
"""Test that get_class correctly sets the name and type hierarchy across namespaces."""
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
baz_spec = GroupSpec(
doc='A test extension',
data_type_def='Baz',
data_type_inc='Bar',
)
create_load_namespace_yaml(namespace_name='ndx-test',
specs=[baz_spec],
output_dir=self.test_dir,
incl_types={CORE_NAMESPACE: ['Bar']},
type_map=self.type_map)
cls = self.type_map.get_dt_container_cls('Baz', 'ndx-test')
self.assertEqual(cls.__name__, 'Baz')
self.assertTrue(issubclass(cls, Bar))
def _build_separate_namespaces(self):
# create an empty extension to test ClassGenerator._get_container_type resolution
# the Bar class has not been mapped yet to the bar spec
qux_spec = DatasetSpec(doc='A test extension', data_type_def='Qux')
spam_spec = DatasetSpec(doc='A test extension', data_type_def='Spam')
create_load_namespace_yaml(namespace_name='ndx-qux',
specs=[qux_spec, spam_spec],
output_dir=self.test_dir,
incl_types={},
type_map=self.type_map)
# resolve Spam first so that ndx-qux is resolved first
self.type_map.get_dt_container_cls('Spam', 'ndx-qux')
baz_spec = GroupSpec(doc='A test extension',
data_type_def='Baz',
data_type_inc='Bar',
groups=[
GroupSpec(data_type_inc='Qux',
doc='a qux',
quantity='?'),
GroupSpec(data_type_inc='Bar',
doc='a bar',
quantity='?')
])
create_load_namespace_yaml(namespace_name='ndx-test',
specs=[baz_spec],
output_dir=self.test_dir,
incl_types={
CORE_NAMESPACE: ['Bar'],
'ndx-qux': ['Qux']
},
type_map=self.type_map)
def _check_classes(self, baz_cls, bar_cls, bar_cls2, qux_cls, qux_cls2):
self.assertEqual(qux_cls.__name__, 'Qux')
self.assertEqual(baz_cls.__name__, 'Baz')
self.assertEqual(bar_cls.__name__, 'Bar')
self.assertIs(bar_cls,
bar_cls2) # same class, two different namespaces
self.assertIs(qux_cls, qux_cls2)
self.assertTrue(issubclass(qux_cls, Data))
self.assertTrue(issubclass(baz_cls, bar_cls))
self.assertTrue(issubclass(bar_cls, Container))
qux_inst = qux_cls(name='qux_name', data=[1])
bar_inst = bar_cls(name='bar_name',
data=100,
attr1='a string',
attr2=10)
baz_inst = baz_cls(name='baz_name',
qux=qux_inst,
bar=bar_inst,
data=100,
attr1='a string',
attr2=10)
self.assertIs(baz_inst.qux, qux_inst)
def test_get_class_include_from_separate_ns_1(self):
"""Test that get_class correctly sets the name and includes types correctly across namespaces.
This is one of multiple tests carried out to ensure that order of which get_dt_container_cls is called
does not impact the results
first use EXTENSION namespace, then use ORIGINAL namespace
"""
self._build_separate_namespaces()
baz_cls = self.type_map.get_dt_container_cls(
'Baz', 'ndx-test') # Qux and Bar are not yet resolved
bar_cls = self.type_map.get_dt_container_cls('Bar', 'ndx-test')
bar_cls2 = self.type_map.get_dt_container_cls('Bar', CORE_NAMESPACE)
qux_cls = self.type_map.get_dt_container_cls('Qux', 'ndx-test')
qux_cls2 = self.type_map.get_dt_container_cls('Qux', 'ndx-qux')
self._check_classes(baz_cls, bar_cls, bar_cls2, qux_cls, qux_cls2)
def test_get_class_include_from_separate_ns_2(self):
"""Test that get_class correctly sets the name and includes types correctly across namespaces.
This is one of multiple tests carried out to ensure that order of which get_dt_container_cls is called
does not impact the results
first use ORIGINAL namespace, then use EXTENSION namespace
"""
self._build_separate_namespaces()
baz_cls = self.type_map.get_dt_container_cls(
'Baz', 'ndx-test') # Qux and Bar are not yet resolved
bar_cls2 = self.type_map.get_dt_container_cls('Bar', CORE_NAMESPACE)
bar_cls = self.type_map.get_dt_container_cls('Bar', 'ndx-test')
qux_cls = self.type_map.get_dt_container_cls('Qux', 'ndx-test')
qux_cls2 = self.type_map.get_dt_container_cls('Qux', 'ndx-qux')
self._check_classes(baz_cls, bar_cls, bar_cls2, qux_cls, qux_cls2)
def test_get_class_include_from_separate_ns_3(self):
"""Test that get_class correctly sets the name and includes types correctly across namespaces.
This is one of multiple tests carried out to ensure that order of which get_dt_container_cls is called
does not impact the results
first use EXTENSION namespace, then use EXTENSION namespace
"""
self._build_separate_namespaces()
baz_cls = self.type_map.get_dt_container_cls(
'Baz', 'ndx-test') # Qux and Bar are not yet resolved
bar_cls = self.type_map.get_dt_container_cls('Bar', 'ndx-test')
bar_cls2 = self.type_map.get_dt_container_cls('Bar', CORE_NAMESPACE)
qux_cls2 = self.type_map.get_dt_container_cls('Qux', 'ndx-qux')
qux_cls = self.type_map.get_dt_container_cls('Qux', 'ndx-test')
self._check_classes(baz_cls, bar_cls, bar_cls2, qux_cls, qux_cls2)
def test_get_class_include_from_separate_ns_4(self):
"""Test that get_class correctly sets the name and includes types correctly across namespaces.
This is one of multiple tests carried out to ensure that order of which get_dt_container_cls is called
does not impact the results
first use ORIGINAL namespace, then use EXTENSION namespace
"""
self._build_separate_namespaces()
baz_cls = self.type_map.get_dt_container_cls(
'Baz', 'ndx-test') # Qux and Bar are not yet resolved
bar_cls2 = self.type_map.get_dt_container_cls('Bar', CORE_NAMESPACE)
bar_cls = self.type_map.get_dt_container_cls('Bar', 'ndx-test')
qux_cls2 = self.type_map.get_dt_container_cls('Qux', 'ndx-qux')
qux_cls = self.type_map.get_dt_container_cls('Qux', 'ndx-test')
self._check_classes(baz_cls, bar_cls, bar_cls2, qux_cls, qux_cls2)
