def test_load_namespace_unversioned_version(self):
"""Test that reading a namespace file with version=unversioned string works but raises a warning."""
# create namespace with version key (remove it later)
ns_dict = {
'doc': 'a test namespace',
'name': 'test_ns',
'schema': [
{'source': self.specs_path}
],
'version': '0.0.1'
}
namespace = SpecNamespace.build_namespace(**ns_dict)
namespace['version'] = str(SpecNamespace.UNVERSIONED) # work around lack of setter to remove version key
# write the namespace to file without version key
to_dump = {'namespaces': [namespace]}
with open(self.namespace_path, 'w') as tmp:
yaml.safe_dump(json.loads(json.dumps(to_dump)), tmp, default_flow_style=False)
# load the namespace from file
ns_catalog = NamespaceCatalog()
msg = "Loaded namespace 'test_ns' is unversioned. Please notify the extension author."
with self.assertWarnsWith(UserWarning, msg):
ns_catalog.load_namespaces(self.namespace_path)
self.assertEqual(ns_catalog.get_namespace('test_ns').version, SpecNamespace.UNVERSIONED)
def setUp(self):
self.bar_spec = GroupSpec(
'A test group specification with a data type',
data_type_def='Bar',
datasets=[
DatasetSpec('an example dataset',
'int',
name='data',
attributes=[
AttributeSpec('attr2',
'an example integer attribute',
'int')
])
],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text')
])
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.type_map.register_map(Bar, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.bar_spec)
def setUpManager(self, specs):
spec_catalog = SpecCatalog()
schema_file = 'test.yaml'
for s in specs:
spec_catalog.register_spec(s, schema_file)
namespace = SpecNamespace(doc='a test namespace',
name=CORE_NAMESPACE,
schema=[{
'source': schema_file
}],
version='0.1.0',
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
type_map = TypeMap(namespace_catalog)
type_map.register_container_type(CORE_NAMESPACE, 'SimpleFoo',
SimpleFoo)
type_map.register_container_type(CORE_NAMESPACE, 'NotSimpleFoo',
NotSimpleFoo)
type_map.register_container_type(CORE_NAMESPACE, 'SimpleQux',
SimpleQux)
type_map.register_container_type(CORE_NAMESPACE, 'NotSimpleQux',
NotSimpleQux)
type_map.register_container_type(CORE_NAMESPACE, 'SimpleBucket',
SimpleBucket)
type_map.register_map(SimpleBucket, self.setUpBucketMapper())
self.manager = BuildManager(type_map)
def setUp(self):
self.foo_spec = GroupSpec(
'A test group specification with data type Foo',
data_type_def='Foo')
self.bar_spec = GroupSpec(
'A test group specification with a data type Bar',
data_type_def='Bar',
datasets=[DatasetSpec('an example dataset', 'int', name='data')],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text'),
AttributeSpec('attr2', 'an example integer attribute', 'int'),
AttributeSpec('foo',
'a referenced foo',
RefSpec('Foo', 'object'),
required=False)
])
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.foo_spec, 'test.yaml')
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.manager = BuildManager(self.type_map)
self.foo_mapper = ObjectMapper(self.foo_spec)
self.bar_mapper = ObjectMapper(self.bar_spec)
def create_test_type_map(specs, container_classes, mappers=None):
"""
Create a TypeMap with the specs registered under a test namespace, and classes and mappers registered to type names.
:param specs: list of specs
:param container_classes: dict of type name to container class
:param mappers: (optional) dict of type name to mapper class
:return: the constructed TypeMap
"""
spec_catalog = SpecCatalog()
schema_file = 'test.yaml'
for s in specs:
spec_catalog.register_spec(s, schema_file)
namespace = SpecNamespace(doc='a test namespace',
name=CORE_NAMESPACE,
schema=[{
'source': schema_file
}],
version='0.1.0',
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
type_map = TypeMap(namespace_catalog)
for type_name, container_cls in container_classes.items():
type_map.register_container_type(CORE_NAMESPACE, type_name,
container_cls)
if mappers:
for type_name, mapper_cls in mappers.items():
container_cls = container_classes[type_name]
type_map.register_map(container_cls, mapper_cls)
return type_map
def setUp(self):
self.foo_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='Foo',
datasets=[
DatasetSpec(doc='an example dataset',
dtype='int',
name='my_data',
attributes=[
AttributeSpec(
name='attr2',
doc='an example integer attribute',
dtype='int')
])
],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text')
])
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.foo_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
self.type_map.register_map(Foo, FooMapper)
self.manager = BuildManager(self.type_map)
def test_load_namespace_none_version(self):
"""Test that reading a namespace file without a version works but raises a warning."""
# create namespace with version key (remove it later)
ns_dict = {
'doc': 'a test namespace',
'name': 'test_ns',
'schema': [{
'source': self.specs_path
}],
'version': '0.0.1'
}
namespace = SpecNamespace.build_namespace(**ns_dict)
namespace[
'version'] = None # work around lack of setter to remove version key
# write the namespace to file without version key
to_dump = {'namespaces': [namespace]}
with open(self.namespace_path, 'w') as tmp:
yaml_obj = yaml.YAML(typ='safe', pure=True)
yaml_obj.default_flow_style = False
yaml_obj.dump(json.loads(json.dumps(to_dump)), tmp)
# load the namespace from file
ns_catalog = NamespaceCatalog()
msg = (
"Loaded namespace 'test_ns' is missing the required key 'version'. Version will be set to "
"'%s'. Please notify the extension author." %
SpecNamespace.UNVERSIONED)
with self.assertWarnsWith(UserWarning, msg):
ns_catalog.load_namespaces(self.namespace_path)
self.assertEqual(
ns_catalog.get_namespace('test_ns').version,
SpecNamespace.UNVERSIONED)
def setUp(self):
spec_catalog = SpecCatalog()
for spec in self.getSpecs():
spec_catalog.register_spec(spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=spec_catalog)
self.vmap = ValidatorMap(self.namespace)
def test_bad_value(self):
"""Test that an error is raised if the container attribute value for a spec with a data type is not a container
or collection of containers.
"""
class Qux(Container):
@docval(
{
'name': 'name',
'type': str,
'doc': 'the name of this Qux'
}, {
'name': 'foo',
'type': int,
'doc': 'a group'
})
def __init__(self, **kwargs):
name, foo = getargs('name', 'foo', kwargs)
super().__init__(name=name)
self.__foo = foo
if isinstance(foo, Foo):
self.__foo.parent = self
@property
def foo(self):
return self.__foo
self.qux_spec = GroupSpec(
doc='A test group specification with data type Qux',
data_type_def='Qux',
groups=[GroupSpec('an example dataset', data_type_inc='Foo')])
self.foo_spec = GroupSpec(
'A test group specification with data type Foo',
data_type_def='Foo')
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.qux_spec, 'test.yaml')
self.spec_catalog.register_spec(self.foo_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Qux', Qux)
self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.qux_spec)
container = Qux('my_qux', foo=1)
msg = "Qux 'my_qux' attribute 'foo' has unexpected type."
with self.assertRaisesWith(ContainerConfigurationError, msg):
self.mapper.build(container, self.manager)
def setUp(self):
self.bar_spec = GroupSpec(
'A test group specification with a data type', data_type_def='Bar')
spec_catalog = SpecCatalog()
spec_catalog.register_spec(self.bar_spec, 'test.yaml')
namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
self.type_map = TypeMap(namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
def customSetUp(self, bar_spec):
spec_catalog = SpecCatalog()
spec_catalog.register_spec(bar_spec, 'test.yaml')
namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
type_map = TypeMap(namespace_catalog)
type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
return type_map
def set_up_spec(self, dtype):
spec_catalog = SpecCatalog()
spec = GroupSpec(
'A test group specification with a data type',
data_type_def='Bar',
datasets=[DatasetSpec('an example dataset', dtype, name='data')],
attributes=[AttributeSpec('attr1', 'an example attribute', dtype)])
spec_catalog.register_spec(spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=spec_catalog)
self.vmap = ValidatorMap(self.namespace)
def test_get_namespace_missing_version(self):
"""Test that SpecNamespace.version returns the constant for a missing version if version gets removed."""
# create namespace with version key (remove it later)
ns_dict = {
'doc': 'a test namespace',
'name': 'test_ns',
'schema': [
{'source': self.specs_path}
],
'version': '0.0.1'
}
namespace = SpecNamespace.build_namespace(**ns_dict)
namespace['version'] = None # work around lack of setter to remove version key
self.assertEqual(namespace.version, SpecNamespace.UNVERSIONED)
def setUp(self):
self.setUpBazSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.baz_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Baz', Baz)
self.type_map.register_map(Baz, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.baz_spec)
def test_build_namespace_missing_version(self):
"""Test that building/creating a SpecNamespace without a version works but raises a warning."""
# create namespace without version key
ns_dict = {
'doc': 'a test namespace',
'name': 'test_ns',
'schema': [
{'source': self.specs_path}
],
}
msg = ("Loaded namespace 'test_ns' is missing the required key 'version'. Version will be set to "
"'%s'. Please notify the extension author." % SpecNamespace.UNVERSIONED)
with self.assertWarnsWith(UserWarning, msg):
namespace = SpecNamespace.build_namespace(**ns_dict)
self.assertEqual(namespace.version, SpecNamespace.UNVERSIONED)
def setUp(self):
self.bar_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='EmptyBar')
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'EmptyBar',
EmptyBar)
def test_register_generator(self):
"""Test TypeMap.register_generator and ClassGenerator.register_generator."""
class MyClassGenerator(CustomClassGenerator):
@classmethod
def apply_generator_to_field(cls, field_spec, bases, type_map):
return True
@classmethod
def process_field_spec(cls, classdict, docval_args, parent_cls,
attr_name, not_inherited_fields, type_map,
spec):
# append attr_name to classdict['__custom_fields__'] list
classdict.setdefault('process_field_spec',
list()).append(attr_name)
@classmethod
def post_process(cls, classdict, bases, docval_args, spec):
classdict['post_process'] = True
spec = GroupSpec(doc='A test group specification with a data type',
data_type_def='Baz',
attributes=[
AttributeSpec(name='attr1',
doc='a string attribute',
dtype='text')
])
spec_catalog = SpecCatalog()
spec_catalog.register_spec(spec, 'test.yaml')
namespace = SpecNamespace(doc='a test namespace',
name=CORE_NAMESPACE,
schema=[{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
type_map = TypeMap(namespace_catalog)
type_map.register_generator(MyClassGenerator)
cls = type_map.get_dt_container_cls('Baz', CORE_NAMESPACE)
self.assertEqual(cls.process_field_spec, ['attr1'])
self.assertTrue(cls.post_process)
def setUp(self):
self.set_up_specs()
spec_catalog = SpecCatalog()
spec_catalog.register_spec(self.bar_data_spec, 'test.yaml')
spec_catalog.register_spec(self.bar_data_holder_spec, 'test.yaml')
namespace = SpecNamespace(doc='a test namespace',
name=CORE_NAMESPACE,
schema=[{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
type_map = TypeMap(namespace_catalog)
type_map.register_container_type(CORE_NAMESPACE, 'BarData', BarData)
type_map.register_container_type(CORE_NAMESPACE, 'BarDataHolder',
BarDataHolder)
type_map.register_map(BarData, ExtBarDataMapper)
type_map.register_map(BarDataHolder, ObjectMapper)
self.manager = BuildManager(type_map)
def setUp(self):
self.attributes = [
AttributeSpec('attribute1', 'my first attribute', 'text'),
AttributeSpec('attribute2', 'my second attribute', 'text')
]
self.dset1_attributes = [
AttributeSpec('attribute3', 'my third attribute', 'text'),
AttributeSpec('attribute4', 'my fourth attribute', 'text')
]
self.dset2_attributes = [
AttributeSpec('attribute5', 'my fifth attribute', 'text'),
AttributeSpec('attribute6', 'my sixth attribute', 'text')
]
self.datasets = [
DatasetSpec('my first dataset', # noqa: F405
'int',
name='dataset1',
attributes=self.dset1_attributes,
linkable=True),
DatasetSpec('my second dataset', # noqa: F405
'int',
name='dataset2',
dims=(None, None),
attributes=self.dset2_attributes,
linkable=True,
data_type_def='VoltageArray')
]
self.spec = GroupSpec('A test group', # noqa: F405
name='root_constructor_datatype',
datasets=self.datasets,
attributes=self.attributes,
linkable=False,
data_type_def='EphysData')
dset1_attributes_ext = [
AttributeSpec('dset1_extra_attribute', 'an extra attribute for the first dataset', 'text')
]
self.ext_datasets = [
DatasetSpec('my first dataset extension', # noqa: F405
'int',
name='dataset1',
attributes=dset1_attributes_ext,
linkable=True),
]
self.ext_attributes = [
AttributeSpec('ext_extra_attribute', 'an extra attribute for the group', 'text'),
]
self.ext_spec = GroupSpec('A test group extension', # noqa: F405
name='root_constructor_datatype',
datasets=self.ext_datasets,
attributes=self.ext_attributes,
linkable=False,
data_type_inc='EphysData',
data_type_def='SpikeData')
to_dump = {'groups': [self.spec, self.ext_spec]}
self.specs_path = 'test_load_namespace.specs.yaml'
self.namespace_path = 'test_load_namespace.namespace.yaml'
with open(self.specs_path, 'w') as tmp:
yaml.safe_dump(json.loads(json.dumps(to_dump)), tmp, default_flow_style=False)
ns_dict = {
'doc': 'a test namespace',
'name': self.NS_NAME,
'schema': [
{'source': self.specs_path}
]
}
self.namespace = SpecNamespace.build_namespace(**ns_dict) # noqa: F405
to_dump = {'namespaces': [self.namespace]}
with open(self.namespace_path, 'w') as tmp:
yaml.safe_dump(json.loads(json.dumps(to_dump)), tmp, default_flow_style=False)
self.ns_catalog = NamespaceCatalog() # noqa: F405
def setUp(self):
self.dt_spec = GroupSpec(
'A test extension that contains a dynamic table',
data_type_def='TestTable',
data_type_inc='DynamicTable',
datasets=[
DatasetSpec(
data_type_inc='VectorData',
name='my_col',
doc='a test column',
dtype='float'
),
DatasetSpec(
data_type_inc='VectorData',
name='indexed_col',
doc='a test column',
dtype='float'
),
DatasetSpec(
data_type_inc='VectorIndex',
name='indexed_col_index',
doc='a test column',
),
DatasetSpec(
data_type_inc='VectorData',
name='optional_col1',
doc='a test column',
dtype='float',
quantity='?',
),
DatasetSpec(
data_type_inc='VectorData',
name='optional_col2',
doc='a test column',
dtype='float',
quantity='?',
)
]
)
self.dt_spec2 = GroupSpec(
'A test extension that contains a dynamic table',
data_type_def='TestDTRTable',
data_type_inc='DynamicTable',
datasets=[
DatasetSpec(
data_type_inc='DynamicTableRegion',
name='ref_col',
doc='a test column',
),
DatasetSpec(
data_type_inc='DynamicTableRegion',
name='indexed_ref_col',
doc='a test column',
),
DatasetSpec(
data_type_inc='VectorIndex',
name='indexed_ref_col_index',
doc='a test column',
),
DatasetSpec(
data_type_inc='DynamicTableRegion',
name='optional_ref_col',
doc='a test column',
quantity='?'
),
DatasetSpec(
data_type_inc='DynamicTableRegion',
name='optional_indexed_ref_col',
doc='a test column',
quantity='?'
),
DatasetSpec(
data_type_inc='VectorIndex',
name='optional_indexed_ref_col_index',
doc='a test column',
quantity='?'
),
DatasetSpec(
data_type_inc='VectorData',
name='optional_col3',
doc='a test column',
dtype='float',
quantity='?',
)
]
)
from hdmf.spec.write import YAMLSpecWriter
writer = YAMLSpecWriter(outdir='.')
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.dt_spec, 'test.yaml')
self.spec_catalog.register_spec(self.dt_spec2, 'test.yaml')
self.namespace = SpecNamespace(
'a test namespace', CORE_NAMESPACE,
[
dict(
namespace='hdmf-common',
),
dict(source='test.yaml'),
],
version='0.1.0',
catalog=self.spec_catalog
)
self.test_dir = tempfile.mkdtemp()
spec_fpath = os.path.join(self.test_dir, 'test.yaml')
namespace_fpath = os.path.join(self.test_dir, 'test-namespace.yaml')
writer.write_spec(dict(groups=[self.dt_spec, self.dt_spec2]), spec_fpath)
writer.write_namespace(self.namespace, namespace_fpath)
self.namespace_catalog = NamespaceCatalog()
hdmf_typemap = get_type_map()
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.merge(hdmf_typemap, ns_catalog=True)
self.type_map.load_namespaces(namespace_fpath)
self.manager = BuildManager(self.type_map)
self.TestTable = self.type_map.get_dt_container_cls('TestTable', CORE_NAMESPACE)
self.TestDTRTable = self.type_map.get_dt_container_cls('TestDTRTable', CORE_NAMESPACE)