class ObjectMapperMixin(metaclass=ABCMeta):
def setUp(self):
self.setUpBarSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.bar_spec)
@abstractmethod
def setUpBarSpec(self):
raise NotImplementedError(
'Cannot run test unless setUpBarSpec is implemented')
def test_default_mapping(self):
attr_map = self.mapper.get_attr_names(self.bar_spec)
keys = set(attr_map.keys())
for key in keys:
with self.subTest(key=key):
self.assertIs(attr_map[key], self.mapper.get_attr_spec(key))
self.assertIs(attr_map[key], self.mapper.get_carg_spec(key))
class TestObjectMapper(with_metaclass(ABCMeta, unittest.TestCase)):
def setUp(self):
self.setUpBarSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.type_map.register_map(Bar, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.bar_spec)
def setUpBarSpec(self):
raise unittest.SkipTest('setUpBarSpec not implemented')
def test_default_mapping(self):
attr_map = self.mapper.get_attr_names(self.bar_spec)
keys = set(attr_map.keys())
for key in keys:
with self.subTest(key=key):
self.assertIs(attr_map[key], self.mapper.get_attr_spec(key))
self.assertIs(attr_map[key], self.mapper.get_carg_spec(key))
def setUp(self):
self.foo_spec = GroupSpec(
'A test group specification with data type Foo',
data_type_def='Foo')
self.bar_spec = GroupSpec(
'A test group specification with a data type Bar',
data_type_def='Bar',
datasets=[DatasetSpec('an example dataset', 'int', name='data')],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text'),
AttributeSpec('attr2', 'an example integer attribute', 'int'),
AttributeSpec('foo',
'a referenced foo',
RefSpec('Foo', 'object'),
required=False)
])
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.foo_spec, 'test.yaml')
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.manager = BuildManager(self.type_map)
self.foo_mapper = ObjectMapper(self.foo_spec)
self.bar_mapper = ObjectMapper(self.bar_spec)
def setUp(self):
self.foo_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='Foo',
datasets=[
DatasetSpec(doc='an example dataset',
dtype='int',
name='my_data',
attributes=[
AttributeSpec(
name='attr2',
doc='an example integer attribute',
dtype='int')
])
],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text')
])
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.foo_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
self.type_map.register_map(Foo, FooMapper)
self.manager = BuildManager(self.type_map)
def setUp(self):
self.bar_spec = GroupSpec(
'A test group specification with a data type',
data_type_def='Bar',
datasets=[
DatasetSpec('an example dataset',
'int',
name='data',
attributes=[
AttributeSpec('attr2',
'an example integer attribute',
'int')
])
],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text')
])
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.type_map.register_map(Bar, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.bar_spec)
def test_bad_generator(self):
"""Test that register_generator raises an error if the generator is not an instance of CustomClassGenerator."""
class NotACustomClassGenerator:
pass
type_map = TypeMap()
msg = 'Generator <.*> must be a subclass of CustomClassGenerator.'
with self.assertRaisesRegex(ValueError, msg):
type_map.register_generator(NotACustomClassGenerator)
class TestObjectMapperBadValue(TestCase):
def test_bad_value(self):
"""Test that an error is raised if the container attribute value for a spec with a data type is not a container
or collection of containers.
"""
class Qux(Container):
@docval(
{
'name': 'name',
'type': str,
'doc': 'the name of this Qux'
}, {
'name': 'foo',
'type': int,
'doc': 'a group'
})
def __init__(self, **kwargs):
name, foo = getargs('name', 'foo', kwargs)
super().__init__(name=name)
self.__foo = foo
if isinstance(foo, Foo):
self.__foo.parent = self
@property
def foo(self):
return self.__foo
self.qux_spec = GroupSpec(
doc='A test group specification with data type Qux',
data_type_def='Qux',
groups=[GroupSpec('an example dataset', data_type_inc='Foo')])
self.foo_spec = GroupSpec(
'A test group specification with data type Foo',
data_type_def='Foo')
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.qux_spec, 'test.yaml')
self.spec_catalog.register_spec(self.foo_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Qux', Qux)
self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.qux_spec)
container = Qux('my_qux', foo=1)
msg = "Qux 'my_qux' attribute 'foo' has unexpected type."
with self.assertRaisesWith(ContainerConfigurationError, msg):
self.mapper.build(container, self.manager)
def setUp(self):
self.bar_spec = GroupSpec(
'A test group specification with a data type', data_type_def='Bar')
spec_catalog = SpecCatalog()
spec_catalog.register_spec(self.bar_spec, 'test.yaml')
namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
self.type_map = TypeMap(namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
def customSetUp(self, bar_spec):
spec_catalog = SpecCatalog()
spec_catalog.register_spec(bar_spec, 'test.yaml')
namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
type_map = TypeMap(namespace_catalog)
type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
return type_map
class TestDataMapScalar(TestCase):
def setUp(self):
self.setUpBazSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.baz_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'BazScalar',
BazScalar)
self.type_map.register_map(BazScalar, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.baz_spec)
def setUpBazSpec(self):
self.baz_spec = DatasetSpec(doc='a BazScalar type',
dtype='int',
name='MyBaz',
data_type_def='BazScalar')
def test_construct_scalar_dataset(self):
"""Test constructing a Data object with an h5py.Dataset with shape (1, ) for scalar spec."""
with h5py.File('test.h5', 'w') as file:
test_ds = file.create_dataset('test_ds', data=[1])
expected = BazScalar(
name='MyBaz',
data=1,
)
builder = DatasetBuilder(
name='MyBaz',
data=test_ds,
attributes={
'data_type': 'BazScalar',
'namespace': CORE_NAMESPACE,
'object_id': expected.object_id
},
)
container = self.mapper.construct(builder, self.manager)
self.assertTrue(np.issubdtype(type(
container.data), np.integer)) # as opposed to h5py.Dataset
self.assertContainerEqual(container, expected)
os.remove('test.h5')
def test_update_docval_default_value_none(self):
"""Test that update_docval_args for an optional field sets default: None."""
spec = GroupSpec(doc='A test group specification with a data type',
data_type_def='Baz',
attributes=[
AttributeSpec(name='attr1',
doc='a string attribute',
dtype='text',
required=False)
])
not_inherited_fields = {'attr1': spec.get_attribute('attr1')}
docval_args = list()
CustomClassGenerator.process_field_spec(
classdict={},
docval_args=docval_args,
parent_cls=EmptyBar, # <-- arbitrary class
attr_name='attr1',
not_inherited_fields=not_inherited_fields,
type_map=TypeMap(),
spec=spec)
expected = [{
'name': 'attr1',
'type': str,
'doc': 'a string attribute',
'default': None
}]
self.assertListEqual(docval_args, expected)
def test_update_docval_dset_shape(self):
"""Test that update_docval_args for a dataset with shape sets the type and shape keys."""
spec = GroupSpec(doc='A test group specification with a data type',
data_type_def='Baz',
datasets=[
DatasetSpec(name='dset1',
doc='a string dataset',
dtype='text',
shape=[None])
])
not_inherited_fields = {'dset1': spec.get_dataset('dset1')}
docval_args = list()
CustomClassGenerator.process_field_spec(
classdict={},
docval_args=docval_args,
parent_cls=EmptyBar, # <-- arbitrary class
attr_name='dset1',
not_inherited_fields=not_inherited_fields,
type_map=TypeMap(),
spec=spec)
expected = [{
'name': 'dset1',
'type': ('array_data', 'data'),
'doc': 'a string dataset',
'shape': [None]
}]
self.assertListEqual(docval_args, expected)
def setUp(self):
self.setUpBazSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.baz_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Baz', Baz)
self.type_map.register_map(Baz, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.baz_spec)
def setUp(self):
self.bar_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='EmptyBar')
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'EmptyBar',
EmptyBar)
def setUpManager(self, specs):
spec_catalog = SpecCatalog()
schema_file = 'test.yaml'
for s in specs:
spec_catalog.register_spec(s, schema_file)
namespace = SpecNamespace(
doc='a test namespace',
name=CORE_NAMESPACE,
schema=[{'source': schema_file}],
version='0.1.0',
catalog=spec_catalog
)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
type_map = TypeMap(namespace_catalog)
type_map.register_container_type(CORE_NAMESPACE, 'BasicBucket', BasicBucket)
self.manager = BuildManager(type_map)
class TestGetSubSpec(unittest.TestCase):
def setUp(self):
self.bar_spec = GroupSpec(
'A test group specification with a data type', data_type_def='Bar')
spec_catalog = SpecCatalog()
spec_catalog.register_spec(self.bar_spec, 'test.yaml')
namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
self.type_map = TypeMap(namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
def test_get_subspec_data_type_noname(self):
parent_spec = GroupSpec('Something to hold a Bar',
'bar_bucket',
groups=[self.bar_spec])
sub_builder = GroupBuilder('my_bar',
attributes={
'data_type': 'Bar',
'namespace': CORE_NAMESPACE
})
builder = GroupBuilder('bar_bucket',
groups={'my_bar': sub_builder}) # noqa: F841
result = self.type_map.get_subspec(parent_spec, sub_builder)
self.assertIs(result, self.bar_spec)
def test_get_subspec_named(self):
child_spec = GroupSpec('A test group specification with a data type',
'my_subgroup')
parent_spec = GroupSpec('Something to hold a Bar',
'my_group',
groups=[child_spec])
sub_builder = GroupBuilder('my_subgroup',
attributes={
'data_type': 'Bar',
'namespace': CORE_NAMESPACE
})
builder = GroupBuilder('my_group', groups={'my_bar':
sub_builder}) # noqa: F841
result = self.type_map.get_subspec(parent_spec, sub_builder)
self.assertIs(result, child_spec)
def test_register_generator(self):
"""Test TypeMap.register_generator and ClassGenerator.register_generator."""
class MyClassGenerator(CustomClassGenerator):
@classmethod
def apply_generator_to_field(cls, field_spec, bases, type_map):
return True
@classmethod
def process_field_spec(cls, classdict, docval_args, parent_cls,
attr_name, not_inherited_fields, type_map,
spec):
# append attr_name to classdict['__custom_fields__'] list
classdict.setdefault('process_field_spec',
list()).append(attr_name)
@classmethod
def post_process(cls, classdict, bases, docval_args, spec):
classdict['post_process'] = True
spec = GroupSpec(doc='A test group specification with a data type',
data_type_def='Baz',
attributes=[
AttributeSpec(name='attr1',
doc='a string attribute',
dtype='text')
])
spec_catalog = SpecCatalog()
spec_catalog.register_spec(spec, 'test.yaml')
namespace = SpecNamespace(doc='a test namespace',
name=CORE_NAMESPACE,
schema=[{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
type_map = TypeMap(namespace_catalog)
type_map.register_generator(MyClassGenerator)
cls = type_map.get_dt_container_cls('Baz', CORE_NAMESPACE)
self.assertEqual(cls.process_field_spec, ['attr1'])
self.assertTrue(cls.post_process)
def setUp(self):
self.set_up_specs()
spec_catalog = SpecCatalog()
spec_catalog.register_spec(self.bar_data_spec, 'test.yaml')
spec_catalog.register_spec(self.bar_data_holder_spec, 'test.yaml')
namespace = SpecNamespace(doc='a test namespace',
name=CORE_NAMESPACE,
schema=[{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
type_map = TypeMap(namespace_catalog)
type_map.register_container_type(CORE_NAMESPACE, 'BarData', BarData)
type_map.register_container_type(CORE_NAMESPACE, 'BarDataHolder',
BarDataHolder)
type_map.register_map(BarData, ExtBarDataMapper)
type_map.register_map(BarDataHolder, ObjectMapper)
self.manager = BuildManager(type_map)
class BazSpecMixin:
def setUp(self):
self.setUpBazSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.baz_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Baz', Baz)
self.type_map.register_map(Baz, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.baz_spec)
def setUpBazSpec(self):
raise NotImplementedError('Test must implement this method.')
def create_test_type_map(specs, container_classes, mappers=None):
"""
Create a TypeMap with the specs registered under a test namespace, and classes and mappers registered to type names.
:param specs: list of specs
:param container_classes: dict of type name to container class
:param mappers: (optional) dict of type name to mapper class
:return: the constructed TypeMap
"""
spec_catalog = SpecCatalog()
schema_file = 'test.yaml'
for s in specs:
spec_catalog.register_spec(s, schema_file)
namespace = SpecNamespace(doc='a test namespace',
name=CORE_NAMESPACE,
schema=[{
'source': schema_file
}],
version='0.1.0',
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
type_map = TypeMap(namespace_catalog)
for type_name, container_cls in container_classes.items():
type_map.register_container_type(CORE_NAMESPACE, type_name,
container_cls)
if mappers:
for type_name, mapper_cls in mappers.items():
container_cls = container_classes[type_name]
type_map.register_map(container_cls, mapper_cls)
return type_map
def test_no_generators(self):
"""Test that a ClassGenerator without registered generators does nothing."""
cg = ClassGenerator()
spec = GroupSpec(doc='A test group spec with a data type',
data_type_def='Baz')
cls = cg.generate_class(data_type='Baz',
spec=spec,
parent_cls=Container,
attr_names={},
type_map=TypeMap())
self.assertEqual(cls.__mro__,
(cls, Container, AbstractContainer, object))
self.assertTrue(hasattr(cls, '__init__'))
class TestDataMap(unittest.TestCase):
def setUp(self):
self.setUpBazSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.baz_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Baz', Baz)
self.type_map.register_map(Baz, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.baz_spec)
def setUpBazSpec(self):
self.baz_spec = DatasetSpec(
'an Baz type',
'int',
name='MyBaz',
data_type_def='Baz',
attributes=[
AttributeSpec('baz_attr', 'an example string attribute',
'text')
])
def test_build(self):
''' Test default mapping functionality when no attributes are nested '''
container = Baz('my_baz', list(range(10)),
'abcdefghijklmnopqrstuvwxyz')
builder = self.mapper.build(container, self.manager)
expected = DatasetBuilder(
'my_baz',
list(range(10)),
attributes={'baz_attr': 'abcdefghijklmnopqrstuvwxyz'})
self.assertDictEqual(builder, expected)
class TestDataIOEdgeCases(TestCase):
def setUp(self):
self.setUpBazSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.baz_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Baz', Baz)
self.type_map.register_map(Baz, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.baz_spec)
def setUpBazSpec(self):
self.baz_spec = DatasetSpec(
doc='an Baz type',
dtype=None,
name='MyBaz',
data_type_def='Baz',
shape=[None],
attributes=[
AttributeSpec('baz_attr', 'an example string attribute',
'text')
])
def test_build_dataio(self):
"""Test building of a dataset with data_type and no dtype with value DataIO."""
container = Baz('my_baz', H5DataIO(['a', 'b', 'c', 'd'], chunks=True),
'value1')
builder = self.type_map.build(container)
self.assertIsInstance(builder.get('data'), H5DataIO)
def test_build_datachunkiterator(self):
"""Test building of a dataset with data_type and no dtype with value DataChunkIterator."""
container = Baz('my_baz', DataChunkIterator(['a', 'b', 'c', 'd']),
'value1')
builder = self.type_map.build(container)
self.assertIsInstance(builder.get('data'), DataChunkIterator)
def test_build_dataio_datachunkiterator(self): # hdmf#512
"""Test building of a dataset with no dtype and no data_type with value DataIO wrapping a DCI."""
container = Baz(
'my_baz',
H5DataIO(DataChunkIterator(['a', 'b', 'c', 'd']), chunks=True),
'value1')
builder = self.type_map.build(container)
self.assertIsInstance(builder.get('data'), H5DataIO)
self.assertIsInstance(builder.get('data').data, DataChunkIterator)
class TestTypeMap(TestCase):
def setUp(self):
self.bar_spec = GroupSpec(
'A test group specification with a data type', data_type_def='Bar')
self.foo_spec = GroupSpec(
'A test group specification with data type Foo',
data_type_def='Foo')
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.spec_catalog.register_spec(self.foo_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
def test_get_map_unique_mappers(self):
bar_inst = Bar('my_bar', list(range(10)), 'value1', 10)
foo_inst = Foo(name='my_foo')
bar_mapper = self.type_map.get_map(bar_inst)
foo_mapper = self.type_map.get_map(foo_inst)
self.assertIsNot(bar_mapper, foo_mapper)
def test_get_map(self):
container_inst = Bar('my_bar', list(range(10)), 'value1', 10)
mapper = self.type_map.get_map(container_inst)
self.assertIsInstance(mapper, ObjectMapper)
self.assertIs(mapper.spec, self.bar_spec)
mapper2 = self.type_map.get_map(container_inst)
self.assertIs(mapper, mapper2)
def test_get_map_register(self):
class MyMap(ObjectMapper):
pass
self.type_map.register_map(Bar, MyMap)
container_inst = Bar('my_bar', list(range(10)), 'value1', 10)
mapper = self.type_map.get_map(container_inst)
self.assertIs(mapper.spec, self.bar_spec)
self.assertIsInstance(mapper, MyMap)
def setUp(self):
self.test_dir = tempfile.mkdtemp()
if os.path.exists(self.test_dir): # start clean
self.tearDown()
os.mkdir(self.test_dir)
self.bar_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='Bar',
datasets=[DatasetSpec(name='data', doc='a dataset', dtype='int')],
attributes=[
AttributeSpec(name='attr1',
doc='a string attribute',
dtype='text'),
AttributeSpec(name='attr2',
doc='an integer attribute',
dtype='int')
])
self.type_map = TypeMap()
create_load_namespace_yaml(namespace_name=CORE_NAMESPACE,
specs=[self.bar_spec],
output_dir=self.test_dir,
incl_types=dict(),
type_map=self.type_map)
def test_process_field_spec_overwrite(self):
"""Test that docval generation overwrites previous docval args."""
spec = GroupSpec(doc='A test group specification with a data type',
data_type_def='Baz',
attributes=[
AttributeSpec(name='attr1',
doc='a string attribute',
dtype='text',
shape=[None])
])
not_inherited_fields = {'attr1': spec.get_attribute('attr1')}
docval_args = [
{
'name': 'attr1',
'type': ('array_data', 'data'),
'doc': 'a string attribute',
'shape': [[None], [None, None]]
}, # this dict will be overwritten below
{
'name': 'attr2',
'type': ('array_data', 'data'),
'doc': 'a string attribute',
'shape': [[None], [None, None]]
}
]
CustomClassGenerator.process_field_spec(
classdict={},
docval_args=docval_args,
parent_cls=EmptyBar, # <-- arbitrary class
attr_name='attr1',
not_inherited_fields=not_inherited_fields,
type_map=TypeMap(),
spec=spec)
expected = [{
'name': 'attr1',
'type': ('array_data', 'data'),
'doc': 'a string attribute',
'shape': [None]
}, {
'name': 'attr2',
'type': ('array_data', 'data'),
'doc': 'a string attribute',
'shape': [[None], [None, None]]
}]
self.assertListEqual(docval_args, expected)
def _get_resources():
# LEGACY: Needed to support legacy implementation.
return __get_resources()
# a global namespace catalog
global __NS_CATALOG
global __TYPE_MAP
__NS_CATALOG = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace)
hdmf_typemap = hdmf.common.get_type_map()
__NS_CATALOG.merge(hdmf_typemap.namespace_catalog)
__TYPE_MAP = TypeMap(__NS_CATALOG)
__TYPE_MAP.merge(hdmf_typemap)
@docval(
{
'name': 'extensions',
'type': (str, TypeMap, list),
'doc':
'a path to a namespace, a TypeMap, or a list consisting of paths to namespaces and TypeMaps',
'default': None
},
returns="the namespaces loaded from the given file",
rtype=tuple,
is_method=False)
def get_type_map(**kwargs):
def __init__(self, **kwargs):
path, mode, manager, extensions, load_namespaces, file_obj, comm =\
popargs('path', 'mode', 'manager', 'extensions', 'load_namespaces', 'file', 'comm', kwargs)
# root group
self.__rgroup = file_obj
chunk_store = getattr(file_obj, 'chunk_store', None)
if chunk_store is not None:
try:
filename = getattr(chunk_store.source, 'path', None)
if filename is None:
filename = chunk_store.source.name
except:
filename = None
if filename is None:
filename = f'{type(file_obj.store).__name__}'
self.__rgroup.filename = filename
file_obj = self.__set_rgroup(file_obj)
self.__built = dict() # keep track of each builder for each dataset/group/link for each file
self.__read = dict() # keep track of which files have been read. Key is the filename value is the builder
self.__file = file_obj
if load_namespaces:
if manager is not None:
warn("loading namespaces from file - ignoring 'manager'")
if extensions is not None:
warn("loading namespaces from file - ignoring 'extensions' argument")
# namespaces are not loaded when creating an NWBZARRHDF5IO object in write mode
if 'w' in mode or mode == 'x':
raise ValueError("cannot load namespaces from file when writing to it")
tm = get_type_map()
self.load_namespaces(tm, path, file=file_obj)
manager = BuildManager(tm)
# XXX: Leaving this here in case we want to revert to this strategy for
# loading cached namespaces
# ns_catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace)
# super(NWBZARRHDF5IO, self).load_namespaces(ns_catalog, path)
# tm = TypeMap(ns_catalog)
# tm.copy_mappers(get_type_map())
else:
if manager is not None and extensions is not None:
raise ValueError("'manager' and 'extensions' cannot be specified together")
elif extensions is not None:
manager = get_manager(extensions=extensions)
elif manager is None:
manager = get_manager()
self.logger = logging.getLogger('%s.%s' % (self.__class__.__module__, self.__class__.__qualname__))
if file_obj is not None:
if path is None:
path = file_obj.filename
elif os.path.abspath(file_obj.filename) != os.path.abspath(path):
msg = 'You argued %s as this object\'s path, ' % path
msg += 'but supplied a file with filename: %s' % file_obj.filename
raise ValueError(msg)
elif path is None:
TypeError("Must supply either 'path' or 'file' arg to HDF5IO.")
if file_obj is None and not os.path.exists(path) and (mode == 'r' or mode == 'r+'):
msg = "Unable to open file %s in '%s' mode. File does not exist." % (path, mode)
raise UnsupportedOperation(msg)
if file_obj is None and os.path.exists(path) and (mode == 'w-' or mode == 'x'):
msg = "Unable to open file %s in '%s' mode. File already exists." % (path, mode)
raise UnsupportedOperation(msg)
if manager is None:
manager = BuildManager(TypeMap(NamespaceCatalog()))
elif isinstance(manager, TypeMap):
manager = BuildManager(manager)
# TO DO #
self._HDF5IO__comm = comm
self._HDF5IO__mode = mode
self._HDF5IO__path = path
self._HDF5IO__file = file_obj
super(_HDF5IO, self).__init__(manager, source=path)
self._HDF5IO__ref_queue = deque() # a queue of the references that need to be added
self._HDF5IO__dci_queue = deque() # a queue of DataChunkIterators that need to be exhausted
return ret
def _get_resources():
# LEGACY: Needed to support legacy implementation.
return __get_resources()
# a global namespace catalog
global __NS_CATALOG
global __TYPE_MAP
from .spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace # noqa E402
__NS_CATALOG = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace)
__TYPE_MAP = TypeMap(__NS_CATALOG)
@docval(
{
'name': 'extensions',
'type': (str, TypeMap, list),
'doc':
'a path to a namespace, a TypeMap, or a list consisting of paths to namespaces and TypeMaps',
'default': None
},
returns="the namespaces loaded from the given file",
rtype=tuple,
is_method=False)
def get_type_map(**kwargs):
'''
def _get_manager():
foo_spec = GroupSpec('A test group specification with a data type',
data_type_def='Foo',
datasets=[DatasetSpec('an example dataset',
'int',
name='my_data',
attributes=[AttributeSpec('attr2',
'an example integer attribute',
'int')])],
attributes=[AttributeSpec('attr1', 'an example string attribute', 'text')])
tmp_spec = GroupSpec('A subgroup for Foos',
name='foo_holder',
groups=[GroupSpec('the Foos in this bucket', data_type_inc='Foo', quantity=ZERO_OR_MANY)])
bucket_spec = GroupSpec('A test group specification for a data type containing data type',
data_type_def='FooBucket',
groups=[tmp_spec])
class BucketMapper(ObjectMapper):
def __init__(self, spec):
super(BucketMapper, self).__init__(spec)
foo_spec = spec.get_group('foo_holder').get_data_type('Foo')
self.map_spec('foos', foo_spec)
file_spec = GroupSpec("A file of Foos contained in FooBuckets",
name='root',
data_type_def='FooFile',
groups=[GroupSpec('Holds the FooBuckets',
name='buckets',
groups=[GroupSpec("One ore more FooBuckets",
data_type_inc='FooBucket',
quantity=ONE_OR_MANY)])])
class FileMapper(ObjectMapper):
def __init__(self, spec):
super(FileMapper, self).__init__(spec)
bucket_spec = spec.get_group('buckets').get_data_type('FooBucket')
self.map_spec('buckets', bucket_spec)
spec_catalog = SpecCatalog()
spec_catalog.register_spec(foo_spec, 'test.yaml')
spec_catalog.register_spec(bucket_spec, 'test.yaml')
spec_catalog.register_spec(file_spec, 'test.yaml')
namespace = SpecNamespace(
'a test namespace',
CORE_NAMESPACE,
[{'source': 'test.yaml'}],
catalog=spec_catalog)
namespace_catalog = NamespaceCatalog()
namespace_catalog.add_namespace(CORE_NAMESPACE, namespace)
type_map = TypeMap(namespace_catalog)
type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
type_map.register_container_type(CORE_NAMESPACE, 'FooBucket', FooBucket)
type_map.register_container_type(CORE_NAMESPACE, 'FooFile', FooFile)
type_map.register_map(FooBucket, BucketMapper)
type_map.register_map(FooFile, FileMapper)
manager = BuildManager(type_map)
return manager
