class ObjectMapperMixin(metaclass=ABCMeta):
def setUp(self):
self.setUpBarSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.bar_spec)
@abstractmethod
def setUpBarSpec(self):
raise NotImplementedError(
'Cannot run test unless setUpBarSpec is implemented')
def test_default_mapping(self):
attr_map = self.mapper.get_attr_names(self.bar_spec)
keys = set(attr_map.keys())
for key in keys:
with self.subTest(key=key):
self.assertIs(attr_map[key], self.mapper.get_attr_spec(key))
self.assertIs(attr_map[key], self.mapper.get_carg_spec(key))
class TestObjectMapper(with_metaclass(ABCMeta, unittest.TestCase)):
def setUp(self):
self.setUpBarSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.type_map.register_map(Bar, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.bar_spec)
def setUpBarSpec(self):
raise unittest.SkipTest('setUpBarSpec not implemented')
def test_default_mapping(self):
attr_map = self.mapper.get_attr_names(self.bar_spec)
keys = set(attr_map.keys())
for key in keys:
with self.subTest(key=key):
self.assertIs(attr_map[key], self.mapper.get_attr_spec(key))
self.assertIs(attr_map[key], self.mapper.get_carg_spec(key))
def test_build_empty_data(self):
"""Test building of a Data object with empty data."""
baz_inc_spec = DatasetSpec(doc='doc',
data_type_inc='Baz',
quantity=ZERO_OR_MANY)
baz_holder_spec = GroupSpec(doc='doc',
data_type_def='BazHolder',
datasets=[baz_inc_spec])
self.spec_catalog.register_spec(baz_holder_spec, 'test.yaml')
self.type_map.register_container_type(CORE_NAMESPACE, 'BazHolder',
BazHolder)
self.holder_mapper = ObjectMapper(baz_holder_spec)
baz = Baz('MyBaz', [], 'abcdefghijklmnopqrstuvwxyz')
holder = BazHolder('holder', [baz])
builder = self.holder_mapper.build(holder, self.manager)
expected = GroupBuilder(
name='holder',
datasets=[
DatasetBuilder(name='MyBaz',
data=[],
attributes={
'baz_attr': 'abcdefghijklmnopqrstuvwxyz',
'data_type': 'Baz',
'namespace': 'test_core',
'object_id': baz.object_id
})
])
self.assertBuilderEqual(builder, expected)
def setUp(self):
self.foo_spec = GroupSpec(
'A test group specification with data type Foo',
data_type_def='Foo')
self.bar_spec = GroupSpec(
'A test group specification with a data type Bar',
data_type_def='Bar',
datasets=[DatasetSpec('an example dataset', 'int', name='data')],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text'),
AttributeSpec('attr2', 'an example integer attribute', 'int'),
AttributeSpec('foo',
'a referenced foo',
RefSpec('Foo', 'object'),
required=False)
])
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.foo_spec, 'test.yaml')
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.manager = BuildManager(self.type_map)
self.foo_mapper = ObjectMapper(self.foo_spec)
self.bar_mapper = ObjectMapper(self.bar_spec)
class TestObjectMapperBadValue(TestCase):
def test_bad_value(self):
"""Test that an error is raised if the container attribute value for a spec with a data type is not a container
or collection of containers.
"""
class Qux(Container):
@docval(
{
'name': 'name',
'type': str,
'doc': 'the name of this Qux'
}, {
'name': 'foo',
'type': int,
'doc': 'a group'
})
def __init__(self, **kwargs):
name, foo = getargs('name', 'foo', kwargs)
super().__init__(name=name)
self.__foo = foo
if isinstance(foo, Foo):
self.__foo.parent = self
@property
def foo(self):
return self.__foo
self.qux_spec = GroupSpec(
doc='A test group specification with data type Qux',
data_type_def='Qux',
groups=[GroupSpec('an example dataset', data_type_inc='Foo')])
self.foo_spec = GroupSpec(
'A test group specification with data type Foo',
data_type_def='Foo')
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.qux_spec, 'test.yaml')
self.spec_catalog.register_spec(self.foo_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Qux', Qux)
self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.qux_spec)
container = Qux('my_qux', foo=1)
msg = "Qux 'my_qux' attribute 'foo' has unexpected type."
with self.assertRaisesWith(ContainerConfigurationError, msg):
self.mapper.build(container, self.manager)
def _test_convert_mismatch_helper(self, spec_type, value_types):
data = 1
spec = DatasetSpec('an example dataset', spec_type, name='data')
for dtype in value_types:
value = self._get_type(dtype)(data) # convert data to given dtype
with self.subTest(dtype=dtype):
s = np.dtype(self._get_type(spec_type))
g = np.dtype(self._get_type(dtype))
msg = "expected %s, received %s - must supply %s" % (s.name, g.name, s.name)
with self.assertRaisesWith(ValueError, msg):
ObjectMapper.convert_dtype(spec, value)
def test_ascii_spec(self):
ascii_spec_types = ['ascii', 'bytes']
for spec_type in ascii_spec_types:
with self.subTest(spec_type=spec_type):
spec = DatasetSpec('an example dataset', spec_type, name='data')
value = 'a'
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, b'a')
self.assertIs(type(ret), bytes)
self.assertEqual(ret_dtype, 'ascii')
value = b'a'
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, b'a')
self.assertIs(type(ret), bytes)
self.assertEqual(ret_dtype, 'ascii')
value = ['a', 'b']
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertListEqual(ret, [b'a', b'b'])
self.assertIs(type(ret[0]), bytes)
self.assertEqual(ret_dtype, 'ascii')
value = np.array(['a', 'b'])
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
np.testing.assert_array_equal(ret, np.array(['a', 'b'], dtype='S1'))
self.assertEqual(ret_dtype, 'ascii')
value = np.array(['a', 'b'], dtype='S1')
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
np.testing.assert_array_equal(ret, value)
self.assertEqual(ret_dtype, 'ascii')
value = []
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertListEqual(ret, value)
self.assertEqual(ret_dtype, 'ascii')
value = 1
msg = "Expected unicode or ascii string, got <class 'int'>"
with self.assertRaisesWith(ValueError, msg):
ObjectMapper.convert_dtype(spec, value)
value = DataChunkIterator(np.array(['a', 'b']))
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) # no conversion
self.assertIs(ret, value)
self.assertEqual(ret_dtype, 'ascii')
value = DataChunkIterator(np.array(['a', 'b'], dtype='S1'))
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) # no conversion
self.assertIs(ret, value)
self.assertEqual(ret_dtype, 'ascii')
def test_numeric_spec(self):
spec_type = 'numeric'
spec = DatasetSpec('an example dataset', spec_type, name='data')
value = np.uint64(4)
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, value)
self.assertIs(type(ret), np.uint64)
self.assertEqual(ret_dtype, np.uint64)
value = DataChunkIterator(data=[1, 2, 3])
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, value)
self.assertIs(ret.dtype.type, np.dtype(int).type)
self.assertIs(type(ret.data[0]), int)
self.assertEqual(ret_dtype, np.dtype(int).type)
value = ['a', 'b']
msg = "Cannot convert from <class 'str'> to 'numeric' specification dtype."
with self.assertRaisesWith(ValueError, msg):
ObjectMapper.convert_dtype(spec, value)
value = np.array(['a', 'b'])
msg = "Cannot convert from <class 'numpy.str_'> to 'numeric' specification dtype."
with self.assertRaisesWith(ValueError, msg):
ObjectMapper.convert_dtype(spec, value)
value = []
msg = "Cannot infer dtype of empty list or tuple. Please use numpy array with specified dtype."
with self.assertRaisesWith(ValueError, msg):
ObjectMapper.convert_dtype(spec, value)
def test_dci_input(self):
spec = DatasetSpec('an example dataset', 'int64', name='data')
value = DataChunkIterator(np.array([1, 2, 3], dtype=np.int32))
msg = "Spec 'data': Value with data type int32 is being converted to data type int64 as specified."
with self.assertWarnsWith(UserWarning, msg):
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) # no conversion
self.assertIs(ret, value)
self.assertEqual(ret_dtype, np.int64)
spec = DatasetSpec('an example dataset', 'int16', name='data')
value = DataChunkIterator(np.array([1, 2, 3], dtype=np.int32))
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value) # no conversion
self.assertIs(ret, value)
self.assertEqual(ret_dtype, np.int32) # increase precision
def test_bool_spec(self):
spec_type = 'bool'
spec = DatasetSpec('an example dataset', spec_type, name='data')
value = np.bool_(True)
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, value)
self.assertIs(type(ret), np.bool_)
self.assertEqual(ret_dtype, np.bool_)
value = True
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, value)
self.assertIs(type(ret), np.bool_)
self.assertEqual(ret_dtype, np.bool_)
def setUp(self):
self.bar_spec = GroupSpec(
'A test group specification with a data type',
data_type_def='Bar',
datasets=[
DatasetSpec('an example dataset',
'int',
name='data',
attributes=[
AttributeSpec('attr2',
'an example integer attribute',
'int')
])
],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text')
])
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.type_map.register_map(Bar, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.bar_spec)
def setUp(self):
self.setUpBazSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.baz_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Baz', Baz)
self.type_map.register_map(Baz, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.baz_spec)
def test_compound_type(self):
"""Test that convert_dtype passes through arguments if spec dtype is a list without any validation."""
spec_type = [DtypeSpec('an int field', 'f1', 'int'), DtypeSpec('a float field', 'f2', 'float')]
spec = DatasetSpec('an example dataset', spec_type, name='data')
value = ['a', 1, 2.2]
res, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertListEqual(res, value)
self.assertListEqual(ret_dtype, spec_type)
def _test_keep_higher_precision_helper(self, spec_type, value_types):
data = 1
spec = DatasetSpec('an example dataset', spec_type, name='data')
for dtype in value_types:
value = self._get_type(dtype)(data)
match = (value, self._get_type(dtype))
with self.subTest(dtype=dtype):
ret = ObjectMapper.convert_dtype(spec, value)
self.assertTupleEqual(ret, match)
self.assertIs(ret[0].dtype.type, match[1])
def _test_convert_alias(self, spec_type, value_types):
data = 1
spec = DatasetSpec('an example dataset', spec_type, name='data')
match = (self._get_type(spec_type)(data), self._get_type(spec_type))
for dtype in value_types:
value = self._get_type(dtype)(data) # convert data to given dtype
with self.subTest(dtype=dtype):
ret = ObjectMapper.convert_dtype(spec, value)
self.assertTupleEqual(ret, match)
self.assertIs(ret[0].dtype.type, match[1])
def test_isodatetime_spec(self):
spec_type = 'isodatetime'
spec = DatasetSpec('an example dataset', spec_type, name='data')
# NOTE: datetime.isoformat is called on all values with a datetime spec before conversion
# see ObjectMapper.get_attr_value
value = datetime.isoformat(datetime(2020, 11, 10))
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, b'2020-11-10T00:00:00')
self.assertIs(type(ret), bytes)
self.assertEqual(ret_dtype, 'ascii')
def _test_convert_higher_precision_helper(self, spec_type, value_types):
data = 1
spec = DatasetSpec('an example dataset', spec_type, name='data')
match = (self._get_type(spec_type)(data), self._get_type(spec_type))
for dtype in value_types:
value = self._get_type(dtype)(data) # convert data to given dtype
with self.subTest(dtype=dtype):
s = np.dtype(self._get_type(spec_type))
g = np.dtype(self._get_type(dtype))
msg = ("Spec 'data': Value with data type %s is being converted to data type %s as specified."
% (g.name, s.name))
with self.assertWarnsWith(UserWarning, msg):
ret = ObjectMapper.convert_dtype(spec, value)
self.assertTupleEqual(ret, match)
self.assertIs(ret[0].dtype.type, match[1])
class TestDataMapScalar(TestCase):
def setUp(self):
self.setUpBazSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.baz_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'BazScalar',
BazScalar)
self.type_map.register_map(BazScalar, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.baz_spec)
def setUpBazSpec(self):
self.baz_spec = DatasetSpec(doc='a BazScalar type',
dtype='int',
name='MyBaz',
data_type_def='BazScalar')
def test_construct_scalar_dataset(self):
"""Test constructing a Data object with an h5py.Dataset with shape (1, ) for scalar spec."""
with h5py.File('test.h5', 'w') as file:
test_ds = file.create_dataset('test_ds', data=[1])
expected = BazScalar(
name='MyBaz',
data=1,
)
builder = DatasetBuilder(
name='MyBaz',
data=test_ds,
attributes={
'data_type': 'BazScalar',
'namespace': CORE_NAMESPACE,
'object_id': expected.object_id
},
)
container = self.mapper.construct(builder, self.manager)
self.assertTrue(np.issubdtype(type(
container.data), np.integer)) # as opposed to h5py.Dataset
self.assertContainerEqual(container, expected)
os.remove('test.h5')
class TestDataMap(unittest.TestCase):
def setUp(self):
self.setUpBazSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.baz_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Baz', Baz)
self.type_map.register_map(Baz, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.baz_spec)
def setUpBazSpec(self):
self.baz_spec = DatasetSpec(
'an Baz type',
'int',
name='MyBaz',
data_type_def='Baz',
attributes=[
AttributeSpec('baz_attr', 'an example string attribute',
'text')
])
def test_build(self):
''' Test default mapping functionality when no attributes are nested '''
container = Baz('my_baz', list(range(10)),
'abcdefghijklmnopqrstuvwxyz')
builder = self.mapper.build(container, self.manager)
expected = DatasetBuilder(
'my_baz',
list(range(10)),
attributes={'baz_attr': 'abcdefghijklmnopqrstuvwxyz'})
self.assertDictEqual(builder, expected)
def test_value_none(self):
spec = DatasetSpec('an example dataset', 'int', name='data')
self.assertTupleEqual(ObjectMapper.convert_dtype(spec, None), (None, 'int'))
spec = DatasetSpec('an example dataset', RefSpec(reftype='object', target_type='int'), name='data')
self.assertTupleEqual(ObjectMapper.convert_dtype(spec, None), (None, 'object'))
class TestDataMap(BazSpecMixin, TestCase):
def setUp(self):
self.setUpBazSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.baz_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Baz', Baz)
self.type_map.register_map(Baz, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.baz_spec)
def setUpBazSpec(self):
self.baz_spec = DatasetSpec(
doc='an Baz type',
dtype='int',
name='MyBaz',
data_type_def='Baz',
shape=[None],
attributes=[
AttributeSpec('baz_attr', 'an example string attribute',
'text')
])
def test_build(self):
''' Test default mapping functionality when no attributes are nested '''
container = Baz('MyBaz', list(range(10)), 'abcdefghijklmnopqrstuvwxyz')
builder = self.mapper.build(container, self.manager)
expected = DatasetBuilder(
'MyBaz',
list(range(10)),
attributes={'baz_attr': 'abcdefghijklmnopqrstuvwxyz'})
self.assertBuilderEqual(builder, expected)
def test_build_empty_data(self):
"""Test building of a Data object with empty data."""
baz_inc_spec = DatasetSpec(doc='doc',
data_type_inc='Baz',
quantity=ZERO_OR_MANY)
baz_holder_spec = GroupSpec(doc='doc',
data_type_def='BazHolder',
datasets=[baz_inc_spec])
self.spec_catalog.register_spec(baz_holder_spec, 'test.yaml')
self.type_map.register_container_type(CORE_NAMESPACE, 'BazHolder',
BazHolder)
self.holder_mapper = ObjectMapper(baz_holder_spec)
baz = Baz('MyBaz', [], 'abcdefghijklmnopqrstuvwxyz')
holder = BazHolder('holder', [baz])
builder = self.holder_mapper.build(holder, self.manager)
expected = GroupBuilder(
name='holder',
datasets=[
DatasetBuilder(name='MyBaz',
data=[],
attributes={
'baz_attr': 'abcdefghijklmnopqrstuvwxyz',
'data_type': 'Baz',
'namespace': 'test_core',
'object_id': baz.object_id
})
])
self.assertBuilderEqual(builder, expected)
def test_append(self):
with h5py.File('test.h5', 'w') as file:
test_ds = file.create_dataset('test_ds',
data=[1, 2, 3],
chunks=True,
maxshape=(None, ))
container = Baz('MyBaz', test_ds, 'abcdefghijklmnopqrstuvwxyz')
container.append(4)
np.testing.assert_array_equal(container[:], [1, 2, 3, 4])
os.remove('test.h5')
def test_extend(self):
with h5py.File('test.h5', 'w') as file:
test_ds = file.create_dataset('test_ds',
data=[1, 2, 3],
chunks=True,
maxshape=(None, ))
container = Baz('MyBaz', test_ds, 'abcdefghijklmnopqrstuvwxyz')
container.extend([4, 5])
np.testing.assert_array_equal(container[:], [1, 2, 3, 4, 5])
os.remove('test.h5')
class TestDataMapScalarCompound(TestCase):
def setUp(self):
self.setUpBazSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.baz_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE,
'BazScalarCompound',
BazScalarCompound)
self.type_map.register_map(BazScalarCompound, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.baz_spec)
def setUpBazSpec(self):
self.baz_spec = DatasetSpec(
doc='a BazScalarCompound type',
dtype=[
DtypeSpec(name='id',
dtype='uint64',
doc='The unique identifier in this table.'),
DtypeSpec(name='attr1', dtype='text', doc='A text attribute.'),
],
name='MyBaz',
data_type_def='BazScalarCompound',
)
def test_construct_scalar_compound_dataset(self):
"""Test construct on a compound h5py.Dataset with shape (1, ) for scalar spec does not resolve the data."""
with h5py.File('test.h5', 'w') as file:
comp_type = np.dtype([('id', np.uint64),
('attr1', h5py.special_dtype(vlen=str))])
test_ds = file.create_dataset(name='test_ds',
data=np.array((1, 'text'),
dtype=comp_type),
shape=(1, ),
dtype=comp_type)
expected = BazScalarCompound(
name='MyBaz',
data=(1, 'text'),
)
builder = DatasetBuilder(
name='MyBaz',
data=test_ds,
attributes={
'data_type': 'BazScalarCompound',
'namespace': CORE_NAMESPACE,
'object_id': expected.object_id
},
)
container = self.mapper.construct(builder, self.manager)
self.assertEqual(type(container.data), h5py.Dataset)
self.assertContainerEqual(container, expected)
os.remove('test.h5')
def test_no_spec(self):
spec_type = None
spec = DatasetSpec('an example dataset', spec_type, name='data')
value = [1, 2, 3]
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertListEqual(ret, value)
self.assertIs(type(ret[0]), int)
self.assertEqual(ret_dtype, int)
value = np.uint64(4)
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, value)
self.assertIs(type(ret), np.uint64)
self.assertEqual(ret_dtype, np.uint64)
value = 'hello'
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, value)
self.assertIs(type(ret), str)
self.assertEqual(ret_dtype, 'utf8')
value = b'hello'
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, value)
self.assertIs(type(ret), bytes)
self.assertEqual(ret_dtype, 'ascii')
value = np.array(['aa', 'bb'])
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
np.testing.assert_array_equal(ret, value)
self.assertEqual(ret_dtype, 'utf8')
value = np.array(['aa', 'bb'], dtype='S2')
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
np.testing.assert_array_equal(ret, value)
self.assertEqual(ret_dtype, 'ascii')
value = DataChunkIterator(data=[1, 2, 3])
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, value)
self.assertIs(ret.dtype.type, np.dtype(int).type)
self.assertIs(type(ret.data[0]), int)
self.assertEqual(ret_dtype, np.dtype(int).type)
value = DataChunkIterator(data=['a', 'b'])
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, value)
self.assertIs(ret.dtype.type, np.str_)
self.assertIs(type(ret.data[0]), str)
self.assertEqual(ret_dtype, 'utf8')
value = H5DataIO(np.arange(30).reshape(5, 2, 3))
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, value)
self.assertIs(ret.data.dtype.type, np.dtype(int).type)
self.assertEqual(ret_dtype, np.dtype(int).type)
value = H5DataIO(['foo', 'bar'])
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, value)
self.assertIs(type(ret.data[0]), str)
self.assertEqual(ret_dtype, 'utf8')
value = H5DataIO([b'foo', b'bar'])
ret, ret_dtype = ObjectMapper.convert_dtype(spec, value)
self.assertEqual(ret, value)
self.assertIs(type(ret.data[0]), bytes)
self.assertEqual(ret_dtype, 'ascii')
value = []
msg = "Cannot infer dtype of empty list or tuple. Please use numpy array with specified dtype."
with self.assertRaisesWith(ValueError, msg):
ObjectMapper.convert_dtype(spec, value)
class TestReference(TestCase):
def setUp(self):
self.foo_spec = GroupSpec(
'A test group specification with data type Foo',
data_type_def='Foo')
self.bar_spec = GroupSpec(
'A test group specification with a data type Bar',
data_type_def='Bar',
datasets=[DatasetSpec('an example dataset', 'int', name='data')],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text'),
AttributeSpec('attr2', 'an example integer attribute', 'int'),
AttributeSpec('foo',
'a referenced foo',
RefSpec('Foo', 'object'),
required=False)
])
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.foo_spec, 'test.yaml')
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.manager = BuildManager(self.type_map)
self.foo_mapper = ObjectMapper(self.foo_spec)
self.bar_mapper = ObjectMapper(self.bar_spec)
def test_build_attr_ref(self):
''' Test default mapping functionality when one container contains an attribute reference to another container.
'''
foo_inst = Foo('my_foo')
bar_inst1 = Bar('my_bar1', list(range(10)), 'value1', 10, foo=foo_inst)
bar_inst2 = Bar('my_bar2', list(range(10)), 'value1', 10)
foo_builder = self.manager.build(foo_inst, root=True)
bar1_builder = self.manager.build(bar_inst1, root=True) # adds refs
bar2_builder = self.manager.build(bar_inst2, root=True)
foo_expected = GroupBuilder('my_foo',
attributes={
'data_type': 'Foo',
'namespace': CORE_NAMESPACE,
'object_id': foo_inst.object_id
})
bar1_expected = GroupBuilder(
'n/a', # name doesn't matter
datasets={'data': DatasetBuilder('data', list(range(10)))},
attributes={
'attr1': 'value1',
'attr2': 10,
'foo': ReferenceBuilder(foo_expected),
'data_type': 'Bar',
'namespace': CORE_NAMESPACE,
'object_id': bar_inst1.object_id
})
bar2_expected = GroupBuilder(
'n/a', # name doesn't matter
datasets={'data': DatasetBuilder('data', list(range(10)))},
attributes={
'attr1': 'value1',
'attr2': 10,
'data_type': 'Bar',
'namespace': CORE_NAMESPACE,
'object_id': bar_inst2.object_id
})
self.assertDictEqual(foo_builder, foo_expected)
self.assertDictEqual(bar1_builder, bar1_expected)
self.assertDictEqual(bar2_builder, bar2_expected)
def test_build_attr_ref_invalid(self):
''' Test default mapping functionality when one container contains an attribute reference to another container.
'''
bar_inst1 = Bar('my_bar1', list(range(10)), 'value1', 10)
bar_inst1._Bar__foo = object() # make foo object a non-container type
msg = "invalid type for reference 'foo' (<class 'object'>) - must be AbstractContainer"
with self.assertRaisesWith(ValueError, msg):
self.bar_mapper.build(bar_inst1, self.manager)
def test_override_type_int_restrict_precision(self):
spec = DatasetSpec('an example dataset', 'int8', name='data')
res = ObjectMapper.convert_dtype(spec, np.int64(1), 'int64')
self.assertTupleEqual(res, (np.int64(1), np.int64))
class TestLinkedContainer(TestCase):
def setUp(self):
self.foo_spec = GroupSpec(
'A test group specification with data type Foo',
data_type_def='Foo')
self.bar_spec = GroupSpec(
'A test group specification with a data type Bar',
data_type_def='Bar',
groups=[self.foo_spec],
datasets=[DatasetSpec('an example dataset', 'int', name='data')],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text'),
AttributeSpec('attr2', 'an example integer attribute', 'int')
])
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.foo_spec, 'test.yaml')
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.manager = BuildManager(self.type_map)
self.foo_mapper = ObjectMapper(self.foo_spec)
self.bar_mapper = ObjectMapper(self.bar_spec)
def test_build_child_link(self):
''' Test default mapping functionality when one container contains a child link to another container '''
foo_inst = Foo('my_foo')
bar_inst1 = Bar('my_bar1', list(range(10)), 'value1', 10, foo=foo_inst)
# bar_inst2.foo should link to bar_inst1.foo
bar_inst2 = Bar('my_bar2', list(range(10)), 'value1', 10, foo=foo_inst)
foo_builder = self.foo_mapper.build(foo_inst, self.manager)
bar1_builder = self.bar_mapper.build(bar_inst1, self.manager)
bar2_builder = self.bar_mapper.build(bar_inst2, self.manager)
foo_expected = GroupBuilder('my_foo')
inner_foo_builder = GroupBuilder('my_foo',
attributes={
'data_type': 'Foo',
'namespace': CORE_NAMESPACE,
'object_id': foo_inst.object_id
})
bar1_expected = GroupBuilder(
'my_bar1',
datasets={'data': DatasetBuilder('data', list(range(10)))},
groups={'foo': inner_foo_builder},
attributes={
'attr1': 'value1',
'attr2': 10
})
link_foo_builder = LinkBuilder(builder=inner_foo_builder)
bar2_expected = GroupBuilder(
'my_bar2',
datasets={'data': DatasetBuilder('data', list(range(10)))},
links={'foo': link_foo_builder},
attributes={
'attr1': 'value1',
'attr2': 10
})
self.assertBuilderEqual(foo_builder, foo_expected)
self.assertBuilderEqual(bar1_builder, bar1_expected)
self.assertBuilderEqual(bar2_builder, bar2_expected)
@unittest.expectedFailure
def test_build_broken_link_parent(self):
''' Test that building a container with a broken link that has a parent raises an error. '''
foo_inst = Foo('my_foo')
Bar('my_bar1', list(range(10)), 'value1', 10,
foo=foo_inst) # foo_inst.parent is this bar
# bar_inst2.foo should link to bar_inst1.foo
bar_inst2 = Bar('my_bar2', list(range(10)), 'value1', 10, foo=foo_inst)
# TODO bar_inst.foo.parent exists but is never built - this is a tricky edge case that should raise an error
with self.assertRaises(OrphanContainerBuildError):
self.bar_mapper.build(bar_inst2, self.manager)
def test_build_broken_link_no_parent(self):
''' Test that building a container with a broken link that has no parent raises an error. '''
foo_inst = Foo('my_foo')
bar_inst1 = Bar('my_bar1', list(range(10)), 'value1', 10,
foo=foo_inst) # foo_inst.parent is this bar
# bar_inst2.foo should link to bar_inst1.foo
bar_inst2 = Bar('my_bar2', list(range(10)), 'value1', 10, foo=foo_inst)
bar_inst1.remove_foo()
msg = (
"my_bar2 (my_bar2): Linked Foo 'my_foo' has no parent. Remove the link or ensure the linked container "
"is added properly.")
with self.assertRaisesWith(OrphanContainerBuildError, msg):
self.bar_mapper.build(bar_inst2, self.manager)
def test_override_type_numeric_to_uint(self):
spec = DatasetSpec('an example dataset', 'numeric', name='data')
res = ObjectMapper.convert_dtype(spec, np.uint32(1), 'uint8')
self.assertTupleEqual(res, (np.uint32(1), np.uint32))
def test_override_type_numeric_to_uint_list(self):
spec = DatasetSpec('an example dataset', 'numeric', name='data')
res = ObjectMapper.convert_dtype(spec, np.uint32((1, 2, 3)), 'uint8')
np.testing.assert_array_equal(res[0], np.uint32((1, 2, 3)))
self.assertEqual(res[1], np.uint32)
def test_override_type_none_to_bool(self):
spec = DatasetSpec('an example dataset', None, name='data')
res = ObjectMapper.convert_dtype(spec, True, 'bool')
self.assertTupleEqual(res, (True, np.bool_))
class TestDataMap(TestCase):
def setUp(self):
self.setUpBazSpec()
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.baz_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Baz', Baz)
self.type_map.register_map(Baz, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.baz_spec)
def setUpBazSpec(self):
self.baz_spec = DatasetSpec(
'an Baz type',
'int',
name='MyBaz',
data_type_def='Baz',
shape=[None],
attributes=[
AttributeSpec('baz_attr', 'an example string attribute',
'text')
])
def test_build(self):
''' Test default mapping functionality when no attributes are nested '''
container = Baz('my_baz', list(range(10)),
'abcdefghijklmnopqrstuvwxyz')
builder = self.mapper.build(container, self.manager)
expected = DatasetBuilder(
'my_baz',
list(range(10)),
attributes={'baz_attr': 'abcdefghijklmnopqrstuvwxyz'})
self.assertDictEqual(builder, expected)
def test_append(self):
with h5py.File('test.h5', 'w') as file:
test_ds = file.create_dataset('test_ds',
data=[1, 2, 3],
chunks=True,
maxshape=(None, ))
container = Baz('my_baz', test_ds, 'abcdefghijklmnopqrstuvwxyz')
container.append(4)
np.testing.assert_array_equal(container[:], [1, 2, 3, 4])
os.remove('test.h5')
def test_extend(self):
with h5py.File('test.h5', 'w') as file:
test_ds = file.create_dataset('test_ds',
data=[1, 2, 3],
chunks=True,
maxshape=(None, ))
container = Baz('my_baz', test_ds, 'abcdefghijklmnopqrstuvwxyz')
container.extend([4, 5])
np.testing.assert_array_equal(container[:], [1, 2, 3, 4, 5])
os.remove('test.h5')