def setUpBucketSpec(self):
tmp_spec = GroupSpec('A subgroup for Foos',
name='foo_holder',
groups=[
GroupSpec('the Foos in this bucket',
data_type_inc='Foo',
quantity=ZERO_OR_MANY)
])
self.bucket_spec = GroupSpec(
'A test group specification for a data type containing data type',
name="test_foo_bucket",
data_type_def='FooBucket',
groups=[tmp_spec])
def test_catch_duplicate_spec_different(self):
spec1 = GroupSpec(
data_type_def='Group1',
doc='This is my new group 1',
)
spec2 = GroupSpec(
data_type_def='Group1',
doc='This is my other group 1',
)
source = 'test_extension.yaml'
self.catalog.register_spec(spec1, source)
msg = "'Group1' - cannot overwrite existing specification"
with self.assertRaisesWith(ValueError, msg):
self.catalog.register_spec(spec2, source)
def test_path_complicated(self):
attribute = AttributeSpec('attribute1', 'my fifth attribute', 'text')
dataset = DatasetSpec('my first dataset',
'int',
name='dataset1',
attributes=[attribute])
subgroup = GroupSpec('A subgroup',
name='subgroup1',
datasets=[dataset])
self.assertEqual(attribute.path, 'subgroup1/dataset1/attribute1')
_ = GroupSpec('A test group', name='root', groups=[subgroup])
self.assertEqual(attribute.path, 'root/subgroup1/dataset1/attribute1')
def test_update_attribute_spec(self):
spec = GroupSpec('A test group',
name='root_constructor',
attributes=[
AttributeSpec('attribute1', 'my first attribute',
'text'),
AttributeSpec('attribute2', 'my second attribute',
'text')
])
spec.set_attribute(
AttributeSpec('attribute2', 'my second attribute', 'int', value=5))
res = spec.get_attribute('attribute2')
self.assertEqual(res.value, 5)
self.assertEqual(res.dtype, 'int')
def test_two_named_unnamed_dataset_same_type(self):
"""Test get_data_type when a group contains two named and one unnamed dataset with type X."""
child0 = DatasetSpec(doc='Group 0',
data_type_inc='Type0',
name='type0')
child1 = DatasetSpec(doc='Group 1',
data_type_inc='Type0',
name='type1')
child2 = DatasetSpec(doc='Group 2', data_type_inc='Type0')
parent_spec = GroupSpec(doc='A test group',
name='parent',
datasets=[child0, child1, child2],
data_type_def='ParentType')
self.assertIs(parent_spec.get_data_type('Type0'), child2)
def test_get_subspec_named(self):
child_spec = GroupSpec('A test group specification with a data type',
'my_subgroup')
parent_spec = GroupSpec('Something to hold a Bar',
'my_group',
groups=[child_spec])
sub_builder = GroupBuilder('my_subgroup',
attributes={
'data_type': 'Bar',
'namespace': CORE_NAMESPACE
})
builder = GroupBuilder('my_group', groups={'my_bar':
sub_builder}) # noqa: F841
result = self.type_map.get_subspec(parent_spec, sub_builder)
self.assertIs(result, child_spec)
def test_dynamic_container_composition_missing_type(self):
baz_spec1 = GroupSpec(
'A composition test outside',
data_type_def='Baz1',
data_type_inc=self.bar_spec,
attributes=[
AttributeSpec('attr3', 'a float attribute', 'float'),
AttributeSpec('attr4', 'another float attribute', 'float')
],
groups=[GroupSpec('A composition inside', data_type_inc='Baz2')])
self.spec_catalog.register_spec(baz_spec1, 'extension.yaml')
msg = "No specification for 'Baz2' in namespace 'test_core'"
with self.assertRaisesWith(ValueError, msg):
self.type_map.get_dt_container_cls('Baz1', CORE_NAMESPACE)
def setUpBarSpec(self):
self.bar_spec = GroupSpec(
doc='A test group specification with a data type Bar',
data_type_def='Bar',
groups=[
GroupSpec(name='empty',
doc='empty group',
quantity='?',
attributes=[
AttributeSpec('attr3',
'an optional float attribute',
'float',
required=False)
])
])
def create_specs(self, quantity):
# Type SimpleBucket contains:
# - [quantity] groups of data_type_inc SimpleFoo and [quantity] group of data_type_inc NotSimpleFoo
# - [quantity] datasets of data_type_inc SimpleQux and [quantity] datasets of data_type_inc NotSimpleQux
# - [quantity] links of target_type SimpleFoo and [quantity] links of target_type NotSimpleFoo
foo_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='SimpleFoo',
)
not_foo_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='NotSimpleFoo',
)
qux_spec = DatasetSpec(
doc='A test group specification with a data type',
data_type_def='SimpleQux',
)
not_qux_spec = DatasetSpec(
doc='A test group specification with a data type',
data_type_def='NotSimpleQux',
)
foo_inc_spec = GroupSpec(doc='the SimpleFoos in this bucket',
data_type_inc='SimpleFoo',
quantity=quantity)
not_foo_inc_spec = GroupSpec(doc='the SimpleFoos in this bucket',
data_type_inc='NotSimpleFoo',
quantity=quantity)
qux_inc_spec = DatasetSpec(doc='the SimpleQuxs in this bucket',
data_type_inc='SimpleQux',
quantity=quantity)
not_qux_inc_spec = DatasetSpec(doc='the SimpleQuxs in this bucket',
data_type_inc='NotSimpleQux',
quantity=quantity)
foo_link_spec = LinkSpec(doc='the links in this bucket',
target_type='SimpleFoo',
quantity=quantity)
not_foo_link_spec = LinkSpec(doc='the links in this bucket',
target_type='NotSimpleFoo',
quantity=quantity)
bucket_spec = GroupSpec(
doc=
'A test group specification for a data type containing data type',
name="test_bucket",
data_type_def='SimpleBucket',
groups=[foo_inc_spec, not_foo_inc_spec],
datasets=[qux_inc_spec, not_qux_inc_spec],
links=[foo_link_spec, not_foo_link_spec])
return [foo_spec, not_foo_spec, qux_spec, not_qux_spec, bucket_spec]
def test_dynamic_container_constructor_name_default_name(self):
# if both name and default_name are specified, name should be used
with self.assertWarns(Warning):
baz_spec = GroupSpec('A test extension with no Container class',
data_type_def='Baz',
data_type_inc=self.bar_spec,
name='A fixed name',
default_name='A default name',
attributes=[
AttributeSpec('attr3',
'a float attribute',
'float'),
AttributeSpec('attr4',
'another float attribute',
'float')
])
self.spec_catalog.register_spec(baz_spec, 'extension.yaml')
cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE)
inst = cls([1, 2, 3, 4],
'string attribute',
1000,
attr3=98.6,
attr4=1.0)
self.assertEqual(inst.name, 'A fixed name')
def test_dynamic_container_constructor_name(self):
# name is specified in spec and cannot be changed
baz_spec = GroupSpec(
'A test extension with no Container class',
data_type_def='Baz',
data_type_inc=self.bar_spec,
name='A fixed name',
attributes=[
AttributeSpec('attr3', 'an example float attribute', 'float'),
AttributeSpec('attr4', 'another example float attribute',
'float')
])
self.spec_catalog.register_spec(baz_spec, 'extension.yaml')
cls = self.type_map.get_container_cls(CORE_NAMESPACE, 'Baz')
with self.assertRaises(TypeError):
inst = cls('My Baz', [1, 2, 3, 4],
'string attribute',
1000,
attr3=98.6,
attr4=1.0)
inst = cls([1, 2, 3, 4],
'string attribute',
1000,
attr3=98.6,
attr4=1.0)
self.assertEqual(inst.name, 'A fixed name')
self.assertEqual(inst.data, [1, 2, 3, 4])
self.assertEqual(inst.attr1, 'string attribute')
self.assertEqual(inst.attr2, 1000)
self.assertEqual(inst.attr3, 98.6)
self.assertEqual(inst.attr4, 1.0)
def setUp(self):
self.bar_spec = GroupSpec(
'A test group specification with a data type',
data_type_def='Bar',
datasets=[
DatasetSpec('an example dataset',
'int',
name='data',
attributes=[
AttributeSpec('attr2',
'an example integer attribute',
'int')
])
],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text')
])
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.bar_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar)
self.type_map.register_map(Bar, ObjectMapper)
self.manager = BuildManager(self.type_map)
self.mapper = ObjectMapper(self.bar_spec)
def test_catch_duplicate_spec_nested(self):
spec1 = GroupSpec(
data_type_def='Group1',
doc='This is my new group 1',
)
spec2 = GroupSpec(
data_type_def='Group2',
doc='This is my new group 2',
groups=[spec1], # nested definition
)
source = 'test_extension.yaml'
self.catalog.register_spec(spec1, source)
self.catalog.register_spec(
spec2, source) # this is OK because Group1 is the same spec
ret = self.catalog.get_spec('Group1')
self.assertIs(ret, spec1)
def test_get_spec_source_file(self):
spikes_spec = GroupSpec('test group', data_type_def='SpikeData')
source_file_path = '/test/myt/test.yaml'
self.catalog.auto_register(spikes_spec, source_file_path)
recorded_source_file_path = self.catalog.get_spec_source_file(
'SpikeData')
self.assertEqual(recorded_source_file_path, source_file_path)
def setUp(self):
self.foo_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='Foo',
datasets=[
DatasetSpec(doc='an example dataset',
dtype='int',
name='my_data',
attributes=[
AttributeSpec(
name='attr2',
doc='an example integer attribute',
dtype='int')
])
],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text')
])
self.spec_catalog = SpecCatalog()
self.spec_catalog.register_spec(self.foo_spec, 'test.yaml')
self.namespace = SpecNamespace('a test namespace',
CORE_NAMESPACE, [{
'source': 'test.yaml'
}],
version='0.1.0',
catalog=self.spec_catalog)
self.namespace_catalog = NamespaceCatalog()
self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace)
self.type_map = TypeMap(self.namespace_catalog)
self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo)
self.type_map.register_map(Foo, FooMapper)
self.manager = BuildManager(self.type_map)
def test_post_process_fixed_name(self):
"""Test that docval generation for a class with a fixed name does not contain a docval arg for name."""
spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='Baz',
name='MyBaz', # <-- fixed name
attributes=[
AttributeSpec(name='attr1',
doc='a string attribute',
dtype='text',
shape=[None])
])
classdict = {}
bases = [Container]
docval_args = [{
'name': 'name',
'type': str,
'doc': 'name'
}, {
'name': 'attr1',
'type': ('array_data', 'data'),
'doc': 'a string attribute',
'shape': [None]
}]
CustomClassGenerator.post_process(classdict, bases, docval_args, spec)
expected = [{
'name': 'attr1',
'type': ('array_data', 'data'),
'doc': 'a string attribute',
'shape': [None]
}]
self.assertListEqual(docval_args, expected)
def getSpecs(self):
ret = GroupSpec(
'A test group specification with a data type',
data_type_def='Bar',
datasets=[
DatasetSpec('an example dataset',
'int',
name='data',
attributes=[
AttributeSpec('attr2',
'an example integer attribute',
'int')
]),
DatasetSpec('an example time dataset',
'isodatetime',
name='time'),
DatasetSpec('an array of times',
'isodatetime',
name='time_array',
dims=('num_times', ),
shape=(None, ))
],
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text')
])
return (ret, )
def test_set_parent_exists(self):
GroupSpec('A test group',
name='root_constructor',
groups=self.subgroups)
msg = 'Cannot re-assign parent.'
with self.assertRaisesWith(AttributeError, msg):
self.subgroups[0].parent = self.subgroups[1]
def test_dynamic_container_constructor(self):
baz_spec = GroupSpec('A test extension with no Container class',
data_type_def='Baz',
data_type_inc=self.bar_spec,
attributes=[
AttributeSpec('attr3', 'a float attribute',
'float'),
AttributeSpec('attr4',
'another float attribute',
'float')
])
self.spec_catalog.register_spec(baz_spec, 'extension.yaml')
cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE)
# TODO: test that constructor works!
inst = cls('My Baz', [1, 2, 3, 4],
'string attribute',
1000,
attr3=98.6,
attr4=1.0)
self.assertEqual(inst.name, 'My Baz')
self.assertEqual(inst.data, [1, 2, 3, 4])
self.assertEqual(inst.attr1, 'string attribute')
self.assertEqual(inst.attr2, 1000)
self.assertEqual(inst.attr3, 98.6)
self.assertEqual(inst.attr4, 1.0)
def test_get_namespace_spec(self):
expected = AttributeSpec(
'namespace',
'the namespace for the data type of this object',
'text',
required=False)
self.assertDictEqual(GroupSpec.get_namespace_spec(), expected)
def setUp(self):
self.attr1 = AttributeSpec(name='attr1',
doc='a string attribute',
dtype='text')
self.attr2 = AttributeSpec(name='attr2',
doc='an integer attribute',
dtype='int')
self.attr3 = AttributeSpec(name='attr3',
doc='an integer attribute',
dtype='int')
self.bar_spec = GroupSpec(
doc='A test group specification with a data type',
data_type_def='Bar',
datasets=[
DatasetSpec(doc='a dataset',
dtype='int',
name='data',
attributes=[self.attr2])
],
attributes=[self.attr1])
specs = [self.bar_spec]
containers = {'Bar': Bar}
self.type_map = create_test_type_map(specs, containers)
self.spec_catalog = self.type_map.namespace_catalog.get_namespace(
CORE_NAMESPACE).catalog
self.cls = self.type_map.get_dt_container_cls(self.bar_spec.data_type)
self.bar = self.cls(name='bar', data=[1], attr1='attr1', attr2=1)
obj_mapper_bar = self.type_map.get_map(self.bar)
obj_mapper_bar.map_spec('attr2', spec=self.attr2)
def test_build_empty_data(self):
"""Test building of a Data object with empty data."""
baz_inc_spec = DatasetSpec(doc='doc',
data_type_inc='Baz',
quantity=ZERO_OR_MANY)
baz_holder_spec = GroupSpec(doc='doc',
data_type_def='BazHolder',
datasets=[baz_inc_spec])
self.spec_catalog.register_spec(baz_holder_spec, 'test.yaml')
self.type_map.register_container_type(CORE_NAMESPACE, 'BazHolder',
BazHolder)
self.holder_mapper = ObjectMapper(baz_holder_spec)
baz = Baz('MyBaz', [], 'abcdefghijklmnopqrstuvwxyz')
holder = BazHolder('holder', [baz])
builder = self.holder_mapper.build(holder, self.manager)
expected = GroupBuilder(
name='holder',
datasets=[
DatasetBuilder(name='MyBaz',
data=[],
attributes={
'baz_attr': 'abcdefghijklmnopqrstuvwxyz',
'data_type': 'Baz',
'namespace': 'test_core',
'object_id': baz.object_id
})
])
self.assertBuilderEqual(builder, expected)
def test_process_field_spec_overwrite(self):
"""Test that docval generation overwrites previous docval args."""
spec = GroupSpec(doc='A test group specification with a data type',
data_type_def='Baz',
attributes=[
AttributeSpec(name='attr1',
doc='a string attribute',
dtype='text',
shape=[None])
])
not_inherited_fields = {'attr1': spec.get_attribute('attr1')}
docval_args = [
{
'name': 'attr1',
'type': ('array_data', 'data'),
'doc': 'a string attribute',
'shape': [[None], [None, None]]
}, # this dict will be overwritten below
{
'name': 'attr2',
'type': ('array_data', 'data'),
'doc': 'a string attribute',
'shape': [[None], [None, None]]
}
]
CustomClassGenerator.process_field_spec(
classdict={},
docval_args=docval_args,
parent_cls=EmptyBar, # <-- arbitrary class
attr_name='attr1',
not_inherited_fields=not_inherited_fields,
type_map=TypeMap(),
spec=spec)
expected = [{
'name': 'attr1',
'type': ('array_data', 'data'),
'doc': 'a string attribute',
'shape': [None]
}, {
'name': 'attr2',
'type': ('array_data', 'data'),
'doc': 'a string attribute',
'shape': [[None], [None, None]]
}]
self.assertListEqual(docval_args, expected)
def test_type_extension(self):
spec = GroupSpec('A test group',
name='parent_type',
datasets=self.datasets,
attributes=self.attributes,
linkable=False,
data_type_def='EphysData')
dset1_attributes_ext = [
AttributeSpec('dset1_extra_attribute', 'an extra attribute for the first dataset', 'text')
]
ext_datasets = [
DatasetSpec('my first dataset extension',
'int',
name='dataset1',
attributes=dset1_attributes_ext,
linkable=True),
]
ext_attributes = [
AttributeSpec('ext_extra_attribute', 'an extra attribute for the group', 'text'),
]
ext = GroupSpec('A test group extension',
name='child_type',
datasets=ext_datasets,
attributes=ext_attributes,
linkable=False,
data_type_inc=spec,
data_type_def='SpikeData')
ext_dset1 = ext.get_dataset('dataset1')
ext_dset1_attrs = ext_dset1.attributes
self.assertDictEqual(ext_dset1_attrs[0], dset1_attributes_ext[0])
self.assertDictEqual(ext_dset1_attrs[1], self.dset1_attributes[0])
self.assertDictEqual(ext_dset1_attrs[2], self.dset1_attributes[1])
self.assertEqual(ext.data_type_def, 'SpikeData')
self.assertEqual(ext.data_type_inc, 'EphysData')
ext_dset2 = ext.get_dataset('dataset2')
self.maxDiff = None
# this will suffice for now, assertDictEqual doesn't do deep equality checks
self.assertEqual(str(ext_dset2), str(self.datasets[1]))
self.assertAttributesEqual(ext_dset2, self.datasets[1])
# self.ns_attr_spec
ndt_attr_spec = AttributeSpec('data_type', 'the data type of this object', # noqa: F841
'text', value='SpikeData')
res_attrs = ext.attributes
self.assertDictEqual(res_attrs[0], ext_attributes[0])
self.assertDictEqual(res_attrs[1], self.attributes[0])
self.assertDictEqual(res_attrs[2], self.attributes[1])
# test that inherited specs are tracked appropriate
for d in self.datasets:
with self.subTest(dataset=d.name):
self.assertTrue(ext.is_inherited_spec(d))
self.assertFalse(spec.is_inherited_spec(d))
json.dumps(spec)
def test_three_named_datasets_same_type(self):
"""Test get_target_type when a group contains three named links with type X."""
child0 = DatasetSpec(doc='Group 0',
data_type_inc='Type0',
name='group0')
child1 = DatasetSpec(doc='Group 1',
data_type_inc='Type0',
name='group1')
child2 = DatasetSpec(doc='Group 2',
data_type_inc='Type0',
name='group2')
parent_spec = GroupSpec(doc='A test group',
name='parent',
datasets=[child0, child1, child2],
data_type_def='ParentType')
self.assertEqual(parent_spec.get_data_type('Type0'),
[child0, child1, child2])
def test_constructor(self):
link0 = LinkSpec(doc='Link 0', target_type='TargetType0')
link1 = LinkSpec(doc='Link 1', target_type='TargetType1')
links = [link0, link1]
spec = GroupSpec(doc='A test group', name='root', links=links)
self.assertIs(spec, links[0].parent)
self.assertIs(spec, links[1].parent)
json.dumps(spec)
def setUpBarHolderSpec(self):
ext_attr = AttributeSpec(
name='ext_attr',
dtype='bool',
doc='A boolean attribute',
)
bar_ext_no_name_spec = GroupSpec(
doc='A Bar extended with attribute ext_attr',
data_type_inc='Bar',
quantity='*',
attributes=[ext_attr],
)
self.bar_holder_spec = GroupSpec(
doc='A container of multiple extended Bar objects',
data_type_def='BarHolder',
groups=[bar_ext_no_name_spec],
)
def setUpBarHolderSpec(self):
int_attr2 = AttributeSpec(
name='attr2',
dtype='int64',
doc='Refine Bar spec from int to int64',
)
bar_ext_no_name_spec = GroupSpec(
doc='A Bar extended with modified attribute attr2',
data_type_inc='Bar',
quantity='*',
attributes=[int_attr2],
)
self.bar_holder_spec = GroupSpec(
doc='A container of multiple extended Bar objects',
data_type_def='BarHolder',
groups=[bar_ext_no_name_spec],
)
def setUpBarSpec(self):
self.bar_spec = GroupSpec(
doc='A test group specification with a data type Bar',
data_type_def='Bar',
attributes=[
AttributeSpec('attr1', 'an example string attribute', 'text'),
AttributeSpec('attr2', 'an example integer attribute', 'int')
])
def setUpBarSpec(self):
self.bar_spec = GroupSpec(
doc='A test group specification with a data type Bar',
data_type_def='Bar',
attributes=[
AttributeSpec('foo', 'a referenced foo',
RefSpec('Foo', 'object'))
])