def setUpBucketSpec(self): tmp_spec = GroupSpec('A subgroup for Foos', name='foo_holder', groups=[ GroupSpec('the Foos in this bucket', data_type_inc='Foo', quantity=ZERO_OR_MANY) ]) self.bucket_spec = GroupSpec( 'A test group specification for a data type containing data type', name="test_foo_bucket", data_type_def='FooBucket', groups=[tmp_spec])
def test_catch_duplicate_spec_different(self): spec1 = GroupSpec( data_type_def='Group1', doc='This is my new group 1', ) spec2 = GroupSpec( data_type_def='Group1', doc='This is my other group 1', ) source = 'test_extension.yaml' self.catalog.register_spec(spec1, source) msg = "'Group1' - cannot overwrite existing specification" with self.assertRaisesWith(ValueError, msg): self.catalog.register_spec(spec2, source)
def test_path_complicated(self): attribute = AttributeSpec('attribute1', 'my fifth attribute', 'text') dataset = DatasetSpec('my first dataset', 'int', name='dataset1', attributes=[attribute]) subgroup = GroupSpec('A subgroup', name='subgroup1', datasets=[dataset]) self.assertEqual(attribute.path, 'subgroup1/dataset1/attribute1') _ = GroupSpec('A test group', name='root', groups=[subgroup]) self.assertEqual(attribute.path, 'root/subgroup1/dataset1/attribute1')
def test_update_attribute_spec(self): spec = GroupSpec('A test group', name='root_constructor', attributes=[ AttributeSpec('attribute1', 'my first attribute', 'text'), AttributeSpec('attribute2', 'my second attribute', 'text') ]) spec.set_attribute( AttributeSpec('attribute2', 'my second attribute', 'int', value=5)) res = spec.get_attribute('attribute2') self.assertEqual(res.value, 5) self.assertEqual(res.dtype, 'int')
def test_two_named_unnamed_dataset_same_type(self): """Test get_data_type when a group contains two named and one unnamed dataset with type X.""" child0 = DatasetSpec(doc='Group 0', data_type_inc='Type0', name='type0') child1 = DatasetSpec(doc='Group 1', data_type_inc='Type0', name='type1') child2 = DatasetSpec(doc='Group 2', data_type_inc='Type0') parent_spec = GroupSpec(doc='A test group', name='parent', datasets=[child0, child1, child2], data_type_def='ParentType') self.assertIs(parent_spec.get_data_type('Type0'), child2)
def test_get_subspec_named(self): child_spec = GroupSpec('A test group specification with a data type', 'my_subgroup') parent_spec = GroupSpec('Something to hold a Bar', 'my_group', groups=[child_spec]) sub_builder = GroupBuilder('my_subgroup', attributes={ 'data_type': 'Bar', 'namespace': CORE_NAMESPACE }) builder = GroupBuilder('my_group', groups={'my_bar': sub_builder}) # noqa: F841 result = self.type_map.get_subspec(parent_spec, sub_builder) self.assertIs(result, child_spec)
def test_dynamic_container_composition_missing_type(self): baz_spec1 = GroupSpec( 'A composition test outside', data_type_def='Baz1', data_type_inc=self.bar_spec, attributes=[ AttributeSpec('attr3', 'a float attribute', 'float'), AttributeSpec('attr4', 'another float attribute', 'float') ], groups=[GroupSpec('A composition inside', data_type_inc='Baz2')]) self.spec_catalog.register_spec(baz_spec1, 'extension.yaml') msg = "No specification for 'Baz2' in namespace 'test_core'" with self.assertRaisesWith(ValueError, msg): self.type_map.get_dt_container_cls('Baz1', CORE_NAMESPACE)
def setUpBarSpec(self): self.bar_spec = GroupSpec( doc='A test group specification with a data type Bar', data_type_def='Bar', groups=[ GroupSpec(name='empty', doc='empty group', quantity='?', attributes=[ AttributeSpec('attr3', 'an optional float attribute', 'float', required=False) ]) ])
def create_specs(self, quantity): # Type SimpleBucket contains: # - [quantity] groups of data_type_inc SimpleFoo and [quantity] group of data_type_inc NotSimpleFoo # - [quantity] datasets of data_type_inc SimpleQux and [quantity] datasets of data_type_inc NotSimpleQux # - [quantity] links of target_type SimpleFoo and [quantity] links of target_type NotSimpleFoo foo_spec = GroupSpec( doc='A test group specification with a data type', data_type_def='SimpleFoo', ) not_foo_spec = GroupSpec( doc='A test group specification with a data type', data_type_def='NotSimpleFoo', ) qux_spec = DatasetSpec( doc='A test group specification with a data type', data_type_def='SimpleQux', ) not_qux_spec = DatasetSpec( doc='A test group specification with a data type', data_type_def='NotSimpleQux', ) foo_inc_spec = GroupSpec(doc='the SimpleFoos in this bucket', data_type_inc='SimpleFoo', quantity=quantity) not_foo_inc_spec = GroupSpec(doc='the SimpleFoos in this bucket', data_type_inc='NotSimpleFoo', quantity=quantity) qux_inc_spec = DatasetSpec(doc='the SimpleQuxs in this bucket', data_type_inc='SimpleQux', quantity=quantity) not_qux_inc_spec = DatasetSpec(doc='the SimpleQuxs in this bucket', data_type_inc='NotSimpleQux', quantity=quantity) foo_link_spec = LinkSpec(doc='the links in this bucket', target_type='SimpleFoo', quantity=quantity) not_foo_link_spec = LinkSpec(doc='the links in this bucket', target_type='NotSimpleFoo', quantity=quantity) bucket_spec = GroupSpec( doc= 'A test group specification for a data type containing data type', name="test_bucket", data_type_def='SimpleBucket', groups=[foo_inc_spec, not_foo_inc_spec], datasets=[qux_inc_spec, not_qux_inc_spec], links=[foo_link_spec, not_foo_link_spec]) return [foo_spec, not_foo_spec, qux_spec, not_qux_spec, bucket_spec]
def test_dynamic_container_constructor_name_default_name(self): # if both name and default_name are specified, name should be used with self.assertWarns(Warning): baz_spec = GroupSpec('A test extension with no Container class', data_type_def='Baz', data_type_inc=self.bar_spec, name='A fixed name', default_name='A default name', attributes=[ AttributeSpec('attr3', 'a float attribute', 'float'), AttributeSpec('attr4', 'another float attribute', 'float') ]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) inst = cls([1, 2, 3, 4], 'string attribute', 1000, attr3=98.6, attr4=1.0) self.assertEqual(inst.name, 'A fixed name')
def test_dynamic_container_constructor_name(self): # name is specified in spec and cannot be changed baz_spec = GroupSpec( 'A test extension with no Container class', data_type_def='Baz', data_type_inc=self.bar_spec, name='A fixed name', attributes=[ AttributeSpec('attr3', 'an example float attribute', 'float'), AttributeSpec('attr4', 'another example float attribute', 'float') ]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_container_cls(CORE_NAMESPACE, 'Baz') with self.assertRaises(TypeError): inst = cls('My Baz', [1, 2, 3, 4], 'string attribute', 1000, attr3=98.6, attr4=1.0) inst = cls([1, 2, 3, 4], 'string attribute', 1000, attr3=98.6, attr4=1.0) self.assertEqual(inst.name, 'A fixed name') self.assertEqual(inst.data, [1, 2, 3, 4]) self.assertEqual(inst.attr1, 'string attribute') self.assertEqual(inst.attr2, 1000) self.assertEqual(inst.attr3, 98.6) self.assertEqual(inst.attr4, 1.0)
def setUp(self): self.bar_spec = GroupSpec( 'A test group specification with a data type', data_type_def='Bar', datasets=[ DatasetSpec('an example dataset', 'int', name='data', attributes=[ AttributeSpec('attr2', 'an example integer attribute', 'int') ]) ], attributes=[ AttributeSpec('attr1', 'an example string attribute', 'text') ]) self.spec_catalog = SpecCatalog() self.spec_catalog.register_spec(self.bar_spec, 'test.yaml') self.namespace = SpecNamespace('a test namespace', CORE_NAMESPACE, [{ 'source': 'test.yaml' }], catalog=self.spec_catalog) self.namespace_catalog = NamespaceCatalog() self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace) self.type_map = TypeMap(self.namespace_catalog) self.type_map.register_container_type(CORE_NAMESPACE, 'Bar', Bar) self.type_map.register_map(Bar, ObjectMapper) self.manager = BuildManager(self.type_map) self.mapper = ObjectMapper(self.bar_spec)
def test_catch_duplicate_spec_nested(self): spec1 = GroupSpec( data_type_def='Group1', doc='This is my new group 1', ) spec2 = GroupSpec( data_type_def='Group2', doc='This is my new group 2', groups=[spec1], # nested definition ) source = 'test_extension.yaml' self.catalog.register_spec(spec1, source) self.catalog.register_spec( spec2, source) # this is OK because Group1 is the same spec ret = self.catalog.get_spec('Group1') self.assertIs(ret, spec1)
def test_get_spec_source_file(self): spikes_spec = GroupSpec('test group', data_type_def='SpikeData') source_file_path = '/test/myt/test.yaml' self.catalog.auto_register(spikes_spec, source_file_path) recorded_source_file_path = self.catalog.get_spec_source_file( 'SpikeData') self.assertEqual(recorded_source_file_path, source_file_path)
def setUp(self): self.foo_spec = GroupSpec( doc='A test group specification with a data type', data_type_def='Foo', datasets=[ DatasetSpec(doc='an example dataset', dtype='int', name='my_data', attributes=[ AttributeSpec( name='attr2', doc='an example integer attribute', dtype='int') ]) ], attributes=[ AttributeSpec('attr1', 'an example string attribute', 'text') ]) self.spec_catalog = SpecCatalog() self.spec_catalog.register_spec(self.foo_spec, 'test.yaml') self.namespace = SpecNamespace('a test namespace', CORE_NAMESPACE, [{ 'source': 'test.yaml' }], version='0.1.0', catalog=self.spec_catalog) self.namespace_catalog = NamespaceCatalog() self.namespace_catalog.add_namespace(CORE_NAMESPACE, self.namespace) self.type_map = TypeMap(self.namespace_catalog) self.type_map.register_container_type(CORE_NAMESPACE, 'Foo', Foo) self.type_map.register_map(Foo, FooMapper) self.manager = BuildManager(self.type_map)
def test_post_process_fixed_name(self): """Test that docval generation for a class with a fixed name does not contain a docval arg for name.""" spec = GroupSpec( doc='A test group specification with a data type', data_type_def='Baz', name='MyBaz', # <-- fixed name attributes=[ AttributeSpec(name='attr1', doc='a string attribute', dtype='text', shape=[None]) ]) classdict = {} bases = [Container] docval_args = [{ 'name': 'name', 'type': str, 'doc': 'name' }, { 'name': 'attr1', 'type': ('array_data', 'data'), 'doc': 'a string attribute', 'shape': [None] }] CustomClassGenerator.post_process(classdict, bases, docval_args, spec) expected = [{ 'name': 'attr1', 'type': ('array_data', 'data'), 'doc': 'a string attribute', 'shape': [None] }] self.assertListEqual(docval_args, expected)
def getSpecs(self): ret = GroupSpec( 'A test group specification with a data type', data_type_def='Bar', datasets=[ DatasetSpec('an example dataset', 'int', name='data', attributes=[ AttributeSpec('attr2', 'an example integer attribute', 'int') ]), DatasetSpec('an example time dataset', 'isodatetime', name='time'), DatasetSpec('an array of times', 'isodatetime', name='time_array', dims=('num_times', ), shape=(None, )) ], attributes=[ AttributeSpec('attr1', 'an example string attribute', 'text') ]) return (ret, )
def test_set_parent_exists(self): GroupSpec('A test group', name='root_constructor', groups=self.subgroups) msg = 'Cannot re-assign parent.' with self.assertRaisesWith(AttributeError, msg): self.subgroups[0].parent = self.subgroups[1]
def test_dynamic_container_constructor(self): baz_spec = GroupSpec('A test extension with no Container class', data_type_def='Baz', data_type_inc=self.bar_spec, attributes=[ AttributeSpec('attr3', 'a float attribute', 'float'), AttributeSpec('attr4', 'another float attribute', 'float') ]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) # TODO: test that constructor works! inst = cls('My Baz', [1, 2, 3, 4], 'string attribute', 1000, attr3=98.6, attr4=1.0) self.assertEqual(inst.name, 'My Baz') self.assertEqual(inst.data, [1, 2, 3, 4]) self.assertEqual(inst.attr1, 'string attribute') self.assertEqual(inst.attr2, 1000) self.assertEqual(inst.attr3, 98.6) self.assertEqual(inst.attr4, 1.0)
def test_get_namespace_spec(self): expected = AttributeSpec( 'namespace', 'the namespace for the data type of this object', 'text', required=False) self.assertDictEqual(GroupSpec.get_namespace_spec(), expected)
def setUp(self): self.attr1 = AttributeSpec(name='attr1', doc='a string attribute', dtype='text') self.attr2 = AttributeSpec(name='attr2', doc='an integer attribute', dtype='int') self.attr3 = AttributeSpec(name='attr3', doc='an integer attribute', dtype='int') self.bar_spec = GroupSpec( doc='A test group specification with a data type', data_type_def='Bar', datasets=[ DatasetSpec(doc='a dataset', dtype='int', name='data', attributes=[self.attr2]) ], attributes=[self.attr1]) specs = [self.bar_spec] containers = {'Bar': Bar} self.type_map = create_test_type_map(specs, containers) self.spec_catalog = self.type_map.namespace_catalog.get_namespace( CORE_NAMESPACE).catalog self.cls = self.type_map.get_dt_container_cls(self.bar_spec.data_type) self.bar = self.cls(name='bar', data=[1], attr1='attr1', attr2=1) obj_mapper_bar = self.type_map.get_map(self.bar) obj_mapper_bar.map_spec('attr2', spec=self.attr2)
def test_build_empty_data(self): """Test building of a Data object with empty data.""" baz_inc_spec = DatasetSpec(doc='doc', data_type_inc='Baz', quantity=ZERO_OR_MANY) baz_holder_spec = GroupSpec(doc='doc', data_type_def='BazHolder', datasets=[baz_inc_spec]) self.spec_catalog.register_spec(baz_holder_spec, 'test.yaml') self.type_map.register_container_type(CORE_NAMESPACE, 'BazHolder', BazHolder) self.holder_mapper = ObjectMapper(baz_holder_spec) baz = Baz('MyBaz', [], 'abcdefghijklmnopqrstuvwxyz') holder = BazHolder('holder', [baz]) builder = self.holder_mapper.build(holder, self.manager) expected = GroupBuilder( name='holder', datasets=[ DatasetBuilder(name='MyBaz', data=[], attributes={ 'baz_attr': 'abcdefghijklmnopqrstuvwxyz', 'data_type': 'Baz', 'namespace': 'test_core', 'object_id': baz.object_id }) ]) self.assertBuilderEqual(builder, expected)
def test_process_field_spec_overwrite(self): """Test that docval generation overwrites previous docval args.""" spec = GroupSpec(doc='A test group specification with a data type', data_type_def='Baz', attributes=[ AttributeSpec(name='attr1', doc='a string attribute', dtype='text', shape=[None]) ]) not_inherited_fields = {'attr1': spec.get_attribute('attr1')} docval_args = [ { 'name': 'attr1', 'type': ('array_data', 'data'), 'doc': 'a string attribute', 'shape': [[None], [None, None]] }, # this dict will be overwritten below { 'name': 'attr2', 'type': ('array_data', 'data'), 'doc': 'a string attribute', 'shape': [[None], [None, None]] } ] CustomClassGenerator.process_field_spec( classdict={}, docval_args=docval_args, parent_cls=EmptyBar, # <-- arbitrary class attr_name='attr1', not_inherited_fields=not_inherited_fields, type_map=TypeMap(), spec=spec) expected = [{ 'name': 'attr1', 'type': ('array_data', 'data'), 'doc': 'a string attribute', 'shape': [None] }, { 'name': 'attr2', 'type': ('array_data', 'data'), 'doc': 'a string attribute', 'shape': [[None], [None, None]] }] self.assertListEqual(docval_args, expected)
def test_type_extension(self): spec = GroupSpec('A test group', name='parent_type', datasets=self.datasets, attributes=self.attributes, linkable=False, data_type_def='EphysData') dset1_attributes_ext = [ AttributeSpec('dset1_extra_attribute', 'an extra attribute for the first dataset', 'text') ] ext_datasets = [ DatasetSpec('my first dataset extension', 'int', name='dataset1', attributes=dset1_attributes_ext, linkable=True), ] ext_attributes = [ AttributeSpec('ext_extra_attribute', 'an extra attribute for the group', 'text'), ] ext = GroupSpec('A test group extension', name='child_type', datasets=ext_datasets, attributes=ext_attributes, linkable=False, data_type_inc=spec, data_type_def='SpikeData') ext_dset1 = ext.get_dataset('dataset1') ext_dset1_attrs = ext_dset1.attributes self.assertDictEqual(ext_dset1_attrs[0], dset1_attributes_ext[0]) self.assertDictEqual(ext_dset1_attrs[1], self.dset1_attributes[0]) self.assertDictEqual(ext_dset1_attrs[2], self.dset1_attributes[1]) self.assertEqual(ext.data_type_def, 'SpikeData') self.assertEqual(ext.data_type_inc, 'EphysData') ext_dset2 = ext.get_dataset('dataset2') self.maxDiff = None # this will suffice for now, assertDictEqual doesn't do deep equality checks self.assertEqual(str(ext_dset2), str(self.datasets[1])) self.assertAttributesEqual(ext_dset2, self.datasets[1]) # self.ns_attr_spec ndt_attr_spec = AttributeSpec('data_type', 'the data type of this object', # noqa: F841 'text', value='SpikeData') res_attrs = ext.attributes self.assertDictEqual(res_attrs[0], ext_attributes[0]) self.assertDictEqual(res_attrs[1], self.attributes[0]) self.assertDictEqual(res_attrs[2], self.attributes[1]) # test that inherited specs are tracked appropriate for d in self.datasets: with self.subTest(dataset=d.name): self.assertTrue(ext.is_inherited_spec(d)) self.assertFalse(spec.is_inherited_spec(d)) json.dumps(spec)
def test_three_named_datasets_same_type(self): """Test get_target_type when a group contains three named links with type X.""" child0 = DatasetSpec(doc='Group 0', data_type_inc='Type0', name='group0') child1 = DatasetSpec(doc='Group 1', data_type_inc='Type0', name='group1') child2 = DatasetSpec(doc='Group 2', data_type_inc='Type0', name='group2') parent_spec = GroupSpec(doc='A test group', name='parent', datasets=[child0, child1, child2], data_type_def='ParentType') self.assertEqual(parent_spec.get_data_type('Type0'), [child0, child1, child2])
def test_constructor(self): link0 = LinkSpec(doc='Link 0', target_type='TargetType0') link1 = LinkSpec(doc='Link 1', target_type='TargetType1') links = [link0, link1] spec = GroupSpec(doc='A test group', name='root', links=links) self.assertIs(spec, links[0].parent) self.assertIs(spec, links[1].parent) json.dumps(spec)
def setUpBarHolderSpec(self): ext_attr = AttributeSpec( name='ext_attr', dtype='bool', doc='A boolean attribute', ) bar_ext_no_name_spec = GroupSpec( doc='A Bar extended with attribute ext_attr', data_type_inc='Bar', quantity='*', attributes=[ext_attr], ) self.bar_holder_spec = GroupSpec( doc='A container of multiple extended Bar objects', data_type_def='BarHolder', groups=[bar_ext_no_name_spec], )
def setUpBarHolderSpec(self): int_attr2 = AttributeSpec( name='attr2', dtype='int64', doc='Refine Bar spec from int to int64', ) bar_ext_no_name_spec = GroupSpec( doc='A Bar extended with modified attribute attr2', data_type_inc='Bar', quantity='*', attributes=[int_attr2], ) self.bar_holder_spec = GroupSpec( doc='A container of multiple extended Bar objects', data_type_def='BarHolder', groups=[bar_ext_no_name_spec], )
def setUpBarSpec(self): self.bar_spec = GroupSpec( doc='A test group specification with a data type Bar', data_type_def='Bar', attributes=[ AttributeSpec('attr1', 'an example string attribute', 'text'), AttributeSpec('attr2', 'an example integer attribute', 'int') ])
def setUpBarSpec(self): self.bar_spec = GroupSpec( doc='A test group specification with a data type Bar', data_type_def='Bar', attributes=[ AttributeSpec('foo', 'a referenced foo', RefSpec('Foo', 'object')) ])