def test_mutually_exclusive_subgroups(self):
gb1 = GroupBuilder('gb1', {'subgroup1': GroupBuilder('subgroup1')})
gb2 = GroupBuilder('gb2', {'subgroup2': GroupBuilder('subgroup2')})
gb1.deep_update(gb2)
self.assertIn('subgroup2', gb1)
gb1sg = gb1['subgroup2']
gb2sg = gb2['subgroup2']
self.assertIs(gb1sg, gb2sg)
def get_two_bucket_test(self):
foos = [SimpleFoo('my_foo1'), SimpleFoo('my_foo2')]
quxs = [
SimpleQux('my_qux1', data=[1, 2, 3]),
SimpleQux('my_qux2', data=[4, 5, 6])
]
bucket = SimpleBucket(name='test_bucket',
foos=foos,
quxs=quxs,
links=foos)
foo1_builder = self._create_builder(bucket.foos['my_foo1'])
foo2_builder = self._create_builder(bucket.foos['my_foo2'])
qux1_builder = self._create_builder(bucket.quxs['my_qux1'])
qux2_builder = self._create_builder(bucket.quxs['my_qux2'])
foo_holder_builder = GroupBuilder(name='foo_holder',
groups={
'my_foo1': foo1_builder,
'my_foo2': foo2_builder
})
qux_holder_builder = GroupBuilder(name='qux_holder',
datasets={
'my_qux1': qux1_builder,
'my_qux2': qux2_builder
})
foo1_link_builder = LinkBuilder(builder=foo1_builder)
foo2_link_builder = LinkBuilder(builder=foo2_builder)
link_holder_builder = GroupBuilder(name='link_holder',
links={
'my_foo1': foo1_link_builder,
'my_foo2': foo2_link_builder
})
bucket_builder = GroupBuilder(name='test_bucket',
groups={
'foos': foo_holder_builder,
'quxs': qux_holder_builder,
'links': link_holder_builder
},
attributes={
'namespace': CORE_NAMESPACE,
'data_type': 'SimpleBucket',
'object_id': bucket.object_id
})
return bucket, bucket_builder
def test_required_group_missing(self):
"""Test mapping when one container is missing a required group."""
bar_inst1 = Bar('my_bar1', list(range(10)), 'value1', 10)
msg = "Bar 'my_bar1' is missing required value for attribute 'foo'."
with self.assertWarnsWith(MissingRequiredBuildWarning, msg):
builder = self.mapper.build(bar_inst1, self.manager)
expected = GroupBuilder(name='my_bar1', )
self.assertBuilderEqual(expected, builder)
def test_invalid_missing_req_type(self):
foo_builder = GroupBuilder('my_foo',
attributes={
'data_type': 'Foo',
'foo_attr': text('example Foo object')
})
results = self.vmap.validate(foo_builder)
self.assertIsInstance(results[0], MissingDataType) # noqa: F405
self.assertEqual(results[0].name, 'Foo')
self.assertEqual(results[0].reason, 'missing data type Bar')
def test_unnamed_group_data_type_inc(self):
"""Test get_subspec on a builder that maps to an unnamed subgroup of the given spec using data_type_inc."""
inc_spec = GroupSpec(doc='This Bar', data_type_inc='Bar')
parent_spec = GroupSpec(
doc='Something to hold a Bar',
name='bar_bucket',
groups=[inc_spec] # using data_type_inc
)
sub_builder = GroupBuilder(name='my_bar',
attributes={
'data_type': 'Bar',
'namespace': CORE_NAMESPACE,
'object_id': -1
})
GroupBuilder(name='bar_bucket',
groups={'my_bar': sub_builder
}) # add sub_builder as a child to bar_bucket
result = self.type_map.get_subspec(parent_spec, sub_builder)
self.assertIs(result, inc_spec)
def setUp(self):
super(TestNestedBase, self).setUp()
self.foo_bucket = FooBucket('test_foo_bucket', [
Foo('my_foo1', list(range(10)), 'value1', 10),
Foo('my_foo2', list(range(10, 20)), 'value2', 20)
])
self.foo_builders = {
'my_foo1':
GroupBuilder('my_foo1',
datasets={
'my_data':
DatasetBuilder('my_data',
list(range(10)),
attributes={'attr2': 10})
},
attributes={
'attr1': 'value1',
'namespace': CORE_NAMESPACE,
'data_type': 'Foo'
}),
'my_foo2':
GroupBuilder('my_foo2',
datasets={
'my_data':
DatasetBuilder('my_data',
list(range(10, 20)),
attributes={'attr2': 20})
},
attributes={
'attr1': 'value2',
'namespace': CORE_NAMESPACE,
'data_type': 'Foo'
})
}
self.setUpBucketBuilder()
self.setUpBucketSpec()
self.spec_catalog.register_spec(self.bucket_spec, 'test.yaml')
self.type_map.register_container_type(CORE_NAMESPACE, 'FooBucket',
FooBucket)
self.type_map.register_map(FooBucket, ObjectMapper)
self.manager = BuildManager(self.type_map)
def test_invalid_wrong_name_req_type(self):
bar_builder = GroupBuilder(
'bad_bar_name',
attributes={
'data_type': 'Bar',
'attr1': 'a string attribute'
},
datasets=[DatasetBuilder('data', 100, attributes={'attr2': 10})])
foo_builder = GroupBuilder('my_foo',
attributes={
'data_type': 'Foo',
'foo_attr': text('example Foo object')
},
groups=[bar_builder])
results = self.vmap.validate(foo_builder)
self.assertEqual(len(results), 1)
self.assertIsInstance(results[0], MissingDataType) # noqa: F405
self.assertEqual(results[0].data_type, 'Bar')
def test_get_ns_dt(self):
bldr = GroupBuilder('my_foo',
attributes={
'attr1': 'value1',
'namespace': 'CORE',
'data_type': 'Foo'
})
dt = self.type_map.get_builder_dt(bldr)
ns = self.type_map.get_builder_ns(bldr)
self.assertEqual(dt, 'Foo')
self.assertEqual(ns, 'CORE')
def setUpBuilder(self):
device_builder = GroupBuilder('dev1',
attributes={
'neurodata_type':
'Device',
'namespace':
'core',
'help':
'A recording device e.g. amplifier'
})
return GroupBuilder(
'elec1',
attributes={
'neurodata_type': 'ElectrodeGroup',
'namespace': 'core',
'help': 'A physical grouping of channels',
'description': 'a test ElectrodeGroup',
'location': 'a nonexistent place'
},
links={'device': LinkBuilder(device_builder, 'device')})
def setUpBucketBuilder(self):
self.bucket_builder = GroupBuilder('test_foo_bucket',
groups=self.foo_builders,
attributes={
'namespace':
CORE_NAMESPACE,
'data_type':
'FooBucket',
'object_id':
self.foo_bucket.object_id
})
def get_zero_bucket_test(self):
bucket = SimpleBucket(
name='test_bucket'
)
bucket_builder = GroupBuilder(
name='test_bucket',
attributes={'namespace': CORE_NAMESPACE,
'data_type': 'SimpleBucket',
'object_id': bucket.object_id}
)
return bucket, bucket_builder
def test_construct_memoization(self):
builder = GroupBuilder(
'my_foo', datasets={'my_data': DatasetBuilder(
'my_data',
list(range(10)),
attributes={'attr2': 10})},
attributes={'attr1': 'value1', 'namespace': CORE_NAMESPACE, 'data_type': 'Foo',
'object_id': -1})
container1 = self.manager.construct(builder)
container2 = self.manager.construct(builder)
self.assertIs(container1, container2)
def test_valid(self):
builder = GroupBuilder(
'my_bar',
attributes={
'data_type': 'Bar',
'attr1': text('a string attribute')
},
datasets=[DatasetBuilder('data', 100, attributes={'attr2': 10})])
validator = self.vmap.get_validator('Bar')
result = validator.validate(builder)
self.assertEqual(len(result), 0)
def test_build_empty(self):
''' Test default mapping functionality when no attributes are nested '''
container = Bar('my_bar', [], 'value1', 10)
builder = self.mapper.build(container, self.manager)
expected = GroupBuilder('my_bar',
datasets={'data': DatasetBuilder('data', [])},
attributes={
'attr1': 'value1',
'attr2': 10
})
self.assertBuilderEqual(builder, expected)
def test_optional_group_not_empty(self):
"""Test mapping when one container has an optional not empty group."""
self.mapper.map_spec(
'attr3',
self.mapper.spec.get_group('empty').get_attribute('attr3'))
bar_inst1 = Bar('my_bar1', list(range(10)), 'value1', 10, attr3=1.23)
builder = self.mapper.build(bar_inst1, self.manager)
expected = GroupBuilder(
name='my_bar1',
groups={
'empty': GroupBuilder(
name='empty',
attributes={'attr3': 1.23},
)
},
)
self.assertBuilderEqual(expected, builder)
def test_named_group(self):
"""Test get_subspec on a builder that maps to a named subgroup of the given spec."""
# NOTE this works despite the fact that child_spec has no data type but the builder has a data type because
# get_subspec acts on the name and not necessarily the data type
child_spec = GroupSpec(doc='A test group specification',
name='my_subgroup')
parent_spec = GroupSpec(doc='Something to hold a Bar',
name='my_group',
groups=[child_spec])
sub_builder = GroupBuilder(name='my_subgroup',
attributes={
'data_type': 'Bar',
'namespace': CORE_NAMESPACE,
'object_id': -1
})
GroupBuilder(name='my_group',
groups={'my_bar': sub_builder
}) # add sub_builder as a child to my_group
result = self.type_map.get_subspec(parent_spec, sub_builder)
self.assertIs(result, child_spec)
def test_build_attr_ref(self):
''' Test default mapping functionality when one container contains an attribute reference to another container.
'''
foo_inst = Foo('my_foo')
bar_inst1 = Bar('my_bar1', list(range(10)), 'value1', 10, foo=foo_inst)
bar_inst2 = Bar('my_bar2', list(range(10)), 'value1', 10)
foo_builder = self.manager.build(foo_inst, root=True)
bar1_builder = self.manager.build(bar_inst1, root=True) # adds refs
bar2_builder = self.manager.build(bar_inst2, root=True)
foo_expected = GroupBuilder('my_foo',
attributes={
'data_type': 'Foo',
'namespace': CORE_NAMESPACE,
'object_id': foo_inst.object_id
})
bar1_expected = GroupBuilder(
'n/a', # name doesn't matter
datasets={'data': DatasetBuilder('data', list(range(10)))},
attributes={
'attr1': 'value1',
'attr2': 10,
'foo': ReferenceBuilder(foo_expected),
'data_type': 'Bar',
'namespace': CORE_NAMESPACE,
'object_id': bar_inst1.object_id
})
bar2_expected = GroupBuilder(
'n/a', # name doesn't matter
datasets={'data': DatasetBuilder('data', list(range(10)))},
attributes={
'attr1': 'value1',
'attr2': 10,
'data_type': 'Bar',
'namespace': CORE_NAMESPACE,
'object_id': bar_inst2.object_id
})
self.assertDictEqual(foo_builder, foo_expected)
self.assertDictEqual(bar1_builder, bar1_expected)
self.assertDictEqual(bar2_builder, bar2_expected)
def test_empty_list(self):
"""Test that validator allows an empty list where an array is specified."""
self.set_up_spec('text')
value = []
bar_builder = GroupBuilder('my_bar',
attributes={
'data_type': 'Bar',
'attr1': value
},
datasets=[DatasetBuilder('data', value)])
results = self.vmap.validate(bar_builder)
self.assertEqual(len(results), 0)
def test_int64_for_numeric(self):
"""Test that validator allows int64 data where numeric is specified."""
self.set_up_spec('numeric')
value = np.int64(1)
bar_builder = GroupBuilder('my_bar',
attributes={
'data_type': 'Bar',
'attr1': value
},
datasets=[DatasetBuilder('data', value)])
results = self.vmap.validate(bar_builder)
self.assertEqual(len(results), 0)
def test_ascii_for_utf8(self):
"""Test that validator allows ASCII data where UTF8 is specified."""
self.set_up_spec('text')
value = b'an ascii string'
bar_builder = GroupBuilder('my_bar',
attributes={
'data_type': 'Bar',
'attr1': value
},
datasets=[DatasetBuilder('data', value)])
results = self.vmap.validate(bar_builder)
self.assertEqual(len(results), 0)
def setUpBuilder(self):
es = self.setUpElectricalSeriesBuilders()
ret = GroupBuilder('FilteredEphys',
attributes={
'namespace': base.CORE_NAMESPACE,
'neurodata_type': 'FilteredEphys'
},
groups={
'test_es1': es[0],
'test_es2': es[1]
})
return ret
def test_mutually_exclusive_links(self):
gb1 = GroupBuilder(
'gb1',
links={'link1': LinkBuilder(GroupBuilder('target1'), 'link1')})
gb2 = GroupBuilder(
'gb2',
links={'link2': LinkBuilder(GroupBuilder('target2'), 'link2')})
gb1.deep_update(gb2)
self.assertIn('link2', gb2)
self.assertEqual(gb1['link2'], gb2['link2'])
def test_required_attr_missing(self):
"""Test mapping when one container is missing a required attribute."""
bar_inst1 = Bar('my_bar1', list(range(10)), 'value1', 10)
bar_inst1._Bar__attr1 = None # make attr1 attribute None
msg = "Bar 'my_bar1' is missing required value for attribute 'attr1'."
with self.assertWarnsWith(MissingRequiredBuildWarning, msg):
builder = self.mapper.build(bar_inst1, self.manager)
expected = GroupBuilder(name='my_bar1', attributes={'attr2': 10})
self.assertBuilderEqual(expected, builder)
def test_intersecting_links(self):
gb1 = GroupBuilder(
'gb1',
links={'link2': LinkBuilder(GroupBuilder('target1'), 'link2')})
gb2 = GroupBuilder(
'gb2',
links={'link2': LinkBuilder(GroupBuilder('target2'), 'link2')})
gb1.deep_update(gb2)
self.assertIn('link2', gb2)
self.assertEqual(gb1['link2'], gb2['link2'])
def test_empty_nparray(self):
"""Test that validator allows an empty numpy array where an array is specified."""
self.set_up_spec('text')
value = np.array([]) # note: dtype is float64
bar_builder = GroupBuilder('my_bar',
attributes={
'data_type': 'Bar',
'attr1': value
},
datasets=[DatasetBuilder('data', value)])
results = self.vmap.validate(bar_builder)
self.assertEqual(len(results), 0)
def test_invalid_incorrect_type_validate(self):
builder = GroupBuilder('my_bar',
attributes={
'data_type': 'Bar',
'attr1': 10
})
result = self.vmap.validate(builder)
self.assertEqual(len(result), 2)
self.assertIsInstance(result[0], DtypeError) # noqa: F405
self.assertEqual(result[0].name, 'Bar/attr1')
self.assertIsInstance(result[1], MissingError) # noqa: F405
self.assertEqual(result[1].name, 'Bar/data')
def test_build_child_link(self):
''' Test default mapping functionality when one container contains a child link to another container '''
foo_inst = Foo('my_foo')
bar_inst1 = Bar('my_bar1', list(range(10)), 'value1', 10, foo=foo_inst)
# bar_inst2.foo should link to bar_inst1.foo
bar_inst2 = Bar('my_bar2', list(range(10)), 'value1', 10, foo=foo_inst)
foo_builder = self.foo_mapper.build(foo_inst, self.manager)
bar1_builder = self.bar_mapper.build(bar_inst1, self.manager)
bar2_builder = self.bar_mapper.build(bar_inst2, self.manager)
foo_expected = GroupBuilder('my_foo')
inner_foo_builder = GroupBuilder('my_foo',
attributes={
'data_type': 'Foo',
'namespace': CORE_NAMESPACE,
'object_id': foo_inst.object_id
})
bar1_expected = GroupBuilder(
'my_bar1',
datasets={'data': DatasetBuilder('data', list(range(10)))},
groups={'foo': inner_foo_builder},
attributes={
'attr1': 'value1',
'attr2': 10
})
link_foo_builder = LinkBuilder(builder=inner_foo_builder)
bar2_expected = GroupBuilder(
'my_bar2',
datasets={'data': DatasetBuilder('data', list(range(10)))},
links={'foo': link_foo_builder},
attributes={
'attr1': 'value1',
'attr2': 10
})
self.assertBuilderEqual(foo_builder, foo_expected)
self.assertBuilderEqual(bar1_builder, bar1_expected)
self.assertBuilderEqual(bar2_builder, bar2_expected)
def test_optional_group_empty(self):
"""Test mapping when one container has an optional empty group."""
self.mapper.map_spec(
'attr3',
self.mapper.spec.get_group('empty').get_attribute('attr3'))
bar_inst1 = Bar('my_bar1', list(range(10)), 'value1', 10)
bar_inst1._Bar__attr3 = None # force attr3 to be None
builder = self.mapper.build(bar_inst1, self.manager)
expected = GroupBuilder(name='my_bar1', )
self.assertBuilderEqual(expected, builder)
def test_build_empty_data(self):
"""Test building a container which contains an untyped empty subgroup and an untyped empty dataset."""
bucket = BasicBucket(name='test_bucket')
# an optional untyped empty group builder will NOT be created by default
# an optional untyped empty dataset builder will NOT be created by default
bucket_builder = GroupBuilder(
name='test_bucket',
attributes={'namespace': CORE_NAMESPACE,
'data_type': 'BasicBucket',
'object_id': bucket.object_id}
)
builder = self.manager.build(bucket)
self.assertDictEqual(builder, bucket_builder)
def test_required_attr_missing(self):
"""Test mapping when one container has a required attribute with a fixed value."""
bar_inst1 = Bar('my_bar1', list(range(10)), 'value1',
10) # attr1=value1 is not processed
builder = self.mapper.build(bar_inst1, self.manager)
expected = GroupBuilder(name='my_bar1',
attributes={
'attr1': 'hi',
'attr2': 10
})
self.assertBuilderEqual(builder, expected)
