def test_group_contained(self):
g = og('',
gg=[og('subgroup', dd=[od('x', 4)],
vv=[ov('sv1', dd=[od('x'), od('y')])])])
add_missing_dims(g)
self.assertEqual(list(g.dimensions), [od('y')])
self.assertEqual(list(g.groups['subgroup'].dimensions), [od('x', 4)])
def _make_complex_group():
g = og(
'temp',
aa=[oa('a_root_attr_num', _long(1)),
oa('c_root_attr_str', 'xyz'),
oa('b_root_attr_vec', np.array([1.2, 3, 4]))],
dd=[od('root_dim_x', 2)],
vv=[ov('root_var_1',
dd=[od('root_dim_x')],
aa=[oa('root_var_attr_1', _long(11))],
data=np.zeros((2))),
ov('root_var_2_scalar',
data=np.array(3.15, dtype=np.float32))],
gg=[og('subgroup',
aa=[oa('subgroup_attr', 'qq')],
dd=[od('subgroup_dim_y', 3)],
vv=[ov('subgroup_var',
dd=[od('root_dim_x'), od('subgroup_dim_y')],
aa=[oa('subgroup_var_attr', 57.5)],
data=np.zeros((2, 3)))],
gg=[og('sub_sub_group',
aa=[oa('sub_sub_group_attr', 'this')],
vv=[ov('sub_sub_group_var',
dd=[od('subgroup_dim_y')],
data=np.zeros((3)))])]),
og('sg_2_empty')])
ncg.complete(g)
return g
def test_grouped_okay(self):
g = og('', dd=[od('x'), od('y')],
vv=[ov('v1', dd=[od('x')]), ov('v2', dd=[od('x'), od('y')])],
gg=[og('subgroup', dd=[od('z')],
vv=[ov('sv1', dd=[od('x')]),
ov('sv2', dd=[od('y'), od('z')])])])
self.assertTrue(has_no_missing_dims(g))
def test_var_vs_grp(self):
g = og('root', vv=[ov('tst1')], gg=[og('tst1')])
with self.assertRaises(NameConflictError) as err_context:
check_names(g)
msg = err_context.exception.message
self.check_all_in_str(msg, ['group "root"', 'both',
'variable', 'group', 'tst1'])
def eg_simple_grouped():
"""Produce a grouped version of the simple containers example."""
d_lat = od('lat', 10)
d_lon = od('lon', 5)
d_time = od('time', u=True)
dims = [d_lat, d_lon, d_time]
c_lat = ov('lat', dd=[d_lat], aa=[oa('units', 'degrees_north')])
c_lon = ov('lon', dd=[d_lon], aa=[oa('units', 'degrees_east')])
c_time = ov('time', dd=[d_time], aa=[oa('units', 'seconds')])
coords = [c_lat, c_lon, c_time]
g = og('foo',
dd=dims,
gg=[og('instrument',
vv=(coords +
[ov('rh', dd=dims, aa=[oa('_FillValue', -1)])]),
aa=[oa('container_type', 'simple'),
oa('measurement_platform', 'aircraft')]),
og('model',
vv=(coords +
[ov('rh', dd=dims, aa=[oa('_FillValue', -1)])]),
aa=[oa('container_type', 'simple'),
oa('measurement_platform',
'global circulation model')])])
_fake_complete(g)
return g
def test_subgroup(self):
g = og('',
gg=[og('subgroup', vv=[ov('v1', dd=[od('y', 2)])])])
self.do_complete(g)
self.assertEqual(len(g.dimensions), 1)
self.assertEqual(list(g.dimensions), [od('y', 2)])
self.assertIs(g.groups['subgroup'].variables['v1'].dimensions[0],
g.dimensions['y'])
def test_okay_override_subgroup(self):
# NOTE: here *two* 'x' dimensions, which do not conflict.
g = og('', dd=[od('x')],
vv=[ov('v1', dd=[od('x', 2)])],
gg=[og('subgroup',
dd=[od('x')],
vv=[ov('v2', dd=[od('x', 3)])])])
check_dims(g)
def test_subgroup_okay(self):
g = og('root', vv=[ov('test_var')],
gg=[og('tst1', vv=[ov('test_var')])])
subgroup = g.groups['tst1']
self.assertFalse(group_is_tagged(g))
self.assertFalse(group_is_tagged(subgroup))
check_names(g)
self.assertFalse(group_is_tagged(g))
self.assertFalse(group_is_tagged(subgroup))
def test_inner_groups(self):
g = og('group_name', gg=[og('sub_group')])
result = cdl(g)
self.assertEqual(result,
'netcdf group_name {\n'
'\n'
'group: sub_group {\n'
'} // group sub_group\n'
'}')
def test_fail_nomatch_subgroup(self):
g = og('', dd=[od('x')],
vv=[ov('v1', dd=[od('x', 2)])],
gg=[og('subgroup', vv=[ov('v2', dd=[od('x', 3)])])])
with self.assertRaises(DimensionConflictError) as err_context:
check_dims(g)
msg = err_context.exception.message
self.check_all_in_str(msg, ['/v1', '"x" = 2',
'/subgroup/v2', '"x" = 3'])
def test_with_varsdata_okay(self):
g = og('', dd=[od('x')],
vv=[ov('v1', dd=[od('x', 2)])],
gg=[og('subgroup', vv=[ov('v2', dd=[od('x', 2)])])])
subgroup = g.groups['subgroup']
tag_group(g)
self.assertTrue(group_is_tagged(g))
self.assertFalse(group_is_tagged(subgroup))
check_dims(g)
self.assertTrue(group_is_tagged(g))
self.assertFalse(group_is_tagged(subgroup))
def test_grouped_fail(self):
g = og('', dd=[od('x'), od('y')],
vv=[ov('v1', dd=[od('x')]), ov('v2', dd=[od('x'), od('y')])],
gg=[og('subgroup', dd=[od('zz')],
vv=[ov('sv1', dd=[od('x')]),
ov('sv2', dd=[od('y'), od('z')])])])
with self.assertRaises(IncompleteStructureError) as err_context:
has_no_missing_dims(g, fail_if_not=True)
msg = err_context.exception.message
self.check_all_in_str(msg, [
'Variable "/subgroup/sv2"', 'dimension "z"',
'no definition exists'])
def test_partial_grouped(self):
# Build a group with one dimension to be found in the subgroup.
g = og('',
gg=[og('subgroup',
dd=[od('x', 4)],
vv=[ov('subvar_1', dd=[od('x')]),
ov('subvar_2', dd=[od('z', 3), od('x')])])])
dim_x_def = g.groups['subgroup'].dimensions['x']
self.do_complete(g)
self.assertEqual(list(g.dimensions), [od('z', 3)])
self.assertEqual(list(g.groups['subgroup'].dimensions), [dim_x_def])
self.assertIs(g.groups['subgroup'].dimensions['x'], dim_x_def)
def test_with_varsdata_fail(self):
g = og('', dd=[od('x')],
vv=[ov('v1', dd=[od('x', 2)])],
gg=[og('subgroup', vv=[ov('v2', dd=[od('x', 3)])])])
subgroup = g.groups['subgroup']
tag_group(g)
self.assertTrue(group_is_tagged(g))
self.assertFalse(group_is_tagged(subgroup))
with self.assertRaises(DimensionConflictError):
check_dims(g)
self.assertTrue(group_is_tagged(g))
self.assertFalse(group_is_tagged(subgroup))
def test_inner_group_attr(self):
g = og('group_name',
gg=[og('sub_group',
aa=[oa('x', _long(2))])])
result = cdl(g)
self.assertEqual(result,
'netcdf group_name {\n'
'\n'
'group: sub_group {\n'
'\n'
'// group attributes:\n'
' :x = 2L ;\n'
'} // group sub_group\n'
'}')
def test_subgroup_mixed(self):
g = og('',
vv=[ov('v1', dd=[od('q')])],
gg=[og('subgroup',
vv=[ov('v1', dd=[od('q', 2)])])])
self.assertEqual(list(g.dimensions), [])
v1 = g.variables['v1']
v2 = g.groups['subgroup'].variables['v1']
self.do_complete(g)
self.assertEqual(len(g.dimensions), 1)
self.assertEqual(list(g.dimensions), [od('q', 2)])
test_dim = g.dimensions['q']
self.assertIs(v1.dimensions[0], test_dim)
self.assertIs(v2.dimensions[0], test_dim)
def test_shared_clash(self):
# NOTE: "complete" won't allow this, but this method doesn't care.
g = og('', vv=[ov('v1', dd=[od('x', 2)]),
ov('v2', dd=[od('x', 3)])])
add_missing_dims(g)
self.assertEqual(len(g.dimensions), 1)
self.assertEqual(g.dimensions.names(), ['x'])
def test_unlimited(self):
g = og('', vv=[ov('v', dd=[od('x', 2, u=True)])])
self.assertEqual(all_dimensions(g), [])
add_missing_dims(g)
self.assertEqual(len(g.dimensions), 1)
self.assertNotEqual(all_dimensions(g), [od('x', 2)])
self.assertEqual(all_dimensions(g), [od('x', 2, u=True)])
def eg_simple_flat():
"""Produce a flat version of the simple containers example."""
d_lat = od('lat', 10)
d_lon = od('lon', 5)
d_time = od('time', u=True)
dims = [d_lat, d_lon, d_time]
collection_vars = [
ov('instrument', data=np.array(0),
aa=[oa('container_type', 'simple'),
oa('members',
('instrument___lat instrument___lon instrument___rh '
'instrument___time')),
oa('measurement_platform', 'aircraft')]),
ov('model', data=np.array(0),
aa=[oa('container_type', 'simple'),
oa('members',
'model___lat model___lon model___rh model___time'),
oa('measurement_platform', 'global circulation model')])]
data_vars = [
ov('instrument___lat', dd=[d_lat], aa=[oa('units', 'degrees_north')]),
ov('instrument___lon', dd=[d_lon], aa=[oa('units', 'degrees_east')]),
ov('instrument___time', dd=[d_time], aa=[oa('units', 'seconds')]),
ov('instrument___rh', dd=dims, aa=[oa('_FillValue', -1)]),
ov('model___lat', dd=[d_lat], aa=[oa('units', 'degrees_north')]),
ov('model___lon', dd=[d_lon], aa=[oa('units', 'degrees_east')]),
ov('model___time', dd=[d_time], aa=[oa('units', 'seconds')]),
ov('model___rh', dd=dims, aa=[oa('_FillValue', -1)])]
g = og('foo', dd=dims, vv=collection_vars + data_vars)
_fake_complete(g)
return g
def test_mixture(self):
g = og('', dd=[od('x')], vv=[ov('v', dd=[od('y')])])
self.assertEqual(all_dimensions(g), [od('x')])
r = add_missing_dims(g)
self.assertEqual(r, [od('y')])
self.assertEqual(len(g.dimensions), 2)
self.assertIs(r[0], g.dimensions['y'])
def test_missing(self):
g = og('', vv=[ov('v', dd=[od('x')])])
self.assertEqual(all_dimensions(g), [])
r = add_missing_dims(g)
self.assertEqual(r, [od('x')])
self.assertEqual(all_dimensions(g), [od('x')])
self.assertIs(g.dimensions['x'], r[0])
def group_flat_containers(group):
"""Example operation, how to group a flat containers representation."""
result = group.detached_copy()
# find variables representing containers.
con_vars = [var for var in ncg.all_variables(result)
if 'container_type' in var.attributes.names()]
# produce a group from each of these
for con_var in con_vars:
# container variables should have no dimensions.
assert not con_var.dimensions
# remove container variable and make a group instead
con_name = con_var.name
result.variables.remove(con_var)
result.groups.add(og(con_name, aa=con_var.attributes))
con_grp = result.groups[con_name]
# remove redundant 'members' attribute inherited from container var
con_grp.attributes.pop('members')
# move member variables into the group (removing prefixes)
prefix = con_name + '___'
memb_names = con_var.attributes['members'].value.split(' ')
for memb_name in memb_names:
# remove member from root
memb_var = result.variables.pop(memb_name)
# strip initial disambiguation prefix, if present
if memb_var.name.startswith(prefix):
memb_name = memb_name[len(prefix):]
memb_var.rename(memb_name)
# place member in group
con_grp.variables.add(memb_var)
return result
def test_fail_error(self):
g = og('', vv=[ov('v', dd=[od('x')])])
with self.assertRaises(IncompleteStructureError) as err_context:
has_no_missing_dims(g, fail_if_not=True)
msg = err_context.exception.message
self.check_all_in_str(msg, [
'Variable "/v"', 'dimension "x"', 'no definition exists'])
def test_simple_data(self):
g = og('', vv=[ov('v', dd=[od('y'), od('x')],
data=_mockdata((15, 20)))])
self.do_complete(g)
self.assertEqual(len(g.dimensions), 2)
self.assertEqual(g.dimensions['x'], od('x', 20))
self.assertEqual(g.dimensions['y'], od('y', 15))
def test_fail_bad_data_shapes(self):
d1 = _mockdata((5, 3, 2))
g = og('', dd=[od('x'), od('y')],
vv=[ov('v1', dd=[od('x'), od('y')], data=d1)])
with self.assertRaises(DimensionConflictError) as err_context:
check_dims(g)
msg = err_context.exception.message
self.check_all_in_str(msg, ['/v1', '3 dimensions', '2 dimensions'])
def test_type_bad(self):
g = og('')
with self.assertRaises(ValueError) as err_context:
fnd(g, 'name', type(None))
msg = err_context.exception.message
self.check_all_in_str(msg, ['<type \'NoneType\'>',
'not recognised',
'or not supported'])
def test_type_unsupported(self):
g = og('')
with self.assertRaises(ValueError) as err_context:
fnd(g, 'name', nco.Group)
msg = err_context.exception.message
self.check_all_in_str(msg, ['ncobj.Group',
'not recognised',
'or not supported'])
def test_nomissing(self):
g = og('', dd=[od('x')], vv=[ov('v', dd=[od('x')])])
test_dim = g.dimensions['x']
str_before = str(g)
r = add_missing_dims(g)
self.assertEqual(r, [])
self.assertIs(g.dimensions['x'], test_dim)
self.assertEqual(str(g), str_before)
def test_length(self):
g = og('', vv=[ov('v', dd=[od('x', 2)])])
self.assertEqual(all_dimensions(g), [])
r = add_missing_dims(g)
self.assertEqual(r, [od('x', 2)])
self.assertEqual(len(g.dimensions), 1)
self.assertNotEqual(all_dimensions(g), [od('x')])
self.assertEqual(all_dimensions(g), [od('x', 2)])
def test_fail_conflicting_data_shapes(self):
d1 = _mockdata((2, 23))
v1 = ov('v1', dd=[od('x'), od('y')], data=d1)
v2 = ov('v2', dd=[od('x', 17), od('y', 23)])
g = og('', dd=[od('x'), od('y')], vv=[v1, v2])
with self.assertRaises(DimensionConflictError) as err_context:
check_dims(g)
msg = err_context.exception.message
self.check_all_in_str(msg, ['/v1', '/v2', '"x" = 17', '"x" = 2'])