def test_group_contained(self): g = og('', gg=[og('subgroup', dd=[od('x', 4)], vv=[ov('sv1', dd=[od('x'), od('y')])])]) add_missing_dims(g) self.assertEqual(list(g.dimensions), [od('y')]) self.assertEqual(list(g.groups['subgroup'].dimensions), [od('x', 4)])
def _make_complex_group(): g = og( 'temp', aa=[oa('a_root_attr_num', _long(1)), oa('c_root_attr_str', 'xyz'), oa('b_root_attr_vec', np.array([1.2, 3, 4]))], dd=[od('root_dim_x', 2)], vv=[ov('root_var_1', dd=[od('root_dim_x')], aa=[oa('root_var_attr_1', _long(11))], data=np.zeros((2))), ov('root_var_2_scalar', data=np.array(3.15, dtype=np.float32))], gg=[og('subgroup', aa=[oa('subgroup_attr', 'qq')], dd=[od('subgroup_dim_y', 3)], vv=[ov('subgroup_var', dd=[od('root_dim_x'), od('subgroup_dim_y')], aa=[oa('subgroup_var_attr', 57.5)], data=np.zeros((2, 3)))], gg=[og('sub_sub_group', aa=[oa('sub_sub_group_attr', 'this')], vv=[ov('sub_sub_group_var', dd=[od('subgroup_dim_y')], data=np.zeros((3)))])]), og('sg_2_empty')]) ncg.complete(g) return g
def test_grouped_okay(self): g = og('', dd=[od('x'), od('y')], vv=[ov('v1', dd=[od('x')]), ov('v2', dd=[od('x'), od('y')])], gg=[og('subgroup', dd=[od('z')], vv=[ov('sv1', dd=[od('x')]), ov('sv2', dd=[od('y'), od('z')])])]) self.assertTrue(has_no_missing_dims(g))
def test_var_vs_grp(self): g = og('root', vv=[ov('tst1')], gg=[og('tst1')]) with self.assertRaises(NameConflictError) as err_context: check_names(g) msg = err_context.exception.message self.check_all_in_str(msg, ['group "root"', 'both', 'variable', 'group', 'tst1'])
def eg_simple_grouped(): """Produce a grouped version of the simple containers example.""" d_lat = od('lat', 10) d_lon = od('lon', 5) d_time = od('time', u=True) dims = [d_lat, d_lon, d_time] c_lat = ov('lat', dd=[d_lat], aa=[oa('units', 'degrees_north')]) c_lon = ov('lon', dd=[d_lon], aa=[oa('units', 'degrees_east')]) c_time = ov('time', dd=[d_time], aa=[oa('units', 'seconds')]) coords = [c_lat, c_lon, c_time] g = og('foo', dd=dims, gg=[og('instrument', vv=(coords + [ov('rh', dd=dims, aa=[oa('_FillValue', -1)])]), aa=[oa('container_type', 'simple'), oa('measurement_platform', 'aircraft')]), og('model', vv=(coords + [ov('rh', dd=dims, aa=[oa('_FillValue', -1)])]), aa=[oa('container_type', 'simple'), oa('measurement_platform', 'global circulation model')])]) _fake_complete(g) return g
def test_subgroup(self): g = og('', gg=[og('subgroup', vv=[ov('v1', dd=[od('y', 2)])])]) self.do_complete(g) self.assertEqual(len(g.dimensions), 1) self.assertEqual(list(g.dimensions), [od('y', 2)]) self.assertIs(g.groups['subgroup'].variables['v1'].dimensions[0], g.dimensions['y'])
def test_okay_override_subgroup(self): # NOTE: here *two* 'x' dimensions, which do not conflict. g = og('', dd=[od('x')], vv=[ov('v1', dd=[od('x', 2)])], gg=[og('subgroup', dd=[od('x')], vv=[ov('v2', dd=[od('x', 3)])])]) check_dims(g)
def test_subgroup_okay(self): g = og('root', vv=[ov('test_var')], gg=[og('tst1', vv=[ov('test_var')])]) subgroup = g.groups['tst1'] self.assertFalse(group_is_tagged(g)) self.assertFalse(group_is_tagged(subgroup)) check_names(g) self.assertFalse(group_is_tagged(g)) self.assertFalse(group_is_tagged(subgroup))
def test_inner_groups(self): g = og('group_name', gg=[og('sub_group')]) result = cdl(g) self.assertEqual(result, 'netcdf group_name {\n' '\n' 'group: sub_group {\n' '} // group sub_group\n' '}')
def test_fail_nomatch_subgroup(self): g = og('', dd=[od('x')], vv=[ov('v1', dd=[od('x', 2)])], gg=[og('subgroup', vv=[ov('v2', dd=[od('x', 3)])])]) with self.assertRaises(DimensionConflictError) as err_context: check_dims(g) msg = err_context.exception.message self.check_all_in_str(msg, ['/v1', '"x" = 2', '/subgroup/v2', '"x" = 3'])
def test_with_varsdata_okay(self): g = og('', dd=[od('x')], vv=[ov('v1', dd=[od('x', 2)])], gg=[og('subgroup', vv=[ov('v2', dd=[od('x', 2)])])]) subgroup = g.groups['subgroup'] tag_group(g) self.assertTrue(group_is_tagged(g)) self.assertFalse(group_is_tagged(subgroup)) check_dims(g) self.assertTrue(group_is_tagged(g)) self.assertFalse(group_is_tagged(subgroup))
def test_grouped_fail(self): g = og('', dd=[od('x'), od('y')], vv=[ov('v1', dd=[od('x')]), ov('v2', dd=[od('x'), od('y')])], gg=[og('subgroup', dd=[od('zz')], vv=[ov('sv1', dd=[od('x')]), ov('sv2', dd=[od('y'), od('z')])])]) with self.assertRaises(IncompleteStructureError) as err_context: has_no_missing_dims(g, fail_if_not=True) msg = err_context.exception.message self.check_all_in_str(msg, [ 'Variable "/subgroup/sv2"', 'dimension "z"', 'no definition exists'])
def test_partial_grouped(self): # Build a group with one dimension to be found in the subgroup. g = og('', gg=[og('subgroup', dd=[od('x', 4)], vv=[ov('subvar_1', dd=[od('x')]), ov('subvar_2', dd=[od('z', 3), od('x')])])]) dim_x_def = g.groups['subgroup'].dimensions['x'] self.do_complete(g) self.assertEqual(list(g.dimensions), [od('z', 3)]) self.assertEqual(list(g.groups['subgroup'].dimensions), [dim_x_def]) self.assertIs(g.groups['subgroup'].dimensions['x'], dim_x_def)
def test_with_varsdata_fail(self): g = og('', dd=[od('x')], vv=[ov('v1', dd=[od('x', 2)])], gg=[og('subgroup', vv=[ov('v2', dd=[od('x', 3)])])]) subgroup = g.groups['subgroup'] tag_group(g) self.assertTrue(group_is_tagged(g)) self.assertFalse(group_is_tagged(subgroup)) with self.assertRaises(DimensionConflictError): check_dims(g) self.assertTrue(group_is_tagged(g)) self.assertFalse(group_is_tagged(subgroup))
def test_inner_group_attr(self): g = og('group_name', gg=[og('sub_group', aa=[oa('x', _long(2))])]) result = cdl(g) self.assertEqual(result, 'netcdf group_name {\n' '\n' 'group: sub_group {\n' '\n' '// group attributes:\n' ' :x = 2L ;\n' '} // group sub_group\n' '}')
def test_subgroup_mixed(self): g = og('', vv=[ov('v1', dd=[od('q')])], gg=[og('subgroup', vv=[ov('v1', dd=[od('q', 2)])])]) self.assertEqual(list(g.dimensions), []) v1 = g.variables['v1'] v2 = g.groups['subgroup'].variables['v1'] self.do_complete(g) self.assertEqual(len(g.dimensions), 1) self.assertEqual(list(g.dimensions), [od('q', 2)]) test_dim = g.dimensions['q'] self.assertIs(v1.dimensions[0], test_dim) self.assertIs(v2.dimensions[0], test_dim)
def test_shared_clash(self): # NOTE: "complete" won't allow this, but this method doesn't care. g = og('', vv=[ov('v1', dd=[od('x', 2)]), ov('v2', dd=[od('x', 3)])]) add_missing_dims(g) self.assertEqual(len(g.dimensions), 1) self.assertEqual(g.dimensions.names(), ['x'])
def test_unlimited(self): g = og('', vv=[ov('v', dd=[od('x', 2, u=True)])]) self.assertEqual(all_dimensions(g), []) add_missing_dims(g) self.assertEqual(len(g.dimensions), 1) self.assertNotEqual(all_dimensions(g), [od('x', 2)]) self.assertEqual(all_dimensions(g), [od('x', 2, u=True)])
def eg_simple_flat(): """Produce a flat version of the simple containers example.""" d_lat = od('lat', 10) d_lon = od('lon', 5) d_time = od('time', u=True) dims = [d_lat, d_lon, d_time] collection_vars = [ ov('instrument', data=np.array(0), aa=[oa('container_type', 'simple'), oa('members', ('instrument___lat instrument___lon instrument___rh ' 'instrument___time')), oa('measurement_platform', 'aircraft')]), ov('model', data=np.array(0), aa=[oa('container_type', 'simple'), oa('members', 'model___lat model___lon model___rh model___time'), oa('measurement_platform', 'global circulation model')])] data_vars = [ ov('instrument___lat', dd=[d_lat], aa=[oa('units', 'degrees_north')]), ov('instrument___lon', dd=[d_lon], aa=[oa('units', 'degrees_east')]), ov('instrument___time', dd=[d_time], aa=[oa('units', 'seconds')]), ov('instrument___rh', dd=dims, aa=[oa('_FillValue', -1)]), ov('model___lat', dd=[d_lat], aa=[oa('units', 'degrees_north')]), ov('model___lon', dd=[d_lon], aa=[oa('units', 'degrees_east')]), ov('model___time', dd=[d_time], aa=[oa('units', 'seconds')]), ov('model___rh', dd=dims, aa=[oa('_FillValue', -1)])] g = og('foo', dd=dims, vv=collection_vars + data_vars) _fake_complete(g) return g
def test_mixture(self): g = og('', dd=[od('x')], vv=[ov('v', dd=[od('y')])]) self.assertEqual(all_dimensions(g), [od('x')]) r = add_missing_dims(g) self.assertEqual(r, [od('y')]) self.assertEqual(len(g.dimensions), 2) self.assertIs(r[0], g.dimensions['y'])
def test_missing(self): g = og('', vv=[ov('v', dd=[od('x')])]) self.assertEqual(all_dimensions(g), []) r = add_missing_dims(g) self.assertEqual(r, [od('x')]) self.assertEqual(all_dimensions(g), [od('x')]) self.assertIs(g.dimensions['x'], r[0])
def group_flat_containers(group): """Example operation, how to group a flat containers representation.""" result = group.detached_copy() # find variables representing containers. con_vars = [var for var in ncg.all_variables(result) if 'container_type' in var.attributes.names()] # produce a group from each of these for con_var in con_vars: # container variables should have no dimensions. assert not con_var.dimensions # remove container variable and make a group instead con_name = con_var.name result.variables.remove(con_var) result.groups.add(og(con_name, aa=con_var.attributes)) con_grp = result.groups[con_name] # remove redundant 'members' attribute inherited from container var con_grp.attributes.pop('members') # move member variables into the group (removing prefixes) prefix = con_name + '___' memb_names = con_var.attributes['members'].value.split(' ') for memb_name in memb_names: # remove member from root memb_var = result.variables.pop(memb_name) # strip initial disambiguation prefix, if present if memb_var.name.startswith(prefix): memb_name = memb_name[len(prefix):] memb_var.rename(memb_name) # place member in group con_grp.variables.add(memb_var) return result
def test_fail_error(self): g = og('', vv=[ov('v', dd=[od('x')])]) with self.assertRaises(IncompleteStructureError) as err_context: has_no_missing_dims(g, fail_if_not=True) msg = err_context.exception.message self.check_all_in_str(msg, [ 'Variable "/v"', 'dimension "x"', 'no definition exists'])
def test_simple_data(self): g = og('', vv=[ov('v', dd=[od('y'), od('x')], data=_mockdata((15, 20)))]) self.do_complete(g) self.assertEqual(len(g.dimensions), 2) self.assertEqual(g.dimensions['x'], od('x', 20)) self.assertEqual(g.dimensions['y'], od('y', 15))
def test_fail_bad_data_shapes(self): d1 = _mockdata((5, 3, 2)) g = og('', dd=[od('x'), od('y')], vv=[ov('v1', dd=[od('x'), od('y')], data=d1)]) with self.assertRaises(DimensionConflictError) as err_context: check_dims(g) msg = err_context.exception.message self.check_all_in_str(msg, ['/v1', '3 dimensions', '2 dimensions'])
def test_type_bad(self): g = og('') with self.assertRaises(ValueError) as err_context: fnd(g, 'name', type(None)) msg = err_context.exception.message self.check_all_in_str(msg, ['<type \'NoneType\'>', 'not recognised', 'or not supported'])
def test_type_unsupported(self): g = og('') with self.assertRaises(ValueError) as err_context: fnd(g, 'name', nco.Group) msg = err_context.exception.message self.check_all_in_str(msg, ['ncobj.Group', 'not recognised', 'or not supported'])
def test_nomissing(self): g = og('', dd=[od('x')], vv=[ov('v', dd=[od('x')])]) test_dim = g.dimensions['x'] str_before = str(g) r = add_missing_dims(g) self.assertEqual(r, []) self.assertIs(g.dimensions['x'], test_dim) self.assertEqual(str(g), str_before)
def test_length(self): g = og('', vv=[ov('v', dd=[od('x', 2)])]) self.assertEqual(all_dimensions(g), []) r = add_missing_dims(g) self.assertEqual(r, [od('x', 2)]) self.assertEqual(len(g.dimensions), 1) self.assertNotEqual(all_dimensions(g), [od('x')]) self.assertEqual(all_dimensions(g), [od('x', 2)])
def test_fail_conflicting_data_shapes(self): d1 = _mockdata((2, 23)) v1 = ov('v1', dd=[od('x'), od('y')], data=d1) v2 = ov('v2', dd=[od('x', 17), od('y', 23)]) g = og('', dd=[od('x'), od('y')], vv=[v1, v2]) with self.assertRaises(DimensionConflictError) as err_context: check_dims(g) msg = err_context.exception.message self.check_all_in_str(msg, ['/v1', '/v2', '"x" = 17', '"x" = 2'])