def test_attrs(self):
result = cdl(ov('x', dd=[od('p'), od('q')],
aa=[oa('n1', _long(3)), oa('r1', 1.24)],
data=np.array(1.0)))
self.assertEqual(result,
'double x(p, q) ;\n'
' x:n1 = 3L ;\n'
' x:r1 = 1.24 ;')
def eg_simple_grouped():
"""Produce a grouped version of the simple containers example."""
d_lat = od('lat', 10)
d_lon = od('lon', 5)
d_time = od('time', u=True)
dims = [d_lat, d_lon, d_time]
c_lat = ov('lat', dd=[d_lat], aa=[oa('units', 'degrees_north')])
c_lon = ov('lon', dd=[d_lon], aa=[oa('units', 'degrees_east')])
c_time = ov('time', dd=[d_time], aa=[oa('units', 'seconds')])
coords = [c_lat, c_lon, c_time]
g = og('foo',
dd=dims,
gg=[og('instrument',
vv=(coords +
[ov('rh', dd=dims, aa=[oa('_FillValue', -1)])]),
aa=[oa('container_type', 'simple'),
oa('measurement_platform', 'aircraft')]),
og('model',
vv=(coords +
[ov('rh', dd=dims, aa=[oa('_FillValue', -1)])]),
aa=[oa('container_type', 'simple'),
oa('measurement_platform',
'global circulation model')])])
_fake_complete(g)
return g
def _inner_flatten_grouped_containers(result):
# represent groups as renamed variables with special container attributes
for grp in list(result.groups): # list() because loop changes the object
# Flatten any inner groups first.
_inner_flatten_grouped_containers(grp)
# Remove group from output, and create a container variable instead.
con_name = grp.name
con_prefix = con_name + '___'
grp.remove()
result.variables.add(ov(con_name, aa=grp.attributes, data=np.array(0)))
con_var = result.variables[con_name]
con_attrs = con_var.attributes
# Add container type if not defined.
if 'container_type' not in con_attrs.names():
con_attrs.add(oa('container_type', 'simple'))
# Check or add container_dims attribute.
con_dims = [_RefTag(None, grp.dimensions[dim_name])
for dim_name in sorted(grp.dimensions.names())]
if 'container_dims' in con_attrs.names():
assert con_attrs['container_dims'].value == con_dims
else:
con_attrs.add(oa('container_dims', con_dims))
# Move dimensions to the parent level, and rename them.
for dim in list(grp.dimensions): # list() because loop changes it
result.dimensions.setitem_reference(con_prefix + dim.name, dim)
# Check or add container_vars (aka "members") attribute.
memb_vars = [_RefTag(None, grp.variables[name])
for name in sorted(grp.variables.names())]
con_vars_attr = con_attrs.get('container_vars', None)
if con_vars_attr:
# Check the list matches the enclosed variables (sorted by name)
role_vars = [_RefTag(None, var)
for role_name, var in con_vars_attr.value]
assert role_vars == memb_vars
else:
con_attrs.add(oa('container_vars', memb_vars))
# Move variables to the parent level, and rename.
for var in list(grp.variables): # list() because loop changes it
result.variables.setitem_reference(con_prefix + var.name, var)
return result
def _make_complex_group():
g = og(
'temp',
aa=[oa('a_root_attr_num', _long(1)),
oa('c_root_attr_str', 'xyz'),
oa('b_root_attr_vec', np.array([1.2, 3, 4]))],
dd=[od('root_dim_x', 2)],
vv=[ov('root_var_1',
dd=[od('root_dim_x')],
aa=[oa('root_var_attr_1', _long(11))],
data=np.zeros((2))),
ov('root_var_2_scalar',
data=np.array(3.15, dtype=np.float32))],
gg=[og('subgroup',
aa=[oa('subgroup_attr', 'qq')],
dd=[od('subgroup_dim_y', 3)],
vv=[ov('subgroup_var',
dd=[od('root_dim_x'), od('subgroup_dim_y')],
aa=[oa('subgroup_var_attr', 57.5)],
data=np.zeros((2, 3)))],
gg=[og('sub_sub_group',
aa=[oa('sub_sub_group_attr', 'this')],
vv=[ov('sub_sub_group_var',
dd=[od('subgroup_dim_y')],
data=np.zeros((3)))])]),
og('sg_2_empty')])
ncg.complete(g)
return g
def setUp(self):
# make a really simple grouped structure
self.g_simple_grouped_plain = og("root", vv=[ov("global_var")], gg=[og("subgroup", vv=[ov("local_var")])])
# make a separate grouped version with ':container_type=simple'
self.g_simple_grouped_explicit = og(
"root",
vv=[ov("global_var")],
gg=[og("subgroup", vv=[ov("local_var")], aa=[oa("container_type", "simple")])],
)
# make an equivalent flat structure
self.g_simple_flat = og(
"root",
vv=[
ov("global_var"),
ov("subgroup___local_var"),
ov("subgroup", data=[0], aa=[oa("container_type", "simple"), oa("members", "subgroup___local_var")]),
],
)
def test_attr(self):
g = og('group_name', aa=[oa('x', _long(2))])
result = cdl(g)
self.assertEqual(result,
'netcdf group_name {\n'
'\n'
'// global attributes:\n'
' :x = 2L ;\n'
'}')
def flatten_grouped_containers(group):
"""Example operation, how to flatten a containers representation."""
result = group.detached_copy()
# get a list of the groups (because we alter while iterating)
result_groups = list(result.groups)
# represent groups as renamed variables with special container attributes
for grp in result_groups:
# Fail sub-sub-groups -- don't know how to deal with these ??
assert not grp.groups
# Also expect all dimensions at the top, not in groups ??for now??
assert not grp.dimensions
# Remove group from output, and add a container variable instead
grp.remove()
con_name = grp.name
container_prefix = con_name + '___'
con_attrs = grp.attributes.detached_contents_copy()
# Add container special attributes
if 'container_type' not in con_attrs.names():
con_attrs.add(oa('container_type', 'simple'))
sorted_names = sorted(grp.variables.names())
if 'members' in con_attrs:
# check the existing attribute matches the expected
# NOTE: we are insisting on sorted order here
var_names = sorted(grp.variables.names())
memb_names = con_attrs['members']
assert var_names == memb_names
# remove it (we will be replacing with prefixed names)
con_attrs.pop('members')
# define (or redefine) the 'members' attribute
con_attrs.add(oa('members',
' '.join(container_prefix + var_name
for var_name in sorted_names)))
result.variables.add(ov(con_name,
aa=con_attrs, # includes type+members
data=np.array(0)))
# Add another output root variable for each of the groups members
for var in grp.variables:
var_tmp = var.detached_copy()
# prepend group name to disambiguate variable names
var_tmp.rename(container_prefix + var_tmp.name)
result.variables.add(var_tmp)
return result
def test_inner_group_attr(self):
g = og('group_name',
gg=[og('sub_group',
aa=[oa('x', _long(2))])])
result = cdl(g)
self.assertEqual(result,
'netcdf group_name {\n'
'\n'
'group: sub_group {\n'
'\n'
'// group attributes:\n'
' :x = 2L ;\n'
'} // group sub_group\n'
'}')
def setUp(self):
self.root = og('',
aa=[oa('test_global_attr', v='test string')],
dd=[od('t', 3)],
vv=[ov('x')],
gg=[og('group_X',
dd=[od('x', 3), od('y', 2)],
vv=[ov('var_A', dd=[od('y'), od('x')],
aa=[oa('var_att_a', 'this')]),
ov('var_B', dd=[od('t')])],
aa=[oa('att_1', 4.3), oa('att_2', 4.7)])])
subgroup = self.root.groups['group_X']
self.all_objs = [
self.root,
self.root.attributes['test_global_attr'],
self.root.dimensions['t'],
self.root.variables['x'],
subgroup,
subgroup.attributes['att_1'],
subgroup.attributes['att_2'],
subgroup.dimensions['x'],
subgroup.dimensions['y'],
subgroup.variables['var_A'],
subgroup.variables['var_B']]
def test_simple_to_path(self):
test_outfile_name = 'test_simple.nc'
test_outfile_path = os.path.join(testdata_dirpath,
test_outfile_name)
array = np.arange(4)
var = ov('v1', dd=[od('x')], aa=[oa('v_att', 'this')], data=array)
g = og('', vv=[var])
self.assertFalse(ncg.has_no_missing_dims(g))
try:
write(test_outfile_path, g)
self.assertTrue(ncg.has_no_missing_dims(g))
# Read it back again and check.
with netCDF4.Dataset(test_outfile_path) as ds:
g_back = read(ds)
self.assertEqual(g_back, g)
finally:
if not leave_output:
# Remove temporary file
if os.path.exists(test_outfile_path):
os.remove(test_outfile_path)
def test_add_to_dataset(self):
test_outfile_name = 'test_add.nc'
test_outfile_path = os.path.join(testdata_dirpath, test_outfile_name)
try:
ds = netCDF4.Dataset(test_outfile_path, 'w')
ds.setncattr('extra', 4.5)
g = og('', vv=[ov('v1', aa=[oa('var_attr', 'this_value')],
data=np.array(3.2))])
write(ds, g)
ds.close()
# Read it back again and check.
with netCDF4.Dataset(test_outfile_path) as ds:
g = read(ds)
self.assertEqual(g.attributes['extra'].value, 4.5)
self.assertEqual(list(g.dimensions), [])
self.assertEqual(
g.variables['v1'].attributes['var_attr'].value,
'this_value')
finally:
if not leave_output:
if os.path.exists(test_outfile_path):
os.remove(test_outfile_path)
def test_string(self):
result = cdl(oa('x', 'this'))
self.assertEqual(result, 'x = "this"')
def eg_simple_flat():
"""Produce a flat version of the simple containers example."""
d_lat = od('lat', 10)
d_lon = od('lon', 5)
d_time = od('time', u=True)
dims = [d_lat, d_lon, d_time]
collection_vars = [
ov('instrument', data=np.array(0),
aa=[oa('container_type', 'simple'),
oa('members',
('instrument___lat instrument___lon instrument___rh '
'instrument___time')),
oa('measurement_platform', 'aircraft')]),
ov('model', data=np.array(0),
aa=[oa('container_type', 'simple'),
oa('members',
'model___lat model___lon model___rh model___time'),
oa('measurement_platform', 'global circulation model')])]
data_vars = [
ov('instrument___lat', dd=[d_lat], aa=[oa('units', 'degrees_north')]),
ov('instrument___lon', dd=[d_lon], aa=[oa('units', 'degrees_east')]),
ov('instrument___time', dd=[d_time], aa=[oa('units', 'seconds')]),
ov('instrument___rh', dd=dims, aa=[oa('_FillValue', -1)]),
ov('model___lat', dd=[d_lat], aa=[oa('units', 'degrees_north')]),
ov('model___lon', dd=[d_lon], aa=[oa('units', 'degrees_east')]),
ov('model___time', dd=[d_time], aa=[oa('units', 'seconds')]),
ov('model___rh', dd=dims, aa=[oa('_FillValue', -1)])]
g = og('foo', dd=dims, vv=collection_vars + data_vars)
_fake_complete(g)
return g
def test_single_type(self):
result = cdl(oa('x', np.array(5, dtype=np.dtype('int32'))))
self.assertEqual(result, 'x = 5')
def test_unsigned_type(self):
result = cdl(oa('x', np.array(5, dtype=np.dtype('uint32'))))
self.assertEqual(result, 'x = 5U')
def test_array(self):
result = cdl(oa('x', [3.21, 1.23]))
self.assertEqual(result, 'x = 3.21, 1.23')
"""
Code to create a netcdf4 file with "all" possible features for CDL testing.
(Supported features, that is).
"""
import netCDF4 as nc4
import numpy as np
import os
import ncobj.grouping as ncg
from ncobj.shorts import og, od, ov, oa
import ncobj.nc_dataset as ncf
g = og(
'rootname',
aa=[oa('root_attr_num', 1),
oa('root_attr_str', 'xyz'),
oa('root_attr_vec', np.array([1.2, 3, 4]))],
dd=[od('root_dim_x', 2)],
vv=[ov('root_var_1',
dd=[od('root_dim_x')],
aa=[oa('root_var_attr_1', 11)],
data=np.zeros((2))),
ov('root_var_2_scalar', data=np.array(3.15, dtype=np.float32))],
gg=[og('subgroup',
aa=[oa('subgroup_attr', 'qq')],
dd=[od('subgroup_dim_y', 3)],
vv=[ov('subgroup_var',
dd=[od('root_dim_x'), od('subgroup_dim_y')],
aa=[oa('subgroup_var_attr', 57.31)],
data=np.zeros((2, 3)))],
def test_int(self):
result = cdl(oa('x', _long(3)))
self.assertEqual(result, 'x = 3L')
def test_ushort_type(self):
result = cdl(oa('x', np.array(5, dtype=np.dtype('uint16'))))
self.assertEqual(result, 'x = 5US')
def test_ulong_type(self):
result = cdl(oa('x', np.array(5, dtype=np.dtype('uint64'))))
self.assertEqual(result, 'x = 5UL')
def test_float_type(self):
result = cdl(oa('x', np.array(1.23, dtype=np.dtype('float32'))))
self.assertEqual(result, 'x = 1.23f')
def test_scalar(self):
result = cdl(oa('x', 3.21))
self.assertEqual(result, 'x = 3.21')
def test_double_type(self):
result = cdl(oa('x', np.array(1.23, dtype=np.dtype('float64'))))
self.assertEqual(result, 'x = 1.23')
def test_isolate_and_attribute(self):
self.assertEqual('att_a', group_path(oa('att_a')))
def test_ubyte_type(self):
result = cdl(oa('x', np.array(5, dtype=np.dtype('uint8'))))
self.assertEqual(result, 'x = 5UB')
