def EG_extract_region():
# subset on dimensions using Coordinate concept.
# NOTE: could be done in Iris, but then also requires CF compliance ??
input_path = os.path.join(basedir, 'test.nc')
output_path = os.path.join(basedir, 'test_out.nc')
depth_start, depth_end = (50.0, 550.0)
input = nc_files.read(input_path)
depth_dim = input.dimensions['depth']
depth_coord = input.variables['depth']
# Work out indexing to the part we want
i_start = np.where(depth_coord[:] >= depth_start)[0][0]
i_end_indices = np.where(depth_coord[:] >= depth_end)[0]
if i_end_indices:
i_end = i_end_indices[0]
n_depths = i_end - i_start + 1
else:
i_end = -1
n_depths = depth_dim.length - i_start
depth_slice = slice(i_start, i_end)
# Adjust the dimension definition.
depth_dim.length = n_depths
# Adjust all the referencing variables (N.B. includes depth coord).
for var in ncg.all_variables(input):
if depth_dim in var.dimensions:
dim_index = var.dimensions.index(depth_dim)
slices = [depth_slice if index == dim_index else slice(None)
for index in range(len(var.dimensions))]
var.data = var[slices]
# Write result
nc_files.write(output, output_path)
def _fake_complete(group):
"""
'Complete' a group which is lacking some dimension lengths or data types.
As the examples in simple_semantic_containers don't contain data, they
don't have data types or lengths of unlimited dimensions (e.g. 'time').
This operation sets missing lengths to 1, and fakes the data.
This allows CDL to be generated (which needs the dtype), and variables to
be compared (via "np.all(x.data == y.data)").
This allows the examples builder functions to omit variables data, which is
neater and saves space.
"""
# Check all dims have definitions defined in the build code (for clarity).
ncg.has_no_missing_dims(group, fail_if_not=True)
# Fix up any top-level dims with no length (e.g. unlimited 'time').
for dim in group.dimensions:
if dim.length is None:
dim.length = 1
# Patch it all together. Should work, now we have all the lengths.
ncg.complete(group)
for var in ncg.all_variables(group):
if var.data is None:
# NOTE: this might be better be done with numpy stride tricks ?
var.data = _dummy_data(dim.length for dim in var.dimensions)
def group_flat_containers(group):
"""Example operation, how to group a flat containers representation."""
result = group.detached_copy()
# find variables representing containers.
con_vars = [var for var in ncg.all_variables(result)
if 'container_type' in var.attributes.names()]
# produce a group from each of these
for con_var in con_vars:
# container variables should have no dimensions.
assert not con_var.dimensions
# remove container variable and make a group instead
con_name = con_var.name
result.variables.remove(con_var)
result.groups.add(og(con_name, aa=con_var.attributes))
con_grp = result.groups[con_name]
# remove redundant 'members' attribute inherited from container var
con_grp.attributes.pop('members')
# move member variables into the group (removing prefixes)
prefix = con_name + '___'
memb_names = con_var.attributes['members'].value.split(' ')
for memb_name in memb_names:
# remove member from root
memb_var = result.variables.pop(memb_name)
# strip initial disambiguation prefix, if present
if memb_var.name.startswith(prefix):
memb_name = memb_name[len(prefix):]
memb_var.rename(memb_name)
# place member in group
con_grp.variables.add(memb_var)
return result
def EG_flatten():
# Copy certain information to output file, losing existing structure.
input_path = os.path.join(basedir, 'test.nc')
output_path = os.path.join(basedir, 'test_out.nc')
# Read input.
input = nc_files.read(input_path)
# Make blank output.
output = Group()
# Partial copy.
output.groups.add(input.groups['grid_dims'])
output.variables.add_all(ncg.all_variables(input.groups['group_a']))
output.variables.add_all(ncg.all_variables(input.groups['group_Q']))
# Write out.
# N.B. relevant top-level dimensions etc. will be re-created.
# but this 'flattens' everything into the root group.
nc_files.write(nco, output_path)
def unlink_attribute_references(group, ref_attr_names_types=None):
# Locate any variable and group attributes previously identified as
# variable references, and reconstruct the attribute string from the names
# of the referenced elements.
groups_and_vars = ncg.all_variables(group) + ncg.all_groups(group)
for elem in groups_and_vars:
for ref_name, _ in REFERENCE_ATTRIBUTES_NAMES_AND_TYPES:
attr = elem.attributes.get(ref_name, None)
if attr:
attr.value = ' '.join(defstring_from_reftag(reftag)
for reftag in attr.value)
def EG_some_variables():
# Copy only certain variables to output file.
input_path = os.path.join(basedir, 'test.nc')
output_path = os.path.join(basedir, 'test_out.nc')
# Read input.
nco = nc_files.read(input_path)
# Delete all-but selected variables.
varnames = ('temp', 'depth')
for var in ncg.all_variables(input):
if var.name not in var_names:
output.variables.add(var)
# Write out.
nc_files.write(nco, output_path)
def link_attribute_references(group):
# Locate all variable and group attributes that contain variable
# references (i.e. names), and replace their values with data containing
# links to the actual variables, so we can reconstruct these attributes
# after possibly renaming variables.
groups_and_vars = ncg.all_variables(group) + ncg.all_groups(group)
for elem in groups_and_vars:
in_group = (elem if isinstance(elem, nco.Group)
else elem.container.in_element)
assert isinstance(in_group, nco.Group)
for ref_name, ref_type in REFERENCE_ATTRIBUTES_NAMES_AND_TYPES:
attr = elem.attributes.get(ref_name, None)
if attr:
refs = split_noempties(attr.value, ' ')
attr.value = [reftag_from_defstring(ref, in_group, ref_type)
for ref in refs]
def EG_filter_variables():
# Filter variables, removing some, and flatten structure.
input_path = os.path.join(basedir, 'test.nc')
output_path = os.path.join(basedir, 'test_out.nc')
# Read input.
input = nc_files.read(input_path)
# Make blank output.
output = Group()
varname_parts_only = ('temp', 'depth')
varname_ends_exclude = ('_QC', '_stats')
# Copy selected variables.
for var in ncg.all_variables(input):
if (any(var.name.find(part) >= 0
for part in varname_parts_only) and
not any(var.name.endswith(part)
for part in varname_ends_exclude)):
output.variables.add(var)
# Write out.
nc_files.write(output, output_path)
def __getitem__(self, keys):
"""
Return an extracted sub-section of the contained group.
The specified extractor dimensions are indexed as required.
The result is a completed group based on the original.
"""
if not isinstance(keys, Iterable):
keys = [keys]
#print 'index by:', keys
group = self._group.detached_copy()
for index, dim in zip(keys, self._dims):
#print 'slice {} by {}'.format(dim, index)
dim_name = dim.name
dim_reduces = not isinstance(index, slice)
# Slice all variables that map this dimension
for var in ncg.all_variables(group):
#print 'var before:', var
var_inds = [index if this_dim.name == dim_name else slice(None)
for this_dim in var.dimensions]
#print 'var_inds = ', var_inds
var.data = var.data[tuple(var_inds)]
if dim_reduces:
var_dims = [this_dim for this_dim in var.dimensions
if this_dim.name != dim_name]
var.dimensions = var_dims
#print 'var_dims = ', var_dims
#print 'var after:', var, 'shape = ', var.data.shape
# Remove the dimension if we indexed it away.
if dim_reduces:
#print 'removed ', dim
group.dimensions.remove(dim)
ncg.complete(group)
return group
def EG_prune_dimensions(group):
dims_used = set(
sum((var.dimensions for var in ncg.all_variables(group)), []))
for dim in ncg.all_dimensions(group):
if dim not in dims_used:
dim.remove()
def test_all_variables(self):
walk_objs = list(all_variables(self.root))
self._check_all_of_type(walk_objs, nco.Variable)
def show_results(extracted):
#print extracted
print cdl(extracted)
print 'vars data: \n' + '\n'.join("{}:{}".format(var.name, var.data)
for var in ncg.all_variables(extracted))