def test_system_spatial_averaging_weights(self):
"""Test creating averaging weights from first principles."""
# x/y coordinate arrays and grid objects. Coordinates may be two-dimensional otherwise,
x = ocgis.Variable(name='xc',
dimensions='dimx',
value=np.arange(5, 360, 10, dtype=float))
y = ocgis.Variable(name='yc',
dimensions='dimy',
value=np.arange(-85, 90, 10, dtype=float))
grid = ocgis.Grid(x, y, crs=ocgis.crs.Spherical())
# Create spatial bounds on the grid coordinates. This allows us to use polygons as opposed to points for the
# spatial averaging.
grid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds')
self.assertEqual(grid.abstraction, ocgis.constants.Topology.POLYGON)
# This is the subset geometry. OCGIS geometry variables may be used to take advantage of wrapping and coordinate
# system conversion.
subset_geom = shapely.geometry.box(52, -70, 83, 10)
# Perform an intersection. First, data is reduced in spatial extent using an intersects operation. A
# clip/intersection is then performed for each geometry object.
sub, slc = grid.get_intersection(subset_geom, return_slice=True)
slc = {dim.name: se
for dim, se in zip(grid.dimensions, slc)
} # Just how to convert to a dictionary slice...
# Weights are computed on demand and is equal to original_area/clipped_area.
weights = sub.weights
self.assertAlmostEqual(weights.sum(), 24.799999999999997)
def _preformatting_(self, i, coll):
"""
Modify in place the collections so they can be saved as discrete
geometries along a new spatial dimension.
"""
# TODO: UGID and GID show up in the output file, but they are equal. Remove one.
if not self.ops or self.ops.aggregate is False:
return coll
# Size of spatial dimension
ncoll = len(self.ops.geom)
udim = DimensionName.UNIONED_GEOMETRY
ugids = coll.properties.keys()
assert len(ugids) == 1
ugid = list(ugids)[0]
# Geometry centroid location
lon, lat = coll.geoms[ugid].centroid.xy
for field in coll.iter_fields():
lon_attrs = field.x.attrs.copy()
lat_attrs = field.y.attrs.copy()
xn = field.x.name
yn = field.y.name
# Removed for now. It'd be nice to find an elegant way to retain those.
field.remove_variable(xn)
field.remove_variable(yn)
# Create new lon and lat variables
field.add_variable(
ocgis.Variable(xn,
value=lon,
dimensions=(udim, ),
attrs=dict(
lon_attrs,
**{'long_name': 'Centroid longitude'})))
field.add_variable(
ocgis.Variable(yn,
value=lat,
dimensions=(udim, ),
attrs=dict(lat_attrs,
**{'long_name':
'Centroid latitude'})))
if VariableName.SPATIAL_MASK in field:
# Remove the spatial_mask.
field.remove_variable(VariableName.SPATIAL_MASK)
field.dimension_map.set_spatial_mask(None)
grid = ocgis.Grid(field[xn],
field[yn],
abstraction='point',
crs=field.crs,
parent=field)
field.set_grid(grid)
# Geometry variables from the geom properties dict
dm = get_data_model(self.ops)
# Some dtypes are not supported by netCDF3. Use the netCDF4
# data model to avoid these issues.
for key, val in coll.properties[ugid].items():
if np.issubdtype(type(val), int):
dt = get_dtype('int', dm)
elif np.issubdtype(type(val), float):
dt = get_dtype('float', dm)
else:
dt = 'auto'
# There is no metadata for those yet, but it could be passed
# using the output_format_options keyword.
field.add_variable(
ocgis.Variable(key,
value=[
val,
],
dtype=dt,
dimensions=(udim, )), )
# Propagate max string length if it is set.
smg = coll.children[ugid][key].string_max_length_global
if smg is not None:
field[key].set_string_max_length_global(value=smg)
# ------------------ Dimension update ------------------------ #
# Modify the dimensions for the number of geometries
gdim = field.dimensions[udim]
gdim.set_size(ncoll)
for var in field.iter_variables_by_dimensions([gdim]):
d = var.dimensions_dict[udim]
d.bounds_local = (i, i + 1)
# ------------------------------------------------------------ #
# CF-Conventions
# Options for cf-role are timeseries_id, profile_id, trajectory_id
gid = field[HeaderName.ID_GEOMETRY]
gid.attrs['cf_role'] = 'timeseries_id'
# Name of spatial dimension
if self.options.get('geom_dim', None):
gdim.set_name(self.options.get('geom_dim', None))
return coll
def write(self):
ocgis_lh('starting write method', self._log, logging.DEBUG)
# Indicates if user geometries should be written to file.
write_ugeom = False
ncoll = len(self.ops.geom)
build = True
for i, coll in enumerate(self):
ugids = coll.properties.keys()
assert len(ugids) == 1
ugid = ugids[0]
# Geometry centroid location
lon, lat = coll.geoms[ugid].centroid.xy
for field in coll.iter_fields():
lon_attrs = field.x.attrs.copy()
lat_attrs = field.y.attrs.copy()
# Removed for now. It'd be nice to find an elegant way to retain those.
field.remove_variable('lat')
field.remove_variable('lon')
# Create new lon and lat variables
field.add_variable(
ocgis.Variable('lon',
value=lon,
dimensions=(DimensionName.UNIONED_GEOMETRY,),
attrs=dict(lon_attrs, **{'long_name':'Centroid longitude'})
)
)
field.add_variable(
ocgis.Variable('lat',
value=lat,
dimensions=(DimensionName.UNIONED_GEOMETRY,),
attrs=dict(lat_attrs, **{'long_name':'Centroid latitude'})
)
)
if 'ocgis_spatial_mask' in field:
# Remove the spatial_mask and replace by new one.
field.remove_variable('ocgis_spatial_mask')
grid = ocgis.Grid(field['lon'], field['lat'], abstraction='point',
crs=field.crs, parent=field)
grid.set_mask([[False,]])
field.set_grid(grid)
# Geometry variables from the geom properties dict
# There is no metadata for those...
dm = get_data_model(self.ops)
for key, val in coll.properties[ugid].items():
if np.issubdtype(type(val), int):
dt = get_dtype('int', dm)
elif np.issubdtype(type(val), float):
dt = get_dtype('float', dm)
else:
dt='auto'
field.add_variable(
ocgis.Variable(key,
value=[val,],
dtype=dt,
dimensions=(DimensionName.UNIONED_GEOMETRY,)))
# ------------------ Dimension update ------------------------ #
# Modify the dimensions for the number of geometries
gdim = field.dimensions[DimensionName.UNIONED_GEOMETRY]
gdim.set_size(ncoll)
for var in field.iter_variables_by_dimensions([gdim]):
d = var.dimensions_dict[DimensionName.UNIONED_GEOMETRY]
d.bounds_local = (i, i+1)
# ------------------------------------------------------------ #
# CF-Conventions
# Can this be anything else than a timeseries_id
# Options are timeseries_id, profile_id, trajectory_id
gid = field[HeaderName.ID_GEOMETRY]
gid.attrs['cf_role'] = 'timeseries_id'
# TODO: Hard-code the name in constants.py
gdim.set_name('region')
# Path to the output object.
# I needed to put it here because _write_archetype pops it, so it's not available after the first loop.
f = {KeywordArgument.PATH: self.path}
# This will be changed to "write" if we are on the build loop.
write_mode = MPIWriteMode.APPEND
if build:
# During a build loop, create the file and write the first series of records. Let the drivers determine
# the appropriate write modes for handling parallelism.
write_mode = None
# Write the user geometries if selected and there is one present on the incoming collection.
if self._add_ugeom and coll.has_container_geometries:
write_ugeom = True
if write_ugeom:
if vm.rank == 0:
# The output file name for the user geometries.
ugid_shp_name = self.prefix + '_ugid.shp'
if self._add_ugeom_nest:
ugeom_fiona_path = os.path.join(self._get_or_create_shp_folder_(), ugid_shp_name)
else:
ugeom_fiona_path = os.path.join(self.outdir, ugid_shp_name)
else:
ugeom_fiona_path = None
build = False
f[KeywordArgument.WRITE_MODE] = write_mode
self._write_coll_(f, coll)
if write_ugeom:
with vm.scoped(SubcommName.UGEOM_WRITE, [0]):
if not vm.is_null:
for subset_field in list(coll.children.values()):
subset_field.write(ugeom_fiona_path, write_mode=write_mode, driver=DriverVector)
# The metadata and dataset descriptor files may only be written if OCGIS operations are present.
ops = self.ops
if ops is not None and self.add_auxiliary_files and MPI_RANK == 0:
# Add OCGIS metadata output if requested.
if self.add_meta:
ocgis_lh('adding OCGIS metadata file', 'conv', logging.DEBUG)
from ocgis.conv.meta import MetaOCGISConverter
lines = MetaOCGISConverter(ops).write()
out_path = os.path.join(self.outdir, self.prefix + '_' + MetaOCGISConverter._meta_filename)
with open(out_path, 'w') as f:
f.write(lines)
# Add the dataset descriptor file if requested.
if self._add_did_file:
ocgis_lh('writing dataset description (DID) file', 'conv', logging.DEBUG)
path = os.path.join(self.outdir, self.prefix + '_did.csv')
_write_dataset_identifier_file_(path, ops)
# Add source metadata if requested.
if self._add_source_meta:
ocgis_lh('writing source metadata file', 'conv', logging.DEBUG)
path = os.path.join(self.outdir, self.prefix + '_source_metadata.txt')
_write_source_meta_(path, ops)
# Return the internal path unless overloaded by subclasses.
ret = self._get_return_()
return ret