def write_regridded_data_to_shapefile(dstfield):
"""
:param dstfield: The ESMF field object containing a mesh to write to shapefile.
:type dstfield: :class:`ESMF.api.field.Field`
:returns: Path to the output shapefile.
:rtype: str
"""
# turn the shapefile into an OCGIS field and get the spatial information
ofield = ocgis.RequestDataset(PATH_SHP).get()
# get the time dimension from the original netCDF file
otime = ocgis.RequestDataset(PATH_PR).get().temporal
# create an OCGIS variable from the regridded data values
pr = ocgis.Variable(name='pr', value=np.array(dstfield.reshape(1, otime.shape[0], 1, 1, ofield.shape[-1])))
# this holds our variables
vc = ocgis.VariableCollection([pr])
# we want to maintain the original shapefile data, but it needs to reshaped to account for the new time dimension.
for var in ofield.variables.itervalues():
newvalue = np.zeros(pr.shape, dtype=var.dtype)
newvalue[:] = var.value
newvar = ocgis.Variable(name=var.name, value=newvalue)
vc[newvar.name] = newvar
# combine the spatial data with time and the regridded values
ofield2 = ocgis.Field(temporal=otime, spatial=ofield.spatial, variables=vc)
# write this to shapefile
path_out_shp = ocgis.OcgOperations(dataset=ofield2, output_format='shp', prefix='pr_catchments',
add_auxiliary_files=False).execute()
return path_out_shp
def test_system_spatial_averaging_weights(self):
"""Test creating averaging weights from first principles."""
# x/y coordinate arrays and grid objects. Coordinates may be two-dimensional otherwise,
x = ocgis.Variable(name='xc',
dimensions='dimx',
value=np.arange(5, 360, 10, dtype=float))
y = ocgis.Variable(name='yc',
dimensions='dimy',
value=np.arange(-85, 90, 10, dtype=float))
grid = ocgis.Grid(x, y, crs=ocgis.crs.Spherical())
# Create spatial bounds on the grid coordinates. This allows us to use polygons as opposed to points for the
# spatial averaging.
grid.set_extrapolated_bounds('xc_bounds', 'yc_bounds', 'bounds')
self.assertEqual(grid.abstraction, ocgis.constants.Topology.POLYGON)
# This is the subset geometry. OCGIS geometry variables may be used to take advantage of wrapping and coordinate
# system conversion.
subset_geom = shapely.geometry.box(52, -70, 83, 10)
# Perform an intersection. First, data is reduced in spatial extent using an intersects operation. A
# clip/intersection is then performed for each geometry object.
sub, slc = grid.get_intersection(subset_geom, return_slice=True)
slc = {dim.name: se
for dim, se in zip(grid.dimensions, slc)
} # Just how to convert to a dictionary slice...
# Weights are computed on demand and is equal to original_area/clipped_area.
weights = sub.weights
self.assertAlmostEqual(weights.sum(), 24.799999999999997)
def create_data():
col = np.linspace(-104., -100., 100)
row = np.linspace(32, 36, 100)
col = VectorDimension(value=col, name='longitude', name_bounds='longitude_bounds', attrs={'standard_name': 'longitude',
'units': 'degrees_east'})
col.set_extrapolated_bounds()
row = VectorDimension(value=row, name='latitude', name_bounds='latitude_bounds', attrs={'standard_name': 'latitude',
'units': 'degrees_north'})
row.set_extrapolated_bounds()
grid = ocgis.SpatialGridDimension(row=row, col=col)
sdim = ocgis.SpatialDimension(grid=grid)
start = datetime.datetime(2000, 1, 1)
stop = datetime.datetime(2000, 12, 31)
days = 1
ret = []
delta = datetime.timedelta(days=days)
check = start
while check <= stop:
ret.append(check)
check += delta
temporal = ocgis.TemporalDimension(value=ret, unlimited=True)
var_value = np.ones((1, temporal.shape[0], 1, row.shape[0], col.shape[0]), dtype=float)
variable = ocgis.Variable(value=var_value, name='pr')
field = ocgis.Field(spatial=sdim, temporal=temporal, variables=variable)
ds = nc.Dataset(PATH_FAKE_DATA, 'w', format='NETCDF3_CLASSIC')
field.write_netcdf(ds)
ds.close()
def do_record_test(exedir, record):
# The record's unique HRU identifier
hruid = record['properties']['hruid']
# The current output directory
curr_outdir = OUTDIR_TEMPLATE.format(hruid)
# Create the directory
os.makedirs(curr_outdir, exist_ok=True)
# Make that the current working directory
os.chdir(curr_outdir)
# We need to transform the coordinate system from WGS84 to Spherical for ESMF
crs = ocgis.crs.CoordinateReferenceSystem(value=record['meta']['crs'])
field = ocgis.Field.from_records([record], crs=crs)
field.update_crs(ocgis.crs.Spherical())
# Convert the field geometry to an unstructured grid format based on the UGRID spec.
gc = field.geom.convert_to(pack=PACK,
node_threshold=NODE_THRESHOLD,
split_interiors=SPLIT_INTERIORS,
remove_self_intersects=False,
allow_splitting_excs=False)
# Path to the output netCDF file for the current element
out_element_nc = os.path.join(curr_outdir,
"esmf-element_hruid-{}.nc".format(hruid))
# Add the center coordinate to make ESMF happy (even though we are not using it)
centerCoords = np.array(
[field.geom.v()[0].centroid.x,
field.geom.v()[0].centroid.y]).reshape(1, 2)
ocgis.Variable(name='centerCoords',
value=centerCoords,
dimensions=['elementCount', 'coordDim'],
attrs={'units': 'degrees'},
parent=gc.parent)
# When writing the data to file, convert to ESMF unstructured format.
gc.parent.write(out_element_nc, driver='netcdf-esmf-unstruct')
# Run the simple regridding test
success = do_esmf(out_element_nc, exedir, curr_outdir)
if success:
# If successful, remove the directory
assert 'hruid-tmp-' in curr_outdir
shutil.rmtree(curr_outdir)
else:
# If it's not successful, leave the directory. Write the shapefile so it's easy to look at. Also send the record
# string to a file.
field.geom.write_vector('02-problem-hruid-{}.shp'.format(hruid))
record_out = '01-problem-record-hruid-{}.out'.format(hruid)
with open(record_out, 'w') as f:
f.write(str(record))
# Change back to the execution directory
os.chdir(exedir)
# Try to remove the log file if it exists.
if os.path.exists('PET0.ESMF_LogFile'):
os.remove('PET0.ESMF_LogFile')
def run():
import ocgis
print('Hello from run(). I am rank: {}'.format(ocgis.vm.rank))
dist = ocgis.vmachine.mpi.OcgDist()
dim = dist.create_dimension('foo', 10, dist=True)
dist.update_dimension_bounds()
var = ocgis.Variable(name='a_var', dimensions=dim)
var.get_value()[:] = ocgis.vm.rank
var.parent.write('a_var_data.nc')
if ocgis.vm.rank == 0:
invar = ocgis.RequestDataset('a_var_data.nc').create_field()['a_var']
print('Value on disk:', invar.get_value())
import os
import ocgis
from ocgis.constants import DimensionMapKey
OUT_NC = os.path.join(os.getcwd(), 'ocgis_example_dimension_map.nc')
########################################################################################################################
# Write some initial data.
var_x = ocgis.Variable(name='nonstandard_x',
value=[10, 20, 30],
dimensions='nsx')
var_y = ocgis.Variable(name='nonstandard_y',
value=[40, 50, 60, 70],
dimensions='nsy')
var_t = ocgis.Variable(name='nonstandard_time',
value=[1, 2, 3],
units='days since 2000-1-1',
attrs={'calendar': 'standard'},
dimensions='nst')
data_dimensions = [
var_t.dimensions[0], var_x.dimensions[0], var_y.dimensions[0]
]
var_data = ocgis.Variable(name='some_data', dimensions=data_dimensions)
vc = ocgis.VariableCollection(variables=[var_x, var_y, var_t, var_data])
vc.write(OUT_NC)
########################################################################################################################
# This metadata is not self-describing. Hence, no data or coordinate variables are interpretable. OpenClimateGIS will
lat = field.get('XLAT_M').get_value()
ydim = field.dimensions["south_north"]
xdim = field.dimensions["west_east"]
#
lon_notime = lon[0,:,:]
lat_notime = lat[0,:,:]
# import ipdb; ipdb.set_trace()
field.remove_variable('XLONG_M')
field.remove_variable('XLAT_M')
attrs = {"units":"degrees_east",
"axis":"x"}
var = ocgis.Variable(name='lon', value=lon_notime, dimensions=[ydim, xdim],
attrs=attrs)
field.add_variable(var)
attrs = {"units":"degrees_north",
"axis":"y"}
var = ocgis.Variable(name='lat', value=lat_notime, dimensions=[ydim, xdim],
attrs=attrs)
field.add_variable(var)
# field["lon_center"].set_name("lon")
# field["lat_center"].set_name("lat")
# now create a grid and write bounds
## grid = field.grid
# field["lon"].set_bounds(field.get('bounds_lon').extract(), force=True)
def _preformatting_(self, i, coll):
"""
Modify in place the collections so they can be saved as discrete
geometries along a new spatial dimension.
"""
# TODO: UGID and GID show up in the output file, but they are equal. Remove one.
if not self.ops or self.ops.aggregate is False:
return coll
# Size of spatial dimension
ncoll = len(self.ops.geom)
udim = DimensionName.UNIONED_GEOMETRY
ugids = coll.properties.keys()
assert len(ugids) == 1
ugid = list(ugids)[0]
# Geometry centroid location
lon, lat = coll.geoms[ugid].centroid.xy
for field in coll.iter_fields():
lon_attrs = field.x.attrs.copy()
lat_attrs = field.y.attrs.copy()
xn = field.x.name
yn = field.y.name
# Removed for now. It'd be nice to find an elegant way to retain those.
field.remove_variable(xn)
field.remove_variable(yn)
# Create new lon and lat variables
field.add_variable(
ocgis.Variable(xn,
value=lon,
dimensions=(udim, ),
attrs=dict(
lon_attrs,
**{'long_name': 'Centroid longitude'})))
field.add_variable(
ocgis.Variable(yn,
value=lat,
dimensions=(udim, ),
attrs=dict(lat_attrs,
**{'long_name':
'Centroid latitude'})))
if VariableName.SPATIAL_MASK in field:
# Remove the spatial_mask.
field.remove_variable(VariableName.SPATIAL_MASK)
field.dimension_map.set_spatial_mask(None)
grid = ocgis.Grid(field[xn],
field[yn],
abstraction='point',
crs=field.crs,
parent=field)
field.set_grid(grid)
# Geometry variables from the geom properties dict
dm = get_data_model(self.ops)
# Some dtypes are not supported by netCDF3. Use the netCDF4
# data model to avoid these issues.
for key, val in coll.properties[ugid].items():
if np.issubdtype(type(val), int):
dt = get_dtype('int', dm)
elif np.issubdtype(type(val), float):
dt = get_dtype('float', dm)
else:
dt = 'auto'
# There is no metadata for those yet, but it could be passed
# using the output_format_options keyword.
field.add_variable(
ocgis.Variable(key,
value=[
val,
],
dtype=dt,
dimensions=(udim, )), )
# Propagate max string length if it is set.
smg = coll.children[ugid][key].string_max_length_global
if smg is not None:
field[key].set_string_max_length_global(value=smg)
# ------------------ Dimension update ------------------------ #
# Modify the dimensions for the number of geometries
gdim = field.dimensions[udim]
gdim.set_size(ncoll)
for var in field.iter_variables_by_dimensions([gdim]):
d = var.dimensions_dict[udim]
d.bounds_local = (i, i + 1)
# ------------------------------------------------------------ #
# CF-Conventions
# Options for cf-role are timeseries_id, profile_id, trajectory_id
gid = field[HeaderName.ID_GEOMETRY]
gid.attrs['cf_role'] = 'timeseries_id'
# Name of spatial dimension
if self.options.get('geom_dim', None):
gdim.set_name(self.options.get('geom_dim', None))
return coll
nodeCoords[:, 0] = esmf_corners_x.flatten()
nodeCoords[:, 1] = esmf_corners_y.flatten()
# Get the center coordinates
centerCoords = np.zeros((elementCount, 2), dtype=np.float32)
centerCoords[:, 0] = grid.x.v().flatten()
centerCoords[:, 1] = grid.y.v().flatten()
# Number of nodes per element. Always four corners in this case
numElementConn = np.ones(elementCount, dtype=np.int32) * 4
# Write out the data
vc = ocgis.VariableCollection()
nodeCoordsV = ocgis.Variable(name='nodeCoords',
value=nodeCoords,
dimensions=['nodeCount', 'coordDim'],
attrs={'units': 'degrees'},
parent=vc)
elementConnV = ocgis.Variable(
name='elementConn',
value=elementConn,
dimensions=['elementCount', 'maxNodePElement'],
attrs={'long_name': 'Node indices that define the element connectivity'},
parent=vc)
numElementConnV = ocgis.Variable(
name='numElementConn',
value=numElementConn,
dimensions=['elementCount'],
attrs={'long_name': 'Number of nodes per element'},
parent=vc)
centerCoordsV = ocgis.Variable(name='centerCoords',
def write(self):
ocgis_lh('starting write method', self._log, logging.DEBUG)
# Indicates if user geometries should be written to file.
write_ugeom = False
ncoll = len(self.ops.geom)
build = True
for i, coll in enumerate(self):
ugids = coll.properties.keys()
assert len(ugids) == 1
ugid = ugids[0]
# Geometry centroid location
lon, lat = coll.geoms[ugid].centroid.xy
for field in coll.iter_fields():
lon_attrs = field.x.attrs.copy()
lat_attrs = field.y.attrs.copy()
# Removed for now. It'd be nice to find an elegant way to retain those.
field.remove_variable('lat')
field.remove_variable('lon')
# Create new lon and lat variables
field.add_variable(
ocgis.Variable('lon',
value=lon,
dimensions=(DimensionName.UNIONED_GEOMETRY,),
attrs=dict(lon_attrs, **{'long_name':'Centroid longitude'})
)
)
field.add_variable(
ocgis.Variable('lat',
value=lat,
dimensions=(DimensionName.UNIONED_GEOMETRY,),
attrs=dict(lat_attrs, **{'long_name':'Centroid latitude'})
)
)
if 'ocgis_spatial_mask' in field:
# Remove the spatial_mask and replace by new one.
field.remove_variable('ocgis_spatial_mask')
grid = ocgis.Grid(field['lon'], field['lat'], abstraction='point',
crs=field.crs, parent=field)
grid.set_mask([[False,]])
field.set_grid(grid)
# Geometry variables from the geom properties dict
# There is no metadata for those...
dm = get_data_model(self.ops)
for key, val in coll.properties[ugid].items():
if np.issubdtype(type(val), int):
dt = get_dtype('int', dm)
elif np.issubdtype(type(val), float):
dt = get_dtype('float', dm)
else:
dt='auto'
field.add_variable(
ocgis.Variable(key,
value=[val,],
dtype=dt,
dimensions=(DimensionName.UNIONED_GEOMETRY,)))
# ------------------ Dimension update ------------------------ #
# Modify the dimensions for the number of geometries
gdim = field.dimensions[DimensionName.UNIONED_GEOMETRY]
gdim.set_size(ncoll)
for var in field.iter_variables_by_dimensions([gdim]):
d = var.dimensions_dict[DimensionName.UNIONED_GEOMETRY]
d.bounds_local = (i, i+1)
# ------------------------------------------------------------ #
# CF-Conventions
# Can this be anything else than a timeseries_id
# Options are timeseries_id, profile_id, trajectory_id
gid = field[HeaderName.ID_GEOMETRY]
gid.attrs['cf_role'] = 'timeseries_id'
# TODO: Hard-code the name in constants.py
gdim.set_name('region')
# Path to the output object.
# I needed to put it here because _write_archetype pops it, so it's not available after the first loop.
f = {KeywordArgument.PATH: self.path}
# This will be changed to "write" if we are on the build loop.
write_mode = MPIWriteMode.APPEND
if build:
# During a build loop, create the file and write the first series of records. Let the drivers determine
# the appropriate write modes for handling parallelism.
write_mode = None
# Write the user geometries if selected and there is one present on the incoming collection.
if self._add_ugeom and coll.has_container_geometries:
write_ugeom = True
if write_ugeom:
if vm.rank == 0:
# The output file name for the user geometries.
ugid_shp_name = self.prefix + '_ugid.shp'
if self._add_ugeom_nest:
ugeom_fiona_path = os.path.join(self._get_or_create_shp_folder_(), ugid_shp_name)
else:
ugeom_fiona_path = os.path.join(self.outdir, ugid_shp_name)
else:
ugeom_fiona_path = None
build = False
f[KeywordArgument.WRITE_MODE] = write_mode
self._write_coll_(f, coll)
if write_ugeom:
with vm.scoped(SubcommName.UGEOM_WRITE, [0]):
if not vm.is_null:
for subset_field in list(coll.children.values()):
subset_field.write(ugeom_fiona_path, write_mode=write_mode, driver=DriverVector)
# The metadata and dataset descriptor files may only be written if OCGIS operations are present.
ops = self.ops
if ops is not None and self.add_auxiliary_files and MPI_RANK == 0:
# Add OCGIS metadata output if requested.
if self.add_meta:
ocgis_lh('adding OCGIS metadata file', 'conv', logging.DEBUG)
from ocgis.conv.meta import MetaOCGISConverter
lines = MetaOCGISConverter(ops).write()
out_path = os.path.join(self.outdir, self.prefix + '_' + MetaOCGISConverter._meta_filename)
with open(out_path, 'w') as f:
f.write(lines)
# Add the dataset descriptor file if requested.
if self._add_did_file:
ocgis_lh('writing dataset description (DID) file', 'conv', logging.DEBUG)
path = os.path.join(self.outdir, self.prefix + '_did.csv')
_write_dataset_identifier_file_(path, ops)
# Add source metadata if requested.
if self._add_source_meta:
ocgis_lh('writing source metadata file', 'conv', logging.DEBUG)
path = os.path.join(self.outdir, self.prefix + '_source_metadata.txt')
_write_source_meta_(path, ops)
# Return the internal path unless overloaded by subclasses.
ret = self._get_return_()
return ret
# Should I be using ESMF corners?
# # Convert ocgis corners to esmf corners
# esmf_corners_x = get_esmf_corners_from_ocgis_corners(grid.x.bounds.v())
# esmf_corners_y = get_esmf_corners_from_ocgis_corners(grid.y.bounds.v())
# # Create one-based global index
# indexing = np.arange(1, esmf_corners_x.size + 1, dtype=np.int32).reshape(esmf_corners_x.shape)
# Write out the data
grid_size = grid.archetype.size
lat, lon = grid.x.shape
grid_corners = grid.x.bounds.shape[2]
vc = ocgis.VariableCollection()
grid_dims = ocgis.Variable(name='grid_dims',
value=grid.x.shape,
dimensions=['grid_rank'],
parent=vc)
latitude = ocgis.Variable(name='grid_center_lat',
value=grid.y.v().flatten(),
dimensions=['grid_size'],
attrs={'units': 'degrees'},
parent=vc)
longitude = ocgis.Variable(name='grid_center_lon',
value=grid.x.v().flatten(),
dimensions=['grid_size'],
attrs={'units': 'degrees'},
parent=vc)
gxbf = np.zeros([grid_size, grid_corners])
for i in range(lat):
gxbf[i * lon:(i + 1) * lon, :] = grid.x.bounds.v()[i, :, :]