def test_load_from_metadata(self):
# Test with greater depth which checks variable attributes.
lat = {'attrs': {CFName.STANDARD_NAME: CFName.StandardName.LATITUDE}}
lon = {'attrs': {CFName.STANDARD_NAME: CFName.StandardName.LONGITUDE}}
var = {'attrs': {}}
meta = {'variables': {'lat': lat, 'lon': lon, 'var': var}}
actual = CFSpherical.load_from_metadata('var', meta, depth=2)
self.assertIsInstance(actual, CFSpherical)
# Test a depth check is not supported.
with self.assertRaises(CRSDepthNotImplemented):
CFSpherical.load_from_metadata('var', meta, depth=-1)
def test_load_from_metadata(self):
# Test with greater depth which checks variable attributes.
lat = {'attrs': {CFName.STANDARD_NAME: CFName.StandardName.LATITUDE}}
lon = {'attrs': {CFName.STANDARD_NAME: CFName.StandardName.LONGITUDE}}
var = {'attrs': {}}
meta = {'variables': {'lat': lat, 'lon': lon, 'var': var}}
actual = CFSpherical.load_from_metadata('var', meta, depth=2)
self.assertIsInstance(actual, CFSpherical)
# Test a depth check is not supported.
with self.assertRaises(CRSDepthNotImplemented):
CFSpherical.load_from_metadata('var', meta, depth=-1)
def test_get_rotated_pole_transformation(self):
"""Test SpatialDimension objects are appropriately transformed."""
rd = self.test_data.get_rd('rotated_pole_ichec')
field = rd.get()
original_coordinate_values = field.grid.get_value_stacked().copy()
field.update_crs(CFSpherical())
self.assertEqual(field.crs, CFSpherical())
field.update_crs(rd.get().crs)
back_to_original_coordinate_values = field.grid.get_value_stacked(
).copy()
self.assertNumpyAllClose(original_coordinate_values,
back_to_original_coordinate_values)
def _prepare_geometry_(self, geom):
"""
Compare ``subset_sdim`` geographic state with ``target`` and perform any necessary transformations to ensure a
smooth subset operation.
:param subset_sdim: The input object to use for subsetting of ``target``.
:type subset_sdim: :class:`ocgis.interface.base.dimension.spatial.SpatialDimension`
:rtype: :class:`ocgis.interface.base.dimension.spatial.SpatialDimension`
"""
assert isinstance(geom, GeometryVariable)
prepared = geom.deepcopy()
if geom.crs is not None:
assert prepared.crs is not None
if isinstance(self.field.crs, CFRotatedPole):
prepared.update_crs(CFSpherical())
else:
if prepared.crs is None and self.field.crs is not None:
msg = "The subset geometry has no assigned CRS and the target field does. Set the subset geometry's " \
"CRS to continue."
raise ValueError(msg)
if prepared.crs is not None and self.field.crs is not None:
prepared.update_crs(self.field.crs)
field_wrapped_state = self.field.wrapped_state
prepared_wrapped_state = prepared.wrapped_state
if field_wrapped_state == WrappedState.UNWRAPPED:
if prepared_wrapped_state == WrappedState.WRAPPED:
prepared.unwrap()
elif field_wrapped_state == WrappedState.WRAPPED:
if prepared_wrapped_state == WrappedState.UNWRAPPED:
prepared.wrap()
return prepared
def test_prepare_geometry(self):
for geometry_record in self.get_subset_geometries():
for ss, k in self:
gvar = GeometryVariable(value=geometry_record['geom'],
dimensions='dim',
crs=WGS84())
self.assertIsNotNone(gvar.crs)
prepared = ss._prepare_geometry_(gvar)
self.assertNotEqual(gvar.get_value()[0].bounds,
prepared.get_value()[0].bounds)
self.assertFalse(
np.may_share_memory(gvar.get_value(),
prepared.get_value()))
try:
self.assertEqual(prepared.crs, ss.field.crs)
except AssertionError:
# Rotated pole on the fields become spherical.
self.assertEqual(prepared.crs, CFSpherical())
self.assertIsInstance(ss.field.crs, CFRotatedPole)
# Test nebraska against an unwrapped dataset.
nebraska = GeometryVariable(value=self.nebraska['geom'],
dimensions='d',
crs=WGS84())
field = self.test_data.get_rd('cancm4_tas').get()
ss = SpatialSubsetOperation(field)
prepared = ss._prepare_geometry_(nebraska)
self.assertEqual(prepared.wrapped_state, WrappedState.UNWRAPPED)
def test_get_field(self):
driver = self.get_drivernetcdf()
field = driver.get_field(format_time=False)
self.assertEqual(field.driver.key, driver.key)
self.assertIsInstance(field.time, TemporalVariable)
with self.assertRaises(CannotFormatTimeError):
assert field.time.value_datetime
self.assertIsInstance(field.crs, CoordinateReferenceSystem)
# Test overloading the coordinate system.
path = self.get_temporary_file_path('foo.nc')
with self.nc_scope(path, 'w') as ds:
v = ds.createVariable('latitude_longitude', np.int)
v.grid_mapping_name = 'latitude_longitude'
# First, test the default is found.
rd = RequestDataset(uri=path)
driver = DriverNetcdfCF(rd)
self.assertEqual(driver.get_crs(driver.metadata_source), CFSpherical())
self.assertEqual(driver.get_field().crs, CFSpherical())
# Second, test the overloaded CRS is found.
desired = CoordinateReferenceSystem(epsg=2136)
rd = RequestDataset(uri=path, crs=desired)
self.assertEqual(rd.crs, desired)
driver = DriverNetcdfCF(rd)
self.assertEqual(driver.get_crs(driver.metadata_source), CFSpherical())
field = driver.get_field()
self.assertEqual(field.crs, desired)
# Test file coordinate system variable is removed.
self.assertNotIn('latitude_longitude', field)
# Test the default coordinate system is used when nothing is in the file.
path = self.get_temporary_file_path('foo.nc')
with self.nc_scope(path, 'w') as ds:
ds.createVariable('nothing', np.int)
rd = RequestDataset(uri=path)
driver = DriverNetcdfCF(rd)
self.assertEqual(rd.crs, env.DEFAULT_COORDSYS)
self.assertEqual(driver.get_crs(driver.rd.metadata),
env.DEFAULT_COORDSYS)
self.assertEqual(driver.get_field().crs, env.DEFAULT_COORDSYS)
def test_get_rotated_pole_transformation(self):
"""Test SpatialDimension objects are appropriately transformed."""
try:
rd = self.test_data.get_rd('rotated_pole_ichec')
field = rd.get()
original_coordinate_values = field.grid.get_value_stacked().copy()
field.update_crs(CFSpherical())
self.assertEqual(field.crs, CFSpherical())
field.update_crs(rd.get().crs)
back_to_original_coordinate_values = field.grid.get_value_stacked(
).copy()
self.assertNumpyAllClose(original_coordinate_values,
back_to_original_coordinate_values)
except RuntimeError as e:
if "HDF error" in str(e):
raise SkipTest('HDF sometimes has trouble reading the dataset')
else:
raise
def get_crs_variable(metadata, rotated_pole_priority, to_search=None):
found = []
variables = metadata['variables']
for vname, var in list(variables.items()):
if to_search is not None:
if vname not in to_search:
continue
for potential in itersubclasses(CFCoordinateReferenceSystem):
try:
crs = potential.load_from_metadata(vname,
metadata,
strict=False)
found.append(crs)
break
except ProjectionDoesNotMatch:
continue
fset = set([f.name for f in found])
if len(fset) > 1:
msg = 'Multiple coordinate systems found. There should be only one.'
raise ValueError(msg)
elif len(fset) == 0:
crs = None
else:
crs = found[0]
# Allow spherical coordinate systems to overload rotated pole if they are present in the metadata.
if not rotated_pole_priority and isinstance(crs, CFRotatedPole):
try:
crs = CFSpherical.load_from_metadata(None, metadata, depth=2)
except ProjectionDoesNotMatch:
# No spherical coordinate system data available. Fall back on previous crs.
pass
# If there is no CRS, check for OCGIS coordinate systems.
if crs is None:
try:
crs = AbstractProj4CRS.load_from_metadata(metadata)
except ProjectionDoesNotMatch:
pass
return crs
def get_crs_variable(metadata, rotated_pole_priority, to_search=None):
found = []
variables = metadata['variables']
for vname, var in list(variables.items()):
if to_search is not None:
if vname not in to_search:
continue
for potential in itersubclasses(CFCoordinateReferenceSystem):
try:
crs = potential.load_from_metadata(vname, metadata, strict=False)
found.append(crs)
break
except ProjectionDoesNotMatch:
continue
fset = set([f.name for f in found])
if len(fset) > 1:
msg = 'Multiple coordinate systems found. There should be only one.'
raise ValueError(msg)
elif len(fset) == 0:
crs = None
else:
crs = found[0]
# Allow spherical coordinate systems to overload rotated pole if they are present in the metadata.
if not rotated_pole_priority and isinstance(crs, CFRotatedPole):
try:
crs = CFSpherical.load_from_metadata(None, metadata, depth=2)
except ProjectionDoesNotMatch:
# No spherical coordinate system data available. Fall back on previous crs.
pass
# If there is no CRS, check for OCGIS coordinate systems.
if crs is None:
try:
crs = AbstractProj4CRS.load_from_metadata(metadata)
except ProjectionDoesNotMatch:
pass
return crs
def _prepare_geometry_(self, geom):
"""
Compare ``geom`` geographic state with the target field and perform any necessary transformations to ensure a
smooth subset operation.
:param geom: The input geometry to use for subsetting.
:type geom: :class:`~ocgis.GeometryVariable`
:rtype: :class:`~ocgis.GeometryVariable`
"""
assert isinstance(geom, GeometryVariable)
# The subset geometry may be modified during this transaction. Use a deep copy to preserve the original
# geometry's state to avoid error accumulations during transformations.
prepared = geom.deepcopy()
if geom.crs is not None:
assert prepared.crs is not None
# Update the subset geometry's coordinate system to match the field's.
if isinstance(self.field.crs, CFRotatedPole):
prepared.update_crs(CFSpherical())
else:
if prepared.crs is None and self.field.crs is not None:
msg = "The subset geometry has no assigned CRS and the target field does. Set the subset geometry's " \
"CRS to continue."
raise ValueError(msg)
if prepared.crs is not None and self.field.crs is not None:
prepared.update_crs(self.field.crs)
# Update the subset geometry's spatial wrapping to match the target field.
field_wrapped_state = self.field.wrapped_state
prepared_wrapped_state = prepared.wrapped_state
if field_wrapped_state == WrappedState.UNWRAPPED:
if prepared_wrapped_state == WrappedState.WRAPPED:
prepared.unwrap()
elif field_wrapped_state == WrappedState.WRAPPED:
if prepared_wrapped_state == WrappedState.UNWRAPPED:
prepared.wrap()
return prepared
def __init__(self):
self.OVERWRITE = EnvParm('OVERWRITE',
False,
formatter=self._format_bool_)
self.DIR_OUTPUT = EnvParm('DIR_OUTPUT', os.getcwd())
self.DIR_GEOMCABINET = EnvParm('DIR_GEOMCABINET', None)
# Left in for backwards compatibility. Performs the same function as DIR_GEOMCABINET.
self.DIR_SHPCABINET = EnvParm('DIR_SHPCABINET', None)
self.DIR_DATA = EnvParm('DIR_DATA', None)
self.DIR_TEST_DATA = EnvParm('DIR_TEST_DATA', None)
self.MELTED = EnvParm('MELTED', None, formatter=self._format_bool_)
self.MODE = EnvParm('MODE', 'raw')
self.PREFIX = EnvParm('PREFIX', 'ocgis_output')
self.FILL_VALUE = EnvParm('FILL_VALUE', 1e20, formatter=float)
self.VERBOSE = EnvParm('VERBOSE', False, formatter=self._format_bool_)
self.OPTIMIZE_FOR_CALC = EnvParm('OPTIMIZE_FOR_CALC',
False,
formatter=self._format_bool_)
self.ENABLE_FILE_LOGGING = EnvParm('ENABLE_FILE_LOGGING',
False,
formatter=self._format_bool_)
self.DEBUG = EnvParm('DEBUG', False, formatter=self._format_bool_)
self.DIR_BIN = EnvParm('DIR_BIN', None)
self.USE_SPATIAL_INDEX = EnvParmImport('USE_SPATIAL_INDEX', None,
'rtree')
self.USE_CFUNITS = EnvParmImport('USE_CFUNITS', None,
('cf_units', 'cfunits'))
self.USE_ESMF = EnvParmImport('USE_ESMF', None, 'ESMF')
self.USE_ICCLIM = EnvParmImport('USE_ICCLIM', None, 'icclim')
self.USE_MPI4PY = EnvParmImport('USE_MPI4PY', None, 'mpi4py')
self.CONF_PATH = EnvParm('CONF_PATH',
os.path.expanduser('~/.config/ocgis.conf'))
self.SUPPRESS_WARNINGS = EnvParm('SUPPRESS_WARNINGS',
True,
formatter=self._format_bool_)
self.DEFAULT_GEOM_UID = EnvParm(
'DEFAULT_GEOM_UID',
constants.OCGIS_UNIQUE_GEOMETRY_IDENTIFIER,
formatter=str)
self.NETCDF_FILE_FORMAT = EnvParm('NETCDF_FILE_FORMAT',
constants.NETCDF_DEFAULT_DATA_MODEL,
formatter=str)
self.NP_INT = EnvParm('NP_INT', constants.DEFAULT_NP_INT)
self.NP_FLOAT = EnvParm('NP_FLOAT', constants.DEFAULT_NP_FLOAT)
self.ADD_OPS_MPI_BARRIER = EnvParm('ADD_OPS_MPI_BARRIER',
True,
formatter=self._format_bool_)
self.PREFER_NETCDFTIME = EnvParm('PREFER_NETCDFTIME',
None,
formatter=self._format_bool_)
self.CLOBBER_UNITS_ON_BOUNDS = EnvParm('CLOBBER_UNITS_ON_BOUNDS',
True,
formatter=self._format_bool_)
if self.PREFER_NETCDFTIME is None:
self.PREFER_NETCDFTIME = get_netcdftime_preference()
from ocgis.variable.crs import CFSpherical
self.DEFAULT_COORDSYS = EnvParm('DEFAULT_COORDSYS', CFSpherical())
self.ops = None
self._optimize_store = {}
def test_eq(self):
other = CFSpherical()
sph = Spherical()
self.assertTrue(sph == other)
def __init__(self):
self.OVERWRITE = EnvParm('OVERWRITE',
False,
formatter=self._format_bool_)
self.DIR_OUTPUT = EnvParm('DIR_OUTPUT', os.getcwd())
self.DIR_GEOMCABINET = EnvParm('DIR_GEOMCABINET', None)
# Left in for backwards compatibility. Performs the same function as DIR_GEOMCABINET.
self.DIR_SHPCABINET = EnvParm('DIR_SHPCABINET', None)
self.DIR_DATA = EnvParm('DIR_DATA', None)
self.DIR_TEST_DATA = EnvParm('DIR_TEST_DATA', None)
self.MELTED = EnvParm('MELTED', None, formatter=self._format_bool_)
self.MODE = EnvParm('MODE', 'raw')
self.PREFIX = EnvParm('PREFIX', 'ocgis_output')
self.FILL_VALUE = EnvParm('FILL_VALUE', 1e20, formatter=float)
self.VERBOSE = EnvParm('VERBOSE', False, formatter=self._format_bool_)
self.OPTIMIZE_FOR_CALC = EnvParm('OPTIMIZE_FOR_CALC',
False,
formatter=self._format_bool_)
self.ENABLE_FILE_LOGGING = EnvParm('ENABLE_FILE_LOGGING',
False,
formatter=self._format_bool_)
self.DEBUG = EnvParm('DEBUG', False, formatter=self._format_bool_)
self.DIR_BIN = EnvParm('DIR_BIN', None)
self.USE_SPATIAL_INDEX = EnvParmImport('USE_SPATIAL_INDEX', None,
'rtree')
self.USE_CFUNITS = EnvParmImport('USE_CFUNITS', None,
('cf_units', 'cfunits'))
self.USE_ESMF = EnvParmImport('USE_ESMF', None, 'ESMF')
self.USE_ICCLIM = EnvParmImport('USE_ICCLIM', None, 'icclim')
self.USE_MPI4PY = EnvParmImport('USE_MPI4PY', None, 'mpi4py')
self.USE_MEMORY_OPTIMIZATIONS = EnvParm('USE_MEMORY_OPTIMIZATIONS',
False,
formatter=self._format_bool_)
self.USE_NETCDF4_MPI = EnvParm('USE_NETCDF4_MPI',
None,
formatter=self._format_bool_)
self.CONF_PATH = EnvParm('CONF_PATH',
os.path.expanduser('~/.config/ocgis.conf'))
self.SUPPRESS_WARNINGS = EnvParm('SUPPRESS_WARNINGS',
True,
formatter=self._format_bool_)
self.DEFAULT_GEOM_UID = EnvParm(
'DEFAULT_GEOM_UID',
constants.OCGIS_UNIQUE_GEOMETRY_IDENTIFIER,
formatter=str)
self.NETCDF_FILE_FORMAT = EnvParm('NETCDF_FILE_FORMAT',
constants.NETCDF_DEFAULT_DATA_MODEL,
formatter=str)
self.NP_INT = EnvParm('NP_INT', constants.DEFAULT_NP_INT)
self.NP_FLOAT = EnvParm('NP_FLOAT', constants.DEFAULT_NP_FLOAT)
# If True, place an MPI barrier following OCGIS operations.
self.ADD_OPS_MPI_BARRIER = EnvParm('ADD_OPS_MPI_BARRIER',
True,
formatter=self._format_bool_)
# If True, prefer using netcdftime if available. If None, automatically detect which to use.
self.PREFER_NETCDFTIME = EnvParm('PREFER_NETCDFTIME',
None,
formatter=self._format_bool_)
# If True, add units and possibly calendar for time variables on bounds variables.
self.CLOBBER_UNITS_ON_BOUNDS = EnvParm('CLOBBER_UNITS_ON_BOUNDS',
True,
formatter=self._format_bool_)
# If True, add axis attributes to grid coordinate variables when the grid is initialized.
self.SET_GRID_AXIS_ATTRS = EnvParm('SET_GRID_AXIS_ATTRS',
True,
formatter=self._format_bool_)
# If a coordinate system object, the current coordinate system is a dummy coordinate system and should be
# removed before writing and replaced with the original.
self.COORDSYS_ACTUAL = EnvParm('COORDSYS_ACTUAL', None)
if self.PREFER_NETCDFTIME is None:
self.PREFER_NETCDFTIME = get_netcdftime_preference()
from ocgis.variable.crs import CFSpherical
self.DEFAULT_COORDSYS = EnvParm('DEFAULT_COORDSYS', CFSpherical())
if self.USE_NETCDF4_MPI is None:
import netCDF4
self.USE_NETCDF4_MPI = False
if hasattr(netCDF4, '__has_nc_par__'):
if netCDF4.__has_nc_par__ == 1:
self.USE_NETCDF4_MPI = True
self.ops = None
self._optimize_store = {}
def DEFAULT_COORDSYS(self):
"""Here to handle circular imports."""
from ocgis.variable.crs import CFSpherical
return EnvParm('DEFAULT_COORDSYS', CFSpherical())
