def validate(self, value):
if value is not None:
try:
get_units_object(value)
except ValueError:
msg = 'Units not recognized by conversion backend: {}'.format(value)
raise DefinitionValidationError(self.__class__, msg)
def test_cfunits_conform(self):
units_kelvin = get_units_object('kelvin')
# Conversion of celsius units to kelvin.
attrs = {k: 1 for k in NETCDF_ATTRIBUTES_TO_REMOVE_ON_VALUE_CHANGE}
var = Variable(name='tas', units='celsius', value=self.value, attrs=attrs)
self.assertEqual(len(var.attrs), 4)
var.cfunits_conform(units_kelvin)
self.assertNumpyAll(var.value, np.ma.array([278.15] * 3))
self.assertEqual(var.cfunits, units_kelvin)
self.assertEqual(var.units, 'kelvin')
self.assertEqual(len(var.attrs), 0)
# If there are no units associated with a variable, conforming the units should fail.
var = Variable(name='tas', units=None, value=self.value)
with self.assertRaises(NoUnitsError):
var.cfunits_conform(units_kelvin)
# Conversion should fail for nonequivalent units.
var = Variable(name='tas', units='kelvin', value=self.value)
with self.assertRaises(ValueError):
var.cfunits_conform(get_units_object('grams'))
# The data type should always be updated to match the output from CF units backend.
av = Variable(value=np.array([4, 5, 6]), dtype=int)
self.assertEqual(av.dtype, np.dtype(int))
with self.assertRaises(NoUnitsError):
av.cfunits_conform('K')
av.units = 'celsius'
av.cfunits_conform('K')
self.assertIsNone(av._dtype)
self.assertEqual(av.dtype, av.value.dtype)
def validate(self, value):
if value is not None:
try:
get_units_object(value)
except ValueError:
msg = 'Units not recognized by conversion backend: {}'.format(
value)
raise DefinitionValidationError(self.__class__, msg)
def test_validate_units_bad_units(self):
field = self.get_field(with_value=True)
field['tmax'].units = 'celsius'
self.assertEqual(field['tmax'].cfunits, get_units_object('celsius'))
fnu = MockNeedsUnits(field=field)
with self.assertRaises(UnitsValidationError):
fnu.execute()
def test_validate_units_bad_units(self):
field = self.get_field(with_value=True)
field.variables['tmax'].units = 'celsius'
self.assertEqual(field.variables['tmax'].cfunits, get_units_object('celsius'))
fnu = FooNeedsUnits(field=field)
with self.assertRaises(UnitsValidationError):
fnu.execute()
def test_init(self):
cc = ConformUnitsTo()
self.assertEqual(cc.value, None)
cc = ConformUnitsTo('kelvin')
self.assertEqual(cc.value, 'kelvin')
cc = ConformUnitsTo('not_a_unit')
self.assertEqual(cc.value, 'not_a_unit')
cc = ConformUnitsTo(get_units_object('celsius'))
target = get_are_units_equal((cc.value, get_units_object('celsius')))
self.assertTrue(target)
cc = ConformUnitsTo('hPa')
target = get_are_units_equal((cc.value, get_units_object('hPa')))
self.assertTrue(target)
def test_validate_units_bad_units(self):
field = self.get_field(with_value=True)
tgd = field.temporal.get_grouping(['month'])
field.variables['tmax'].units = 'celsius'
self.assertEqual(field.variables['tmax'].cfunits, get_units_object('celsius'))
fnu = FooNeedsUnitsSet(field=field,tgd=tgd)
with self.assertRaises(UnitsValidationError):
fnu.execute()
def test_validate_units_bad_units(self):
field = self.get_field(with_value=True)
tgd = field.temporal.get_grouping(['month'])
field['tmax'].units = 'celsius'
self.assertEqual(field['tmax'].cfunits, get_units_object('celsius'))
fnu = MockNeedsUnitsSet(field=field, tgd=tgd)
with self.assertRaises(UnitsValidationError):
fnu.execute()
def test_execute_units_are_maintained(self):
field = self.get_field(with_value=True, month_count=2)
units_kelvin = get_units_object('kelvin')
self.assertEqual(field['tmax'].cfunits, units_kelvin)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field, tgd=tgd, alias='my_mean', calc_sample_size=False, dtype=np.float64)
dvc = mu.execute()
self.assertEqual(dvc['my_mean'].cfunits, units_kelvin)
def test_cfunits_conform(self):
vdim = VectorDimension(value=[5., 10., 15.], units='celsius')
vdim.set_extrapolated_bounds()
vdim.cfunits_conform(get_units_object('kelvin'))
self.assertNumpyAll(vdim.bounds, np.array([[275.65, 280.65], [280.65, 285.65], [285.65, 290.65]]))
# Test conforming without bounds.
vdim = VectorDimension(value=[5., 10., 15.], units='celsius')
vdim.cfunits_conform('kelvin')
self.assertNumpyAll(vdim.value, np.array([278.15, 283.15, 288.15]))
def test_get_field_t_conform_units_to(self):
"""
Test conforming time units is appropriately passed to field object.
"""
uri = self.test_data.get_uri('cancm4_tas')
target = get_units_object('days since 1949-1-1', calendar='365_day')
rd = RequestDataset(uri=uri, t_conform_units_to=target)
field = rd.get()
self.assertEqual(field.temporal.conform_units_to, target)
def test_get_are_units_equal(self):
units = [get_units_object('celsius'), get_units_object('kelvin'), get_units_object('fahrenheit')]
self.assertFalse(get_are_units_equal(units))
units = [get_units_object('celsius'), get_units_object('celsius'), get_units_object('celsius')]
self.assertTrue(get_are_units_equal(units))
units = [get_units_object('celsius')]
with self.assertRaises(ValueError):
get_are_units_equal(units)
def test_init(self):
cc = ConformUnitsTo()
self.assertEqual(cc.value, None)
if env.USE_CFUNITS:
cc = ConformUnitsTo('kelvin')
self.assertEqual(cc.value, 'kelvin')
with self.assertRaises(DefinitionValidationError):
ConformUnitsTo('not_a_unit')
cc = ConformUnitsTo(get_units_object('celsius'))
target = get_are_units_equal((cc.value, get_units_object('celsius')))
self.assertTrue(target)
cc = ConformUnitsTo('hPa')
target = get_are_units_equal((cc.value, get_units_object('hPa')))
self.assertTrue(target)
else:
with self.assertRaises(ImportError):
ConformUnitsTo('celsius')
def test_init(self):
self.assertEqual(AbstractVariable.__bases__, (Attributes,))
av = AbstractVariable(value=self.value)
self.assertNumpyAll(av.value, self.value)
self.assertIsNone(av.alias)
self.assertIsNone(av.conform_units_to)
fav = AbstractVariable(name='foo')
self.assertEqual(fav.alias, 'foo')
# Test data types also pulled from value if present.
dtype = float
fav = AbstractVariable(value=self.value, dtype=dtype)
self.assertEqual(fav.dtype, self.value.dtype)
self.assertIsNone(fav._dtype)
# Use string-based units.
var = Variable(name='tas', units='celsius', value=self.value)
self.assertEqual(var.units, 'celsius')
self.assertEqual(var.cfunits, get_units_object('celsius'))
self.assertNotEqual(var.cfunits, get_units_object('kelvin'))
self.assertTrue(get_are_units_equivalent((var.cfunits, get_units_object('kelvin'))))
# Use constructor with units objects v. string.
var = Variable(name='tas', units=get_units_object('celsius'), value=self.value)
self.assertEqual(var.units, 'celsius')
self.assertEqual(var.cfunits, get_units_object('celsius'))
# Test without units.
var = Variable(name='tas', units=None, value=self.value)
self.assertEqual(var.units, None)
self.assertEqual(var.cfunits, get_units_object(None))
def test_init(self):
cc = ConformUnitsTo()
self.assertEqual(cc.value, None)
if env.USE_CFUNITS:
cc = ConformUnitsTo('kelvin')
self.assertEqual(cc.value, 'kelvin')
with self.assertRaises(DefinitionValidationError):
ConformUnitsTo('not_a_unit')
cc = ConformUnitsTo(get_units_object('celsius'))
target = get_are_units_equal(
(cc.value, get_units_object('celsius')))
self.assertTrue(target)
cc = ConformUnitsTo('hPa')
target = get_are_units_equal((cc.value, get_units_object('hPa')))
self.assertTrue(target)
else:
with self.assertRaises(ImportError):
ConformUnitsTo('celsius')
def test_execute_units_are_maintained(self):
field = self.get_field(with_value=True, month_count=2)
units_kelvin = get_units_object('kelvin')
self.assertEqual(field['tmax'].cfunits, units_kelvin)
grouping = ['month']
tgd = field.temporal.get_grouping(grouping)
mu = Mean(field=field,
tgd=tgd,
alias='my_mean',
calc_sample_size=False,
dtype=np.float64)
dvc = mu.execute()
self.assertEqual(dvc['my_mean'].cfunits, units_kelvin)
def test_get_are_units_equal(self):
units = [
get_units_object('celsius'),
get_units_object('kelvin'),
get_units_object('fahrenheit')
]
self.assertFalse(get_are_units_equal(units))
units = [
get_units_object('celsius'),
get_units_object('celsius'),
get_units_object('celsius')
]
self.assertTrue(get_are_units_equal(units))
units = [get_units_object('celsius')]
with self.assertRaises(ValueError):
get_are_units_equal(units)
def test_calculate(self):
klasses = list(itersubclasses(AbstractIcclimPercentileArrayIndice))
# There are six classes to test.
self.assertEqual(len(klasses), 6)
for mod in (1, 2):
field = self.get_field(ntime=365)
# Values less than 1 mm/day will be masked inside icclim.
# var = field.variables.first()
var = field.get_by_tag(TagName.DATA_VARIABLES)[0]
var.get_value()[:] = var.get_value() * mod
tgd = field.temporal.get_grouping(['month'])
for klass in klasses:
c = klass(field=field, tgd=tgd)
res = c.execute()
self.assertIsInstance(res, VariableCollection)
dv = res.first()
# Output units are always mm/day.
if isinstance(c, IcclimR75pTOT):
self.assertTrue(get_are_units_equivalent((dv.cfunits, get_units_object('mm/day'))))
def test_get_field_with_overloaded_units(self):
rd = self.test_data.get_rd('cancm4_tas', kwds={'conform_units_to': 'celsius'})
preload = [False, True]
for pre in preload:
field = rd.get()
# Conform units argument needs to be attached to a field variable.
units_celsius = get_units_object('celsius')
self.assertEqual(field.variables['tas']._conform_units_to, units_celsius)
sub = field.get_time_region({'year': [2009], 'month': [5]})
if pre:
# If we wanted to load the data prior to subset then do so and manually perform the units conversion.
to_test = sub.variables['tas'].value.copy()
get_conformed_units(to_test, sub.variables['tas'].cfunits, units_celsius)
# Assert the conform attribute makes it though the subset
self.assertEqual(sub.variables['tas']._conform_units_to, units_celsius)
value = sub.variables['tas'].value
self.assertAlmostEqual(np.ma.mean(value), 5.921925206338206)
self.assertAlmostEqual(np.ma.median(value), 10.745431900024414)
if pre:
# Assert the manually converted array matches the loaded value.
self.assertNumpyAll(to_test, value)
def test_cfunits_conform_masked_array(self):
# Assert mask is respected by unit conversion.
value = np.ma.array(data=[5, 5, 5], mask=[False, True, False])
var = Variable(name='tas', units=get_units_object('celsius'), value=value)
var.cfunits_conform(get_units_object('kelvin'))
self.assertNumpyAll(np.ma.array([278.15, 278.15, 278.15], mask=[False, True, False]), var.value)
def test_get_field_units_read_from_file(self):
rd = self.test_data.get_rd('cancm4_tas')
field = rd.get()
self.assertEqual(field.variables['tas'].cfunits, get_units_object('K'))
def test_cfunits(self):
units = get_units_object('K')
self.assertEqual(str(units), 'K')
def cfunits(self):
return get_units_object(self.units)
def test_cfunits(self):
units = get_units_object('K')
self.assertEqual(str(units), 'K')
def test_conform_units_to(self):
v = AbstractVariable(value=self.value, units='kelvin', conform_units_to='celsius')
self.assertEqual(v.conform_units_to, get_units_object('celsius'))
def conform_units_to(self, value):
if value is not None:
value = get_units_object(value)
self._conform_units_to = value
def test_units_with_bounds(self):
value = [5., 10., 15.]
vdim = VectorDimension(value=value, units='celsius',
bounds=get_bounds_from_1d(np.array(value)))
vdim.cfunits_conform(get_units_object('kelvin'))
self.assertNumpyAll(vdim.bounds, np.array([[275.65, 280.65], [280.65, 285.65], [285.65, 290.65]]))
def __init__(self,
uri=None,
variable=None,
units=None,
time_range=None,
time_region=None,
time_subset_func=None,
level_range=None,
conform_units_to=None,
crs='auto',
t_units=None,
t_calendar=None,
t_conform_units_to=None,
grid_abstraction='auto',
grid_is_isomorphic='auto',
dimension_map=None,
field_name=None,
driver=None,
regrid_source=True,
regrid_destination=False,
metadata=None,
format_time=True,
opened=None,
uid=None,
rename_variable=None,
predicate=None,
rotated_pole_priority=False,
driver_kwargs=None):
self._is_init = True
self._field_name = field_name
self._level_range = None
self._time_range = None
self._time_region = None
self._time_subset_func = None
self._driver_kwargs = driver_kwargs
if rename_variable is not None:
rename_variable = get_tuple(rename_variable)
self._rename_variable = rename_variable
self.rotated_pole_priority = rotated_pole_priority
self.predicate = predicate
if dimension_map is not None and isinstance(dimension_map, dict):
dimension_map = DimensionMap.from_dict(dimension_map)
self._dimension_map = dimension_map
self._metadata = deepcopy(metadata)
self._uri = None
self.uid = uid
# This is an "open" file-like object that may be passed in-place of file location parameters.
self._opened = opened
if opened is not None and driver is None:
msg = 'If "opened" is not None, then a "driver" must be provided.'
ocgis_lh(logger='request',
exc=RequestValidationError('driver', msg))
# Field creation options.
self.format_time = format_time
self.grid_abstraction = grid_abstraction
self.grid_is_isomorphic = grid_is_isomorphic
# Flag used for regridding to determine if the coordinate system was assigned during initialization.
self._has_assigned_coordinate_system = False if crs == 'auto' else True
if uri is None:
# Fields may be created from pure metadata.
if metadata is not None:
# The default OCGIS driver is NetCDF.
if driver is None:
driver = DriverKey.NETCDF_CF
elif opened is None:
ocgis_lh(logger='request',
exc=RequestValidationError('uri', 'Cannot be None'))
else:
self._uri = get_uri(uri)
if driver is None:
klass = get_autodiscovered_driver(uri)
else:
klass = get_driver(driver)
self._driver = klass(self)
if variable is not None:
variable = get_tuple(variable)
self._variable = variable
self.time_range = time_range
self.time_region = time_region
self.time_subset_func = time_subset_func
self.level_range = level_range
self._crs = deepcopy(crs)
self.regrid_source = regrid_source
self.regrid_destination = regrid_destination
self.units = units
self.conform_units_to = conform_units_to
self._is_init = False
self._validate_time_subset_()
# Update metadata for time variable.
tvar = self.dimension_map.get_variable(DMK.TIME)
if tvar is not None:
m = self.metadata['variables'][tvar]
if t_units is not None:
m['attrs']['units'] = t_units
if t_calendar is not None:
m['attrs']['calendar'] = t_calendar
if t_conform_units_to is not None:
from ocgis.util.units import get_units_object
t_calendar = m['attrs'].get(
'calendar', constants.DEFAULT_TEMPORAL_CALENDAR)
t_conform_units_to = get_units_object(t_conform_units_to,
calendar=t_calendar)
m['conform_units_to'] = t_conform_units_to
def __init__(self, uri=None, variable=None, units=None, time_range=None, time_region=None,
time_subset_func=None, level_range=None, conform_units_to=None, crs='auto', t_units=None,
t_calendar=None, t_conform_units_to=None, grid_abstraction='auto', grid_is_isomorphic='auto',
dimension_map=None, field_name=None, driver=None, regrid_source=True, regrid_destination=False,
metadata=None, format_time=True, opened=None, uid=None, rename_variable=None, predicate=None,
rotated_pole_priority=False, driver_kwargs=None, decomp_type=DecompositionType.OCGIS):
self._is_init = True
self._field_name = field_name
self._level_range = None
self._time_range = None
self._time_region = None
self._time_subset_func = None
self._driver_kwargs = driver_kwargs
self._decomp_type = decomp_type
if rename_variable is not None:
rename_variable = get_tuple(rename_variable)
self._rename_variable = rename_variable
self.rotated_pole_priority = rotated_pole_priority
self.predicate = predicate
if dimension_map is not None and isinstance(dimension_map, dict):
dimension_map = DimensionMap.from_dict(dimension_map)
self._dimension_map = dimension_map
self._metadata = deepcopy(metadata)
self._uri = None
self.uid = uid
# This is an "open" file-like object that may be passed in-place of file location parameters.
self._opened = opened
if opened is not None and driver is None:
msg = 'If "opened" is not None, then a "driver" must be provided.'
ocgis_lh(logger='request', exc=RequestValidationError('driver', msg))
# Field creation options.
self.format_time = format_time
self.grid_abstraction = grid_abstraction
self.grid_is_isomorphic = grid_is_isomorphic
# Flag used for regridding to determine if the coordinate system was assigned during initialization.
self._has_assigned_coordinate_system = False if crs == 'auto' else True
if uri is None:
# Fields may be created from pure metadata.
if metadata is not None:
# The default OCGIS driver is NetCDF.
if driver is None:
driver = DriverKey.NETCDF_CF
elif opened is None:
ocgis_lh(logger='request', exc=RequestValidationError('uri', 'Cannot be None'))
else:
self._uri = get_uri(uri)
if driver is None:
klass = get_autodiscovered_driver(uri)
else:
klass = get_driver(driver)
self._driver = klass(self)
if variable is not None:
variable = get_tuple(variable)
self._variable = variable
self.time_range = time_range
self.time_region = time_region
self.time_subset_func = time_subset_func
self.level_range = level_range
self._crs = deepcopy(crs)
self.regrid_source = regrid_source
self.regrid_destination = regrid_destination
self.units = units
self.conform_units_to = conform_units_to
self._is_init = False
self._validate_time_subset_()
# Update metadata for time variable.
tvar = self.dimension_map.get_variable(DMK.TIME)
if tvar is not None:
m = self.metadata['variables'][tvar]
if t_units is not None:
m['attrs']['units'] = t_units
if t_calendar is not None:
m['attrs']['calendar'] = t_calendar
if t_conform_units_to is not None:
from ocgis.util.units import get_units_object
t_calendar = m['attrs'].get('calendar', constants.DEFAULT_TEMPORAL_CALENDAR)
t_conform_units_to = get_units_object(t_conform_units_to, calendar=t_calendar)
m['conform_units_to'] = t_conform_units_to
