def test_new_variable(self, mock_get_var_dims, mock_query_func):
mock_get_var_dims.return_value = [
'time', 'depth', 'latitude', 'longitude'
]
mock_query_func.return_value = VariableList([
Variable('votemper', 'Sea water potential temperature', 'Kelvin',
sorted(['time', 'depth', 'latitude', 'longitude']))
])
calculated = {
'votemper_new': {
'equation': 'votemper * 2',
'long_name': 'Temperature',
'dims': ('time', 'depth', 'latitude', 'longitude'),
'units': 'degree_C',
'valid_min': -273.15,
'valid_max': 999.0,
}
}
with CalculatedImpl('tests/testdata/mercator_test.nc',
calculated=calculated) as data:
self.assertEqual(len(data.variables), 2)
v = data.get_dataset_variable("votemper_new")
self.assertAlmostEqual(v[0, 0, 17, 816].values, 2.0 * 271.1796875)
self.assertEqual(v.attrs.long_name, "Temperature")
self.assertEqual(v.shape, (1, 50, 850, 1800))
def _get_netcdf4_data_variables(ds):
result = []
required_attrs = {"long_name", "units"}
for var in ds.variables:
if set(ds[var].ncattrs()).intersection(
required_attrs) != required_attrs:
continue
try:
valid_min = ds[var].getncattr("valid_min")
except AttributeError:
valid_min = None
try:
valid_max = ds[var].getncattr("valid_max")
except AttributeError:
valid_max = None
result.append(
Variable(
var,
ds[var].getncattr("long_name"),
ds[var].getncattr("units"),
tuple([dim for dim in ds.dimensions]),
valid_min,
valid_max,
))
return VariableList(result)
def variables(self):
"""
Returns a list of all data variables and their
attributes in the dataset.
"""
if self._calculated_variable_list is None:
variable_list = super().variables
temp_list = list(variable_list)
for name, calculated_var in self._calculated.items():
if name in variable_list:
continue
# New Variable
temp_list.append(
Variable(
name,
calculated_var.get("long_name", name),
calculated_var.get("units", "1"),
self.__get_calculated_dims(name),
calculated_var.get("valid_min",
np.finfo(np.float64).min),
calculated_var.get("valid_max",
np.finfo(np.float64).max),
))
self._calculated_variable_list = VariableList(temp_list)
return self._calculated_variable_list
def test_override(self, mock_query_func):
mock_query_func.return_value = VariableList(
[
Variable(
"votemper",
"Sea water potential temperature",
"Kelvin",
sorted(["time", "depth", "latitude", "longitude"]),
)
]
)
calculated = {
"votemper": {
"equation": "votemper -273.15",
"units": "degree_C",
"dims": ("time", "depth", "latitude", "longitude"),
}
}
with CalculatedImpl(
"tests/testdata/mercator_test.nc", calculated=calculated
) as data:
self.assertEqual(len(data.variables), 1)
v = data.get_dataset_variable("votemper")
self.assertAlmostEqual(v[0, 0, 17, 816].values, 271.1796875 - 273.15)
self.assertEqual(v.attrs.long_name, "Sea water potential temperature")
self.assertEqual(v.shape, (1, 50, 850, 1800))
def setUp(self):
self.variable_list_mock = VariableList([
Variable(
"votemper",
"Water temperature at CMC",
"Kelvins",
sorted(["deptht", "time_counter", "y", "x"]),
)
])
def setUp(self):
self.app = app.test_client()
with open('tests/testdata/datasetconfigpatch.json') as dataPatch:
self.patch_dataset_config_ret_val = json.load(dataPatch)
self.patch_data_vars_ret_val = VariableList([
Variable('votemper', 'Water temperature at CMC',
'Kelvins', sorted(["deptht", "time_counter", "y", "x"]))
])
def setUp(self):
self.variable_list_mock = VariableList(
[
Variable(
"votemper",
"Sea water potential temperature",
"Kelvin",
sorted(["time", "depth", "latitude", "longitude"]),
)
]
)
def test_nothing(self, mock_query_func):
mock_query_func.return_value = VariableList([
Variable('votemper', 'Sea water potential temperature', 'Kelvin',
sorted(['time', 'depth', 'latitude', 'longitude']))
])
with CalculatedImpl('tests/testdata/mercator_test.nc') as data:
self.assertEqual(len(data.variables), 1)
v = data.get_dataset_variable("votemper")
self.assertAlmostEqual(v[0, 0, 17, 816].values, 271.1796875)
def variables(self) -> VariableList:
"""Returns a list of all data variables and their
attributes in the dataset.
Returns:
VariableList -- contains all the data variables (no coordinates)
"""
# Check if variable list has been created yet.
# This saves approx 3 lookups per tile, and
# over a dozen when a new dataset is loaded.
if self._variable_list is not None:
return self._variable_list
# Handle possible list of URLs for staggered grid velocity field datasets
url = self.url if not isinstance(self.url, list) else self.url[0]
if url.endswith(".sqlite3"):
with SQLiteDatabase(url) as db:
self._variable_list = db.get_data_variables() # Cache the list for later
elif url.endswith(".zarr"):
ds_zarr = xarray.open_zarr(url)
var_list =[]
for var in list(ds_zarr.data_vars):
name = var
units = ds_zarr.variables[var].attrs['units'] if ds_zarr.variables[var].attrs['units'] else None
long_name = ds_zarr.variables[var].attrs['long_name'] if ds_zarr.variables[var].attrs['long_name'] else name
valid_min = ds_zarr.variables[var].attrs['valid_min'] if ds_zarr.variables[var].attrs['valid_min'] else None
valid_max = ds_zarr.variables[var].attrs['valid_max'] if ds_zarr.variables[var].attrs['valid_max'] else None
var_list.append(Variable(name, long_name, units, list(ds_zarr[name].dims), valid_min, valid_max))
self._variable_list = var_list
else:
try:
# Handle possible list of URLs for staggered grid velocity field datasets
url = self.url if isinstance(self.url, list) else [self.url]
# This will raise a FutureWarning for xarray>=0.12.2.
# That warning should be resolvable by changing to:
# with xarray.open_mfdataset(url, combine="by_coords", decode_times=False) as ds:
with xarray.open_mfdataset(url, decode_times=False) as ds:
self._variable_list = self._get_xarray_data_variables(ds) # Cache the list for later
except xarray.core.variable.MissingDimensionsError:
# xarray won't open FVCOM files due to dimension/coordinate/variable label
# duplication issue, so fall back to using netCDF4.Dataset()
with netCDF4.Dataset(self.url) as ds:
self._variable_list = self._get_netcdf4_data_variables(ds) # Cache the list for later
return self._variable_list
class TestVariable(unittest.TestCase):
def setUp(self):
self.variable = Variable("my_key", "my_name", "my_unit",
('depth', 'dim2'), 0, 100)
def test_properties(self):
self.assertEqual(self.variable.key, "my_key")
self.assertEqual(self.variable.name, "my_name")
self.assertEqual(self.variable.unit, "my_unit")
self.assertTupleEqual(self.variable.dimensions, ('depth', 'dim2'))
self.assertEqual(self.variable.valid_min, 0)
self.assertEqual(self.variable.valid_max, 100)
def test__str__(self):
self.assertEqual(str(self.variable), "my_key")
def test_has_depth(self):
self.assertTrue(self.variable.has_depth())
def test_is_surface_only(self):
self.assertFalse(self.variable.is_surface_only())
def _get_xarray_data_variables(ds):
result = []
required_attrs = {"long_name", "units"}
for var in ds.data_vars:
if set(ds[var].attrs).intersection(required_attrs) != required_attrs:
continue
result.append(
Variable(
var, ds[var].attrs["long_name"], ds[var].attrs["units"],
tuple([dim for dim in ds.dims]),
# Use .get() here to provide None if variable metadata lacks
# valid_min or valid_max
ds[var].attrs.get("valid_min"), ds[var].attrs.get("valid_max"))
)
return VariableList(result)
def test_nothing(self, mock_query_func):
mock_query_func.return_value = VariableList(
[
Variable(
"votemper",
"Sea water potential temperature",
"Kelvin",
sorted(["time", "depth", "latitude", "longitude"]),
)
]
)
with CalculatedImpl("tests/testdata/mercator_test.nc") as data:
self.assertEqual(len(data.variables), 1)
v = data.get_dataset_variable("votemper")
self.assertEqual(xr.DataArray, type(v))
self.assertAlmostEqual(v[0, 0, 17, 816].values, 271.1796875)
def __build_variable_wrapper(self, var_result: list) -> Variable:
"""Builds a Variable object from a given row list.
Arguments:
var_result {list} -- A row from the Variable table in the sqlite database.
Returns:
[Variable] -- constructed Variable object.
"""
# Look at the SELECT statement in get_all_variables to see the order of values
name = var_result[0]
units = var_result[1] if var_result[1] else None
long_name = var_result[2] if var_result[2] else name
valid_min = var_result[3] if var_result[3] != 1.17549e-38 else None
valid_max = var_result[4] if var_result[4] != 3.40282e+38 else None
return Variable(name, long_name, units, self.get_variable_dims(name),
valid_min, valid_max)
def test_override(self, mock_query_func):
mock_query_func.return_value = VariableList([
Variable('votemper', 'Sea water potential temperature', 'Kelvin',
sorted(['time', 'depth', 'latitude', 'longitude']))
])
calculated = {
'votemper': {
'equation': 'votemper -273.15',
'units': 'degree_C',
'dims': ('time', 'depth', 'latitude', 'longitude')
}
}
with CalculatedImpl('tests/testdata/mercator_test.nc',
calculated=calculated) as data:
self.assertEqual(len(data.variables), 1)
v = data.get_dataset_variable("votemper")
self.assertAlmostEqual(v[0, 0, 17, 816].values,
271.1796875 - 273.15)
self.assertEqual(v.attrs.long_name,
"Sea water potential temperature")
self.assertEqual(v.shape, (1, 50, 850, 1800))
def setUp(self):
self.variable_list = VariableList([
Variable("my_key", "my_name", "my_unit", ("depth", "dim2"), 0, 100)
])
def __make_surface_variable(self):
return Variable("my_key", "my_name", "my_unit", ("dim2"), 0, 100)
def __make_3d_variable(self):
return Variable("my_key", "my_name", "my_unit", ("depth", "dim2"), 0,
100)
def setUp(self):
self.variable_list = VariableList([Variable(
"my_key", "my_name", "my_unit", ('depth', 'dim2'), 0, 100)])
def setUp(self):
self.variable_list_mock = VariableList([
Variable('h', 'Bathymetry', 'm', ["node"]),
Variable('zeta', "Water elevation", "m", []),
Variable('temp', "Temp", "Kelvin", [])
])
def setUp(self):
self.variable_list_mock = VariableList([
Variable("h", "Bathymetry", "m", ["node"]),
Variable("zeta", "Water elevation", "m", []),
Variable("temp", "Temp", "Kelvin", []),
])
def setUp(self):
self.variable_list_mock = VariableList([
Variable('votemper', 'Sea water potential temperature', 'Kelvin',
sorted(['time', 'depth', 'latitude', 'longitude']))
])
def setUp(self):
self.variable_list_mock = VariableList([
Variable('votemper', 'Water temperature at CMC', 'Kelvins',
sorted(["deptht", "time_counter", "y", "x"]))
])