def _get_netcdf4_data_variables(ds):
result = []
required_attrs = {"long_name", "units"}
for var in ds.variables:
if set(ds[var].ncattrs()).intersection(
required_attrs) != required_attrs:
continue
try:
valid_min = ds[var].getncattr("valid_min")
except AttributeError:
valid_min = None
try:
valid_max = ds[var].getncattr("valid_max")
except AttributeError:
valid_max = None
result.append(
Variable(
var,
ds[var].getncattr("long_name"),
ds[var].getncattr("units"),
tuple([dim for dim in ds.dimensions]),
valid_min,
valid_max,
))
return VariableList(result)
def variables(self):
"""
Returns a list of all data variables and their
attributes in the dataset.
"""
if self._calculated_variable_list is None:
variable_list = super().variables
temp_list = list(variable_list)
for name, calculated_var in self._calculated.items():
if name in variable_list:
continue
# New Variable
temp_list.append(
Variable(
name,
calculated_var.get("long_name", name),
calculated_var.get("units", "1"),
self.__get_calculated_dims(name),
calculated_var.get("valid_min",
np.finfo(np.float64).min),
calculated_var.get("valid_max",
np.finfo(np.float64).max),
))
self._calculated_variable_list = VariableList(temp_list)
return self._calculated_variable_list
def test_new_variable(self, mock_get_var_dims, mock_query_func):
mock_get_var_dims.return_value = [
'time', 'depth', 'latitude', 'longitude'
]
mock_query_func.return_value = VariableList([
Variable('votemper', 'Sea water potential temperature', 'Kelvin',
sorted(['time', 'depth', 'latitude', 'longitude']))
])
calculated = {
'votemper_new': {
'equation': 'votemper * 2',
'long_name': 'Temperature',
'dims': ('time', 'depth', 'latitude', 'longitude'),
'units': 'degree_C',
'valid_min': -273.15,
'valid_max': 999.0,
}
}
with CalculatedImpl('tests/testdata/mercator_test.nc',
calculated=calculated) as data:
self.assertEqual(len(data.variables), 2)
v = data.get_dataset_variable("votemper_new")
self.assertAlmostEqual(v[0, 0, 17, 816].values, 2.0 * 271.1796875)
self.assertEqual(v.attrs.long_name, "Temperature")
self.assertEqual(v.shape, (1, 50, 850, 1800))
def test_override(self, mock_query_func):
mock_query_func.return_value = VariableList(
[
Variable(
"votemper",
"Sea water potential temperature",
"Kelvin",
sorted(["time", "depth", "latitude", "longitude"]),
)
]
)
calculated = {
"votemper": {
"equation": "votemper -273.15",
"units": "degree_C",
"dims": ("time", "depth", "latitude", "longitude"),
}
}
with CalculatedImpl(
"tests/testdata/mercator_test.nc", calculated=calculated
) as data:
self.assertEqual(len(data.variables), 1)
v = data.get_dataset_variable("votemper")
self.assertAlmostEqual(v[0, 0, 17, 816].values, 271.1796875 - 273.15)
self.assertEqual(v.attrs.long_name, "Sea water potential temperature")
self.assertEqual(v.shape, (1, 50, 850, 1800))
def setUp(self):
self.variable_list_mock = VariableList([
Variable(
"votemper",
"Water temperature at CMC",
"Kelvins",
sorted(["deptht", "time_counter", "y", "x"]),
)
])
def setUp(self):
self.app = app.test_client()
with open('tests/testdata/datasetconfigpatch.json') as dataPatch:
self.patch_dataset_config_ret_val = json.load(dataPatch)
self.patch_data_vars_ret_val = VariableList([
Variable('votemper', 'Water temperature at CMC',
'Kelvins', sorted(["deptht", "time_counter", "y", "x"]))
])
def setUp(self):
self.variable_list_mock = VariableList(
[
Variable(
"votemper",
"Sea water potential temperature",
"Kelvin",
sorted(["time", "depth", "latitude", "longitude"]),
)
]
)
def test_nothing(self, mock_query_func):
mock_query_func.return_value = VariableList([
Variable('votemper', 'Sea water potential temperature', 'Kelvin',
sorted(['time', 'depth', 'latitude', 'longitude']))
])
with CalculatedImpl('tests/testdata/mercator_test.nc') as data:
self.assertEqual(len(data.variables), 1)
v = data.get_dataset_variable("votemper")
self.assertAlmostEqual(v[0, 0, 17, 816].values, 271.1796875)
def get_data_variables(self) -> VariableList:
"""Retrieves all data variables from the open database (i.e. depth, time, etc. are filtered out).
Returns:
[VariableList] -- VariableList of all data variable names.
"""
all_vars = self.get_all_variables()
regex = re.compile(r'^(.)*(time|depth|lat|lon|polar|^x|^y)+(.)*$')
result = list(filter(lambda i: not regex.match(i.key), all_vars))
return VariableList(result)
def _get_xarray_data_variables(ds):
result = []
required_attrs = {"long_name", "units"}
for var in ds.data_vars:
if set(ds[var].attrs).intersection(required_attrs) != required_attrs:
continue
result.append(
Variable(
var, ds[var].attrs["long_name"], ds[var].attrs["units"],
tuple([dim for dim in ds.dims]),
# Use .get() here to provide None if variable metadata lacks
# valid_min or valid_max
ds[var].attrs.get("valid_min"), ds[var].attrs.get("valid_max"))
)
return VariableList(result)
def get_all_variables(self) -> VariableList:
"""Retrieves all variables from the open database (including depth, time, etc.)
Returns:
[list] -- List of all variable names for this database.
"""
self.c.execute(
"SELECT variable, units, longName, validMin, validMax FROM Variables"
)
result = self.c.fetchall()
l = [self.__build_variable_wrapper(v) for v in result]
return VariableList(l)
def test_nothing(self, mock_query_func):
mock_query_func.return_value = VariableList(
[
Variable(
"votemper",
"Sea water potential temperature",
"Kelvin",
sorted(["time", "depth", "latitude", "longitude"]),
)
]
)
with CalculatedImpl("tests/testdata/mercator_test.nc") as data:
self.assertEqual(len(data.variables), 1)
v = data.get_dataset_variable("votemper")
self.assertEqual(xr.DataArray, type(v))
self.assertAlmostEqual(v[0, 0, 17, 816].values, 271.1796875)
def test_override(self, mock_query_func):
mock_query_func.return_value = VariableList([
Variable('votemper', 'Sea water potential temperature', 'Kelvin',
sorted(['time', 'depth', 'latitude', 'longitude']))
])
calculated = {
'votemper': {
'equation': 'votemper -273.15',
'units': 'degree_C',
'dims': ('time', 'depth', 'latitude', 'longitude')
}
}
with CalculatedImpl('tests/testdata/mercator_test.nc',
calculated=calculated) as data:
self.assertEqual(len(data.variables), 1)
v = data.get_dataset_variable("votemper")
self.assertAlmostEqual(v[0, 0, 17, 816].values,
271.1796875 - 273.15)
self.assertEqual(v.attrs.long_name,
"Sea water potential temperature")
self.assertEqual(v.shape, (1, 50, 850, 1800))
def setUp(self):
self.variable_list = VariableList([
Variable("my_key", "my_name", "my_unit", ("depth", "dim2"), 0, 100)
])
def setUp(self):
self.variable_list = VariableList([Variable(
"my_key", "my_name", "my_unit", ('depth', 'dim2'), 0, 100)])
def setUp(self):
self.variable_list_mock = VariableList([
Variable('h', 'Bathymetry', 'm', ["node"]),
Variable('zeta', "Water elevation", "m", []),
Variable('temp', "Temp", "Kelvin", [])
])
def setUp(self):
self.variable_list_mock = VariableList([
Variable("h", "Bathymetry", "m", ["node"]),
Variable("zeta", "Water elevation", "m", []),
Variable("temp", "Temp", "Kelvin", []),
])
def setUp(self):
self.variable_list_mock = VariableList([
Variable('votemper', 'Sea water potential temperature', 'Kelvin',
sorted(['time', 'depth', 'latitude', 'longitude']))
])
def setUp(self):
self.variable_list_mock = VariableList([
Variable('votemper', 'Water temperature at CMC', 'Kelvins',
sorted(["deptht", "time_counter", "y", "x"]))
])