def _get_netcdf4_data_variables(ds): result = [] required_attrs = {"long_name", "units"} for var in ds.variables: if set(ds[var].ncattrs()).intersection( required_attrs) != required_attrs: continue try: valid_min = ds[var].getncattr("valid_min") except AttributeError: valid_min = None try: valid_max = ds[var].getncattr("valid_max") except AttributeError: valid_max = None result.append( Variable( var, ds[var].getncattr("long_name"), ds[var].getncattr("units"), tuple([dim for dim in ds.dimensions]), valid_min, valid_max, )) return VariableList(result)
def variables(self): """ Returns a list of all data variables and their attributes in the dataset. """ if self._calculated_variable_list is None: variable_list = super().variables temp_list = list(variable_list) for name, calculated_var in self._calculated.items(): if name in variable_list: continue # New Variable temp_list.append( Variable( name, calculated_var.get("long_name", name), calculated_var.get("units", "1"), self.__get_calculated_dims(name), calculated_var.get("valid_min", np.finfo(np.float64).min), calculated_var.get("valid_max", np.finfo(np.float64).max), )) self._calculated_variable_list = VariableList(temp_list) return self._calculated_variable_list
def test_new_variable(self, mock_get_var_dims, mock_query_func): mock_get_var_dims.return_value = [ 'time', 'depth', 'latitude', 'longitude' ] mock_query_func.return_value = VariableList([ Variable('votemper', 'Sea water potential temperature', 'Kelvin', sorted(['time', 'depth', 'latitude', 'longitude'])) ]) calculated = { 'votemper_new': { 'equation': 'votemper * 2', 'long_name': 'Temperature', 'dims': ('time', 'depth', 'latitude', 'longitude'), 'units': 'degree_C', 'valid_min': -273.15, 'valid_max': 999.0, } } with CalculatedImpl('tests/testdata/mercator_test.nc', calculated=calculated) as data: self.assertEqual(len(data.variables), 2) v = data.get_dataset_variable("votemper_new") self.assertAlmostEqual(v[0, 0, 17, 816].values, 2.0 * 271.1796875) self.assertEqual(v.attrs.long_name, "Temperature") self.assertEqual(v.shape, (1, 50, 850, 1800))
def test_override(self, mock_query_func): mock_query_func.return_value = VariableList( [ Variable( "votemper", "Sea water potential temperature", "Kelvin", sorted(["time", "depth", "latitude", "longitude"]), ) ] ) calculated = { "votemper": { "equation": "votemper -273.15", "units": "degree_C", "dims": ("time", "depth", "latitude", "longitude"), } } with CalculatedImpl( "tests/testdata/mercator_test.nc", calculated=calculated ) as data: self.assertEqual(len(data.variables), 1) v = data.get_dataset_variable("votemper") self.assertAlmostEqual(v[0, 0, 17, 816].values, 271.1796875 - 273.15) self.assertEqual(v.attrs.long_name, "Sea water potential temperature") self.assertEqual(v.shape, (1, 50, 850, 1800))
def setUp(self): self.variable_list_mock = VariableList([ Variable( "votemper", "Water temperature at CMC", "Kelvins", sorted(["deptht", "time_counter", "y", "x"]), ) ])
def setUp(self): self.app = app.test_client() with open('tests/testdata/datasetconfigpatch.json') as dataPatch: self.patch_dataset_config_ret_val = json.load(dataPatch) self.patch_data_vars_ret_val = VariableList([ Variable('votemper', 'Water temperature at CMC', 'Kelvins', sorted(["deptht", "time_counter", "y", "x"])) ])
def setUp(self): self.variable_list_mock = VariableList( [ Variable( "votemper", "Sea water potential temperature", "Kelvin", sorted(["time", "depth", "latitude", "longitude"]), ) ] )
def test_nothing(self, mock_query_func): mock_query_func.return_value = VariableList([ Variable('votemper', 'Sea water potential temperature', 'Kelvin', sorted(['time', 'depth', 'latitude', 'longitude'])) ]) with CalculatedImpl('tests/testdata/mercator_test.nc') as data: self.assertEqual(len(data.variables), 1) v = data.get_dataset_variable("votemper") self.assertAlmostEqual(v[0, 0, 17, 816].values, 271.1796875)
def get_data_variables(self) -> VariableList: """Retrieves all data variables from the open database (i.e. depth, time, etc. are filtered out). Returns: [VariableList] -- VariableList of all data variable names. """ all_vars = self.get_all_variables() regex = re.compile(r'^(.)*(time|depth|lat|lon|polar|^x|^y)+(.)*$') result = list(filter(lambda i: not regex.match(i.key), all_vars)) return VariableList(result)
def _get_xarray_data_variables(ds): result = [] required_attrs = {"long_name", "units"} for var in ds.data_vars: if set(ds[var].attrs).intersection(required_attrs) != required_attrs: continue result.append( Variable( var, ds[var].attrs["long_name"], ds[var].attrs["units"], tuple([dim for dim in ds.dims]), # Use .get() here to provide None if variable metadata lacks # valid_min or valid_max ds[var].attrs.get("valid_min"), ds[var].attrs.get("valid_max")) ) return VariableList(result)
def get_all_variables(self) -> VariableList: """Retrieves all variables from the open database (including depth, time, etc.) Returns: [list] -- List of all variable names for this database. """ self.c.execute( "SELECT variable, units, longName, validMin, validMax FROM Variables" ) result = self.c.fetchall() l = [self.__build_variable_wrapper(v) for v in result] return VariableList(l)
def test_nothing(self, mock_query_func): mock_query_func.return_value = VariableList( [ Variable( "votemper", "Sea water potential temperature", "Kelvin", sorted(["time", "depth", "latitude", "longitude"]), ) ] ) with CalculatedImpl("tests/testdata/mercator_test.nc") as data: self.assertEqual(len(data.variables), 1) v = data.get_dataset_variable("votemper") self.assertEqual(xr.DataArray, type(v)) self.assertAlmostEqual(v[0, 0, 17, 816].values, 271.1796875)
def test_override(self, mock_query_func): mock_query_func.return_value = VariableList([ Variable('votemper', 'Sea water potential temperature', 'Kelvin', sorted(['time', 'depth', 'latitude', 'longitude'])) ]) calculated = { 'votemper': { 'equation': 'votemper -273.15', 'units': 'degree_C', 'dims': ('time', 'depth', 'latitude', 'longitude') } } with CalculatedImpl('tests/testdata/mercator_test.nc', calculated=calculated) as data: self.assertEqual(len(data.variables), 1) v = data.get_dataset_variable("votemper") self.assertAlmostEqual(v[0, 0, 17, 816].values, 271.1796875 - 273.15) self.assertEqual(v.attrs.long_name, "Sea water potential temperature") self.assertEqual(v.shape, (1, 50, 850, 1800))
def setUp(self): self.variable_list = VariableList([ Variable("my_key", "my_name", "my_unit", ("depth", "dim2"), 0, 100) ])
def setUp(self): self.variable_list = VariableList([Variable( "my_key", "my_name", "my_unit", ('depth', 'dim2'), 0, 100)])
def setUp(self): self.variable_list_mock = VariableList([ Variable('h', 'Bathymetry', 'm', ["node"]), Variable('zeta', "Water elevation", "m", []), Variable('temp', "Temp", "Kelvin", []) ])
def setUp(self): self.variable_list_mock = VariableList([ Variable("h", "Bathymetry", "m", ["node"]), Variable("zeta", "Water elevation", "m", []), Variable("temp", "Temp", "Kelvin", []), ])
def setUp(self): self.variable_list_mock = VariableList([ Variable('votemper', 'Sea water potential temperature', 'Kelvin', sorted(['time', 'depth', 'latitude', 'longitude'])) ])
def setUp(self): self.variable_list_mock = VariableList([ Variable('votemper', 'Water temperature at CMC', 'Kelvins', sorted(["deptht", "time_counter", "y", "x"])) ])