def test_fill_ds(self): ds = Dataset() ds["array_variable1"] = DatasetUtil.create_variable([7, 8], np.float32) ds["array_variable2"] = DatasetUtil.create_variable([7, 8], np.float32) ds["array_variable1"][2, 3] = np.nan ds["array_variable2"][2, 3] = np.nan HypernetsWriter.fill_ds(ds) self.assertTrue(np.all(ds["array_variable1"] == 9.96921E36)) self.assertTrue(np.all(ds["array_variable2"] == 9.96921E36))
def test__get_flag_encoding_not_flag_var(self): ds = Dataset() ds["array_variable"] = DatasetUtil.create_variable( [7, 8, 3], np.int8, attributes={"standard_name": "std"}) self.assertRaises(KeyError, DatasetUtil._get_flag_encoding, ds["array_variable"])
def test_create_variable_1D_int(self): vector_variable = DatasetUtil.create_variable([5], np.int8) self.assertIsNotNone(vector_variable) self.assertEqual(Variable, type(vector_variable)) self.assertEqual((5, ), vector_variable.shape) self.assertEqual(np.int8, vector_variable.dtype) self.assertEqual(-127, vector_variable[2])
def test_create_variable_3D_int(self): array_variable = DatasetUtil.create_variable([7, 8, 3], np.int8) self.assertIsNotNone(array_variable) self.assertEqual(Variable, type(array_variable)) self.assertEqual((7, 8, 3), array_variable.shape) self.assertEqual(np.int8, array_variable.dtype) self.assertEqual(-127, array_variable[2, 4, 2])
def test_create_variable_3D_int_attributes(self): array_variable = DatasetUtil.create_variable( [7, 8, 3], np.int8, attributes={"standard_name": "std"}) self.assertIsNotNone(array_variable) self.assertEqual(Variable, type(array_variable)) self.assertEqual((7, 8, 3), array_variable.shape) self.assertEqual(np.int8, array_variable.dtype) self.assertEqual(-127, array_variable[2, 4, 2]) self.assertEqual("std", array_variable.attrs["standard_name"])
def test_create_variable_1D_int_attributes(self): vector_variable = DatasetUtil.create_variable( [5], np.int8, attributes={"standard_name": "std"}) self.assertIsNotNone(vector_variable) self.assertEqual(Variable, type(vector_variable)) self.assertEqual((5, ), vector_variable.shape) self.assertEqual(np.int8, vector_variable.dtype) self.assertEqual(-127, vector_variable[2]) self.assertEqual("std", vector_variable.attrs["standard_name"])
def test_create_variable_1D_int_dims(self): vector_variable = DatasetUtil.create_variable([5], np.int8, dim_names=["dim1"]) self.assertIsNotNone(vector_variable) self.assertEqual(Variable, type(vector_variable)) self.assertEqual((5, ), vector_variable.shape) self.assertEqual(np.int8, vector_variable.dtype) self.assertEqual(-127, vector_variable[2]) self.assertEqual(("dim1", ), vector_variable.dims)
def add_variables(ds, variables_dict, dim_sizes_dict): """ Adds defined variables dataset :type ds: xarray.Dataset :param ds: dataset :type variables_dict: dict :type variables_dict: dictionary defining variables :type dim_sizes_dict: dict :param dim_sizes_dict: entry per dataset dimension with value of size as int :return: dataset with defined variables :rtype: xarray.Dataset """ du = DatasetUtil() for variable_name in variables_dict.keys(): variable_attrs = variables_dict[variable_name] # Check variable definition TemplateUtil._check_variable_definition(variable_name, variable_attrs) # Unpack variable attributes dtype = variable_attrs["dtype"] dim_names = variable_attrs["dim"] attributes = deepcopy(variable_attrs["attributes"]) if "attributes" in variable_attrs else None # Determine variable shape from dims try: dim_sizes = TemplateUtil._return_variable_shape(dim_names, dim_sizes_dict) except KeyError: raise KeyError("Dim Name Error - Variable " + variable_name + " defined with dim not in dim_sizes_dict") # Create variable and add to dataset if dtype == "flag": flag_meanings = attributes.pop("flag_meanings") variable = du.create_flags_variable(dim_sizes, meanings=flag_meanings, dim_names=dim_names, attributes=attributes) else: variable = du.create_variable(dim_sizes, dim_names=dim_names, dtype=dtype, attributes=attributes) if "encoding" in variable_attrs: du.add_encoding(variable, **variable_attrs["encoding"]) ds[variable_name] = variable return ds
def test_create_variable_3D_int_dims(self): array_variable = DatasetUtil.create_variable( [7, 8, 3], np.int8, dim_names=["dim1", "dim2", "dim3"]) self.assertIsNotNone(array_variable) self.assertEqual(Variable, type(array_variable)) self.assertEqual((7, 8, 3), array_variable.shape) self.assertEqual(np.int8, array_variable.dtype) self.assertEqual(-127, array_variable[2, 4, 2]) self.assertEqual(( "dim1", "dim2", "dim3", ), array_variable.dims)
def test_add_encoding(self): vector_variable = DatasetUtil.create_variable([5], np.int8) DatasetUtil.add_encoding(vector_variable, np.int32, scale_factor=10, offset=23, fill_value=11, chunksizes=12) self.assertIsNotNone(vector_variable) self.assertEqual(Variable, type(vector_variable)) self.assertEqual((5, ), vector_variable.shape) self.assertEqual(np.int8, vector_variable.dtype) self.assertEqual(-127, vector_variable[2]) self.assertEqual(np.int32, vector_variable.encoding["dtype"]) self.assertEqual(10, vector_variable.encoding["scale_factor"]) self.assertEqual(23, vector_variable.encoding["add_offset"]) self.assertEqual(11, vector_variable.encoding["_FillValue"]) self.assertEqual(12, vector_variable.encoding["chunksizes"])
def preprocess_l0(self, datasetl0, datasetl0_bla, dataset_calib): """ Identifies and removes faulty measurements (e.g. due to cloud cover). :param dataset_l0: :type dataset_l0: :return: :rtype: """ wavs = dataset_calib["wavelength"].values wavpix = dataset_calib["wavpix"].values datasetl0 = datasetl0.isel(wavelength=slice(int(wavpix[0]), int(wavpix[-1]) + 1)) datasetl0_bla = datasetl0_bla.isel( wavelength=slice(int(wavpix[0]), int(wavpix[-1]) + 1)) mask = self.clip_and_mask(datasetl0, datasetl0_bla) datasetl0 = datasetl0.assign_coords(wavelength=wavs) datasetl0_bla = datasetl0_bla.assign_coords(wavelength=wavs) datasetl0["quality_flag"][np.where(mask == 1)] = DatasetUtil.set_flag( datasetl0["quality_flag"][np.where(mask == 1)], "outliers") #for i in range(len(mask))] DN_rand = DatasetUtil.create_variable( [len(datasetl0["wavelength"]), len(datasetl0["scan"])], dim_names=["wavelength", "scan"], dtype=np.uint32, fill_value=0) datasetl0["u_random_digital_number"] = DN_rand rand = np.zeros_like(DN_rand.values) series_ids = np.unique(datasetl0['series_id']) for i in range(len(series_ids)): ids = np.where(datasetl0['series_id'] == series_ids[i])[0] ids_masked = np.where((datasetl0['series_id'] == series_ids[i]) & (mask == 0))[0] dark_signals = np.zeros_like( datasetl0['digital_number'].values[:, ids_masked]) for ii, id in enumerate(ids_masked): dark_signals[:, ii] = self.find_nearest_black( datasetl0_bla, datasetl0['acquisition_time'].values[id], datasetl0['integration_time'].values[id]) std = np.std((datasetl0['digital_number'].values[:, ids_masked] - dark_signals), axis=1) for ii, id in enumerate(ids): rand[:, id] = std datasetl0["u_random_digital_number"].values = rand DN_dark = DatasetUtil.create_variable( [len(datasetl0["wavelength"]), len(datasetl0["scan"])], dim_names=["wavelength", "scan"], dtype=np.uint32, fill_value=0) datasetl0["dark_signal"] = DN_dark dark_signals = [] acqui = datasetl0['acquisition_time'].values inttimes = datasetl0['integration_time'].values for i in range(len(acqui)): dark_signals.append( self.find_nearest_black(datasetl0_bla, acqui[i], inttimes[i])) datasetl0["dark_signal"].values = np.array(dark_signals).T return datasetl0