def test_unpack_flags(self): ds = Dataset() meanings = [ "flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7", "flag8" ] masks = [1, 2, 4, 8, 16, 32, 64, 128] flags_vector_variable = DatasetUtil.create_flags_variable( [2, 3], meanings, dim_names=["dim1", "dim2"], attributes={"standard_name": "std"}) ds["flags"] = flags_vector_variable ds["flags"][0, 0] = ds["flags"][0, 0] | 8 empty = np.zeros((2, 3), bool) flag4 = np.zeros((2, 3), bool) flag4[0, 0] = True flags = DatasetUtil.unpack_flags(ds["flags"]) self.assertTrue((flags["flag1"].data == empty).all()) self.assertTrue((flags["flag2"].data == empty).all()) self.assertTrue((flags["flag3"].data == empty).all()) self.assertTrue((flags["flag4"].data == flag4).all()) self.assertTrue((flags["flag5"].data == empty).all()) self.assertTrue((flags["flag6"].data == empty).all()) self.assertTrue((flags["flag7"].data == empty).all()) self.assertTrue((flags["flag8"].data == empty).all())
def test_fill_ds(self): ds = Dataset() ds["array_variable1"] = DatasetUtil.create_variable([7, 8], np.float32) ds["array_variable2"] = DatasetUtil.create_variable([7, 8], np.float32) ds["array_variable1"][2, 3] = np.nan ds["array_variable2"][2, 3] = np.nan HypernetsWriter.fill_ds(ds) self.assertTrue(np.all(ds["array_variable1"] == 9.96921E36)) self.assertTrue(np.all(ds["array_variable2"] == 9.96921E36))
def test__get_flag_encoding_not_flag_var(self): ds = Dataset() ds["array_variable"] = DatasetUtil.create_variable( [7, 8, 3], np.int8, attributes={"standard_name": "std"}) self.assertRaises(KeyError, DatasetUtil._get_flag_encoding, ds["array_variable"])
def get_wind(self, l1b): lat = l1b.attrs['site_latitude'] lon = l1b.attrs['site_latitude'] wind = [] for i in range(len(l1b.scan)): wa = self.context.get_config_value("wind_ancillary") if not wa: l1b["quality_flag"][l1b["scan"] == i] = du.set_flag( l1b["quality_flag"][l1b["scan"] == i], "def_wind_flag") self.context.logger.info("Default wind speed {}".format( self.context.get_config_value("wind_default"))) wind.append(self.context.get_config_value("wind_default")) else: isodate = datetime.utcfromtimestamp( l1b['acquisition_time'].values[i]).strftime('%Y-%m-%d') isotime = datetime.utcfromtimestamp( l1b['acquisition_time'].values[i]).strftime('%H:%M:%S') anc_wind = self.rhymeranc.get_wind(isodate, lon, lat, isotime=isotime) if anc_wind is not None: wind.append(anc_wind) l1b['fresnel_wind'].values = wind return l1b
def test_create_variable_1D_int(self): vector_variable = DatasetUtil.create_variable([5], np.int8) self.assertIsNotNone(vector_variable) self.assertEqual(Variable, type(vector_variable)) self.assertEqual((5, ), vector_variable.shape) self.assertEqual(np.int8, vector_variable.dtype) self.assertEqual(-127, vector_variable[2])
def test_create_variable_3D_int(self): array_variable = DatasetUtil.create_variable([7, 8, 3], np.int8) self.assertIsNotNone(array_variable) self.assertEqual(Variable, type(array_variable)) self.assertEqual((7, 8, 3), array_variable.shape) self.assertEqual(np.int8, array_variable.dtype) self.assertEqual(-127, array_variable[2, 4, 2])
def test_create_default_array_3D_int(self): default_array = DatasetUtil.create_default_array([7, 8, 3], np.int8) self.assertIsNotNone(default_array) self.assertEqual(DataArray, type(default_array)) self.assertEqual((7, 8, 3), default_array.shape) self.assertEqual(np.int8, default_array.dtype) self.assertEqual(-127, default_array[2, 4, 2])
def test_check_flag_set_false(self): ds = Dataset() meanings = [ "flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7", "flag8" ] flags_vector_variable = DatasetUtil.create_flags_variable( [5], meanings, dim_names=["dim1"], attributes={"standard_name": "std"}) ds["flags"] = flags_vector_variable ds["flags"][3] = ds["flags"][3] | 8 flag_set = DatasetUtil.check_flag_set(ds["flags"][3], "flag6") self.assertFalse(flag_set)
def test_set_flag(self): ds = Dataset() meanings = [ "flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7", "flag8" ] flags_vector_variable = DatasetUtil.create_flags_variable( [5, 4], meanings, dim_names=["dim1", "dim2"], attributes={"standard_name": "std"}) ds["flags"] = flags_vector_variable ds["flags"] = DatasetUtil.set_flag(ds["flags"], "flag4") flags = np.full(ds["flags"].shape, 0 | 8) self.assertTrue((ds["flags"].data == flags).all())
def test_create_variable_3D_int_attributes(self): array_variable = DatasetUtil.create_variable( [7, 8, 3], np.int8, attributes={"standard_name": "std"}) self.assertIsNotNone(array_variable) self.assertEqual(Variable, type(array_variable)) self.assertEqual((7, 8, 3), array_variable.shape) self.assertEqual(np.int8, array_variable.dtype) self.assertEqual(-127, array_variable[2, 4, 2]) self.assertEqual("std", array_variable.attrs["standard_name"])
def test_create_variable_1D_int_attributes(self): vector_variable = DatasetUtil.create_variable( [5], np.int8, attributes={"standard_name": "std"}) self.assertIsNotNone(vector_variable) self.assertEqual(Variable, type(vector_variable)) self.assertEqual((5, ), vector_variable.shape) self.assertEqual(np.int8, vector_variable.dtype) self.assertEqual(-127, vector_variable[2]) self.assertEqual("std", vector_variable.attrs["standard_name"])
def test_get_set_flags(self): ds = Dataset() meanings = [ "flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7", "flag8" ] flags_vector_variable = DatasetUtil.create_flags_variable( [5], meanings, dim_names=["dim1"], attributes={"standard_name": "std"}) ds["flags"] = flags_vector_variable ds["flags"][3] = ds["flags"][3] | 8 ds["flags"][3] = ds["flags"][3] | 32 set_flags = DatasetUtil.get_set_flags(ds["flags"][3]) self.assertCountEqual(set_flags, ["flag4", "flag6"])
def calc_mean_masked(self, dataset, var, flags, rand_unc=False, corr=False): series_id = np.unique(dataset['series_id']) if corr: out = np.empty\ ((len(series_id), len(dataset['wavelength']), len(dataset['wavelength']))) for i in range(len(series_id)): flagged = np.any([ DatasetUtil.unpack_flags(dataset['quality_flag'])[x] for x in flags ], axis=0) ids = np.where((dataset['series_id'] == series_id[i]) & (flagged == False)) out[i] = np.mean(dataset[var].values[:, :, ids], axis=3)[:, :, 0] out = np.mean(out, axis=0) else: out = np.empty((len(series_id), len(dataset['wavelength']))) for i in range(len(series_id)): flagged = np.any([ DatasetUtil.unpack_flags(dataset['quality_flag'])[x] for x in flags ], axis=0) ids = np.where((dataset['series_id'] == series_id[i]) & (flagged == False)) out[i] = np.mean(dataset[var].values[:, ids], axis=2)[:, 0] if rand_unc: out[i] = (np.sum(dataset[var].values[:, ids]**2, axis=2)[:, 0])**0.5 / len(ids[0]) return out.T
def test_add_encoding(self): vector_variable = DatasetUtil.create_variable([5], np.int8) DatasetUtil.add_encoding(vector_variable, np.int32, scale_factor=10, offset=23, fill_value=11, chunksizes=12) self.assertIsNotNone(vector_variable) self.assertEqual(Variable, type(vector_variable)) self.assertEqual((5, ), vector_variable.shape) self.assertEqual(np.int8, vector_variable.dtype) self.assertEqual(-127, vector_variable[2]) self.assertEqual(np.int32, vector_variable.encoding["dtype"]) self.assertEqual(10, vector_variable.encoding["scale_factor"]) self.assertEqual(23, vector_variable.encoding["add_offset"]) self.assertEqual(11, vector_variable.encoding["_FillValue"]) self.assertEqual(12, vector_variable.encoding["chunksizes"])
def test_create_variable_1D_int_dims(self): vector_variable = DatasetUtil.create_variable([5], np.int8, dim_names=["dim1"]) self.assertIsNotNone(vector_variable) self.assertEqual(Variable, type(vector_variable)) self.assertEqual((5, ), vector_variable.shape) self.assertEqual(np.int8, vector_variable.dtype) self.assertEqual(-127, vector_variable[2]) self.assertEqual(("dim1", ), vector_variable.dims)
def test__get_flag_encoding(self): ds = Dataset() meanings = [ "flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7", "flag8" ] masks = [1, 2, 4, 8, 16, 32, 64, 128] flags_vector_variable = DatasetUtil.create_flags_variable( [5], meanings, dim_names=["dim1"], attributes={"standard_name": "std"}) ds["flags"] = flags_vector_variable meanings_out, masks_out = DatasetUtil._get_flag_encoding(ds["flags"]) self.assertCountEqual(meanings, meanings_out) self.assertCountEqual(masks, masks_out)
def test_create_default_array_1D_int_fill_value(self): default_array = DatasetUtil.create_default_array([5], np.int8, fill_value=1) self.assertIsNotNone(default_array) self.assertEqual(DataArray, type(default_array)) self.assertEqual((5, ), default_array.shape) self.assertEqual(np.int8, default_array.dtype) self.assertEqual(1, default_array[2])
def test_create_default_array_1D_int_dims(self): default_array = DatasetUtil.create_default_array([5], np.int8, dim_names=["dim1"]) self.assertIsNotNone(default_array) self.assertEqual(DataArray, type(default_array)) self.assertEqual((5, ), default_array.shape) self.assertEqual(np.int8, default_array.dtype) self.assertEqual(-127, default_array[2]) self.assertEqual(("dim1", ), default_array.dims)
def test_get_flags_mask_and(self): ds = Dataset() meanings = [ "flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7", "flag8" ] flags_vector_variable = DatasetUtil.create_flags_variable( [2, 3], meanings, dim_names=["dim1", "dim2"], attributes={"standard_name": "std"}) ds["flags"] = flags_vector_variable ds["flags"] = DatasetUtil.set_flag(ds["flags"], "flag4") ds["flags"][0, 1] = DatasetUtil.set_flag(ds["flags"][0, 1], "flag5") ds["flags"][1, 1] = DatasetUtil.set_flag(ds["flags"][1, 1], "flag2") ds["flags"][1, 1] = DatasetUtil.set_flag(ds["flags"][1, 1], "flag7") flags_mask = DatasetUtil.get_flags_mask_and(ds["flags"], flags=["flag2", "flag7"]) expected_flags_mask = np.array( [[False, False, False], [False, True, False]], dtype=bool) np.testing.assert_array_almost_equal(flags_mask, expected_flags_mask)
def get_fresnelrefl(self, l1b): ## read mobley rho lut fresnel_coeff = np.zeros(len(l1b.scan)) fresnel_vza = np.zeros(len(l1b.scan)) fresnel_raa = np.zeros(len(l1b.scan)) fresnel_sza = np.zeros(len(l1b.scan)) wind = l1b["fresnel_wind"].values for i in range(len(l1b.scan)): fresnel_vza[i] = l1b['viewing_zenith_angle'][i].values fresnel_sza[i] = l1b['solar_zenith_angle'][i].values diffa = l1b['viewing_azimuth_angle'][i].values - l1b[ 'solar_azimuth_angle'][i].values if diffa >= 360: diffa = diffa - 360 elif 0 <= diffa < 360: diffa = diffa else: diffa = diffa + 360 fresnel_raa[i] = abs((diffa - 180)) ## get fresnel reflectance if self.context.get_config_value("fresnel_option") == 'Mobley': if (fresnel_sza[i] is not None) & (fresnel_raa[i] is not None): sza = min(fresnel_sza[i], 79.999) rhof = self.rhymerproc.mobley_lut_interp(sza, fresnel_vza[i], fresnel_raa[i], wind=wind[i]) else: l1b["quality_flag"][l1b["scan"] == i] = du.set_flag( l1b["quality_flag"][l1b["scan"] == i], "fresnel_default") rhof = self.context.get_config_value("rhof_default") if self.context.get_config_value( "fresnel_option") == 'Ruddick2006': rhof = self.context.get_config_value("rhof_default") self.context.logger.info("Apply Ruddick et al., 2006") if wind[i] is not None: rhof = rhof + 0.00039 * wind[i] + 0.000034 * wind[i]**2 fresnel_coeff[i] = rhof l1b["rhof"].values = fresnel_coeff l1b["fresnel_vza"].values = fresnel_vza l1b["fresnel_raa"].values = fresnel_raa l1b["fresnel_sza"].values = fresnel_sza return l1b
def process_l1c(self, dataset): dataset_l1c = self.templ.l1c_from_l1b_dataset(dataset) dataset_l1c = self.rh.get_wind(dataset_l1c) dataset_l1c = self.rh.get_fresnelrefl(dataset_l1c) l1ctol1b_function = self._measurement_function_factory.get_measurement_function( self.context.get_config_value( "measurement_function_surface_reflectance")) input_vars = l1ctol1b_function.get_argument_names() input_qty = self.prop.find_input(input_vars, dataset_l1c) u_random_input_qty = self.prop.find_u_random_input( input_vars, dataset_l1c) u_systematic_input_qty, corr_systematic_input_qty = \ self.prop.find_u_systematic_input(input_vars, dataset_l1c) L1c = self.prop.process_measurement_function_l2( [ "water_leaving_radiance", "reflectance_nosc", "reflectance", "epsilon" ], dataset_l1c, l1ctol1b_function.function, input_qty, u_random_input_qty, u_systematic_input_qty, corr_systematic_input_qty, param_fixed=[False, False, False, False, True]) failSimil = self.rh.qc_similarity(L1c) L1c["quality_flag"][np.where(failSimil == 1)] = DatasetUtil.set_flag( L1c["quality_flag"][np.where(failSimil == 1)], "simil_fail") # for i in range(len(mask))] if self.context.get_config_value("write_l1c"): self.writer.write(L1c, overwrite=True) for measurandstring in [ "water_leaving_radiance", "reflectance_nosc", "reflectance", "epsilon" ]: try: if self.context.get_config_value("plot_l1c"): self.plot.plot_series_in_sequence(measurandstring, L1c) if self.context.get_config_value("plot_uncertainty"): self.plot.plot_relative_uncertainty(measurandstring, L1c, L2=True) except: print("not plotting ", measurandstring) return L1c
def l1b_template_from_l1a_dataset_water(self, measurandstring, dataset_l1a): """ Makes all L1 templates for the data, and propagates the appropriate keywords from the L0 datasets. :param datasetl0: :type datasetl0: :return: :rtype: """ print(np.unique(dataset_l1a['series_id'])) l1b_dim_sizes_dict = { "wavelength": len(dataset_l1a["wavelength"]), "series": len(np.unique(dataset_l1a['series_id'])) } if measurandstring == "radiance": dataset_l1b = self.hdsb.create_ds_template( l1b_dim_sizes_dict, "W_L1B_RAD", propagate_ds=dataset_l1a, ds=dataset_l1a) elif measurandstring == "irradiance": dataset_l1b = self.hdsb.create_ds_template( l1b_dim_sizes_dict, "W_L1B_IRR", propagate_ds=dataset_l1a, ds=dataset_l1a) dataset_l1b = dataset_l1b.assign_coords( wavelength=dataset_l1a.wavelength) series_id = np.unique(dataset_l1a['series_id']) dataset_l1b["series_id"].values = series_id for variablestring in [ "acquisition_time", "viewing_azimuth_angle", "viewing_zenith_angle", "solar_azimuth_angle", "solar_zenith_angle" ]: temp_arr = np.empty(len(series_id)) for i in range(len(series_id)): ids = np.where( (dataset_l1a['series_id'] == series_id[i]) & np.invert( DatasetUtil.unpack_flags(dataset_l1a["quality_flag"]) ["outliers"])) temp_arr[i] = np.mean(dataset_l1a[variablestring].values[ids]) dataset_l1b[variablestring].values = temp_arr return dataset_l1b
def test_create_variable_3D_int_dims(self): array_variable = DatasetUtil.create_variable( [7, 8, 3], np.int8, dim_names=["dim1", "dim2", "dim3"]) self.assertIsNotNone(array_variable) self.assertEqual(Variable, type(array_variable)) self.assertEqual((7, 8, 3), array_variable.shape) self.assertEqual(np.int8, array_variable.dtype) self.assertEqual(-127, array_variable[2, 4, 2]) self.assertEqual(( "dim1", "dim2", "dim3", ), array_variable.dims)
def plot_diff_scans(self, measurandstring, dataset, dataset_avg=None): series_id = np.unique(dataset['series_id']) for i in range(len(series_id)): plotpath = os.path.join( self.path, "plot_diff_" + measurandstring + "_" + dataset.attrs['product_name'] + "_series_" + str(series_id[i]) + "." + self.context.get_config_value("plotting_format")) ids = np.where(dataset['series_id'] == series_id[i])[0] ydata_subset = dataset[measurandstring].values[:, ids] mask = DatasetUtil.unpack_flags( dataset["quality_flag"])["outliers"] mask = mask[ids] if dataset_avg is None: ids_used = np.where( (dataset['series_id'] == series_id[i]) & np.invert( DatasetUtil.unpack_flags(dataset["quality_flag"]) ["outliers"]))[0] ydata_subset_used = dataset[measurandstring].values[:, ids_used] avgs = np.tile( np.mean(ydata_subset_used, axis=1)[..., None], len(ids)) else: avg_ids = np.where( (dataset_avg['series_id'] == series_id[i]))[0] avgs = np.tile(dataset_avg[measurandstring].values[:, avg_ids], len(ids)) self.plot_variable("relative difference", plotpath, dataset["wavelength"].values, (ydata_subset - avgs) / avgs, ylim=[-0.3, 0.3], mask=mask)
def test_check_flag_set_2d(self): ds = Dataset() meanings = [ "flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7", "flag8" ] flags_vector_variable = DatasetUtil.create_flags_variable( [5], meanings, dim_names=["dim1"], attributes={"standard_name": "std"}) ds["flags"] = flags_vector_variable self.assertRaises(ValueError, DatasetUtil.check_flag_set, ds["flags"], "flag6")
def test_get_default_fill_value(self): self.assertEqual(-127, DatasetUtil.get_default_fill_value(np.int8)) self.assertEqual(-32767, DatasetUtil.get_default_fill_value(np.int16)) self.assertEqual(np.uint16(-1), DatasetUtil.get_default_fill_value(np.uint16)) self.assertEqual(-2147483647, DatasetUtil.get_default_fill_value(np.int32)) self.assertEqual(-9223372036854775806, DatasetUtil.get_default_fill_value(np.int64)) self.assertEqual(np.float32(9.96921E36), DatasetUtil.get_default_fill_value(np.float32)) self.assertEqual(9.969209968386869E36, DatasetUtil.get_default_fill_value(np.float64))
def add_variables(ds, variables_dict, dim_sizes_dict): """ Adds defined variables dataset :type ds: xarray.Dataset :param ds: dataset :type variables_dict: dict :type variables_dict: dictionary defining variables :type dim_sizes_dict: dict :param dim_sizes_dict: entry per dataset dimension with value of size as int :return: dataset with defined variables :rtype: xarray.Dataset """ du = DatasetUtil() for variable_name in variables_dict.keys(): variable_attrs = variables_dict[variable_name] # Check variable definition TemplateUtil._check_variable_definition(variable_name, variable_attrs) # Unpack variable attributes dtype = variable_attrs["dtype"] dim_names = variable_attrs["dim"] attributes = deepcopy(variable_attrs["attributes"]) if "attributes" in variable_attrs else None # Determine variable shape from dims try: dim_sizes = TemplateUtil._return_variable_shape(dim_names, dim_sizes_dict) except KeyError: raise KeyError("Dim Name Error - Variable " + variable_name + " defined with dim not in dim_sizes_dict") # Create variable and add to dataset if dtype == "flag": flag_meanings = attributes.pop("flag_meanings") variable = du.create_flags_variable(dim_sizes, meanings=flag_meanings, dim_names=dim_names, attributes=attributes) else: variable = du.create_variable(dim_sizes, dim_names=dim_names, dtype=dtype, attributes=attributes) if "encoding" in variable_attrs: du.add_encoding(variable, **variable_attrs["encoding"]) ds[variable_name] = variable return ds
def test_set_flag_error_if_set(self): ds = Dataset() meanings = [ "flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7", "flag8" ] flags_vector_variable = DatasetUtil.create_flags_variable( [5], meanings, dim_names=["dim1"], attributes={"standard_name": "std"}) ds["flags"] = flags_vector_variable ds["flags"][3] = ds["flags"][3] | 8 self.assertRaises(ValueError, DatasetUtil.set_flag, ds["flags"], "flag4", error_if_set=True)
def test_create_flags_variable_3D(self): meanings = [ "flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7", "flag8" ] meanings_txt = "flag1 flag2 flag3 flag4 flag5 flag6 flag7 flag8" masks = "1, 2, 4, 8, 16, 32, 64, 128" flags_array_variable = DatasetUtil.create_flags_variable( [7, 8, 3], meanings, dim_names=["dim1", "dim2", "dim3"], attributes={"standard_name": "std"}) self.assertIsNotNone(flags_array_variable) self.assertEqual(Variable, type(flags_array_variable)) self.assertEqual((7, 8, 3), flags_array_variable.shape) self.assertEqual(np.uint8, flags_array_variable.dtype) self.assertEqual(flags_array_variable.attrs['flag_masks'], masks) self.assertEqual(flags_array_variable.attrs['flag_meanings'], meanings_txt) self.assertEqual(0, flags_array_variable[2, 4, 2]) self.assertEqual("std", flags_array_variable.attrs["standard_name"]) self.assertEqual(("dim1", "dim2", "dim3"), flags_array_variable.dims)
def test_return_flags_dtype_32(self): data_type = DatasetUtil.return_flags_dtype(32) self.assertEqual(data_type, np.uint32)