def test_unpack_flags(self): ds = Dataset() meanings = [ "flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7", "flag8" ] masks = [1, 2, 4, 8, 16, 32, 64, 128] flags_vector_variable = DatasetUtil.create_flags_variable( [2, 3], meanings, dim_names=["dim1", "dim2"], attributes={"standard_name": "std"}) ds["flags"] = flags_vector_variable ds["flags"][0, 0] = ds["flags"][0, 0] | 8 empty = np.zeros((2, 3), bool) flag4 = np.zeros((2, 3), bool) flag4[0, 0] = True flags = DatasetUtil.unpack_flags(ds["flags"]) self.assertTrue((flags["flag1"].data == empty).all()) self.assertTrue((flags["flag2"].data == empty).all()) self.assertTrue((flags["flag3"].data == empty).all()) self.assertTrue((flags["flag4"].data == flag4).all()) self.assertTrue((flags["flag5"].data == empty).all()) self.assertTrue((flags["flag6"].data == empty).all()) self.assertTrue((flags["flag7"].data == empty).all()) self.assertTrue((flags["flag8"].data == empty).all())
def calc_mean_masked(self, dataset, var, flags, rand_unc=False, corr=False): series_id = np.unique(dataset['series_id']) if corr: out = np.empty\ ((len(series_id), len(dataset['wavelength']), len(dataset['wavelength']))) for i in range(len(series_id)): flagged = np.any([ DatasetUtil.unpack_flags(dataset['quality_flag'])[x] for x in flags ], axis=0) ids = np.where((dataset['series_id'] == series_id[i]) & (flagged == False)) out[i] = np.mean(dataset[var].values[:, :, ids], axis=3)[:, :, 0] out = np.mean(out, axis=0) else: out = np.empty((len(series_id), len(dataset['wavelength']))) for i in range(len(series_id)): flagged = np.any([ DatasetUtil.unpack_flags(dataset['quality_flag'])[x] for x in flags ], axis=0) ids = np.where((dataset['series_id'] == series_id[i]) & (flagged == False)) out[i] = np.mean(dataset[var].values[:, ids], axis=2)[:, 0] if rand_unc: out[i] = (np.sum(dataset[var].values[:, ids]**2, axis=2)[:, 0])**0.5 / len(ids[0]) return out.T
def l1b_template_from_l1a_dataset_water(self, measurandstring, dataset_l1a): """ Makes all L1 templates for the data, and propagates the appropriate keywords from the L0 datasets. :param datasetl0: :type datasetl0: :return: :rtype: """ print(np.unique(dataset_l1a['series_id'])) l1b_dim_sizes_dict = { "wavelength": len(dataset_l1a["wavelength"]), "series": len(np.unique(dataset_l1a['series_id'])) } if measurandstring == "radiance": dataset_l1b = self.hdsb.create_ds_template( l1b_dim_sizes_dict, "W_L1B_RAD", propagate_ds=dataset_l1a, ds=dataset_l1a) elif measurandstring == "irradiance": dataset_l1b = self.hdsb.create_ds_template( l1b_dim_sizes_dict, "W_L1B_IRR", propagate_ds=dataset_l1a, ds=dataset_l1a) dataset_l1b = dataset_l1b.assign_coords( wavelength=dataset_l1a.wavelength) series_id = np.unique(dataset_l1a['series_id']) dataset_l1b["series_id"].values = series_id for variablestring in [ "acquisition_time", "viewing_azimuth_angle", "viewing_zenith_angle", "solar_azimuth_angle", "solar_zenith_angle" ]: temp_arr = np.empty(len(series_id)) for i in range(len(series_id)): ids = np.where( (dataset_l1a['series_id'] == series_id[i]) & np.invert( DatasetUtil.unpack_flags(dataset_l1a["quality_flag"]) ["outliers"])) temp_arr[i] = np.mean(dataset_l1a[variablestring].values[ids]) dataset_l1b[variablestring].values = temp_arr return dataset_l1b
def plot_diff_scans(self, measurandstring, dataset, dataset_avg=None): series_id = np.unique(dataset['series_id']) for i in range(len(series_id)): plotpath = os.path.join( self.path, "plot_diff_" + measurandstring + "_" + dataset.attrs['product_name'] + "_series_" + str(series_id[i]) + "." + self.context.get_config_value("plotting_format")) ids = np.where(dataset['series_id'] == series_id[i])[0] ydata_subset = dataset[measurandstring].values[:, ids] mask = DatasetUtil.unpack_flags( dataset["quality_flag"])["outliers"] mask = mask[ids] if dataset_avg is None: ids_used = np.where( (dataset['series_id'] == series_id[i]) & np.invert( DatasetUtil.unpack_flags(dataset["quality_flag"]) ["outliers"]))[0] ydata_subset_used = dataset[measurandstring].values[:, ids_used] avgs = np.tile( np.mean(ydata_subset_used, axis=1)[..., None], len(ids)) else: avg_ids = np.where( (dataset_avg['series_id'] == series_id[i]))[0] avgs = np.tile(dataset_avg[measurandstring].values[:, avg_ids], len(ids)) self.plot_variable("relative difference", plotpath, dataset["wavelength"].values, (ydata_subset - avgs) / avgs, ylim=[-0.3, 0.3], mask=mask)