def test_unpack_flags(self):
ds = Dataset()
meanings = [
"flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7",
"flag8"
]
masks = [1, 2, 4, 8, 16, 32, 64, 128]
flags_vector_variable = DatasetUtil.create_flags_variable(
[2, 3],
meanings,
dim_names=["dim1", "dim2"],
attributes={"standard_name": "std"})
ds["flags"] = flags_vector_variable
ds["flags"][0, 0] = ds["flags"][0, 0] | 8
empty = np.zeros((2, 3), bool)
flag4 = np.zeros((2, 3), bool)
flag4[0, 0] = True
flags = DatasetUtil.unpack_flags(ds["flags"])
self.assertTrue((flags["flag1"].data == empty).all())
self.assertTrue((flags["flag2"].data == empty).all())
self.assertTrue((flags["flag3"].data == empty).all())
self.assertTrue((flags["flag4"].data == flag4).all())
self.assertTrue((flags["flag5"].data == empty).all())
self.assertTrue((flags["flag6"].data == empty).all())
self.assertTrue((flags["flag7"].data == empty).all())
self.assertTrue((flags["flag8"].data == empty).all())
def calc_mean_masked(self,
dataset,
var,
flags,
rand_unc=False,
corr=False):
series_id = np.unique(dataset['series_id'])
if corr:
out = np.empty\
((len(series_id), len(dataset['wavelength']), len(dataset['wavelength'])))
for i in range(len(series_id)):
flagged = np.any([
DatasetUtil.unpack_flags(dataset['quality_flag'])[x]
for x in flags
],
axis=0)
ids = np.where((dataset['series_id'] == series_id[i])
& (flagged == False))
out[i] = np.mean(dataset[var].values[:, :, ids], axis=3)[:, :,
0]
out = np.mean(out, axis=0)
else:
out = np.empty((len(series_id), len(dataset['wavelength'])))
for i in range(len(series_id)):
flagged = np.any([
DatasetUtil.unpack_flags(dataset['quality_flag'])[x]
for x in flags
],
axis=0)
ids = np.where((dataset['series_id'] == series_id[i])
& (flagged == False))
out[i] = np.mean(dataset[var].values[:, ids], axis=2)[:, 0]
if rand_unc:
out[i] = (np.sum(dataset[var].values[:, ids]**2,
axis=2)[:, 0])**0.5 / len(ids[0])
return out.T
def l1b_template_from_l1a_dataset_water(self, measurandstring,
dataset_l1a):
"""
Makes all L1 templates for the data, and propagates the appropriate keywords from the L0 datasets.
:param datasetl0:
:type datasetl0:
:return:
:rtype:
"""
print(np.unique(dataset_l1a['series_id']))
l1b_dim_sizes_dict = {
"wavelength": len(dataset_l1a["wavelength"]),
"series": len(np.unique(dataset_l1a['series_id']))
}
if measurandstring == "radiance":
dataset_l1b = self.hdsb.create_ds_template(
l1b_dim_sizes_dict,
"W_L1B_RAD",
propagate_ds=dataset_l1a,
ds=dataset_l1a)
elif measurandstring == "irradiance":
dataset_l1b = self.hdsb.create_ds_template(
l1b_dim_sizes_dict,
"W_L1B_IRR",
propagate_ds=dataset_l1a,
ds=dataset_l1a)
dataset_l1b = dataset_l1b.assign_coords(
wavelength=dataset_l1a.wavelength)
series_id = np.unique(dataset_l1a['series_id'])
dataset_l1b["series_id"].values = series_id
for variablestring in [
"acquisition_time", "viewing_azimuth_angle",
"viewing_zenith_angle", "solar_azimuth_angle",
"solar_zenith_angle"
]:
temp_arr = np.empty(len(series_id))
for i in range(len(series_id)):
ids = np.where(
(dataset_l1a['series_id'] == series_id[i]) & np.invert(
DatasetUtil.unpack_flags(dataset_l1a["quality_flag"])
["outliers"]))
temp_arr[i] = np.mean(dataset_l1a[variablestring].values[ids])
dataset_l1b[variablestring].values = temp_arr
return dataset_l1b
def plot_diff_scans(self, measurandstring, dataset, dataset_avg=None):
series_id = np.unique(dataset['series_id'])
for i in range(len(series_id)):
plotpath = os.path.join(
self.path, "plot_diff_" + measurandstring + "_" +
dataset.attrs['product_name'] + "_series_" +
str(series_id[i]) + "." +
self.context.get_config_value("plotting_format"))
ids = np.where(dataset['series_id'] == series_id[i])[0]
ydata_subset = dataset[measurandstring].values[:, ids]
mask = DatasetUtil.unpack_flags(
dataset["quality_flag"])["outliers"]
mask = mask[ids]
if dataset_avg is None:
ids_used = np.where(
(dataset['series_id'] == series_id[i]) & np.invert(
DatasetUtil.unpack_flags(dataset["quality_flag"])
["outliers"]))[0]
ydata_subset_used = dataset[measurandstring].values[:,
ids_used]
avgs = np.tile(
np.mean(ydata_subset_used, axis=1)[..., None], len(ids))
else:
avg_ids = np.where(
(dataset_avg['series_id'] == series_id[i]))[0]
avgs = np.tile(dataset_avg[measurandstring].values[:, avg_ids],
len(ids))
self.plot_variable("relative difference",
plotpath,
dataset["wavelength"].values,
(ydata_subset - avgs) / avgs,
ylim=[-0.3, 0.3],
mask=mask)
