def test_unpack_flags(self):
ds = Dataset()
meanings = [
"flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7",
"flag8"
]
masks = [1, 2, 4, 8, 16, 32, 64, 128]
flags_vector_variable = DatasetUtil.create_flags_variable(
[2, 3],
meanings,
dim_names=["dim1", "dim2"],
attributes={"standard_name": "std"})
ds["flags"] = flags_vector_variable
ds["flags"][0, 0] = ds["flags"][0, 0] | 8
empty = np.zeros((2, 3), bool)
flag4 = np.zeros((2, 3), bool)
flag4[0, 0] = True
flags = DatasetUtil.unpack_flags(ds["flags"])
self.assertTrue((flags["flag1"].data == empty).all())
self.assertTrue((flags["flag2"].data == empty).all())
self.assertTrue((flags["flag3"].data == empty).all())
self.assertTrue((flags["flag4"].data == flag4).all())
self.assertTrue((flags["flag5"].data == empty).all())
self.assertTrue((flags["flag6"].data == empty).all())
self.assertTrue((flags["flag7"].data == empty).all())
self.assertTrue((flags["flag8"].data == empty).all())
def test_fill_ds(self):
ds = Dataset()
ds["array_variable1"] = DatasetUtil.create_variable([7, 8], np.float32)
ds["array_variable2"] = DatasetUtil.create_variable([7, 8], np.float32)
ds["array_variable1"][2, 3] = np.nan
ds["array_variable2"][2, 3] = np.nan
HypernetsWriter.fill_ds(ds)
self.assertTrue(np.all(ds["array_variable1"] == 9.96921E36))
self.assertTrue(np.all(ds["array_variable2"] == 9.96921E36))
def test__get_flag_encoding_not_flag_var(self):
ds = Dataset()
ds["array_variable"] = DatasetUtil.create_variable(
[7, 8, 3], np.int8, attributes={"standard_name": "std"})
self.assertRaises(KeyError, DatasetUtil._get_flag_encoding,
ds["array_variable"])
def get_wind(self, l1b):
lat = l1b.attrs['site_latitude']
lon = l1b.attrs['site_latitude']
wind = []
for i in range(len(l1b.scan)):
wa = self.context.get_config_value("wind_ancillary")
if not wa:
l1b["quality_flag"][l1b["scan"] == i] = du.set_flag(
l1b["quality_flag"][l1b["scan"] == i], "def_wind_flag")
self.context.logger.info("Default wind speed {}".format(
self.context.get_config_value("wind_default")))
wind.append(self.context.get_config_value("wind_default"))
else:
isodate = datetime.utcfromtimestamp(
l1b['acquisition_time'].values[i]).strftime('%Y-%m-%d')
isotime = datetime.utcfromtimestamp(
l1b['acquisition_time'].values[i]).strftime('%H:%M:%S')
anc_wind = self.rhymeranc.get_wind(isodate,
lon,
lat,
isotime=isotime)
if anc_wind is not None:
wind.append(anc_wind)
l1b['fresnel_wind'].values = wind
return l1b
def test_create_variable_1D_int(self):
vector_variable = DatasetUtil.create_variable([5], np.int8)
self.assertIsNotNone(vector_variable)
self.assertEqual(Variable, type(vector_variable))
self.assertEqual((5, ), vector_variable.shape)
self.assertEqual(np.int8, vector_variable.dtype)
self.assertEqual(-127, vector_variable[2])
def test_create_variable_3D_int(self):
array_variable = DatasetUtil.create_variable([7, 8, 3], np.int8)
self.assertIsNotNone(array_variable)
self.assertEqual(Variable, type(array_variable))
self.assertEqual((7, 8, 3), array_variable.shape)
self.assertEqual(np.int8, array_variable.dtype)
self.assertEqual(-127, array_variable[2, 4, 2])
def test_create_default_array_3D_int(self):
default_array = DatasetUtil.create_default_array([7, 8, 3], np.int8)
self.assertIsNotNone(default_array)
self.assertEqual(DataArray, type(default_array))
self.assertEqual((7, 8, 3), default_array.shape)
self.assertEqual(np.int8, default_array.dtype)
self.assertEqual(-127, default_array[2, 4, 2])
def test_check_flag_set_false(self):
ds = Dataset()
meanings = [
"flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7",
"flag8"
]
flags_vector_variable = DatasetUtil.create_flags_variable(
[5],
meanings,
dim_names=["dim1"],
attributes={"standard_name": "std"})
ds["flags"] = flags_vector_variable
ds["flags"][3] = ds["flags"][3] | 8
flag_set = DatasetUtil.check_flag_set(ds["flags"][3], "flag6")
self.assertFalse(flag_set)
def test_set_flag(self):
ds = Dataset()
meanings = [
"flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7",
"flag8"
]
flags_vector_variable = DatasetUtil.create_flags_variable(
[5, 4],
meanings,
dim_names=["dim1", "dim2"],
attributes={"standard_name": "std"})
ds["flags"] = flags_vector_variable
ds["flags"] = DatasetUtil.set_flag(ds["flags"], "flag4")
flags = np.full(ds["flags"].shape, 0 | 8)
self.assertTrue((ds["flags"].data == flags).all())
def test_create_variable_3D_int_attributes(self):
array_variable = DatasetUtil.create_variable(
[7, 8, 3], np.int8, attributes={"standard_name": "std"})
self.assertIsNotNone(array_variable)
self.assertEqual(Variable, type(array_variable))
self.assertEqual((7, 8, 3), array_variable.shape)
self.assertEqual(np.int8, array_variable.dtype)
self.assertEqual(-127, array_variable[2, 4, 2])
self.assertEqual("std", array_variable.attrs["standard_name"])
def test_create_variable_1D_int_attributes(self):
vector_variable = DatasetUtil.create_variable(
[5], np.int8, attributes={"standard_name": "std"})
self.assertIsNotNone(vector_variable)
self.assertEqual(Variable, type(vector_variable))
self.assertEqual((5, ), vector_variable.shape)
self.assertEqual(np.int8, vector_variable.dtype)
self.assertEqual(-127, vector_variable[2])
self.assertEqual("std", vector_variable.attrs["standard_name"])
def test_get_set_flags(self):
ds = Dataset()
meanings = [
"flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7",
"flag8"
]
flags_vector_variable = DatasetUtil.create_flags_variable(
[5],
meanings,
dim_names=["dim1"],
attributes={"standard_name": "std"})
ds["flags"] = flags_vector_variable
ds["flags"][3] = ds["flags"][3] | 8
ds["flags"][3] = ds["flags"][3] | 32
set_flags = DatasetUtil.get_set_flags(ds["flags"][3])
self.assertCountEqual(set_flags, ["flag4", "flag6"])
def calc_mean_masked(self,
dataset,
var,
flags,
rand_unc=False,
corr=False):
series_id = np.unique(dataset['series_id'])
if corr:
out = np.empty\
((len(series_id), len(dataset['wavelength']), len(dataset['wavelength'])))
for i in range(len(series_id)):
flagged = np.any([
DatasetUtil.unpack_flags(dataset['quality_flag'])[x]
for x in flags
],
axis=0)
ids = np.where((dataset['series_id'] == series_id[i])
& (flagged == False))
out[i] = np.mean(dataset[var].values[:, :, ids], axis=3)[:, :,
0]
out = np.mean(out, axis=0)
else:
out = np.empty((len(series_id), len(dataset['wavelength'])))
for i in range(len(series_id)):
flagged = np.any([
DatasetUtil.unpack_flags(dataset['quality_flag'])[x]
for x in flags
],
axis=0)
ids = np.where((dataset['series_id'] == series_id[i])
& (flagged == False))
out[i] = np.mean(dataset[var].values[:, ids], axis=2)[:, 0]
if rand_unc:
out[i] = (np.sum(dataset[var].values[:, ids]**2,
axis=2)[:, 0])**0.5 / len(ids[0])
return out.T
def test_add_encoding(self):
vector_variable = DatasetUtil.create_variable([5], np.int8)
DatasetUtil.add_encoding(vector_variable,
np.int32,
scale_factor=10,
offset=23,
fill_value=11,
chunksizes=12)
self.assertIsNotNone(vector_variable)
self.assertEqual(Variable, type(vector_variable))
self.assertEqual((5, ), vector_variable.shape)
self.assertEqual(np.int8, vector_variable.dtype)
self.assertEqual(-127, vector_variable[2])
self.assertEqual(np.int32, vector_variable.encoding["dtype"])
self.assertEqual(10, vector_variable.encoding["scale_factor"])
self.assertEqual(23, vector_variable.encoding["add_offset"])
self.assertEqual(11, vector_variable.encoding["_FillValue"])
self.assertEqual(12, vector_variable.encoding["chunksizes"])
def test_create_variable_1D_int_dims(self):
vector_variable = DatasetUtil.create_variable([5],
np.int8,
dim_names=["dim1"])
self.assertIsNotNone(vector_variable)
self.assertEqual(Variable, type(vector_variable))
self.assertEqual((5, ), vector_variable.shape)
self.assertEqual(np.int8, vector_variable.dtype)
self.assertEqual(-127, vector_variable[2])
self.assertEqual(("dim1", ), vector_variable.dims)
def test__get_flag_encoding(self):
ds = Dataset()
meanings = [
"flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7",
"flag8"
]
masks = [1, 2, 4, 8, 16, 32, 64, 128]
flags_vector_variable = DatasetUtil.create_flags_variable(
[5],
meanings,
dim_names=["dim1"],
attributes={"standard_name": "std"})
ds["flags"] = flags_vector_variable
meanings_out, masks_out = DatasetUtil._get_flag_encoding(ds["flags"])
self.assertCountEqual(meanings, meanings_out)
self.assertCountEqual(masks, masks_out)
def test_create_default_array_1D_int_fill_value(self):
default_array = DatasetUtil.create_default_array([5],
np.int8,
fill_value=1)
self.assertIsNotNone(default_array)
self.assertEqual(DataArray, type(default_array))
self.assertEqual((5, ), default_array.shape)
self.assertEqual(np.int8, default_array.dtype)
self.assertEqual(1, default_array[2])
def test_create_default_array_1D_int_dims(self):
default_array = DatasetUtil.create_default_array([5],
np.int8,
dim_names=["dim1"])
self.assertIsNotNone(default_array)
self.assertEqual(DataArray, type(default_array))
self.assertEqual((5, ), default_array.shape)
self.assertEqual(np.int8, default_array.dtype)
self.assertEqual(-127, default_array[2])
self.assertEqual(("dim1", ), default_array.dims)
def test_get_flags_mask_and(self):
ds = Dataset()
meanings = [
"flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7",
"flag8"
]
flags_vector_variable = DatasetUtil.create_flags_variable(
[2, 3],
meanings,
dim_names=["dim1", "dim2"],
attributes={"standard_name": "std"})
ds["flags"] = flags_vector_variable
ds["flags"] = DatasetUtil.set_flag(ds["flags"], "flag4")
ds["flags"][0, 1] = DatasetUtil.set_flag(ds["flags"][0, 1], "flag5")
ds["flags"][1, 1] = DatasetUtil.set_flag(ds["flags"][1, 1], "flag2")
ds["flags"][1, 1] = DatasetUtil.set_flag(ds["flags"][1, 1], "flag7")
flags_mask = DatasetUtil.get_flags_mask_and(ds["flags"],
flags=["flag2", "flag7"])
expected_flags_mask = np.array(
[[False, False, False], [False, True, False]], dtype=bool)
np.testing.assert_array_almost_equal(flags_mask, expected_flags_mask)
def get_fresnelrefl(self, l1b):
## read mobley rho lut
fresnel_coeff = np.zeros(len(l1b.scan))
fresnel_vza = np.zeros(len(l1b.scan))
fresnel_raa = np.zeros(len(l1b.scan))
fresnel_sza = np.zeros(len(l1b.scan))
wind = l1b["fresnel_wind"].values
for i in range(len(l1b.scan)):
fresnel_vza[i] = l1b['viewing_zenith_angle'][i].values
fresnel_sza[i] = l1b['solar_zenith_angle'][i].values
diffa = l1b['viewing_azimuth_angle'][i].values - l1b[
'solar_azimuth_angle'][i].values
if diffa >= 360:
diffa = diffa - 360
elif 0 <= diffa < 360:
diffa = diffa
else:
diffa = diffa + 360
fresnel_raa[i] = abs((diffa - 180))
## get fresnel reflectance
if self.context.get_config_value("fresnel_option") == 'Mobley':
if (fresnel_sza[i] is not None) & (fresnel_raa[i] is not None):
sza = min(fresnel_sza[i], 79.999)
rhof = self.rhymerproc.mobley_lut_interp(sza,
fresnel_vza[i],
fresnel_raa[i],
wind=wind[i])
else:
l1b["quality_flag"][l1b["scan"] == i] = du.set_flag(
l1b["quality_flag"][l1b["scan"] == i],
"fresnel_default")
rhof = self.context.get_config_value("rhof_default")
if self.context.get_config_value(
"fresnel_option") == 'Ruddick2006':
rhof = self.context.get_config_value("rhof_default")
self.context.logger.info("Apply Ruddick et al., 2006")
if wind[i] is not None:
rhof = rhof + 0.00039 * wind[i] + 0.000034 * wind[i]**2
fresnel_coeff[i] = rhof
l1b["rhof"].values = fresnel_coeff
l1b["fresnel_vza"].values = fresnel_vza
l1b["fresnel_raa"].values = fresnel_raa
l1b["fresnel_sza"].values = fresnel_sza
return l1b
def process_l1c(self, dataset):
dataset_l1c = self.templ.l1c_from_l1b_dataset(dataset)
dataset_l1c = self.rh.get_wind(dataset_l1c)
dataset_l1c = self.rh.get_fresnelrefl(dataset_l1c)
l1ctol1b_function = self._measurement_function_factory.get_measurement_function(
self.context.get_config_value(
"measurement_function_surface_reflectance"))
input_vars = l1ctol1b_function.get_argument_names()
input_qty = self.prop.find_input(input_vars, dataset_l1c)
u_random_input_qty = self.prop.find_u_random_input(
input_vars, dataset_l1c)
u_systematic_input_qty, corr_systematic_input_qty = \
self.prop.find_u_systematic_input(input_vars, dataset_l1c)
L1c = self.prop.process_measurement_function_l2(
[
"water_leaving_radiance", "reflectance_nosc", "reflectance",
"epsilon"
],
dataset_l1c,
l1ctol1b_function.function,
input_qty,
u_random_input_qty,
u_systematic_input_qty,
corr_systematic_input_qty,
param_fixed=[False, False, False, False, True])
failSimil = self.rh.qc_similarity(L1c)
L1c["quality_flag"][np.where(failSimil == 1)] = DatasetUtil.set_flag(
L1c["quality_flag"][np.where(failSimil == 1)],
"simil_fail") # for i in range(len(mask))]
if self.context.get_config_value("write_l1c"):
self.writer.write(L1c, overwrite=True)
for measurandstring in [
"water_leaving_radiance", "reflectance_nosc", "reflectance",
"epsilon"
]:
try:
if self.context.get_config_value("plot_l1c"):
self.plot.plot_series_in_sequence(measurandstring, L1c)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty(measurandstring,
L1c,
L2=True)
except:
print("not plotting ", measurandstring)
return L1c
def l1b_template_from_l1a_dataset_water(self, measurandstring,
dataset_l1a):
"""
Makes all L1 templates for the data, and propagates the appropriate keywords from the L0 datasets.
:param datasetl0:
:type datasetl0:
:return:
:rtype:
"""
print(np.unique(dataset_l1a['series_id']))
l1b_dim_sizes_dict = {
"wavelength": len(dataset_l1a["wavelength"]),
"series": len(np.unique(dataset_l1a['series_id']))
}
if measurandstring == "radiance":
dataset_l1b = self.hdsb.create_ds_template(
l1b_dim_sizes_dict,
"W_L1B_RAD",
propagate_ds=dataset_l1a,
ds=dataset_l1a)
elif measurandstring == "irradiance":
dataset_l1b = self.hdsb.create_ds_template(
l1b_dim_sizes_dict,
"W_L1B_IRR",
propagate_ds=dataset_l1a,
ds=dataset_l1a)
dataset_l1b = dataset_l1b.assign_coords(
wavelength=dataset_l1a.wavelength)
series_id = np.unique(dataset_l1a['series_id'])
dataset_l1b["series_id"].values = series_id
for variablestring in [
"acquisition_time", "viewing_azimuth_angle",
"viewing_zenith_angle", "solar_azimuth_angle",
"solar_zenith_angle"
]:
temp_arr = np.empty(len(series_id))
for i in range(len(series_id)):
ids = np.where(
(dataset_l1a['series_id'] == series_id[i]) & np.invert(
DatasetUtil.unpack_flags(dataset_l1a["quality_flag"])
["outliers"]))
temp_arr[i] = np.mean(dataset_l1a[variablestring].values[ids])
dataset_l1b[variablestring].values = temp_arr
return dataset_l1b
def test_create_variable_3D_int_dims(self):
array_variable = DatasetUtil.create_variable(
[7, 8, 3], np.int8, dim_names=["dim1", "dim2", "dim3"])
self.assertIsNotNone(array_variable)
self.assertEqual(Variable, type(array_variable))
self.assertEqual((7, 8, 3), array_variable.shape)
self.assertEqual(np.int8, array_variable.dtype)
self.assertEqual(-127, array_variable[2, 4, 2])
self.assertEqual((
"dim1",
"dim2",
"dim3",
), array_variable.dims)
def plot_diff_scans(self, measurandstring, dataset, dataset_avg=None):
series_id = np.unique(dataset['series_id'])
for i in range(len(series_id)):
plotpath = os.path.join(
self.path, "plot_diff_" + measurandstring + "_" +
dataset.attrs['product_name'] + "_series_" +
str(series_id[i]) + "." +
self.context.get_config_value("plotting_format"))
ids = np.where(dataset['series_id'] == series_id[i])[0]
ydata_subset = dataset[measurandstring].values[:, ids]
mask = DatasetUtil.unpack_flags(
dataset["quality_flag"])["outliers"]
mask = mask[ids]
if dataset_avg is None:
ids_used = np.where(
(dataset['series_id'] == series_id[i]) & np.invert(
DatasetUtil.unpack_flags(dataset["quality_flag"])
["outliers"]))[0]
ydata_subset_used = dataset[measurandstring].values[:,
ids_used]
avgs = np.tile(
np.mean(ydata_subset_used, axis=1)[..., None], len(ids))
else:
avg_ids = np.where(
(dataset_avg['series_id'] == series_id[i]))[0]
avgs = np.tile(dataset_avg[measurandstring].values[:, avg_ids],
len(ids))
self.plot_variable("relative difference",
plotpath,
dataset["wavelength"].values,
(ydata_subset - avgs) / avgs,
ylim=[-0.3, 0.3],
mask=mask)
def test_check_flag_set_2d(self):
ds = Dataset()
meanings = [
"flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7",
"flag8"
]
flags_vector_variable = DatasetUtil.create_flags_variable(
[5],
meanings,
dim_names=["dim1"],
attributes={"standard_name": "std"})
ds["flags"] = flags_vector_variable
self.assertRaises(ValueError, DatasetUtil.check_flag_set, ds["flags"],
"flag6")
def test_get_default_fill_value(self):
self.assertEqual(-127, DatasetUtil.get_default_fill_value(np.int8))
self.assertEqual(-32767, DatasetUtil.get_default_fill_value(np.int16))
self.assertEqual(np.uint16(-1),
DatasetUtil.get_default_fill_value(np.uint16))
self.assertEqual(-2147483647,
DatasetUtil.get_default_fill_value(np.int32))
self.assertEqual(-9223372036854775806,
DatasetUtil.get_default_fill_value(np.int64))
self.assertEqual(np.float32(9.96921E36),
DatasetUtil.get_default_fill_value(np.float32))
self.assertEqual(9.969209968386869E36,
DatasetUtil.get_default_fill_value(np.float64))
def add_variables(ds, variables_dict, dim_sizes_dict):
"""
Adds defined variables dataset
:type ds: xarray.Dataset
:param ds: dataset
:type variables_dict: dict
:type variables_dict: dictionary defining variables
:type dim_sizes_dict: dict
:param dim_sizes_dict: entry per dataset dimension with value of size as int
:return: dataset with defined variables
:rtype: xarray.Dataset
"""
du = DatasetUtil()
for variable_name in variables_dict.keys():
variable_attrs = variables_dict[variable_name]
# Check variable definition
TemplateUtil._check_variable_definition(variable_name, variable_attrs)
# Unpack variable attributes
dtype = variable_attrs["dtype"]
dim_names = variable_attrs["dim"]
attributes = deepcopy(variable_attrs["attributes"]) if "attributes" in variable_attrs else None
# Determine variable shape from dims
try:
dim_sizes = TemplateUtil._return_variable_shape(dim_names, dim_sizes_dict)
except KeyError:
raise KeyError("Dim Name Error - Variable " + variable_name + " defined with dim not in dim_sizes_dict")
# Create variable and add to dataset
if dtype == "flag":
flag_meanings = attributes.pop("flag_meanings")
variable = du.create_flags_variable(dim_sizes, meanings=flag_meanings,
dim_names=dim_names, attributes=attributes)
else:
variable = du.create_variable(dim_sizes, dim_names=dim_names,
dtype=dtype, attributes=attributes)
if "encoding" in variable_attrs:
du.add_encoding(variable, **variable_attrs["encoding"])
ds[variable_name] = variable
return ds
def test_set_flag_error_if_set(self):
ds = Dataset()
meanings = [
"flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7",
"flag8"
]
flags_vector_variable = DatasetUtil.create_flags_variable(
[5],
meanings,
dim_names=["dim1"],
attributes={"standard_name": "std"})
ds["flags"] = flags_vector_variable
ds["flags"][3] = ds["flags"][3] | 8
self.assertRaises(ValueError,
DatasetUtil.set_flag,
ds["flags"],
"flag4",
error_if_set=True)
def test_create_flags_variable_3D(self):
meanings = [
"flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7",
"flag8"
]
meanings_txt = "flag1 flag2 flag3 flag4 flag5 flag6 flag7 flag8"
masks = "1, 2, 4, 8, 16, 32, 64, 128"
flags_array_variable = DatasetUtil.create_flags_variable(
[7, 8, 3],
meanings,
dim_names=["dim1", "dim2", "dim3"],
attributes={"standard_name": "std"})
self.assertIsNotNone(flags_array_variable)
self.assertEqual(Variable, type(flags_array_variable))
self.assertEqual((7, 8, 3), flags_array_variable.shape)
self.assertEqual(np.uint8, flags_array_variable.dtype)
self.assertEqual(flags_array_variable.attrs['flag_masks'], masks)
self.assertEqual(flags_array_variable.attrs['flag_meanings'],
meanings_txt)
self.assertEqual(0, flags_array_variable[2, 4, 2])
self.assertEqual("std", flags_array_variable.attrs["standard_name"])
self.assertEqual(("dim1", "dim2", "dim3"), flags_array_variable.dims)
def test_return_flags_dtype_32(self):
data_type = DatasetUtil.return_flags_dtype(32)
self.assertEqual(data_type, np.uint32)
