def test_get_flags_mask_and(self):
ds = Dataset()
meanings = [
"flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7",
"flag8"
]
flags_vector_variable = DatasetUtil.create_flags_variable(
[2, 3],
meanings,
dim_names=["dim1", "dim2"],
attributes={"standard_name": "std"})
ds["flags"] = flags_vector_variable
ds["flags"] = DatasetUtil.set_flag(ds["flags"], "flag4")
ds["flags"][0, 1] = DatasetUtil.set_flag(ds["flags"][0, 1], "flag5")
ds["flags"][1, 1] = DatasetUtil.set_flag(ds["flags"][1, 1], "flag2")
ds["flags"][1, 1] = DatasetUtil.set_flag(ds["flags"][1, 1], "flag7")
flags_mask = DatasetUtil.get_flags_mask_and(ds["flags"],
flags=["flag2", "flag7"])
expected_flags_mask = np.array(
[[False, False, False], [False, True, False]], dtype=bool)
np.testing.assert_array_almost_equal(flags_mask, expected_flags_mask)
def get_wind(self, l1b):
lat = l1b.attrs['site_latitude']
lon = l1b.attrs['site_latitude']
wind = []
for i in range(len(l1b.scan)):
wa = self.context.get_config_value("wind_ancillary")
if not wa:
l1b["quality_flag"][l1b["scan"] == i] = du.set_flag(
l1b["quality_flag"][l1b["scan"] == i], "def_wind_flag")
self.context.logger.info("Default wind speed {}".format(
self.context.get_config_value("wind_default")))
wind.append(self.context.get_config_value("wind_default"))
else:
isodate = datetime.utcfromtimestamp(
l1b['acquisition_time'].values[i]).strftime('%Y-%m-%d')
isotime = datetime.utcfromtimestamp(
l1b['acquisition_time'].values[i]).strftime('%H:%M:%S')
anc_wind = self.rhymeranc.get_wind(isodate,
lon,
lat,
isotime=isotime)
if anc_wind is not None:
wind.append(anc_wind)
l1b['fresnel_wind'].values = wind
return l1b
def get_fresnelrefl(self, l1b):
## read mobley rho lut
fresnel_coeff = np.zeros(len(l1b.scan))
fresnel_vza = np.zeros(len(l1b.scan))
fresnel_raa = np.zeros(len(l1b.scan))
fresnel_sza = np.zeros(len(l1b.scan))
wind = l1b["fresnel_wind"].values
for i in range(len(l1b.scan)):
fresnel_vza[i] = l1b['viewing_zenith_angle'][i].values
fresnel_sza[i] = l1b['solar_zenith_angle'][i].values
diffa = l1b['viewing_azimuth_angle'][i].values - l1b[
'solar_azimuth_angle'][i].values
if diffa >= 360:
diffa = diffa - 360
elif 0 <= diffa < 360:
diffa = diffa
else:
diffa = diffa + 360
fresnel_raa[i] = abs((diffa - 180))
## get fresnel reflectance
if self.context.get_config_value("fresnel_option") == 'Mobley':
if (fresnel_sza[i] is not None) & (fresnel_raa[i] is not None):
sza = min(fresnel_sza[i], 79.999)
rhof = self.rhymerproc.mobley_lut_interp(sza,
fresnel_vza[i],
fresnel_raa[i],
wind=wind[i])
else:
l1b["quality_flag"][l1b["scan"] == i] = du.set_flag(
l1b["quality_flag"][l1b["scan"] == i],
"fresnel_default")
rhof = self.context.get_config_value("rhof_default")
if self.context.get_config_value(
"fresnel_option") == 'Ruddick2006':
rhof = self.context.get_config_value("rhof_default")
self.context.logger.info("Apply Ruddick et al., 2006")
if wind[i] is not None:
rhof = rhof + 0.00039 * wind[i] + 0.000034 * wind[i]**2
fresnel_coeff[i] = rhof
l1b["rhof"].values = fresnel_coeff
l1b["fresnel_vza"].values = fresnel_vza
l1b["fresnel_raa"].values = fresnel_raa
l1b["fresnel_sza"].values = fresnel_sza
return l1b
def process_l1c(self, dataset):
dataset_l1c = self.templ.l1c_from_l1b_dataset(dataset)
dataset_l1c = self.rh.get_wind(dataset_l1c)
dataset_l1c = self.rh.get_fresnelrefl(dataset_l1c)
l1ctol1b_function = self._measurement_function_factory.get_measurement_function(
self.context.get_config_value(
"measurement_function_surface_reflectance"))
input_vars = l1ctol1b_function.get_argument_names()
input_qty = self.prop.find_input(input_vars, dataset_l1c)
u_random_input_qty = self.prop.find_u_random_input(
input_vars, dataset_l1c)
u_systematic_input_qty, corr_systematic_input_qty = \
self.prop.find_u_systematic_input(input_vars, dataset_l1c)
L1c = self.prop.process_measurement_function_l2(
[
"water_leaving_radiance", "reflectance_nosc", "reflectance",
"epsilon"
],
dataset_l1c,
l1ctol1b_function.function,
input_qty,
u_random_input_qty,
u_systematic_input_qty,
corr_systematic_input_qty,
param_fixed=[False, False, False, False, True])
failSimil = self.rh.qc_similarity(L1c)
L1c["quality_flag"][np.where(failSimil == 1)] = DatasetUtil.set_flag(
L1c["quality_flag"][np.where(failSimil == 1)],
"simil_fail") # for i in range(len(mask))]
if self.context.get_config_value("write_l1c"):
self.writer.write(L1c, overwrite=True)
for measurandstring in [
"water_leaving_radiance", "reflectance_nosc", "reflectance",
"epsilon"
]:
try:
if self.context.get_config_value("plot_l1c"):
self.plot.plot_series_in_sequence(measurandstring, L1c)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty(measurandstring,
L1c,
L2=True)
except:
print("not plotting ", measurandstring)
return L1c
def test_set_flag(self):
ds = Dataset()
meanings = [
"flag1", "flag2", "flag3", "flag4", "flag5", "flag6", "flag7",
"flag8"
]
flags_vector_variable = DatasetUtil.create_flags_variable(
[5, 4],
meanings,
dim_names=["dim1", "dim2"],
attributes={"standard_name": "std"})
ds["flags"] = flags_vector_variable
ds["flags"] = DatasetUtil.set_flag(ds["flags"], "flag4")
flags = np.full(ds["flags"].shape, 0 | 8)
self.assertTrue((ds["flags"].data == flags).all())
def qc_scan(self, dataset, measurandstring, dataset_l1b):
## no inclination
## difference at 550 nm < 25% with neighbours
##
## QV July 2018
## Last modifications: 2019-07-10 (QV) renamed from PANTR, integrated in rhymer
# Modified 10/09/2020 by CG for the PANTHYR
verbosity = self.context.get_config_value("verbosity")
series_id = np.unique(dataset['series_id'])
wave = dataset['wavelength'].values
flags = np.zeros(shape=len(dataset['scan']))
id = 0
for s in series_id:
scans = dataset['scan'][dataset['series_id'] == s]
##
n = len(scans)
## get pixel index for wavelength
iref, wref = self.rhymershared.closest_idx(
wave, self.context.get_config_value("diff_wave"))
cos_sza = []
for i in dataset['solar_zenith_angle'].sel(scan=scans).values:
cos_sza.append(math.cos(math.radians(i)))
## go through the current set of scans
for i in range(n):
## test inclination
## not done
if measurandstring == 'irradiance':
data = dataset['irradiance'].sel(scan=scans).T.values
## test variability at 550 nm
if i == 0:
v = abs(1 - ((data[i][iref] / cos_sza[i]) /
(data[i + 1][iref] / cos_sza[i + 1])))
elif i < n - 1:
v = max(
abs(1 - ((data[i][iref] / cos_sza[i]) /
(data[i + 1][iref] / cos_sza[i + 1]))),
abs(1 - ((data[i][iref] / cos_sza[i]) /
(data[i - 1][iref] / cos_sza[i - 1]))))
else:
v = abs(1 - ((data[i][iref] / cos_sza[i]) /
(data[i - 1][iref] / cos_sza[i - 1])))
else:
data = dataset['radiance'].sel(scan=scans).T.values
## test variability at 550 nm
if i == 0:
v = abs(1 - (data[i][iref] / data[i + 1][iref]))
elif i < n - 1:
v = max(abs(1 - (data[i][iref] / data[i + 1][iref])),
abs(1 - (data[i][iref] / data[i - 1][iref])))
else:
v = abs(1 - (data[i][iref] / data[i - 1][iref]))
## continue if value exceeds the cv threshold
if v > self.context.get_config_value("diff_threshold"):
# get flag value for the temporal variability
if measurandstring == 'irradiance':
flags[id] = 1
dataset_l1b['quality_flag'][range(
len(dataset_l1b['scan']))] = du.set_flag(
dataset_l1b["quality_flag"][range(
len(dataset_l1b['scan']))],
"temp_variability_ed")
else:
flags[id] = 1
dataset_l1b['quality_flag'][range(
len(dataset_l1b['scan']))] = du.set_flag(
dataset_l1b["quality_flag"][range(
len(dataset_l1b['scan']))],
"temp_variability_lu")
seq = dataset.attrs["sequence_id"]
ts = datetime.utcfromtimestamp(
dataset['acquisition_time'][i])
if verbosity > 2:
self.context.logger.info(
'Temporal jump: in {}: Aquisition time {}, {}'.
format(
seq, ts, ', '.join([
'{}:{}'.format(k,
dataset[k][scans[i]].values)
for k in ['scan', 'quality_flag']
])))
id += 1
return dataset_l1b, flags
def cycleparse(self, rad, irr, dataset_l1b):
protocol = self.context.get_config_value(
"measurement_function_surface_reflectance")
self.context.logger.debug(protocol)
nbrlu = self.context.get_config_value("n_upwelling_rad")
nbred = self.context.get_config_value("n_upwelling_irr")
nbrlsky = self.context.get_config_value("n_downwelling_rad")
if protocol != 'WaterNetworkProtocol':
# here we should simply provide surface reflectance?
# what about a non-standard protocol but that includes the required standard series?
self.context.logger.error(
'Unknown measurement protocol: {}'.format(protocol))
else:
uprad = []
downrad = []
for i in rad['scan']:
scani = rad.sel(scan=i)
senz = scani["viewing_zenith_angle"].values
if senz < 90:
measurement = 'upwelling_radiance'
uprad.append(int(i))
if senz >= 90:
measurement = 'downwelling_radiance'
downrad.append(int(i))
if measurement is None: continue
lu = rad.sel(scan=uprad)
lsky = rad.sel(scan=downrad)
for i in lu['scan']:
scani = lu.sel(scan=i)
sena = scani["viewing_azimuth_angle"].values
senz = scani["viewing_zenith_angle"].values
self.context.logger.debug(scani['acquisition_time'].values)
ts = datetime.utcfromtimestamp(
int(scani['acquisition_time'].values))
# not fromtimestamp?
if (senz != 'NULL') & (sena != 'NULL'):
senz = float(senz)
sena = abs(float(sena))
else:
dataset_l1b['quality_flag'] = du.set_flag(
dataset_l1b.sel(scan=i)['quality_flag'],
"angles_missing")
self.context.logger.info(
'NULL angles: Aquisition time {}, {}'.format(
ts, ', '.join([
'{}:{}'.format(k, scani[k].values)
for k in ['scan', 'quality_flag']
])))
continue
# check if we have the same azimuth for lu and lsky
sena_lu = np.unique(lu["viewing_azimuth_angle"].values)
sena_lsky = np.unique(lsky["viewing_azimuth_angle"].values)
for i in sena_lu:
if i not in sena_lsky:
dataset_l1b["quality_flag"][
dataset_l1b["viewing_azimuth_angle"] ==
i] = du.set_flag(
dataset_l1b["quality_flag"][
dataset_l1b["viewing_azimuth_angle"] == i],
"lu_eq_missing")
if self.context.get_config_value("verbosity") > 2:
ts = [
datetime.utcfromtimestamp(x)
for x in lu['acquisition_time'][
lu["viewing_azimuth_angle"] == i].values
]
self.context.logger.info(
'No azimuthal equivalent downwelling radiance measurement: Aquisition time {}, {}'
.format(
ts, ', '.join([
'{}:{}'.format(
k, lu[k][lu["viewing_azimuth_angle"] ==
i].values)
for k in ['scan', 'quality_flag']
])))
# check if we have the required fresnel angle for lsky
senz_lu = np.unique(lu["viewing_zenith_angle"].values)
senz_lsky = 180 - np.unique(lsky["viewing_zenith_angle"].values)
for i in senz_lu:
if i not in senz_lsky:
dataset_l1b["quality_flag"][
dataset_l1b["viewing_azimuth_angle"] ==
i] = du.set_flag(
dataset_l1b["quality_flag"][
dataset_l1b["viewing_azimuth_angle"] == i],
"fresnel_angle_missing")
ts = [
datetime.utcfromtimestamp(x)
for x in lu['acquisition_time'][
lu["viewing_zenith_angle"] == i].values
]
self.context.logger.info(
'No downwelling radiance measurement at appropriate fresnel angle: Aquisition time {}, {}'
.format(
ts, ', '.join([
'{}:{}'.format(
k, lu[k][lu["viewing_azimuth_angle"] ==
i].values)
for k in ['scan', 'quality_flag']
])))
# check if correct number of radiance and irradiance data
if lu.scan[lu['quality_flag'] <= 0].count() < nbrlu:
for i in range(len(dataset_l1b["scan"])):
dataset_l1b["quality_flag"][
dataset_l1b["scan"] == i] = du.set_flag(
dataset_l1b["quality_flag"][dataset_l1b["scan"] ==
i], "min_nbrlu")
self.context.logger.info(
"No enough upwelling radiance data for sequence {}".format(
lu.attrs['sequence_id']))
if lsky.scan[lsky['quality_flag'] <= 1].count() < nbrlsky:
for i in range(len(dataset_l1b["scan"])):
dataset_l1b["quality_flag"][
dataset_l1b["scan"] == i] = du.set_flag(
dataset_l1b["quality_flag"][dataset_l1b["scan"] ==
i], "min_nbrlsky")
self.context.logger.info(
"No enough downwelling radiance data for sequence {}".
format(lsky.attrs['sequence_id']))
if irr.scan[irr['quality_flag'] <= 1].count() < nbred:
for i in range(len(dataset_l1b["scan"])):
dataset_l1b["quality_flag"][
dataset_l1b["scan"] == i] = du.set_flag(
dataset_l1b["quality_flag"][dataset_l1b["scan"] ==
i], "min_nbred")
self.context.logger.info(
"No enough downwelling irradiance data for sequence {}".
format(irr.attrs['sequence_id']))
return lu, lsky, irr, dataset_l1b
def preprocess_l0(self, datasetl0, datasetl0_bla, dataset_calib):
"""
Identifies and removes faulty measurements (e.g. due to cloud cover).
:param dataset_l0:
:type dataset_l0:
:return:
:rtype:
"""
wavs = dataset_calib["wavelength"].values
wavpix = dataset_calib["wavpix"].values
datasetl0 = datasetl0.isel(wavelength=slice(int(wavpix[0]),
int(wavpix[-1]) + 1))
datasetl0_bla = datasetl0_bla.isel(
wavelength=slice(int(wavpix[0]),
int(wavpix[-1]) + 1))
mask = self.clip_and_mask(datasetl0, datasetl0_bla)
datasetl0 = datasetl0.assign_coords(wavelength=wavs)
datasetl0_bla = datasetl0_bla.assign_coords(wavelength=wavs)
datasetl0["quality_flag"][np.where(mask == 1)] = DatasetUtil.set_flag(
datasetl0["quality_flag"][np.where(mask == 1)],
"outliers") #for i in range(len(mask))]
DN_rand = DatasetUtil.create_variable(
[len(datasetl0["wavelength"]),
len(datasetl0["scan"])],
dim_names=["wavelength", "scan"],
dtype=np.uint32,
fill_value=0)
datasetl0["u_random_digital_number"] = DN_rand
rand = np.zeros_like(DN_rand.values)
series_ids = np.unique(datasetl0['series_id'])
for i in range(len(series_ids)):
ids = np.where(datasetl0['series_id'] == series_ids[i])[0]
ids_masked = np.where((datasetl0['series_id'] == series_ids[i])
& (mask == 0))[0]
dark_signals = np.zeros_like(
datasetl0['digital_number'].values[:, ids_masked])
for ii, id in enumerate(ids_masked):
dark_signals[:, ii] = self.find_nearest_black(
datasetl0_bla, datasetl0['acquisition_time'].values[id],
datasetl0['integration_time'].values[id])
std = np.std((datasetl0['digital_number'].values[:, ids_masked] -
dark_signals),
axis=1)
for ii, id in enumerate(ids):
rand[:, id] = std
datasetl0["u_random_digital_number"].values = rand
DN_dark = DatasetUtil.create_variable(
[len(datasetl0["wavelength"]),
len(datasetl0["scan"])],
dim_names=["wavelength", "scan"],
dtype=np.uint32,
fill_value=0)
datasetl0["dark_signal"] = DN_dark
dark_signals = []
acqui = datasetl0['acquisition_time'].values
inttimes = datasetl0['integration_time'].values
for i in range(len(acqui)):
dark_signals.append(
self.find_nearest_black(datasetl0_bla, acqui[i], inttimes[i]))
datasetl0["dark_signal"].values = np.array(dark_signals).T
return datasetl0