class WaterNetworkProtocol:
def __init__(self, context, MCsteps=1000, parallel_cores=1):
self.context = context
self.rh = RhymerHypstar(context)
self.rhp = RhymerProcessing(context)
self.rhs = RhymerShared(context)
def function(self, upwelling_radiance, downwelling_radiance, irradiance,
rhof, wavelength):
'''
This function implements the measurement function.
Each of the arguments can be either a scalar or a vector (1D-array).
'''
# default water network processing
water_leaving_radiance = [
(upwelling_radiance[w] - (rhof * downwelling_radiance[w]))
for w in range(len(downwelling_radiance))
]
reflectance_nosc = [
np.pi * (upwelling_radiance[w] -
(rhof * downwelling_radiance[w])) / irradiance[w]
for w in range(len(downwelling_radiance))
]
# NIR SIMIL CORRECTION
# retrieve variables for NIR SIMIL correction
w1 = self.context.get_config_value("similarity_w1")
w2 = self.context.get_config_value("similarity_w2")
alpha = self.context.get_config_value("similarity_alpha")
iref1, wref1 = self.rhs.closest_idx(wavelength, w1)
iref2, wref2 = self.rhs.closest_idx(wavelength, w2)
## get pixel index for similarity
if alpha is None:
ssd = self.rhp.similarity_read()
id1, w1 = self.rhs.closest_idx(ssd['wave'], w1 / 1000.)
id2, w2 = self.rhs.closest_idx(ssd['wave'], w2 / 1000.)
alpha = ssd['ave'][id1] / ssd['ave'][id2]
epsilon = (alpha * reflectance_nosc[iref1] -
reflectance_nosc[iref2]) / (alpha - 1.0)
reflectance = [r - epsilon for r in reflectance_nosc]
return water_leaving_radiance, reflectance_nosc, reflectance, epsilon
@staticmethod
def get_name():
return "WaterNetworkProtocol"
def get_argument_names(self):
return [
"upwelling_radiance", "downwelling_radiance", "irradiance", "rhof",
"wavelength"
]
def __init__(self, context):
self.context = context
self.templ = DataTemplates(context=context)
self.writer = HypernetsWriter(context)
self.avg = Average(context)
self.intp = Interpolate(context, MCsteps=1000)
self.plot = Plotting(context)
self.rhymeranc = RhymerAncillary(context)
self.rhymerproc = RhymerProcessing(context)
self.rhymershared = RhymerShared(context)
def __init__(self, context, MCsteps=1000, parallel_cores=1):
self._measurement_function_factory = ProtocolFactory(context=context)
self.prop = PropagateUnc(context,
MCsteps,
parallel_cores=parallel_cores)
self.templ = DataTemplates(context=context)
self.writer = HypernetsWriter(context)
self.avg = Average(context)
self.calibrate = Calibrate(context)
self.plot = Plotting(context)
self.context = context
self.rh = RhymerHypstar(context)
self.rhp = RhymerProcessing(context)
self.rhs = RhymerShared(context)
def ancillary_download(
self,
date=None,
ancillary_files=None,
time=None,
file_types=[
'TOAST', 'O3_AURAOMI_24h', 'MET_NCEP_6h', 'MET_NCEP_NEXT'
],
local_dir="/storage/Data/MET",
download=True,
override=False,
verbosity=0,
get_url="https://oceandata.sci.gsfc.nasa.gov/cgi/getfile"):
local_files = []
for basefile in ancillary_files:
yjd = basefile[1:8]
year = yjd[0:4]
jday = yjd[4:7]
url_file = '{}/{}'.format(get_url, basefile)
local_file = '{}/{}/{}/{}'.format(local_dir, year, jday, basefile)
if download:
## download file
if os.path.exists(local_file) & (not override):
if verbosity > 1: print('File {} exists'.format(basefile))
local_files.append(local_file)
else:
if os.path.exists(os.path.dirname(local_file)) is False:
os.makedirs(os.path.dirname(local_file))
if verbosity > 0:
print('Downloading file {}'.format(basefile))
try:
# urllib.request.urlretrieve(url_file, local_file)
RhymerShared(self).download_file(url_file, local_file)
if verbosity > 0:
print('Finished downloading file {}'.format(
basefile))
local_files.append(local_file)
except:
print('Downloading file {} failed'.format(basefile))
return (local_files)
def __init__(self, context):
self.context = context
self.rhymeranc = RhymerAncillary(context)
self.rhymershared = RhymerShared(context)
class RhymerProcessing:
def __init__(self, context):
self.context = context
self.rhymeranc = RhymerAncillary(context)
self.rhymershared = RhymerShared(context)
def mobley_lut_interp(self, ths, thv, phi, wind=2.):
rholut = self.mobley_lut_read()
dval = rholut['header']
# find wind speeds
wind_id, wind_br = self.rhymershared.lutpos(dval['Wind'], wind)
wind_w = wind_id - wind_br[0]
# find geometry
ths_id, ths_br = self.rhymershared.lutpos(dval['Theta-sun'], ths)
thv_id, thv_br = self.rhymershared.lutpos(dval['Theta'], thv)
phi_id, phi_br = self.rhymershared.lutpos(dval['Phi'], abs(phi))
## interpolate for both bracketing wind speeds
it1 = self.rhymershared.interp3d(rholut['data'][wind_br[0], :, :, :],
ths_id, thv_id, phi_id)
it2 = self.rhymershared.interp3d(rholut['data'][wind_br[1], :, :, :],
ths_id, thv_id, phi_id)
## and weigh to wind speed
rint = it2 * wind_w + it1 * (1. - wind_w)
return (rint)
## read Mobley sky reflectance LUT
## QV 2018-07-18
## Last modifications: 2019-07-10 (QV) integrated in rhymer
def mobley_lut_read(self):
vf = self.context.get_config_value("rholut")
ifile = '{}/{}'.format(
self.context.get_config_value("rhymer_data_dir"),
'Shared/Mobley/{}.txt'.format(vf))
header = ' I J Theta Phi Phi-view rho'.split()
data, cur = {}, None
with open(ifile, 'r') as f:
for li, line in enumerate(f.readlines()):
line = line.strip()
if 'rho for' in line:
sp = line.split()
ws = [
sp[v] for v in range(1,
len(sp) - 1)
if (sp[v - 2] == 'SPEED') & (sp[v + 1] == 'm/s')
]
ws = float(ws[0])
ths = [
sp[v] for v in range(1,
len(sp) - 1)
if (sp[v - 2] == 'THETA_SUN') & (sp[v + 1] == 'deg')
]
ths = float(ths[0])
cur = {'ws': ws, 'ths': ths}
else:
if cur is None: continue
if cur['ws'] not in data:
data[cur['ws']] = {}
if cur['ths'] not in data[cur['ws']]:
data[cur['ws']][cur['ths']] = {h: [] for h in header}
sp = [float(v) for v in line.split()]
for ih, h in enumerate(header):
data[cur['ws']][cur['ths']][h].append(sp[ih])
## set up dimensions
dval = {'Wind': list(data.keys())}
dval['Theta-sun'] = list(data[dval['Wind'][0]])
dim = [len(dval['Wind']), len(dval['Theta-sun'])]
skip = ['I', 'J', 'Phi-view', 'rho']
for k in data[dval['Wind'][0]][dval['Theta-sun'][0]].keys():
if k in skip: continue
tmp = data[dval['Wind'][0]][dval['Theta-sun'][0]][k]
dval[k] = list(set(tmp))
dval[k].sort()
dim += [len(dval[k])]
## generate LUT array
rholut = np.zeros(dim)
for wi, wv in enumerate(dval['Wind']):
for szi, sza in enumerate(dval['Theta-sun']):
cd = data[wv][sza]
for vzi, vza in enumerate(dval['Theta']):
for vai, vaa in enumerate(dval['Phi']):
if vzi == 0:
idx = 0
else:
idx = 1 + (vzi - 1) * (len(dval['Phi'])) + vai
rholut[wi, szi, vzi, vai] = cd['rho'][idx]
return ({'header': dval, 'data': rholut})
## def similarity_read
## reads similarity spectrum
## written by Quinten Vanhellemont, RBINS for the PONDER project
## 2017-12
## modifications: 2018-04-17 (QV) added to acolite function
## 2019-07-10 (QV) integrated in rhymer
def similarity_read(self, ifile=None):
if ifile is None:
ifile = '{}/Shared/REMSEM/similarityspectrumtable.txt'.format(
self.context.get_config_value("rhymer_data_dir"))
ss_data = {
'wave': np.ndarray(0),
'ave': np.ndarray(0),
'std': np.ndarray(0),
'cv': np.ndarray(0)
}
with open(ifile, 'r') as f:
for i, line in enumerate(f.readlines()):
line = line.strip()
sp = line.split()
if i == 0:
continue
else:
j = 0
while j < len(sp):
ss_data['wave'] = np.append(ss_data['wave'],
float(sp[j]) / 1000.)
ss_data['ave'] = np.append(ss_data['ave'],
float(sp[j + 1]))
ss_data['std'] = np.append(ss_data['std'],
float(sp[j + 2]))
ss_data['cv'] = np.append(ss_data['cv'],
float(sp[j + 3]))
j += 4
## sort indices
idx = np.argsort(ss_data['wave'])
for k in ss_data.keys():
ss_data[k] = ss_data[k][idx]
return (ss_data)
class RhymerHypstar:
def __init__(self, context):
self.context = context
self.templ = DataTemplates(context=context)
self.writer = HypernetsWriter(context)
self.avg = Average(context)
self.intp = Interpolate(context, MCsteps=1000)
self.plot = Plotting(context)
self.rhymeranc = RhymerAncillary(context)
self.rhymerproc = RhymerProcessing(context)
self.rhymershared = RhymerShared(context)
def qc_scan(self, dataset, measurandstring, dataset_l1b):
## no inclination
## difference at 550 nm < 25% with neighbours
##
## QV July 2018
## Last modifications: 2019-07-10 (QV) renamed from PANTR, integrated in rhymer
# Modified 10/09/2020 by CG for the PANTHYR
verbosity = self.context.get_config_value("verbosity")
series_id = np.unique(dataset['series_id'])
wave = dataset['wavelength'].values
flags = np.zeros(shape=len(dataset['scan']))
id = 0
for s in series_id:
scans = dataset['scan'][dataset['series_id'] == s]
##
n = len(scans)
## get pixel index for wavelength
iref, wref = self.rhymershared.closest_idx(
wave, self.context.get_config_value("diff_wave"))
cos_sza = []
for i in dataset['solar_zenith_angle'].sel(scan=scans).values:
cos_sza.append(math.cos(math.radians(i)))
## go through the current set of scans
for i in range(n):
## test inclination
## not done
if measurandstring == 'irradiance':
data = dataset['irradiance'].sel(scan=scans).T.values
## test variability at 550 nm
if i == 0:
v = abs(1 - ((data[i][iref] / cos_sza[i]) /
(data[i + 1][iref] / cos_sza[i + 1])))
elif i < n - 1:
v = max(
abs(1 - ((data[i][iref] / cos_sza[i]) /
(data[i + 1][iref] / cos_sza[i + 1]))),
abs(1 - ((data[i][iref] / cos_sza[i]) /
(data[i - 1][iref] / cos_sza[i - 1]))))
else:
v = abs(1 - ((data[i][iref] / cos_sza[i]) /
(data[i - 1][iref] / cos_sza[i - 1])))
else:
data = dataset['radiance'].sel(scan=scans).T.values
## test variability at 550 nm
if i == 0:
v = abs(1 - (data[i][iref] / data[i + 1][iref]))
elif i < n - 1:
v = max(abs(1 - (data[i][iref] / data[i + 1][iref])),
abs(1 - (data[i][iref] / data[i - 1][iref])))
else:
v = abs(1 - (data[i][iref] / data[i - 1][iref]))
## continue if value exceeds the cv threshold
if v > self.context.get_config_value("diff_threshold"):
# get flag value for the temporal variability
if measurandstring == 'irradiance':
flags[id] = 1
dataset_l1b['quality_flag'][range(
len(dataset_l1b['scan']))] = du.set_flag(
dataset_l1b["quality_flag"][range(
len(dataset_l1b['scan']))],
"temp_variability_ed")
else:
flags[id] = 1
dataset_l1b['quality_flag'][range(
len(dataset_l1b['scan']))] = du.set_flag(
dataset_l1b["quality_flag"][range(
len(dataset_l1b['scan']))],
"temp_variability_lu")
seq = dataset.attrs["sequence_id"]
ts = datetime.utcfromtimestamp(
dataset['acquisition_time'][i])
if verbosity > 2:
self.context.logger.info(
'Temporal jump: in {}: Aquisition time {}, {}'.
format(
seq, ts, ', '.join([
'{}:{}'.format(k,
dataset[k][scans[i]].values)
for k in ['scan', 'quality_flag']
])))
id += 1
return dataset_l1b, flags
def cycleparse(self, rad, irr, dataset_l1b):
protocol = self.context.get_config_value(
"measurement_function_surface_reflectance")
self.context.logger.debug(protocol)
nbrlu = self.context.get_config_value("n_upwelling_rad")
nbred = self.context.get_config_value("n_upwelling_irr")
nbrlsky = self.context.get_config_value("n_downwelling_rad")
if protocol != 'WaterNetworkProtocol':
# here we should simply provide surface reflectance?
# what about a non-standard protocol but that includes the required standard series?
self.context.logger.error(
'Unknown measurement protocol: {}'.format(protocol))
else:
uprad = []
downrad = []
for i in rad['scan']:
scani = rad.sel(scan=i)
senz = scani["viewing_zenith_angle"].values
if senz < 90:
measurement = 'upwelling_radiance'
uprad.append(int(i))
if senz >= 90:
measurement = 'downwelling_radiance'
downrad.append(int(i))
if measurement is None: continue
lu = rad.sel(scan=uprad)
lsky = rad.sel(scan=downrad)
for i in lu['scan']:
scani = lu.sel(scan=i)
sena = scani["viewing_azimuth_angle"].values
senz = scani["viewing_zenith_angle"].values
self.context.logger.debug(scani['acquisition_time'].values)
ts = datetime.utcfromtimestamp(
int(scani['acquisition_time'].values))
# not fromtimestamp?
if (senz != 'NULL') & (sena != 'NULL'):
senz = float(senz)
sena = abs(float(sena))
else:
dataset_l1b['quality_flag'] = du.set_flag(
dataset_l1b.sel(scan=i)['quality_flag'],
"angles_missing")
self.context.logger.info(
'NULL angles: Aquisition time {}, {}'.format(
ts, ', '.join([
'{}:{}'.format(k, scani[k].values)
for k in ['scan', 'quality_flag']
])))
continue
# check if we have the same azimuth for lu and lsky
sena_lu = np.unique(lu["viewing_azimuth_angle"].values)
sena_lsky = np.unique(lsky["viewing_azimuth_angle"].values)
for i in sena_lu:
if i not in sena_lsky:
dataset_l1b["quality_flag"][
dataset_l1b["viewing_azimuth_angle"] ==
i] = du.set_flag(
dataset_l1b["quality_flag"][
dataset_l1b["viewing_azimuth_angle"] == i],
"lu_eq_missing")
if self.context.get_config_value("verbosity") > 2:
ts = [
datetime.utcfromtimestamp(x)
for x in lu['acquisition_time'][
lu["viewing_azimuth_angle"] == i].values
]
self.context.logger.info(
'No azimuthal equivalent downwelling radiance measurement: Aquisition time {}, {}'
.format(
ts, ', '.join([
'{}:{}'.format(
k, lu[k][lu["viewing_azimuth_angle"] ==
i].values)
for k in ['scan', 'quality_flag']
])))
# check if we have the required fresnel angle for lsky
senz_lu = np.unique(lu["viewing_zenith_angle"].values)
senz_lsky = 180 - np.unique(lsky["viewing_zenith_angle"].values)
for i in senz_lu:
if i not in senz_lsky:
dataset_l1b["quality_flag"][
dataset_l1b["viewing_azimuth_angle"] ==
i] = du.set_flag(
dataset_l1b["quality_flag"][
dataset_l1b["viewing_azimuth_angle"] == i],
"fresnel_angle_missing")
ts = [
datetime.utcfromtimestamp(x)
for x in lu['acquisition_time'][
lu["viewing_zenith_angle"] == i].values
]
self.context.logger.info(
'No downwelling radiance measurement at appropriate fresnel angle: Aquisition time {}, {}'
.format(
ts, ', '.join([
'{}:{}'.format(
k, lu[k][lu["viewing_azimuth_angle"] ==
i].values)
for k in ['scan', 'quality_flag']
])))
# check if correct number of radiance and irradiance data
if lu.scan[lu['quality_flag'] <= 0].count() < nbrlu:
for i in range(len(dataset_l1b["scan"])):
dataset_l1b["quality_flag"][
dataset_l1b["scan"] == i] = du.set_flag(
dataset_l1b["quality_flag"][dataset_l1b["scan"] ==
i], "min_nbrlu")
self.context.logger.info(
"No enough upwelling radiance data for sequence {}".format(
lu.attrs['sequence_id']))
if lsky.scan[lsky['quality_flag'] <= 1].count() < nbrlsky:
for i in range(len(dataset_l1b["scan"])):
dataset_l1b["quality_flag"][
dataset_l1b["scan"] == i] = du.set_flag(
dataset_l1b["quality_flag"][dataset_l1b["scan"] ==
i], "min_nbrlsky")
self.context.logger.info(
"No enough downwelling radiance data for sequence {}".
format(lsky.attrs['sequence_id']))
if irr.scan[irr['quality_flag'] <= 1].count() < nbred:
for i in range(len(dataset_l1b["scan"])):
dataset_l1b["quality_flag"][
dataset_l1b["scan"] == i] = du.set_flag(
dataset_l1b["quality_flag"][dataset_l1b["scan"] ==
i], "min_nbred")
self.context.logger.info(
"No enough downwelling irradiance data for sequence {}".
format(irr.attrs['sequence_id']))
return lu, lsky, irr, dataset_l1b
def get_wind(self, l1b):
lat = l1b.attrs['site_latitude']
lon = l1b.attrs['site_latitude']
wind = []
for i in range(len(l1b.scan)):
wa = self.context.get_config_value("wind_ancillary")
if not wa:
l1b["quality_flag"][l1b["scan"] == i] = du.set_flag(
l1b["quality_flag"][l1b["scan"] == i], "def_wind_flag")
self.context.logger.info("Default wind speed {}".format(
self.context.get_config_value("wind_default")))
wind.append(self.context.get_config_value("wind_default"))
else:
isodate = datetime.utcfromtimestamp(
l1b['acquisition_time'].values[i]).strftime('%Y-%m-%d')
isotime = datetime.utcfromtimestamp(
l1b['acquisition_time'].values[i]).strftime('%H:%M:%S')
anc_wind = self.rhymeranc.get_wind(isodate,
lon,
lat,
isotime=isotime)
if anc_wind is not None:
wind.append(anc_wind)
l1b['fresnel_wind'].values = wind
return l1b
def get_fresnelrefl(self, l1b):
## read mobley rho lut
fresnel_coeff = np.zeros(len(l1b.scan))
fresnel_vza = np.zeros(len(l1b.scan))
fresnel_raa = np.zeros(len(l1b.scan))
fresnel_sza = np.zeros(len(l1b.scan))
wind = l1b["fresnel_wind"].values
for i in range(len(l1b.scan)):
fresnel_vza[i] = l1b['viewing_zenith_angle'][i].values
fresnel_sza[i] = l1b['solar_zenith_angle'][i].values
diffa = l1b['viewing_azimuth_angle'][i].values - l1b[
'solar_azimuth_angle'][i].values
if diffa >= 360:
diffa = diffa - 360
elif 0 <= diffa < 360:
diffa = diffa
else:
diffa = diffa + 360
fresnel_raa[i] = abs((diffa - 180))
## get fresnel reflectance
if self.context.get_config_value("fresnel_option") == 'Mobley':
if (fresnel_sza[i] is not None) & (fresnel_raa[i] is not None):
sza = min(fresnel_sza[i], 79.999)
rhof = self.rhymerproc.mobley_lut_interp(sza,
fresnel_vza[i],
fresnel_raa[i],
wind=wind[i])
else:
l1b["quality_flag"][l1b["scan"] == i] = du.set_flag(
l1b["quality_flag"][l1b["scan"] == i],
"fresnel_default")
rhof = self.context.get_config_value("rhof_default")
if self.context.get_config_value(
"fresnel_option") == 'Ruddick2006':
rhof = self.context.get_config_value("rhof_default")
self.context.logger.info("Apply Ruddick et al., 2006")
if wind[i] is not None:
rhof = rhof + 0.00039 * wind[i] + 0.000034 * wind[i]**2
fresnel_coeff[i] = rhof
l1b["rhof"].values = fresnel_coeff
l1b["fresnel_vza"].values = fresnel_vza
l1b["fresnel_raa"].values = fresnel_raa
l1b["fresnel_sza"].values = fresnel_sza
return l1b
def qc_similarity(self, L1c):
wave = L1c["wavelength"]
wr = L1c.attrs["similarity_waveref"]
wp = L1c.attrs["similarity_wavethres"]
epsilon = L1c["epsilon"]
## get pixel index for wavelength
irefr, wrefr = self.rhymershared.closest_idx(wave, wr)
failSimil = []
scans = L1c['scan']
for i in range(len(scans)):
data = L1c['reflectance_nosc'].sel(scan=i).values
if abs(epsilon[i]) > wp * data[irefr]:
failSimil.append(1)
else:
failSimil.append(0)
return failSimil
def process_l1c_int(self, l1a_rad, l1a_irr):
# because we average to Lu scan we propagate values from radiance!
dataset_l1b = self.templ.l1c_int_template_from_l1a_dataset_water(
l1a_rad)
# QUALITY CHECK: TEMPORAL VARIABILITY IN ED AND LSKY -> ASSIGN FLAG
dataset_l1b, flags_rad = self.qc_scan(l1a_rad, "radiance", dataset_l1b)
dataset_l1b, flags_irr = self.qc_scan(l1a_irr, "irradiance",
dataset_l1b)
# QUALITY CHECK: MIN NBR OF SCANS -> ASSIGN FLAG
# remove temporal variability scans before average
l1a_rad = l1a_rad.sel(scan=np.where(np.array(flags_rad) != 1)[0])
l1a_irr = l1a_irr.sel(scan=np.where(np.array(flags_irr) != 1)[0])
# check number of scans per cycle for up, down radiance and irradiance
L1a_uprad, L1a_downrad, L1a_irr, dataset_l1b = self.cycleparse(
l1a_rad, l1a_irr, dataset_l1b)
L1b_downrad = self.avg.average_l1b("radiance", L1a_downrad)
L1b_irr = self.avg.average_l1b("irradiance", L1a_irr)
# INTERPOLATE Lsky and Ed FOR EACH Lu SCAN! Threshold in time -> ASSIGN FLAG
# interpolate_l1b_w calls interpolate_irradiance which includes interpolation of the
# irradiance wavelength to the radiance wavelength
L1c_int = self.intp.interpolate_l1b_w(dataset_l1b, L1a_uprad,
L1b_downrad, L1b_irr)
return L1c_int
def __init__(self, context, MCsteps=1000, parallel_cores=1):
self.context = context
self.rh = RhymerHypstar(context)
self.rhp = RhymerProcessing(context)
self.rhs = RhymerShared(context)