def __init__(self,context,MCsteps=1000,parallel_cores=1):
self._measurement_function_factory = InterpolationFactory()
self.prop = PropagateUnc(context, MCsteps, parallel_cores=parallel_cores)
self.templ = DataTemplates(context=context)
self.writer=HypernetsWriter(context)
self.plot=Plotting(context)
self.context=context
def __init__(self, context):
self.context = context
self.templ = DataTemplates(context=context)
self.writer = HypernetsWriter(context)
self.avg = Average(context)
self.intp = Interpolate(context, MCsteps=1000)
self.plot = Plotting(context)
self.rhymeranc = RhymerAncillary(context)
self.rhymerproc = RhymerProcessing(context)
self.rhymershared = RhymerShared(context)
def __init__(self, context):
dir_path = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
self.path_ascii = os.path.join(dir_path, 'calibration_files_ascii',
'HYPSTAR_cal')
self.path_netcdf = os.path.join(
dir_path, 'hypernets_processor/calibration/calibration_files',
'HYPSTAR_cal')
context.set_config_value("product_format", "netcdf")
self.templ = DataTemplates(context)
self.writer = HypernetsWriter(context)
self.context = context
def __init__(self, context, MCsteps=1000, parallel_cores=1):
self._measurement_function_factory = ProtocolFactory(context=context)
self.prop = PropagateUnc(context,
MCsteps,
parallel_cores=parallel_cores)
self.templ = DataTemplates(context=context)
self.writer = HypernetsWriter(context)
self.avg = Average(context)
self.calibrate = Calibrate(context)
self.plot = Plotting(context)
self.context = context
self.rh = RhymerHypstar(context)
self.rhp = RhymerProcessing(context)
self.rhs = RhymerShared(context)
def __init__(self, context):
self.context = context
self.model = self.context.get_config_value("model").split(',')
self.templ = DataTemplates(context)
self.writer = HypernetsWriter(context=context)
cckeys = [
'mapping_vis_a', 'mapping_vis_b', 'mapping_vis_c', 'mapping_vis_d',
'mapping_vis_e', 'mapping_vis_f'
]
ccvalues = []
for i in range(len(cckeys)):
ccvalues.append(self.context.get_config_value(cckeys[i]))
self.cc_vis = dict(zip(cckeys, ccvalues))
class CalibrationConverter:
def __init__(self, context):
dir_path = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
self.path_ascii = os.path.join(dir_path, 'calibration_files_ascii',
'HYPSTAR_cal')
self.path_netcdf = os.path.join(
dir_path, 'hypernets_processor/calibration/calibration_files',
'HYPSTAR_cal')
context.set_config_value("product_format", "netcdf")
self.templ = DataTemplates(context)
self.writer = HypernetsWriter(context)
self.context = context
def read_calib_files(self):
hypstar = "hypstar_" + str(
self.context.get_config_value("hypstar_cal_number"))
hypstar_path = os.path.join(self.path_netcdf, hypstar)
name = "HYPERNETS_CAL_" + hypstar.upper() + "_RAD_v" + str(
version) + ".nc"
calibration_data_rad = xarray.open_dataset(
os.path.join(hypstar_path, name))
name = "HYPERNETS_CAL_" + hypstar.upper() + "_IRR_v" + str(
version) + ".nc"
calibration_data_irr = xarray.open_dataset(
os.path.join(hypstar_path, name))
calibration_data_times = calibration_data_rad[
"calibrationdates"].values
nonlin_times = calibration_data_rad["nonlineardates"].values
wav_times = calibration_data_rad["wavdates"].values
calibration_data_rad = calibration_data_rad.sel(
calibrationdates=calibration_data_times[-1])
calibration_data_rad = calibration_data_rad.sel(
nonlineardates=nonlin_times[-1])
calibration_data_rad = calibration_data_rad.sel(wavdates=wav_times[-1])
calibration_data_irr = calibration_data_irr.sel(
calibrationdates=calibration_data_times[-1])
calibration_data_irr = calibration_data_irr.sel(
nonlineardates=nonlin_times[-1])
calibration_data_irr = calibration_data_irr.sel(wavdates=wav_times[-1])
if self.context.get_config_value("network") == "l":
name = "HYPERNETS_CAL_" + hypstar.upper() + "_RAD_SWIR_v" + str(
version) + ".nc"
calibration_data_rad_swir = xarray.open_dataset(
os.path.join(hypstar_path, name))
name = "HYPERNETS_CAL_" + hypstar.upper() + "_IRR_SWIR_v" + str(
version) + ".nc"
calibration_data_irr_swir = xarray.open_dataset(
os.path.join(hypstar_path, name))
calibration_data_times = calibration_data_rad_swir[
"calibrationdates"].values
nonlin_times = calibration_data_rad_swir["nonlineardates"].values
calibration_data_rad_swir = calibration_data_rad_swir.sel(
calibrationdates=calibration_data_times[-1])
calibration_data_rad_swir = calibration_data_rad_swir.sel(
nonlineardates=nonlin_times[-1])
calibration_data_rad_swir = calibration_data_rad_swir.sel(
wavdates=wav_times[-1])
calibration_data_irr_swir = calibration_data_irr_swir.sel(
calibrationdates=calibration_data_times[-1])
calibration_data_irr_swir = calibration_data_irr_swir.sel(
nonlineardates=nonlin_times[-1])
calibration_data_irr_swir = calibration_data_irr_swir.sel(
wavdates=wav_times[-1])
return (calibration_data_rad, calibration_data_irr,
calibration_data_rad_swir, calibration_data_irr_swir)
else:
return calibration_data_rad, calibration_data_irr
def convert_all_calibration_data(self):
measurandstrings = ["radiance", "irradiance"]
hypstars = [
os.path.basename(path)
for path in glob.glob(os.path.join(self.path_ascii, "hypstar_*"))
]
for hypstar in hypstars:
print("processing " + hypstar)
hypstar_path = os.path.join(self.path_netcdf, hypstar)
if not os.path.exists(hypstar_path):
os.makedirs(hypstar_path)
for measurandstring in measurandstrings:
if measurandstring == "radiance":
tag = "_RAD_"
else:
tag = "_IRR_"
calib_data = self.prepare_calibration_data(
measurandstring, hypstar=hypstar[8::])
calib_data.attrs["product_name"] = "HYPERNETS_CAL_"+hypstar.upper()\
+tag+"v"+str(version)
self.writer.write(calib_data,
directory=hypstar_path,
overwrite=True)
if hypstar[8] == "2":
tag = tag + "SWIR_"
calib_data = self.prepare_calibration_data(
measurandstring, hypstar=hypstar[8::], swir=True)
calib_data.attrs["product_name"] = "HYPERNETS_CAL_"+\
hypstar.upper()+tag+"v"+str(version)
self.writer.write(calib_data,
directory=hypstar_path,
overwrite=True)
def prepare_calibration_data(self, measurandstring, hypstar, swir=False):
if swir:
sensortag = "swir"
else:
sensortag = "vnir"
directory = self.path_ascii
caldatepaths = [
os.path.basename(path) for path in glob.glob(
os.path.join(directory, "hypstar_" + str(hypstar) +
"/radiometric/*"))
]
nonlindates = []
caldates = []
for caldatepath in caldatepaths:
caldate = caldatepath
nonlinpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) + "\\radiometric\\" +
str(caldatepath) + "\\hypstar_" + str(hypstar) +
"_nonlin_corr_coefs_*.dat"))[0]
if os.path.exists(nonlinpath):
nonlindates = np.append(nonlindates, caldate)
non_linear_cals = np.genfromtxt(nonlinpath)[:, 0]
if measurandstring == "radiance":
calpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) +
"\\radiometric\\" + str(caldatepath) + "\\hypstar_" +
str(hypstar) + "_radcal_L_*_%s.dat" % (sensortag)))[0]
else:
calpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) +
"\\radiometric\\" + str(caldatepath) + "\\hypstar_" +
str(hypstar) + "_radcal_E_*_%s.dat" % (sensortag)))[0]
if os.path.exists(calpath):
caldates = np.append(caldates, caldate)
gains = np.genfromtxt(calpath)
wavs = gains[:, 1]
wavcaldatepaths = [
os.path.basename(path) for path in glob.glob(
os.path.join(directory, "hypstar_" + str(hypstar) +
"/wavelength/*"))
]
wavcaldates = []
for wavcaldatepath in wavcaldatepaths:
wavcaldate = wavcaldatepath
wavcalpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) + "\\wavelength\\" +
str(wavcaldatepath) + "\\hypstar_" + str(hypstar) +
"_wl_coefs_*.dat"))[0]
if os.path.exists(wavcalpath):
wavcaldates = np.append(wavcaldates, wavcaldate)
wav_cals = np.genfromtxt(wavcalpath)[:, 0]
calibration_data = self.templ.calibration_dataset(
wavs, non_linear_cals, wav_cals, caldates, nonlindates,
wavcaldates)
i_nonlin = 0
for caldatepath in caldatepaths:
nonlinpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) + "\\radiometric\\" +
str(caldatepath) + "\\hypstar_" + str(hypstar) +
"_nonlin_corr_coefs_*.dat"))[0]
if os.path.exists(nonlinpath):
non_linear_cals = np.genfromtxt(nonlinpath)[:, 0]
calibration_data["non_linearity_coefficients"].values[
i_nonlin] = non_linear_cals
i_nonlin += 1
i_wavcoef = 0
for wavcaldatepath in wavcaldatepaths:
wavcaldate = wavcaldatepath
wavcalpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) + "\\wavelength\\" +
str(wavcaldatepath) + "\\hypstar_" + str(hypstar) +
"_wl_coefs_*.dat"))[0]
if os.path.exists(wavcalpath):
wav_cals = np.genfromtxt(wavcalpath)
if measurandstring == "radiance" and not swir:
wav_cals = wav_cals[:, 0]
if measurandstring == "irradiance" and not swir:
wav_cals = wav_cals[:, 1]
if measurandstring == "radiance" and swir:
wav_cals = wav_cals[:, 2]
if measurandstring == "irradiance" and swir:
wav_cals = wav_cals[:, 3]
calibration_data["wavelength_coefficients"].values[
i_wavcoef] = wav_cals
i_wavcoef += 1
i_cal = 0
for caldatepath in caldatepaths:
if measurandstring == "radiance":
calpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) +
"\\radiometric\\" + str(caldatepath) + "\\hypstar_" +
str(hypstar) + "_radcal_L_*_%s.dat" % (sensortag)))[0]
else:
calpath = glob.glob(
os.path.join(
directory, "hypstar_" + str(hypstar) +
"\\radiometric\\" + str(caldatepath) + "\\hypstar_" +
str(hypstar) + "_radcal_E_*_%s.dat" % (sensortag)))[0]
if os.path.exists(calpath):
caldates = np.append(caldates, caldate)
gains = np.genfromtxt(calpath)
calibration_data["wavelengths"].values[i_cal] = gains[:, 1]
calibration_data["wavpix"].values[i_cal] = gains[:, 0]
calibration_data["gains"].values[i_cal] = gains[:, 2]
#calibration_data["u_random_gains"].values = None
#calibration_data["corr_random_gains"].values = None
calibration_data["u_systematic_indep_gains"].values[
i_cal] = gains[:, 2] * (
gains[:, 6]**2 + gains[:, 7]**2 + gains[:, 8]**2 +
gains[:, 9]**2 + gains[:, 10]**2 + gains[:, 11]**2 +
gains[:, 12]**2 + gains[:, 13]**2 + gains[:, 14]**2 +
gains[:, 15]**2 + gains[:, 16]**2 + gains[:, 17]**2 +
gains[:, 19]**2)**0.5 / 100
cov_diag = punpy.convert_corr_to_cov(
np.eye(len(gains[:, 2])),
gains[:, 2] * (gains[:, 19]) / 100)
cov_other = punpy.convert_corr_to_cov(
np.eye(len(gains[:, 2])), gains[:, 2] *
(gains[:, 8]**2 + gains[:, 9]**2 + gains[:, 11]**2 +
gains[:, 16]**2 + gains[:, 17]**2)**0.5 / 100)
cov_full = punpy.convert_corr_to_cov(
np.ones((len(gains[:, 2]), len(gains[:,
2]))), gains[:, 2] *
(gains[:, 7]**2 + gains[:, 10]**2 + gains[:, 12]**2 +
gains[:, 13]**2 + gains[:, 14]**2 + gains[:, 15]**2)**0.5
/ 100)
cov_filament = punpy.convert_corr_to_cov(
np.ones((len(gains[:, 2]), len(gains[:, 2]))),
gains[:, 2] * (gains[:, 6]**2)**0.5 / 100)
calibration_data["corr_systematic_indep_gains"].values[i_cal] = \
punpy.correlation_from_covariance(cov_diag+cov_other+cov_full+cov_filament)
calibration_data["u_systematic_corr_rad_irr_gains"].values[
i_cal] = gains[:, 2] * (gains[:, 4]**2 + gains[:, 5]**2 +
gains[:, 18]**2)**0.5 / 100
cov_other = punpy.convert_corr_to_cov(
np.eye(len(gains[:, 2])), gains[:, 2] *
(gains[:, 4]**2 + gains[:, 18]**2)**0.5 / 100)
cov_filament = punpy.convert_corr_to_cov(
np.ones((len(gains[:, 2]), len(gains[:, 2]))),
gains[:, 2] * (gains[:, 5]**2)**0.5 / 100)
calibration_data["corr_systematic_corr_rad_irr_gains"].values[i_cal] = \
punpy.correlation_from_covariance(cov_other+cov_filament)
i_cal += 1
return calibration_data
class CombineSWIR:
def __init__(self, context, MCsteps=1000, parallel_cores=1):
self._measurement_function_factory = CombineFactory()
self.prop = PropagateUnc(context,
MCsteps,
parallel_cores=parallel_cores)
self.avg = Average(context=context)
self.templ = DataTemplates(context)
self.writer = HypernetsWriter(context)
self.plot = Plotting(context)
self.context = context
def combine(self, measurandstring, dataset_l1a, dataset_l1a_swir):
dataset_l1a = self.perform_checks(dataset_l1a)
dataset_l1b = self.avg.average_l1b(measurandstring, dataset_l1a)
dataset_l1b_swir = self.avg.average_l1b(measurandstring,
dataset_l1a_swir)
combine_function = self._measurement_function_factory.get_measurement_function(
self.context.get_config_value("measurement_function_combine"))
input_vars = combine_function.get_argument_names()
input_qty = [
dataset_l1b["wavelength"].values,
dataset_l1b[measurandstring].values,
dataset_l1b_swir["wavelength"].values,
dataset_l1b_swir[measurandstring].values,
self.context.get_config_value("combine_lim_wav")
]
u_random_input_qty = [
None, dataset_l1b["u_random_" + measurandstring].values, None,
dataset_l1b_swir["u_random_" + measurandstring].values, None
]
u_systematic_input_qty_indep = [
None, dataset_l1b["u_systematic_indep_" + measurandstring].values,
None,
dataset_l1b_swir["u_systematic_indep_" + measurandstring].values,
None
]
u_systematic_input_qty_corr = [
None,
dataset_l1b["u_systematic_corr_rad_irr_" + measurandstring].values,
None, dataset_l1b_swir["u_systematic_corr_rad_irr_" +
measurandstring].values, None
]
corr_systematic_input_qty_indep = [
None,
dataset_l1b["corr_systematic_indep_" + measurandstring].values,
None, dataset_l1b_swir["corr_systematic_indep_" +
measurandstring].values, None
]
corr_systematic_input_qty_corr = [
None, dataset_l1b["corr_systematic_corr_rad_irr_" +
measurandstring].values, None,
dataset_l1b_swir["corr_systematic_corr_rad_irr_" +
measurandstring].values, None
]
#todo do this more consistently with other modules, and do a direct copy for ranges that don't overlap
dataset_l1b_comb = self.templ.l1b_template_from_combine(
measurandstring, dataset_l1b, dataset_l1b_swir)
self.prop.process_measurement_function_l1(
measurandstring,
dataset_l1b_comb,
combine_function.function,
input_qty,
u_random_input_qty,
u_systematic_input_qty_indep,
u_systematic_input_qty_corr,
corr_systematic_input_qty_indep,
corr_systematic_input_qty_corr,
param_fixed=[True, False, True, False, True])
if self.context.get_config_value("write_l1b"):
self.writer.write(dataset_l1b_comb, overwrite=True)
if self.context.get_config_value("plot_l1b"):
self.plot.plot_series_in_sequence(measurandstring,
dataset_l1b_comb)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty(measurandstring,
dataset_l1b_comb)
if self.context.get_config_value("plot_correlation"):
self.plot.plot_correlation(measurandstring, dataset_l1b_comb)
# if self.context.get_config_value("plot_diff"):
# self.plot.plot_diff_scans(measurandstring,dataset_l1a,dataset_l1b)
return dataset_l1b_comb
def perform_checks(self, dataset_l1):
"""
Identifies and removes faulty measurements (e.g. due to cloud cover).
:param dataset_l0:
:type dataset_l0:
:return:
:rtype:
"""
return dataset_l1
class SurfaceReflectance:
def __init__(self, context, MCsteps=1000, parallel_cores=1):
self._measurement_function_factory = ProtocolFactory(context=context)
self.prop = PropagateUnc(context,
MCsteps,
parallel_cores=parallel_cores)
self.templ = DataTemplates(context=context)
self.writer = HypernetsWriter(context)
self.avg = Average(context)
self.calibrate = Calibrate(context)
self.plot = Plotting(context)
self.context = context
self.rh = RhymerHypstar(context)
self.rhp = RhymerProcessing(context)
self.rhs = RhymerShared(context)
def process_l1c(self, dataset):
dataset_l1c = self.templ.l1c_from_l1b_dataset(dataset)
dataset_l1c = self.rh.get_wind(dataset_l1c)
dataset_l1c = self.rh.get_fresnelrefl(dataset_l1c)
l1ctol1b_function = self._measurement_function_factory.get_measurement_function(
self.context.get_config_value(
"measurement_function_surface_reflectance"))
input_vars = l1ctol1b_function.get_argument_names()
input_qty = self.prop.find_input(input_vars, dataset_l1c)
u_random_input_qty = self.prop.find_u_random_input(
input_vars, dataset_l1c)
u_systematic_input_qty, corr_systematic_input_qty = \
self.prop.find_u_systematic_input(input_vars, dataset_l1c)
L1c = self.prop.process_measurement_function_l2(
[
"water_leaving_radiance", "reflectance_nosc", "reflectance",
"epsilon"
],
dataset_l1c,
l1ctol1b_function.function,
input_qty,
u_random_input_qty,
u_systematic_input_qty,
corr_systematic_input_qty,
param_fixed=[False, False, False, False, True])
failSimil = self.rh.qc_similarity(L1c)
L1c["quality_flag"][np.where(failSimil == 1)] = DatasetUtil.set_flag(
L1c["quality_flag"][np.where(failSimil == 1)],
"simil_fail") # for i in range(len(mask))]
if self.context.get_config_value("write_l1c"):
self.writer.write(L1c, overwrite=True)
for measurandstring in [
"water_leaving_radiance", "reflectance_nosc", "reflectance",
"epsilon"
]:
try:
if self.context.get_config_value("plot_l1c"):
self.plot.plot_series_in_sequence(measurandstring, L1c)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty(measurandstring,
L1c,
L2=True)
except:
print("not plotting ", measurandstring)
return L1c
def process_l2(self, dataset):
dataset = self.perform_checks(dataset)
l1tol2_function = self._measurement_function_factory.get_measurement_function(
self.context.get_config_value(
"measurement_function_surface_reflectance"))
input_vars = l1tol2_function.get_argument_names()
input_qty = self.prop.find_input(input_vars, dataset)
u_random_input_qty = self.prop.find_u_random_input(input_vars, dataset)
u_systematic_input_qty, cov_systematic_input_qty = \
self.prop.find_u_systematic_input(input_vars, dataset)
if self.context.get_config_value("network").lower() == "w":
dataset_l2a = self.avg.average_L2(dataset)
for measurandstring in [
"water_leaving_radiance", "reflectance_nosc",
"reflectance", "epsilon"
]:
try:
if self.context.get_config_value("plot_l2a"):
self.plot.plot_series_in_sequence(
measurandstring, dataset_l2a)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty(measurandstring,
dataset_l2a,
L2=True)
if self.context.get_config_value("plot_correlation"):
self.plot.plot_correlation(measurandstring,
dataset_l2a,
L2=True)
except:
print("not plotting ", measurandstring)
elif self.context.get_config_value("network").lower() == "l":
dataset_l2a = self.templ.l2_from_l1c_dataset(dataset)
dataset_l2a = self.prop.process_measurement_function_l2(
["reflectance"], dataset_l2a, l1tol2_function.function,
input_qty, u_random_input_qty, u_systematic_input_qty,
cov_systematic_input_qty)
if self.context.get_config_value("plot_l2a"):
self.plot.plot_series_in_sequence("reflectance", dataset_l2a)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty("reflectance",
dataset_l2a,
L2=True)
if self.context.get_config_value("plot_correlation"):
self.plot.plot_correlation("reflectance", dataset_l2a, L2=True)
else:
self.context.logger.error("network is not correctly defined")
if self.context.get_config_value("write_l2a"):
self.writer.write(dataset_l2a, overwrite=True)
return dataset_l2a
def perform_checks(self, dataset_l1):
"""
Identifies and removes faulty measurements (e.g. due to cloud cover).
:param dataset_l0:
:type dataset_l0:
:return:
:rtype:
"""
return dataset_l1
class Interpolate:
def __init__(self,context,MCsteps=1000,parallel_cores=1):
self._measurement_function_factory = InterpolationFactory()
self.prop = PropagateUnc(context, MCsteps, parallel_cores=parallel_cores)
self.templ = DataTemplates(context=context)
self.writer=HypernetsWriter(context)
self.plot=Plotting(context)
self.context=context
def interpolate_l1b_w(self, dataset_l1b, dataset_l1a_uprad,dataset_l1b_downrad, dataset_l1b_irr):
# chek for upwelling radiance
upscan = [i for i, e in enumerate(dataset_l1a_uprad['viewing_zenith_angle'].values) if e < 90]
dataset_l1b=self.templ.l1c_int_template_from_l1a_dataset_water(dataset_l1a_uprad)
dataset_l1b["wavelength"] = dataset_l1a_uprad["wavelength"]
dataset_l1b["upwelling_radiance"] = dataset_l1a_uprad["radiance"].sel(scan=upscan)
dataset_l1b["acquisition_time"] = dataset_l1a_uprad["acquisition_time"].sel(scan=upscan)
# is this correct????
dataset_l1b["u_random_upwelling_radiance"] = dataset_l1a_uprad["u_random_radiance"].sel(scan=upscan)
dataset_l1b["u_systematic_indep_upwelling_radiance"] = dataset_l1a_uprad["u_systematic_indep_radiance"].sel(scan=upscan)
dataset_l1b["u_systematic_corr_rad_irr_upwelling_radiance"] = dataset_l1a_uprad["u_systematic_corr_rad_irr_radiance"].sel(scan=upscan)
dataset_l1b["corr_random_upwelling_radiance"] = dataset_l1a_uprad["corr_random_radiance"]
dataset_l1b["corr_systematic_indep_upwelling_radiance"] = dataset_l1a_uprad["corr_systematic_indep_radiance"]
dataset_l1b["corr_systematic_corr_rad_irr_upwelling_radiance"] = dataset_l1a_uprad["corr_systematic_corr_rad_irr_radiance"]
self.context.logger.info("interpolate sky radiance")
dataset_l1b=self.interpolate_skyradiance(dataset_l1b, dataset_l1b_downrad)
self.context.logger.info("interpolate irradiances")
dataset_l1b=self.interpolate_irradiance(dataset_l1b, dataset_l1b_irr)
return dataset_l1b
def interpolate_l1c(self,dataset_l1b_rad,dataset_l1b_irr):
dataset_l1c=self.templ.l1c_from_l1b_dataset(dataset_l1b_rad)
dataset_l1c["acquisition_time"].values = dataset_l1b_rad["acquisition_time"].values
dataset_l1c=self.interpolate_irradiance(dataset_l1c,dataset_l1b_irr)
if self.context.get_config_value("write_l1c"):
self.writer.write(dataset_l1c,overwrite=True)
if self.context.get_config_value("plot_l1c"):
self.plot.plot_series_in_sequence("irradiance",dataset_l1c)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty("irradiance",dataset_l1c)
if self.context.get_config_value("plot_correlation"):
self.plot.plot_correlation("irradiance",dataset_l1c)
return dataset_l1c
def interpolate_irradiance(self,dataset_l1c,dataset_l1b_irr):
measurement_function_interpolate_wav = self.context.get_config_value(
'measurement_function_interpolate_wav')
interpolation_function_wav = self._measurement_function_factory\
.get_measurement_function(measurement_function_interpolate_wav)
measurement_function_interpolate_time = self.context.get_config_value(
'measurement_function_interpolate_time')
interpolation_function_time = self._measurement_function_factory\
.get_measurement_function(measurement_function_interpolate_time)
# Interpolate in wavelength to radiance wavelengths
wavs_rad=dataset_l1c["wavelength"].values
wavs_irr=dataset_l1b_irr["wavelength"].values
dataset_l1c_temp = self.templ.l1ctemp_dataset(dataset_l1c,dataset_l1b_irr)
dataset_l1c_temp = self.prop.process_measurement_function_l1("irradiance",
dataset_l1c_temp,interpolation_function_wav.function,
[wavs_rad,wavs_irr,dataset_l1b_irr['irradiance'].values],
[None,None,dataset_l1b_irr['u_random_irradiance'].values],
[None,None,dataset_l1b_irr['u_systematic_indep_irradiance'].values],
[None,None,dataset_l1b_irr['u_systematic_corr_rad_irr_irradiance'].values],
[None,None,dataset_l1b_irr["corr_systematic_indep_irradiance"].values],
[None,None,dataset_l1b_irr["corr_systematic_corr_rad_irr_irradiance"].values],
)
# Interpolate in time to radiance times
acqui_irr = dataset_l1b_irr['acquisition_time'].values
acqui_rad = dataset_l1c['acquisition_time'].values
dataset_l1c = self.prop.process_measurement_function_l1("irradiance",
dataset_l1c,interpolation_function_time.function,
[acqui_rad,acqui_irr,dataset_l1c_temp['irradiance'].values],
[None,None,dataset_l1c_temp['u_random_irradiance'].values],
[None,None,dataset_l1c_temp['u_systematic_indep_irradiance'].values],
[None,None,dataset_l1c_temp['u_systematic_corr_rad_irr_irradiance'].values],
[None,None,dataset_l1c_temp["corr_systematic_indep_irradiance"].values],
[None,None,dataset_l1c_temp["corr_systematic_corr_rad_irr_irradiance"].values],
param_fixed=[False,True,True])
return dataset_l1c
def interpolate_skyradiance(self,dataset_l1c,dataset_l1a_skyrad):
measurement_function_interpolate_time = self.context.get_config_value(
'measurement_function_interpolate_time')
interpolation_function_time = self._measurement_function_factory\
.get_measurement_function(measurement_function_interpolate_time)
acqui_irr = dataset_l1a_skyrad['acquisition_time'].values
acqui_rad = dataset_l1c['acquisition_time'].values
dataset_l1c = self.prop.process_measurement_function_l1("downwelling_radiance",dataset_l1c,
interpolation_function_time.function,
[acqui_rad,acqui_irr,
dataset_l1a_skyrad[
'radiance'].values],
[None,None,dataset_l1a_skyrad[
'u_random_radiance'].values],
[None,None,dataset_l1a_skyrad[
'u_systematic_indep_radiance'].values],
[None,None,dataset_l1a_skyrad[
'u_systematic_corr_rad_irr_radiance'].values],
[None,None,dataset_l1a_skyrad["corr_systematic_indep_radiance"].values],
[None,None,dataset_l1a_skyrad["corr_systematic_corr_rad_irr_radiance"].values],
param_fixed=[False,True,True])
return dataset_l1c
class RhymerHypstar:
def __init__(self, context):
self.context = context
self.templ = DataTemplates(context=context)
self.writer = HypernetsWriter(context)
self.avg = Average(context)
self.intp = Interpolate(context, MCsteps=1000)
self.plot = Plotting(context)
self.rhymeranc = RhymerAncillary(context)
self.rhymerproc = RhymerProcessing(context)
self.rhymershared = RhymerShared(context)
def qc_scan(self, dataset, measurandstring, dataset_l1b):
## no inclination
## difference at 550 nm < 25% with neighbours
##
## QV July 2018
## Last modifications: 2019-07-10 (QV) renamed from PANTR, integrated in rhymer
# Modified 10/09/2020 by CG for the PANTHYR
verbosity = self.context.get_config_value("verbosity")
series_id = np.unique(dataset['series_id'])
wave = dataset['wavelength'].values
flags = np.zeros(shape=len(dataset['scan']))
id = 0
for s in series_id:
scans = dataset['scan'][dataset['series_id'] == s]
##
n = len(scans)
## get pixel index for wavelength
iref, wref = self.rhymershared.closest_idx(
wave, self.context.get_config_value("diff_wave"))
cos_sza = []
for i in dataset['solar_zenith_angle'].sel(scan=scans).values:
cos_sza.append(math.cos(math.radians(i)))
## go through the current set of scans
for i in range(n):
## test inclination
## not done
if measurandstring == 'irradiance':
data = dataset['irradiance'].sel(scan=scans).T.values
## test variability at 550 nm
if i == 0:
v = abs(1 - ((data[i][iref] / cos_sza[i]) /
(data[i + 1][iref] / cos_sza[i + 1])))
elif i < n - 1:
v = max(
abs(1 - ((data[i][iref] / cos_sza[i]) /
(data[i + 1][iref] / cos_sza[i + 1]))),
abs(1 - ((data[i][iref] / cos_sza[i]) /
(data[i - 1][iref] / cos_sza[i - 1]))))
else:
v = abs(1 - ((data[i][iref] / cos_sza[i]) /
(data[i - 1][iref] / cos_sza[i - 1])))
else:
data = dataset['radiance'].sel(scan=scans).T.values
## test variability at 550 nm
if i == 0:
v = abs(1 - (data[i][iref] / data[i + 1][iref]))
elif i < n - 1:
v = max(abs(1 - (data[i][iref] / data[i + 1][iref])),
abs(1 - (data[i][iref] / data[i - 1][iref])))
else:
v = abs(1 - (data[i][iref] / data[i - 1][iref]))
## continue if value exceeds the cv threshold
if v > self.context.get_config_value("diff_threshold"):
# get flag value for the temporal variability
if measurandstring == 'irradiance':
flags[id] = 1
dataset_l1b['quality_flag'][range(
len(dataset_l1b['scan']))] = du.set_flag(
dataset_l1b["quality_flag"][range(
len(dataset_l1b['scan']))],
"temp_variability_ed")
else:
flags[id] = 1
dataset_l1b['quality_flag'][range(
len(dataset_l1b['scan']))] = du.set_flag(
dataset_l1b["quality_flag"][range(
len(dataset_l1b['scan']))],
"temp_variability_lu")
seq = dataset.attrs["sequence_id"]
ts = datetime.utcfromtimestamp(
dataset['acquisition_time'][i])
if verbosity > 2:
self.context.logger.info(
'Temporal jump: in {}: Aquisition time {}, {}'.
format(
seq, ts, ', '.join([
'{}:{}'.format(k,
dataset[k][scans[i]].values)
for k in ['scan', 'quality_flag']
])))
id += 1
return dataset_l1b, flags
def cycleparse(self, rad, irr, dataset_l1b):
protocol = self.context.get_config_value(
"measurement_function_surface_reflectance")
self.context.logger.debug(protocol)
nbrlu = self.context.get_config_value("n_upwelling_rad")
nbred = self.context.get_config_value("n_upwelling_irr")
nbrlsky = self.context.get_config_value("n_downwelling_rad")
if protocol != 'WaterNetworkProtocol':
# here we should simply provide surface reflectance?
# what about a non-standard protocol but that includes the required standard series?
self.context.logger.error(
'Unknown measurement protocol: {}'.format(protocol))
else:
uprad = []
downrad = []
for i in rad['scan']:
scani = rad.sel(scan=i)
senz = scani["viewing_zenith_angle"].values
if senz < 90:
measurement = 'upwelling_radiance'
uprad.append(int(i))
if senz >= 90:
measurement = 'downwelling_radiance'
downrad.append(int(i))
if measurement is None: continue
lu = rad.sel(scan=uprad)
lsky = rad.sel(scan=downrad)
for i in lu['scan']:
scani = lu.sel(scan=i)
sena = scani["viewing_azimuth_angle"].values
senz = scani["viewing_zenith_angle"].values
self.context.logger.debug(scani['acquisition_time'].values)
ts = datetime.utcfromtimestamp(
int(scani['acquisition_time'].values))
# not fromtimestamp?
if (senz != 'NULL') & (sena != 'NULL'):
senz = float(senz)
sena = abs(float(sena))
else:
dataset_l1b['quality_flag'] = du.set_flag(
dataset_l1b.sel(scan=i)['quality_flag'],
"angles_missing")
self.context.logger.info(
'NULL angles: Aquisition time {}, {}'.format(
ts, ', '.join([
'{}:{}'.format(k, scani[k].values)
for k in ['scan', 'quality_flag']
])))
continue
# check if we have the same azimuth for lu and lsky
sena_lu = np.unique(lu["viewing_azimuth_angle"].values)
sena_lsky = np.unique(lsky["viewing_azimuth_angle"].values)
for i in sena_lu:
if i not in sena_lsky:
dataset_l1b["quality_flag"][
dataset_l1b["viewing_azimuth_angle"] ==
i] = du.set_flag(
dataset_l1b["quality_flag"][
dataset_l1b["viewing_azimuth_angle"] == i],
"lu_eq_missing")
if self.context.get_config_value("verbosity") > 2:
ts = [
datetime.utcfromtimestamp(x)
for x in lu['acquisition_time'][
lu["viewing_azimuth_angle"] == i].values
]
self.context.logger.info(
'No azimuthal equivalent downwelling radiance measurement: Aquisition time {}, {}'
.format(
ts, ', '.join([
'{}:{}'.format(
k, lu[k][lu["viewing_azimuth_angle"] ==
i].values)
for k in ['scan', 'quality_flag']
])))
# check if we have the required fresnel angle for lsky
senz_lu = np.unique(lu["viewing_zenith_angle"].values)
senz_lsky = 180 - np.unique(lsky["viewing_zenith_angle"].values)
for i in senz_lu:
if i not in senz_lsky:
dataset_l1b["quality_flag"][
dataset_l1b["viewing_azimuth_angle"] ==
i] = du.set_flag(
dataset_l1b["quality_flag"][
dataset_l1b["viewing_azimuth_angle"] == i],
"fresnel_angle_missing")
ts = [
datetime.utcfromtimestamp(x)
for x in lu['acquisition_time'][
lu["viewing_zenith_angle"] == i].values
]
self.context.logger.info(
'No downwelling radiance measurement at appropriate fresnel angle: Aquisition time {}, {}'
.format(
ts, ', '.join([
'{}:{}'.format(
k, lu[k][lu["viewing_azimuth_angle"] ==
i].values)
for k in ['scan', 'quality_flag']
])))
# check if correct number of radiance and irradiance data
if lu.scan[lu['quality_flag'] <= 0].count() < nbrlu:
for i in range(len(dataset_l1b["scan"])):
dataset_l1b["quality_flag"][
dataset_l1b["scan"] == i] = du.set_flag(
dataset_l1b["quality_flag"][dataset_l1b["scan"] ==
i], "min_nbrlu")
self.context.logger.info(
"No enough upwelling radiance data for sequence {}".format(
lu.attrs['sequence_id']))
if lsky.scan[lsky['quality_flag'] <= 1].count() < nbrlsky:
for i in range(len(dataset_l1b["scan"])):
dataset_l1b["quality_flag"][
dataset_l1b["scan"] == i] = du.set_flag(
dataset_l1b["quality_flag"][dataset_l1b["scan"] ==
i], "min_nbrlsky")
self.context.logger.info(
"No enough downwelling radiance data for sequence {}".
format(lsky.attrs['sequence_id']))
if irr.scan[irr['quality_flag'] <= 1].count() < nbred:
for i in range(len(dataset_l1b["scan"])):
dataset_l1b["quality_flag"][
dataset_l1b["scan"] == i] = du.set_flag(
dataset_l1b["quality_flag"][dataset_l1b["scan"] ==
i], "min_nbred")
self.context.logger.info(
"No enough downwelling irradiance data for sequence {}".
format(irr.attrs['sequence_id']))
return lu, lsky, irr, dataset_l1b
def get_wind(self, l1b):
lat = l1b.attrs['site_latitude']
lon = l1b.attrs['site_latitude']
wind = []
for i in range(len(l1b.scan)):
wa = self.context.get_config_value("wind_ancillary")
if not wa:
l1b["quality_flag"][l1b["scan"] == i] = du.set_flag(
l1b["quality_flag"][l1b["scan"] == i], "def_wind_flag")
self.context.logger.info("Default wind speed {}".format(
self.context.get_config_value("wind_default")))
wind.append(self.context.get_config_value("wind_default"))
else:
isodate = datetime.utcfromtimestamp(
l1b['acquisition_time'].values[i]).strftime('%Y-%m-%d')
isotime = datetime.utcfromtimestamp(
l1b['acquisition_time'].values[i]).strftime('%H:%M:%S')
anc_wind = self.rhymeranc.get_wind(isodate,
lon,
lat,
isotime=isotime)
if anc_wind is not None:
wind.append(anc_wind)
l1b['fresnel_wind'].values = wind
return l1b
def get_fresnelrefl(self, l1b):
## read mobley rho lut
fresnel_coeff = np.zeros(len(l1b.scan))
fresnel_vza = np.zeros(len(l1b.scan))
fresnel_raa = np.zeros(len(l1b.scan))
fresnel_sza = np.zeros(len(l1b.scan))
wind = l1b["fresnel_wind"].values
for i in range(len(l1b.scan)):
fresnel_vza[i] = l1b['viewing_zenith_angle'][i].values
fresnel_sza[i] = l1b['solar_zenith_angle'][i].values
diffa = l1b['viewing_azimuth_angle'][i].values - l1b[
'solar_azimuth_angle'][i].values
if diffa >= 360:
diffa = diffa - 360
elif 0 <= diffa < 360:
diffa = diffa
else:
diffa = diffa + 360
fresnel_raa[i] = abs((diffa - 180))
## get fresnel reflectance
if self.context.get_config_value("fresnel_option") == 'Mobley':
if (fresnel_sza[i] is not None) & (fresnel_raa[i] is not None):
sza = min(fresnel_sza[i], 79.999)
rhof = self.rhymerproc.mobley_lut_interp(sza,
fresnel_vza[i],
fresnel_raa[i],
wind=wind[i])
else:
l1b["quality_flag"][l1b["scan"] == i] = du.set_flag(
l1b["quality_flag"][l1b["scan"] == i],
"fresnel_default")
rhof = self.context.get_config_value("rhof_default")
if self.context.get_config_value(
"fresnel_option") == 'Ruddick2006':
rhof = self.context.get_config_value("rhof_default")
self.context.logger.info("Apply Ruddick et al., 2006")
if wind[i] is not None:
rhof = rhof + 0.00039 * wind[i] + 0.000034 * wind[i]**2
fresnel_coeff[i] = rhof
l1b["rhof"].values = fresnel_coeff
l1b["fresnel_vza"].values = fresnel_vza
l1b["fresnel_raa"].values = fresnel_raa
l1b["fresnel_sza"].values = fresnel_sza
return l1b
def qc_similarity(self, L1c):
wave = L1c["wavelength"]
wr = L1c.attrs["similarity_waveref"]
wp = L1c.attrs["similarity_wavethres"]
epsilon = L1c["epsilon"]
## get pixel index for wavelength
irefr, wrefr = self.rhymershared.closest_idx(wave, wr)
failSimil = []
scans = L1c['scan']
for i in range(len(scans)):
data = L1c['reflectance_nosc'].sel(scan=i).values
if abs(epsilon[i]) > wp * data[irefr]:
failSimil.append(1)
else:
failSimil.append(0)
return failSimil
def process_l1c_int(self, l1a_rad, l1a_irr):
# because we average to Lu scan we propagate values from radiance!
dataset_l1b = self.templ.l1c_int_template_from_l1a_dataset_water(
l1a_rad)
# QUALITY CHECK: TEMPORAL VARIABILITY IN ED AND LSKY -> ASSIGN FLAG
dataset_l1b, flags_rad = self.qc_scan(l1a_rad, "radiance", dataset_l1b)
dataset_l1b, flags_irr = self.qc_scan(l1a_irr, "irradiance",
dataset_l1b)
# QUALITY CHECK: MIN NBR OF SCANS -> ASSIGN FLAG
# remove temporal variability scans before average
l1a_rad = l1a_rad.sel(scan=np.where(np.array(flags_rad) != 1)[0])
l1a_irr = l1a_irr.sel(scan=np.where(np.array(flags_irr) != 1)[0])
# check number of scans per cycle for up, down radiance and irradiance
L1a_uprad, L1a_downrad, L1a_irr, dataset_l1b = self.cycleparse(
l1a_rad, l1a_irr, dataset_l1b)
L1b_downrad = self.avg.average_l1b("radiance", L1a_downrad)
L1b_irr = self.avg.average_l1b("irradiance", L1a_irr)
# INTERPOLATE Lsky and Ed FOR EACH Lu SCAN! Threshold in time -> ASSIGN FLAG
# interpolate_l1b_w calls interpolate_irradiance which includes interpolation of the
# irradiance wavelength to the radiance wavelength
L1c_int = self.intp.interpolate_l1b_w(dataset_l1b, L1a_uprad,
L1b_downrad, L1b_irr)
return L1c_int
def __init__(self, context):
self.templ = DataTemplates(context=context)
self.context = context
self.writer = HypernetsWriter(context)
class Average:
def __init__(self, context):
self.templ = DataTemplates(context=context)
self.context = context
self.writer = HypernetsWriter(context)
def average_l1b(self, measurandstring, dataset_l1a):
if self.context.get_config_value("network") == "w":
dataset_l1b = self.templ.l1b_template_from_l1a_dataset_water(
measurandstring, dataset_l1a)
else:
dataset_l1b = self.templ.l1b_template_from_l1a_dataset_land(
measurandstring, dataset_l1a)
if self.context.get_config_value("network") == "l":
flags = ["outliers"]
else:
flags = []
dataset_l1b[measurandstring].values = self.calc_mean_masked(
dataset_l1a, measurandstring, flags)
dataset_l1b["u_random_" + measurandstring].values = self.calc_mean_masked(\
dataset_l1a,"u_random_" + measurandstring,flags,rand_unc=True)
dataset_l1b["u_systematic_indep_"+measurandstring].values = self.calc_mean_masked\
(dataset_l1a,"u_systematic_indep_"+measurandstring,flags)
dataset_l1b["u_systematic_corr_rad_irr_"+measurandstring].values = self.calc_mean_masked\
(dataset_l1a,"u_systematic_corr_rad_irr_"+measurandstring,flags)
dataset_l1b["corr_random_" + measurandstring].values = np.eye(
len(dataset_l1b["u_random_" + measurandstring].values))
dataset_l1b["corr_systematic_indep_"+measurandstring].values = \
dataset_l1a["corr_systematic_indep_"+measurandstring].values
dataset_l1b["corr_systematic_corr_rad_irr_"+measurandstring].values = \
dataset_l1a["corr_systematic_corr_rad_irr_"+measurandstring].values
return dataset_l1b
def average_L2(self, dataset):
dataset_l2a = self.templ.l2_from_l1d_dataset(dataset)
flags = [
"saturation", "nonlinearity", "bad_pointing", "outliers",
"angles_missing", "lu_eq_missing", "fresnel_angle_missing",
"fresnel_default", "temp_variability_ed", "temp_variability_lu",
"min_nbred", "min_nbrlu", "min_nbrlsky"
]
for measurandstring in [
"water_leaving_radiance", "reflectance_nosc", "reflectance"
]:
dataset_l2a[measurandstring].values = self.calc_mean_masked(
dataset, measurandstring, flags)
dataset_l2a["u_random_" +
measurandstring].values = self.calc_mean_masked(
dataset,
"u_random_" + measurandstring,
flags,
rand_unc=True)
dataset_l2a["u_systematic_" +
measurandstring].values = self.calc_mean_masked(
dataset, "u_systematic_" + measurandstring, flags)
dataset_l2a["corr_random_" + measurandstring].values = np.eye(
len(dataset_l2a["u_systematic_" + measurandstring].values))
dataset_l2a["corr_systematic_"+measurandstring].values = \
dataset["corr_systematic_"+measurandstring].values
return dataset_l2a
def calc_mean_masked(self,
dataset,
var,
flags,
rand_unc=False,
corr=False):
series_id = np.unique(dataset['series_id'])
if corr:
out = np.empty\
((len(series_id), len(dataset['wavelength']), len(dataset['wavelength'])))
for i in range(len(series_id)):
flagged = np.any([
DatasetUtil.unpack_flags(dataset['quality_flag'])[x]
for x in flags
],
axis=0)
ids = np.where((dataset['series_id'] == series_id[i])
& (flagged == False))
out[i] = np.mean(dataset[var].values[:, :, ids], axis=3)[:, :,
0]
out = np.mean(out, axis=0)
else:
out = np.empty((len(series_id), len(dataset['wavelength'])))
for i in range(len(series_id)):
flagged = np.any([
DatasetUtil.unpack_flags(dataset['quality_flag'])[x]
for x in flags
],
axis=0)
ids = np.where((dataset['series_id'] == series_id[i])
& (flagged == False))
out[i] = np.mean(dataset[var].values[:, ids], axis=2)[:, 0]
if rand_unc:
out[i] = (np.sum(dataset[var].values[:, ids]**2,
axis=2)[:, 0])**0.5 / len(ids[0])
return out.T
class Calibrate:
def __init__(self, context, MCsteps=1000, parallel_cores=0):
self._measurement_function_factory = MeasurementFunctionFactory()
self.prop = PropagateUnc(context,
MCsteps,
parallel_cores=parallel_cores)
self.templ = DataTemplates(context)
self.writer = HypernetsWriter(context)
self.plot = Plotting(context)
self.context = context
def calibrate_l1a(self,
measurandstring,
dataset_l0,
dataset_l0_bla,
calibration_data,
swir=False):
if measurandstring != "radiance" and measurandstring != "irradiance":
self.context.logger.error(
"the measurandstring needs to be either 'radiance' or 'irradiance"
)
exit()
if self.context.get_config_value("plot_l0"):
self.plot.plot_scans_in_series("digital_number", dataset_l0)
calibrate_function = self._measurement_function_factory.get_measurement_function(
self.context.get_config_value("measurement_function_calibrate"))
input_vars = calibrate_function.get_argument_names()
dataset_l0 = self.preprocess_l0(dataset_l0, dataset_l0_bla,
calibration_data)
dataset_l1a = self.templ.l1a_template_from_l0_dataset(
measurandstring, dataset_l0, swir)
input_qty = self.prop.find_input_l1a(input_vars, dataset_l0,
calibration_data)
u_random_input_qty = self.prop.find_u_random_input_l1a(
input_vars, dataset_l0, calibration_data)
u_systematic_input_qty_indep,u_systematic_input_qty_corr,\
corr_systematic_input_qty_indep,corr_systematic_input_qty_corr = self.prop.find_u_systematic_input_l1a(input_vars, dataset_l0, calibration_data)
dataset_l1a = self.prop.process_measurement_function_l1a(
measurandstring, dataset_l1a, calibrate_function.function,
input_qty, u_random_input_qty, u_systematic_input_qty_indep,
u_systematic_input_qty_corr, corr_systematic_input_qty_indep,
corr_systematic_input_qty_corr)
if self.context.get_config_value("write_l1a"):
self.writer.write(dataset_l1a, overwrite=True)
if self.context.get_config_value("plot_l1a"):
self.plot.plot_scans_in_series(measurandstring, dataset_l1a)
if self.context.get_config_value("plot_l1a_diff"):
self.plot.plot_diff_scans(measurandstring, dataset_l1a)
if self.context.get_config_value("plot_uncertainty"):
self.plot.plot_relative_uncertainty(measurandstring, dataset_l1a)
if self.context.get_config_value("plot_correlation"):
self.plot.plot_correlation(measurandstring, dataset_l1a)
return dataset_l1a
def find_nearest_black(self, dataset, acq_time, int_time):
ids = np.where((abs(dataset['acquisition_time'] - acq_time) == min(
abs(dataset['acquisition_time'] - acq_time)))
& (dataset['integration_time'] == int_time))
#todo check if interation time alwasy has to be same
return np.mean(dataset["digital_number"].values[:, ids], axis=2)[:, 0]
def preprocess_l0(self, datasetl0, datasetl0_bla, dataset_calib):
"""
Identifies and removes faulty measurements (e.g. due to cloud cover).
:param dataset_l0:
:type dataset_l0:
:return:
:rtype:
"""
wavs = dataset_calib["wavelength"].values
wavpix = dataset_calib["wavpix"].values
datasetl0 = datasetl0.isel(wavelength=slice(int(wavpix[0]),
int(wavpix[-1]) + 1))
datasetl0_bla = datasetl0_bla.isel(
wavelength=slice(int(wavpix[0]),
int(wavpix[-1]) + 1))
mask = self.clip_and_mask(datasetl0, datasetl0_bla)
datasetl0 = datasetl0.assign_coords(wavelength=wavs)
datasetl0_bla = datasetl0_bla.assign_coords(wavelength=wavs)
datasetl0["quality_flag"][np.where(mask == 1)] = DatasetUtil.set_flag(
datasetl0["quality_flag"][np.where(mask == 1)],
"outliers") #for i in range(len(mask))]
DN_rand = DatasetUtil.create_variable(
[len(datasetl0["wavelength"]),
len(datasetl0["scan"])],
dim_names=["wavelength", "scan"],
dtype=np.uint32,
fill_value=0)
datasetl0["u_random_digital_number"] = DN_rand
rand = np.zeros_like(DN_rand.values)
series_ids = np.unique(datasetl0['series_id'])
for i in range(len(series_ids)):
ids = np.where(datasetl0['series_id'] == series_ids[i])[0]
ids_masked = np.where((datasetl0['series_id'] == series_ids[i])
& (mask == 0))[0]
dark_signals = np.zeros_like(
datasetl0['digital_number'].values[:, ids_masked])
for ii, id in enumerate(ids_masked):
dark_signals[:, ii] = self.find_nearest_black(
datasetl0_bla, datasetl0['acquisition_time'].values[id],
datasetl0['integration_time'].values[id])
std = np.std((datasetl0['digital_number'].values[:, ids_masked] -
dark_signals),
axis=1)
for ii, id in enumerate(ids):
rand[:, id] = std
datasetl0["u_random_digital_number"].values = rand
DN_dark = DatasetUtil.create_variable(
[len(datasetl0["wavelength"]),
len(datasetl0["scan"])],
dim_names=["wavelength", "scan"],
dtype=np.uint32,
fill_value=0)
datasetl0["dark_signal"] = DN_dark
dark_signals = []
acqui = datasetl0['acquisition_time'].values
inttimes = datasetl0['integration_time'].values
for i in range(len(acqui)):
dark_signals.append(
self.find_nearest_black(datasetl0_bla, acqui[i], inttimes[i]))
datasetl0["dark_signal"].values = np.array(dark_signals).T
return datasetl0
def clip_and_mask(self, dataset, dataset_bla, k_unc=3):
mask = []
# check if zeros, max, fillvalue:
# check if integrated signal is outlier
series_ids = np.unique(dataset['series_id'])
for i in range(len(series_ids)):
ids = np.where(dataset['series_id'] == series_ids[i])
dark_signals = self.find_nearest_black(
dataset_bla, np.mean(dataset['acquisition_time'].values[ids]),
np.mean(dataset['integration_time'].values[ids]))
intsig = np.nanmean((dataset["digital_number"].values[:, ids] -
dark_signals[:, None, None]),
axis=0)[0]
noisestd, noiseavg = self.sigma_clip(
intsig) # calculate std and avg for non NaN columns
maski = np.zeros_like(intsig) # mask the columns that have NaN
maski[np.where(np.abs(intsig - noiseavg) >= k_unc * noisestd)] = 1
mask = np.append(mask, maski)
# check if 10% of pixels are outiers
# mask_wvl = np.zeros((len(datasetl0["wavelength"]),len(datasetl0["scan"])))
# for i in range(len(dataset["wavelength"])):
return mask
def sigma_clip(self,
values,
tolerance=0.01,
median=True,
sigma_thresh=3.0):
# Remove NaNs from input values
values = np.array(values)
values = values[np.where(np.isnan(values) == False)]
values_original = np.copy(values)
# Continue loop until result converges
diff = 10E10
while diff > tolerance:
# Assess current input iteration
if median == False:
average = np.mean(values)
elif median == True:
average = np.median(values)
sigma_old = np.std(values)
# Mask those pixels that lie more than 3 stdev away from mean
check = np.zeros([len(values)])
check[np.where(values > (average +
(sigma_thresh * sigma_old)))] = 1
# check[ np.where( values<(average-(sigma_thresh*sigma_old)) ) ] = 1
values = values[np.where(check < 1)]
# Re-measure sigma and test for convergence
sigma_new = np.std(values)
diff = abs(sigma_old - sigma_new) / sigma_old
# Perform final mask
check = np.zeros([len(values)])
check[np.where(values > (average + (sigma_thresh * sigma_old)))] = 1
check[np.where(values < (average - (sigma_thresh * sigma_old)))] = 1
values = values[np.where(check < 1)]
# Return results
return sigma_new, average
class HypernetsReader:
def __init__(self, context):
self.context = context
self.model = self.context.get_config_value("model").split(',')
self.templ = DataTemplates(context)
self.writer = HypernetsWriter(context=context)
cckeys = [
'mapping_vis_a', 'mapping_vis_b', 'mapping_vis_c', 'mapping_vis_d',
'mapping_vis_e', 'mapping_vis_f'
]
ccvalues = []
for i in range(len(cckeys)):
ccvalues.append(self.context.get_config_value(cckeys[i]))
self.cc_vis = dict(zip(cckeys, ccvalues))
# READ METADATA
# CG - 20200331
# old functions: gen2dict, extract_metadata, read_metadata, read_metadata 2, read_spectra -> NOT USED TO REMOVE
# CG - 20200604
# new functions: read_header, read_data, read_footer, read_seq, read_wavelength
def plot_spectra(self, spectra, dataSpectra):
plt.clf()
plt.title(spectra)
plt.plot([i for i in range(len(dataSpectra))], dataSpectra)
plt.show()
# def save(self, path):
# with open(path, 'w') as f:
# f.write('Dataset length: {} bytes\n'
# 'Timestamp: {} ms\n'
# 'CRC32: {} \n'
# 'Entrance: {}\n'
# 'Radiometer: {}\n'
# 'Exposure time: {} ms\n'
# 'Sensor temperature: {} \'C\n'
# 'Pixel count: {}\n'
# 'Tilt:\n'
# '\tx:{}\u00B1{}\n'
# '\t y:{}\u00B1{}\n'
# '\t z:{}\u00B1{}\n'.format(self.header.total_length, self.header.timestamp, hex(self.crc32[0]),
# self.header.spectrum_type.optics.name,
# self.header.spectrum_type.radiometer.name,
# self.header.exposure_time, self.header.temperature,
# self.header.pixel_count,
# self.header.accel_stats.mean_x,
# self.header.accel_stats.std_x,
# self.header.accel_stats.mean_y,
# self.header.accel_stats.std_y,
# self.header.accel_stats.mean_z,
# self.header.accel_stats.std_z))
def read_header(self, f, headerDef):
header = {}
for headLen, headName, headFormat in headerDef:
data = f.read(headLen)
if len(data) != headLen:
self.context.logger.error(
"Spectra length not similar to header length")
break
continue
# if version_info > (3, 0):
# print("%02X " * headLen % (tuple([b for b in data])))
# else:
# print("%02X " * headLen % (tuple([ord(b) for b in data])))
var, = unpack(headFormat, data)
if headName == "Pixel Count": pixel_count = var
if headName == "Spectrum Type Information":
specInfo = format(ord(data), '#010b')
specInfo = ['1' == a for a in reversed(specInfo[2:])]
# bit 7 for VIS radiometer,
# bit 6 for SWIR,
# bit 4 for radiance,
# bit 3 for irradiance,
# bits 4 and 3 for dark;
strInfo = ""
if specInfo[7]: strInfo += "VIS " # noqa
if specInfo[6]: strInfo += "SWIR " # noqa
if not specInfo[3] and not specInfo[4]:
strInfo += "Dark" # noqa
if specInfo[3] and not specInfo[4]: strInfo += "Irr" # noqa
if specInfo[4] and not specInfo[3]: strInfo += "Rad" # noqa
if specInfo[3] and specInfo[4]: strInfo += "Error" # noqa
self.context.logger.debug("Spectrum Type Info : %s " % strInfo)
header[headName] = var
return header
def read_data(self, f, data_len):
prev = 0
self.context.logger.debug("Reading Data spectra ...")
dataSpectra = []
for i in range(int(data_len)): # Last read data is count
data = f.read(2)
if len(data) != 2:
self.context.logger.error(
"Warning : impossible to read 2 bytes")
break
continue
# Read data as unsigned short
unpackData, = unpack('<H', data)
dataSpectra.append(unpackData)
prev = unpackData
return dataSpectra
def read_footer(self, f, datalength):
# print(f)
self.context.logger.debug("Reading CRC32 ...")
data = f.read(datalength)
unpackData, = unpack('<I', data)
def read_wavelength(self, pixcount, cal_data):
pix = range(pixcount)
wav_coef = cal_data["wavelength_coefficients"]
wav_coef_func = np.poly1d(np.flip(wav_coef))
wvl = wav_coef_func(pix)
self.context.logger.debug("Wavelength range: %s -%s" %
(min(wvl), max(wvl)))
return wvl
def read_series_L(self, seq_dir, series, lat, lon, metadata, flag,
fileformat, cal_data, cal_data_swir):
model_name = self.model
# 1. Read header to create template dataset (including wvl and scan dimensions + end of file!!)
# ----------------------------------------
# scan dimension - to have total number of dimensions
index_scan_total = model_name.index("scan_total")
# series id
# ------------------------------------------
# added to consider concanated files
scanDim = sum(
[int(re.split('_|\.', i)[index_scan_total]) for i in series])
# ------------------------------------------
# added to consider non concanated files
# scanDims = [int(re.split('_|\.', i)[index_scan_total]) for i in series]
# scanDim = sum(scanDims)
## rewrite series
# baseName=["_".join(seriesname.split("_", 7)[:7]) for seriesname in series]
# print(baseName)
# newSeries = []
# for i in series:
# # create dictionary from filename
# seriesAttr = re.split('_|\.', i)[:-1] # remove spe extension
# model = dict(zip(model_name, seriesAttr))
# baseName = '_'.join(seriesAttr)
# nbrScans = int(model["scan_total"])
# n = 1
# while n <= nbrScans:
# new_fname = "{}_{}{}".format(baseName, n, '.spe')
# newSeries.append(new_fname)
# n += 1
# series = newSeries
# -----------------------------------------
# wvl dimensions
FOLDER_NAME = os.path.join(seq_dir, "RADIOMETER/")
f = open(FOLDER_NAME + series[0], "rb")
# Header definition with length, description and decoding format
header = self.read_header(f, HEADER_DEF)
self.context.logger.debug(header)
pixCount = header['Pixel Count']
# if bool(header) == False:
# print("Data corrupt go to next line")
# header = self.read_header(f, HEADER_DEF)
if pixCount == 2048:
wvl = self.read_wavelength(pixCount, cal_data)
elif pixCount == 256:
wvl = self.read_wavelength(pixCount, cal_data_swir)
else:
self.context.logger.error(
"The number of wavelength pixels does not match "
"the expected values for VNIR or SWIR.")
# look for the maximum number of lines to read-- maybe not an elegant way to do?
f.seek(0, 2) # go to end of file
eof = f.tell()
f.close()
# 2. Create template dataset
# -----------------------------------
# use template from variables and metadata in format
ds = self.templ.l0_template_dataset(wvl, scanDim, fileformat)
# Keep track of scan number!
scan_number = 0
# read all spectra (== spe files with concanated files) in a series
for spectra in series:
model = dict(zip(model_name, spectra.split('_')[:-1]))
specBlock = model['series_rep'] + '_' + model[
'series_id'] + '_' + model['vaa'] + '_' + model[
'azimuth_ref'] + '_' + model['vza']
# spectra attributes from metadata file
specattr = dict(metadata[specBlock])
# name of spectra file
acquisitionTime = specattr[spectra]
acquisitionTime = datetime.datetime.strptime(
acquisitionTime + "UTC", '%Y%m%dT%H%M%S%Z')
acquisitionTime = acquisitionTime.replace(tzinfo=timezone.utc)
# -------------------------------------
# # account for non concanated files
# spec = "_".join(spectra.split('_')[:-1]) + ".spe"
# acquisitionTime = specattr[spec]
# print(acquisitionTime)
# acquisitionTime = datetime.datetime.strptime(acquisitionTime + "UTC", '%Y%m%dT%H%M%S%Z')
#
# acquisitionTime = acquisitionTime.replace(tzinfo=timezone.utc)
# model = dict(zip(model_name, str.split(spectra, "_")))
# ________________________________________
# -----------------------
# read the file
# -----------------------
with open(FOLDER_NAME + spectra, "rb") as f:
f.seek(0, 2)
file_size = f.tell()
f.seek(0)
print('file size: {}'.format(file_size))
byte_pointer = 0
chunk_size = 1
chunk_counter = 1
while file_size - byte_pointer:
print('Parsing chunk No {}, size {} bytes, bytes left: {}'.
format(chunk_counter, chunk_size,
file_size - byte_pointer))
chunk_size = unpack('<H', f.read(2))[0]
if chunk_size == 4119:
chunk_size = 4131
f.seek(byte_pointer)
chunk_body = f.read(chunk_size)
spectrum = Spectrum.parse_raw(chunk_body)
spectrum.print_header()
print(spectra, scan_number, pixCount, chunk_size,
len(spectrum.body))
byte_pointer = f.tell()
chunk_counter += 1
# if no header comment those lines
# header = self.read_header(f, HEADER_DEF)
# if bool(header) == False:
# self.context.logger.error("Data corrupt go to next line")
# break
# continue
# # -------------------------------------------------------
#pixCount = spectrum.header.pixel_count
scan = spectrum.body
# should include this back again when crc32 is in the headers!
#crc32 = self.read_footer(f, 4)
# HypernetsReader(self.context).plot_spectra(spectra, scan)
# fill in dataset
# maybe xarray has a better way to do - check merge, concat, ...
series_id = model['series_id']
ds["series_id"][scan_number] = series_id
ds["viewing_azimuth_angle"][scan_number] = model['vaa']
ds["viewing_zenith_angle"][scan_number] = model['vza']
# estimate time based on timestamp
ds["acquisition_time"][
scan_number] = datetime.datetime.timestamp(
acquisitionTime)
# #print(datetime.fromtimestamp(acquisitionTime))
# # didn't use acquisition time from instrument
# # possibility that acquisition time is time since reboot, but how to now reboot time?
# # if we use the metadata time header
# timestamp=header['acquisition_time']
# ts = int(timestamp)/1000
# date_time_str = timereboot+'UTC'
# print(date_time_str)
# date_time_obj = datetime.strptime(date_time_str, '%Y%m%dT%H%M%S%Z')
# print(date_time_obj)
# timereboot = datetime.timestamp(date_time_obj)
# print("timereboot =", timereboot)
# print(datetime.fromtimestamp(timereboot))
# print(datetime.fromtimestamp(int(ts+timereboot)))
# print(datetime.fromtimestamp(int(ts+timereboot))-date_time_obj)
if lat is not None:
ds.attrs["site_latitude"] = lat
ds.attrs["site_longitude"] = lon
ds["solar_zenith_angle"][scan_number] = get_altitude(
float(lat), float(lon), acquisitionTime)
ds["solar_azimuth_angle"][scan_number] = get_azimuth(
float(lat), float(lon), acquisitionTime)
else:
self.context.logger.error(
"Lattitude is not found, using default values instead for lat, lon, sza and saa."
)
ds['quality_flag'][scan_number] = flag
ds['integration_time'][scan_number] = header[
'integration_time']
ds['temperature'][scan_number] = header['temperature']
# accelaration:
# Reference acceleration data contains 3x 16 bit signed integers with X, Y and Z
# acceleration measurements respectively. These are factory-calibrated steady-state
# reference acceleration measurements of the gravity vector when instrument is in
# horizontal position. Due to device manufacturing tolerances, these are
# device-specific and should be applied, when estimating tilt from the measured
# acceleration data. Each measurement is bit count of full range ±19.6 m s−2 .
# Acceleration for each axis can be calculated per Eq. (4).
a = 19.6
b = 2**15
ds['acceleration_x_mean'][
scan_number] = header['acceleration_x_mean'] * a / b
ds['acceleration_x_std'][
scan_number] = header['acceleration_x_std'] * a / b
ds['acceleration_y_mean'][
scan_number] = header['acceleration_y_mean'] * a / b
ds['acceleration_y_std'][
scan_number] = header['acceleration_y_std'] * a / b
ds['acceleration_z_mean'][
scan_number] = header['acceleration_z_mean'] * a / b
ds['acceleration_z_std'][
scan_number] = header['acceleration_z_std'] * a / b
#ds['digital_number'][0:pixCount, scan_number] = scan
scan_number += 1
if f.tell() == eof:
nextLine = False
return ds
def read_series(self,
seq_dir,
series,
lat,
lon,
metadata,
flag,
fileformat,
cal_data=None,
cal_data_swir=None):
model_name = self.model
# 1. Read header to create template dataset (including wvl and scan dimensions + end of file!!)
# ----------------------------------------
# scan dimension - to have total number of dimensions
index_scan_total = model_name.index("scan_total")
# series id
# ------------------------------------------
# added to consider concanated files
scanDim = sum(
[int(re.split('_|\.', i)[index_scan_total]) for i in series])
# ------------------------------------------
# added to consider non concanated files
# scanDims = [int(re.split('_|\.', i)[index_scan_total]) for i in series]
# scanDim = sum(scanDims)
## rewrite series
# baseName=["_".join(seriesname.split("_", 7)[:7]) for seriesname in series]
# print(baseName)
# newSeries = []
# for i in series:
# # create dictionary from filename
# seriesAttr = re.split('_|\.', i)[:-1] # remove spe extension
# model = dict(zip(model_name, seriesAttr))
# baseName = '_'.join(seriesAttr)
# nbrScans = int(model["scan_total"])
# n = 1
# while n <= nbrScans:
# new_fname = "{}_{}{}".format(baseName, n, '.spe')
# newSeries.append(new_fname)
# n += 1
# series = newSeries
# -----------------------------------------
# wvl dimensions
FOLDER_NAME = os.path.join(seq_dir, "RADIOMETER/")
f = open(FOLDER_NAME + series[1], "rb")
# Header definition with length, description and decoding format
header = self.read_header(f, HEADER_DEF)
self.context.logger.debug(header)
pixCount = header['Pixel Count']
# if bool(header) == False:
# print("Data corrupt go to next line")
# header = self.read_header(f, HEADER_DEF)
if pixCount == 2048:
wvl = self.read_wavelength(pixCount, cal_data)
# 2. Create template dataset
# -----------------------------------
# use template from variables and metadata in format
ds = self.templ.l0_template_dataset(wvl, scanDim, fileformat)
else:
self.context.logger.error(
"The number of wavelength pixels does not match "
"the expected values for VNIR.")
# look for the maximum number of lines to read-- maybe not an elegant way to do?
f.seek(0, 2) # go to end of file
eof = f.tell()
f.close()
ds.attrs["source_file"] = str(os.path.basename(seq_dir))
ds["wavelength"] = wvl
# ds["bandwidth"]=wvl
ds["scan"] = np.linspace(1, scanDim, scanDim)
# Keep track of scan number!
scan_number = 0
# read all spectra (== spe files with concanated files) in a series
for spectra in series:
model = dict(zip(model_name, spectra.split('_')[:-1]))
specBlock = model['series_rep'] + '_' + model[
'series_id'] + '_' + model['vaa'] + '_' + model[
'azimuth_ref'] + '_' + model['vza']
# spectra attributes from metadata file
specattr = dict(metadata[specBlock])
# name of spectra file
acquisitionTime = specattr[spectra]
acquisitionTime = datetime.datetime.strptime(
acquisitionTime + "UTC", '%Y%m%dT%H%M%S%Z')
acquisitionTime = acquisitionTime.replace(tzinfo=timezone.utc)
# -------------------------------------
# # account for non concanated files
# spec = "_".join(spectra.split('_')[:-1]) + ".spe"
# acquisitionTime = specattr[spec]
# print(acquisitionTime)
# acquisitionTime = datetime.datetime.strptime(acquisitionTime + "UTC", '%Y%m%dT%H%M%S%Z')
#
# acquisitionTime = acquisitionTime.replace(tzinfo=timezone.utc)
# model = dict(zip(model_name, str.split(spectra, "_")))
# ________________________________________
# -----------------------
# read the file
# -----------------------
f = open(FOLDER_NAME + spectra, "rb")
nextLine = True
while nextLine:
# if no header comment those lines
header = self.read_header(f, HEADER_DEF)
if bool(header) == False:
self.context.logger.error("Data corrupt go to next line")
break
continue
# -------------------------------------------------------
pixCount = header['Pixel Count']
scan = self.read_data(f, pixCount)
# should include this back again when crc32 is in the headers!
crc32 = self.read_footer(f, 4)
# HypernetsReader(self.context).plot_spectra(spectra, scan)
# fill in dataset
# maybe xarray has a better way to do - check merge, concat, ...
series_id = model['series_id']
ds["series_id"][scan_number] = series_id
ds["viewing_azimuth_angle"][scan_number] = model['vaa']
ds["viewing_zenith_angle"][scan_number] = model['vza']
# estimate time based on timestamp
ds["acquisition_time"][
scan_number] = datetime.datetime.timestamp(acquisitionTime)
# #print(datetime.fromtimestamp(acquisitionTime))
# # didn't use acquisition time from instrument
# # possibility that acquisition time is time since reboot, but how to now reboot time?
# # if we use the metadata time header
# timestamp=header['acquisition_time']
# ts = int(timestamp)/1000
# date_time_str = timereboot+'UTC'
# print(date_time_str)
# date_time_obj = datetime.strptime(date_time_str, '%Y%m%dT%H%M%S%Z')
# print(date_time_obj)
# timereboot = datetime.timestamp(date_time_obj)
# print("timereboot =", timereboot)
# print(datetime.fromtimestamp(timereboot))
# print(datetime.fromtimestamp(int(ts+timereboot)))
# print(datetime.fromtimestamp(int(ts+timereboot))-date_time_obj)
if lat is not None:
ds.attrs["site_latitude"] = lat
ds.attrs["site_longitude"] = lon
ds["solar_zenith_angle"][scan_number] = get_altitude(
float(lat), float(lon), acquisitionTime)
ds["solar_azimuth_angle"][scan_number] = get_azimuth(
float(lat), float(lon), acquisitionTime)
else:
self.context.logger.error(
"Lattitude is not found, using default values instead for lat, lon, sza and saa."
)
ds['quality_flag'][scan_number] = flag
ds['integration_time'][scan_number] = header[
'integration_time']
ds['temperature'][scan_number] = header['temperature']
# accelaration:
# Reference acceleration data contains 3x 16 bit signed integers with X, Y and Z
# acceleration measurements respectively. These are factory-calibrated steady-state
# reference acceleration measurements of the gravity vector when instrument is in
# horizontal position. Due to device manufacturing tolerances, these are
# device-specific and should be applied, when estimating tilt from the measured
# acceleration data. Each measurement is bit count of full range ±19.6 m s−2 .
# Acceleration for each axis can be calculated per Eq. (4).
a = 19.6
b = 2**15
ds['acceleration_x_mean'][
scan_number] = header['acceleration_x_mean'] * a / b
ds['acceleration_x_std'][
scan_number] = header['acceleration_x_std'] * a / b
ds['acceleration_y_mean'][
scan_number] = header['acceleration_y_mean'] * a / b
ds['acceleration_y_std'][
scan_number] = header['acceleration_y_std'] * a / b
ds['acceleration_z_mean'][
scan_number] = header['acceleration_z_mean'] * a / b
ds['acceleration_z_std'][
scan_number] = header['acceleration_z_std'] * a / b
ds['digital_number'][0:pixCount, scan_number] = scan
scan_number += 1
if f.tell() == eof:
nextLine = False
return ds
def read_series_L(self, seq_dir, series, lat, lon, metadata, flag,
fileformat, cal_data, cal_data_swir):
FOLDER_NAME = os.path.join(seq_dir, "RADIOMETER/")
model_name = self.model
# read all spectra (== spe files with concanated files) in a series
vnir = []
swir = []
for spectra in series:
self.context.logger.debug("processing " + spectra)
model = dict(zip(model_name, spectra.split('_')[:-1]))
specBlock = model['series_rep']+'_'+model['series_id']+'_'+model['vaa']+'_'+\
model['azimuth_ref']+'_'+model['vza']
# spectra attributes from metadata file
specattr = dict(metadata[specBlock])
# name of spectra file
acquisitionTime = specattr[spectra]
acquisitionTime = datetime.datetime.strptime(
acquisitionTime + "UTC", '%Y%m%dT%H%M%S%Z')
acquisitionTime = acquisitionTime.replace(tzinfo=timezone.utc)
# -----------------------
# read the file
# -----------------------
with open(FOLDER_NAME + spectra, "rb") as f:
f.seek(0, 2)
file_size = f.tell()
f.seek(0)
byte_pointer = 0
chunk_size = 1
chunk_counter = 1
while file_size - byte_pointer:
self.context.logger.debug(
'Parsing chunk No {}, size {} bytes, bytes left: {}'.
format(chunk_counter, chunk_size,
file_size - byte_pointer))
chunk_size = unpack('<H', f.read(2))[0]
if chunk_size == 4119:
chunk_size = 4131
f.seek(byte_pointer)
chunk_body = f.read(chunk_size)
spectrum = Spectrum.parse_raw(chunk_body)
#spectrum.print_header()
if len(spectrum.body) > 500:
if len(vnir) == 0:
vnir = spectrum.body
else:
vnir = np.vstack([vnir, spectrum.body])
else:
if len(swir) == 0:
swir = spectrum.body
else:
swir = np.vstack([swir, spectrum.body])
byte_pointer = f.tell()
chunk_counter += 1
self.context.logger.debug(
"vnir data shape in combined raw files: %s \n "
"swir data shape in combined raw files: %s" %
(vnir.shape, swir.shape))
scanDim = vnir.shape[0]
wvl = self.read_wavelength(vnir.shape[1], cal_data)
ds = self.templ.l0_template_dataset(wvl, scanDim, fileformat)
scanDim = swir.shape[0]
wvl = self.read_wavelength(swir.shape[1], cal_data_swir)
ds_swir = self.templ.l0_template_dataset(wvl,
scanDim,
fileformat,
swir=True)
scan_number = 0
scan_number_swir = 0
for spectra in series:
model = dict(zip(model_name, spectra.split('_')[:-1]))
specBlock = model['series_rep'] + '_' + model[
'series_id'] + '_' + model['vaa'] + '_' + model[
'azimuth_ref'] + '_' + model['vza']
# spectra attributes from metadata file
specattr = dict(metadata[specBlock])
# name of spectra file
acquisitionTime = specattr[spectra]
acquisitionTime = datetime.datetime.strptime(
acquisitionTime + "UTC", '%Y%m%dT%H%M%S%Z')
acquisitionTime = acquisitionTime.replace(tzinfo=timezone.utc)
# -----------------------
# read the file
# -----------------------
with open(FOLDER_NAME + spectra, "rb") as f:
f.seek(0, 2)
file_size = f.tell()
f.seek(0)
byte_pointer = 0
chunk_size = 1
chunk_counter = 1
while file_size - byte_pointer:
chunk_size = unpack('<H', f.read(2))[0]
if chunk_size == 4119:
chunk_size = 4131
f.seek(byte_pointer)
chunk_body = f.read(chunk_size)
spectrum = Spectrum.parse_raw(chunk_body)
#spectrum.print_header()
if len(spectrum.body) > 500:
scan = spectrum.body # should include this back again when crc32 is in the headers! #crc32 = self.read_footer(f, 4)
# HypernetsReader(self.context).plot_spectra(spectra, scan)
# fill in dataset # maybe xarray has a better way to do - check merge, concat, ...
series_id = model['series_id']
ds["series_id"][scan_number] = series_id
ds["viewing_azimuth_angle"][scan_number] = model['vaa']
ds["viewing_zenith_angle"][scan_number] = model['vza']
# estimate time based on timestamp
ds["acquisition_time"][
scan_number] = datetime.datetime.timestamp(
acquisitionTime)
# #print(datetime.fromtimestamp(acquisitionTime))
# # didn't use acquisition time from instrument
# # possibility that acquisition time is time since reboot, but how to now reboot time?
# # if we use the metadata time header
# timestamp=header['acquisition_time']
# ts = int(timestamp)/1000
# date_time_str = timereboot+'UTC'
# print(date_time_str)
# date_time_obj = datetime.strptime(date_time_str, '%Y%m%dT%H%M%S%Z')
# print(date_time_obj)
# timereboot = datetime.timestamp(date_time_obj)
# print("timereboot =", timereboot)
# print(datetime.fromtimestamp(timereboot))
# print(datetime.fromtimestamp(int(ts+timereboot)))
# print(datetime.fromtimestamp(int(ts+timereboot))-date_time_obj)
if lat is not None:
ds.attrs["site_latitude"] = lat
ds.attrs["site_longitude"] = lon
ds["solar_zenith_angle"][
scan_number] = get_altitude(
float(lat), float(lon), acquisitionTime)
ds["solar_azimuth_angle"][
scan_number] = get_azimuth(
float(lat), float(lon), acquisitionTime)
else:
self.context.logger.error(
"Lattitude is not found, using default values instead for lat, lon, sza and saa."
)
ds['quality_flag'][scan_number] = flag
ds['integration_time'][
scan_number] = spectrum.header.exposure_time
ds['temperature'][
scan_number] = spectrum.header.temperature
# accelaration:
# Reference acceleration data contains 3x 16 bit signed integers with X, Y and Z
# acceleration measurements respectively. These are factory-calibrated steady-state
# reference acceleration measurements of the gravity vector when instrument is in
# horizontal position. Due to device manufacturing tolerances, these are
# device-specific and should be applied, when estimating tilt from the measured
# acceleration data. Each measurement is bit count of full range ±19.6 m s−2 .
# Acceleration for each axis can be calculated per Eq. (4).
a = 19.6
b = 2**15
ds['acceleration_x_mean'][
scan_number] = spectrum.header.accel_stats.mean_x * a / b
ds['acceleration_x_std'][
scan_number] = spectrum.header.accel_stats.std_x * a / b
ds['acceleration_y_mean'][
scan_number] = spectrum.header.accel_stats.mean_y * a / b
ds['acceleration_y_std'][
scan_number] = spectrum.header.accel_stats.std_y * a / b
ds['acceleration_z_mean'][
scan_number] = spectrum.header.accel_stats.mean_z * a / b
ds['acceleration_z_std'][
scan_number] = spectrum.header.accel_stats.std_z * a / b
ds['digital_number'][:, scan_number] = scan
scan_number += 1
else:
scan = spectrum.body # should include this back again when crc32 is in the headers! #crc32 = self.read_footer(f, 4)
# HypernetsReader(self.context).plot_spectra(spectra, scan)
# fill in dataset # maybe xarray has a better way to do - check merge, concat, ...
series_id = model['series_id']
ds_swir["series_id"][scan_number_swir] = series_id
ds_swir["viewing_azimuth_angle"][
scan_number_swir] = model['vaa']
ds_swir["viewing_zenith_angle"][
scan_number_swir] = model['vza']
# estimate time based on timestamp
ds_swir["acquisition_time"][
scan_number_swir] = datetime.datetime.timestamp(
acquisitionTime)
# #print(datetime.fromtimestamp(acquisitionTime))
# # didn't use acquisition time from instrument
# # possibility that acquisition time is time since reboot, but how to now reboot time?
# # if we use the metadata time header
# timestamp=header['acquisition_time']
# ts = int(timestamp)/1000
# date_time_str = timereboot+'UTC'
# print(date_time_str)
# date_time_obj = datetime.strptime(date_time_str, '%Y%m%dT%H%M%S%Z')
# print(date_time_obj)
# timereboot = datetime.timestamp(date_time_obj)
# print("timereboot =", timereboot)
# print(datetime.fromtimestamp(timereboot))
# print(datetime.fromtimestamp(int(ts+timereboot)))
# print(datetime.fromtimestamp(int(ts+timereboot))-date_time_obj)
if lat is not None:
ds_swir.attrs["site_latitude"] = lat
ds_swir.attrs["site_longitude"] = lon
ds_swir["solar_zenith_angle"][
scan_number_swir] = get_altitude(
float(lat), float(lon), acquisitionTime)
ds_swir["solar_azimuth_angle"][
scan_number_swir] = get_azimuth(
float(lat), float(lon), acquisitionTime)
else:
self.context.logger.error(
"Lattitude is not found, using default values instead for lat, lon, sza and saa."
)
ds_swir['quality_flag'][scan_number_swir] = flag
ds_swir['integration_time'][
scan_number_swir] = spectrum.header.exposure_time
ds_swir['temperature'][
scan_number_swir] = spectrum.header.temperature
# accelaration:
# Reference acceleration data contains 3x 16 bit signed integers with X, Y and Z
# acceleration measurements respectively. These are factory-calibrated steady-state
# reference acceleration measurements of the gravity vector when instrument is in
# horizontal position. Due to device manufacturing tolerances, these are
# device-specific and should be applied, when estimating tilt from the measured
# acceleration data. Each measurement is bit count of full range ±19.6 m s−2 .
# Acceleration for each axis can be calculated per Eq. (4).
a = 19.6
b = 2**15
ds_swir['acceleration_x_mean'][
scan_number_swir] = spectrum.header.accel_stats.mean_x * a / b
ds_swir['acceleration_x_std'][
scan_number_swir] = spectrum.header.accel_stats.std_x * a / b
ds_swir['acceleration_y_mean'][
scan_number_swir] = spectrum.header.accel_stats.mean_y * a / b
ds_swir['acceleration_y_std'][
scan_number_swir] = spectrum.header.accel_stats.std_y * a / b
ds_swir['acceleration_z_mean'][
scan_number_swir] = spectrum.header.accel_stats.mean_z * a / b
ds_swir['acceleration_z_std'][
scan_number_swir] = spectrum.header.accel_stats.std_z * a / b
ds_swir['digital_number'][:, scan_number_swir] = scan
scan_number_swir += 1
byte_pointer = f.tell()
chunk_counter += 1
return ds, ds_swir
def read_metadata(self, seq_dir):
model_name = self.model
flag = 0
# Spectra name : AA_BBB_CCCC_D_EEEE_FFF_GG_HHHH_II_JJJJ.spe
# A : iterator over "the sequence repeat time"
# B : Number of the line in the sequence file (csv file)
# C : azimuth pointing angle
# D : reference for the azimuth angle
# E : zenith pointing angle
# F : mode
# G : action
# H : integration time
# I : number of scan in the serie
# J : serie time
# .spe : extension
# D (reference) :
# 0 = abs
# 1 = nor
# 2 = sun
# F (mode) :
# MODE_NONE : 0x00 (000)
# MODE_SWIR : 0X40 (064)
# MODE_VIS : 0x80 (128)
# MODE_BOTH : 0xC0 (192)
# G (action) :
# ACTION_BLACK : 0x00 (00)
# ACTION_RAD : 0x10 (16)
# ACTION_IRR : 0x08 (08)
# ACTION_CAL : 0x01 (01)
# ACTION_PIC : 0x02 (02)
# ACTION_NONE : 0x03 (03)
metadata = ConfigParser()
print("seq", os.path.join(seq_dir, "metadata.txt"))
if os.path.exists(os.path.join(seq_dir, "metadata.txt")):
metadata.read(os.path.join(seq_dir, "metadata.txt"))
# ------------------------------
# global attributes + wavelengths -> need to check for swir
# ----------------------------------
globalattr = dict(metadata['Metadata'])
seq = globalattr['datetime']
# reboot time if we want to use acquisition time
# timereboot=globalattr['datetime']
# look for latitude and longitude or lat and lon , more elegant way??
if 'latitude' in (globalattr.keys()):
lat = float(globalattr['latitude'])
elif 'lat' in (globalattr.keys()):
lat = float(globalattr['lat'])
else:
# self.context.logger.error("Latitude is not given, use default")
lat = self.context.get_config_value("lat")
flag = flag + 2**self.context.get_config_value(
"lat_default") # du.set_flag(flag, "lat_default") #
if 'longitude' in (globalattr.keys()):
lon = float(globalattr['longitude'])
elif 'lon' in (globalattr.keys()):
lon = float(globalattr['lon'])
else:
# self.context.logger.error("Longitude is not given, use default")
lon = self.context.get_config_value("lon")
flag = flag + 2**self.context.get_config_value(
"lon_default") # du.set_flag(flag, "lon_default") #
# 2. Estimate wavelengths - NEED TO CHANGE HERE!!!!!!
# ----------------------
# from 1 to 14 cause only able to read the visible wavelengths.... how to read the swir once?
# to change!!!!
if 'cc' not in globalattr:
cc = self.cc_vis
else:
cc = list(str.split(globalattr['cc'], "\n"))
cc = {
k.strip(): float(v.strip())
for k, v in (i.split(":") for i in cc[1:14])
}
# 3. Read series
# ---------------------------
# check for radiance and irradiance series within the metadata
series_all = metadata.sections()[1:len(metadata)]
seriesName = []
seriesPict = []
for i in series_all:
seriesattr = dict(metadata[i])
seriesName.extend(
list(name for name in seriesattr if '.spe' in name))
seriesPict.extend(
list(name for name in seriesattr if '.jpg' in name))
# ----------------
# Make list per action
# ----------------
# ACTION_BLACK : 0x00 (00)
# ACTION_RAD : 0x10 (16)
# ACTION_IRR : 0x08 (08)
# ACTION_CAL : 0x01 (01)
# ACTION_PIC : 0x02 (02) - NOT IN THE FILENAME!
# ACTION_NONE : 0x03 (03)
index_action = model_name.index("action")
action = [re.split('_|\.', i)[index_action] for i in seriesName]
# self.context.logger.info(action)
# this is slow????
seriesIrr = [x for x, y in zip(seriesName, action) if int(y) == 8]
seriesBlack = [
x for x, y in zip(seriesName, action) if int(y) == 0
]
seriesRad = [x for x, y in zip(seriesName, action) if int(y) == 16]
else:
self.context.logger.error(
"Missing metadata file in sequence directory - check sequence directory"
)
self.context.anomaly_db.add_anomaly("s")
return seq, lat, lon, cc, metadata, seriesIrr, seriesRad, seriesBlack, seriesPict, flag
def read_sequence(self,
seq_dir,
calibration_data_rad,
calibration_data_irr,
calibration_data_swir_rad=None,
calibration_data_swir_irr=None):
# define data to return none at end of method if does not exist
l0_irr = None
l0_rad = None
l0_bla = None
seq, lat, lon, cc, metadata, seriesIrr, seriesRad, seriesBlack, seriesPict, flag = self.read_metadata(
seq_dir)
if seriesIrr:
if self.context.get_config_value("network") == "w":
l0_irr = self.read_series(seq_dir, seriesIrr, lat, lon,
metadata, flag, "L0_IRR",
calibration_data_irr)
if self.context.get_config_value("write_l0"):
self.writer.write(l0_irr, overwrite=True)
else:
l0_irr, l0_swir_irr = self.read_series_L(
seq_dir, seriesIrr, lat, lon, metadata, flag, "L0_IRR",
calibration_data_irr, calibration_data_swir_irr)
if self.context.get_config_value("write_l0"):
self.writer.write(l0_irr, overwrite=True)
self.writer.write(l0_swir_irr, overwrite=True)
else:
self.context.logger.error("No irradiance data for this sequence")
if seriesRad:
if self.context.get_config_value("network") == "w":
l0_rad = self.read_series(seq_dir, seriesRad, lat, lon,
metadata, flag, "L0_RAD",
calibration_data_rad)
if self.context.get_config_value("write_l0"):
self.writer.write(l0_rad, overwrite=True)
else:
l0_rad, l0_swir_rad = self.read_series_L(
seq_dir, seriesRad, lat, lon, metadata, flag, "L0_RAD",
calibration_data_rad, calibration_data_swir_rad)
if self.context.get_config_value("write_l0"):
self.writer.write(l0_rad, overwrite=True)
self.writer.write(l0_swir_rad, overwrite=True)
else:
self.context.logger.error("No radiance data for this sequence")
if seriesBlack:
if self.context.get_config_value("network") == "w":
l0_bla = self.read_series(seq_dir, seriesBlack, lat, lon,
metadata, flag, "L0_BLA",
calibration_data_rad)
if self.context.get_config_value("write_l0"):
self.writer.write(l0_bla, overwrite=True)
else:
l0_bla, l0_swir_bla = self.read_series_L(
seq_dir, seriesBlack, lat, lon, metadata, flag, "L0_BLA",
calibration_data_rad, calibration_data_swir_rad)
if self.context.get_config_value("write_l0"):
self.writer.write(l0_bla, overwrite=True)
self.writer.write(l0_swir_bla, overwrite=True)
else:
self.context.logger.error("No black data for this sequence")
if seriesPict:
print("Here we should move the pictures to some place???")
else:
self.context.logger.error("No pictures for this sequence")
if self.context.get_config_value("network") == "w":
return l0_irr, l0_rad, l0_bla
else:
return l0_irr, l0_rad, l0_bla, l0_swir_irr, l0_swir_rad, l0_swir_bla