def _transfer_l1p_vars(self, l1b, l2):
""" Transfer variables from l1p to l2 object"""
# Make this a backward compatible feature (should work without tag in l2 processor definition file)
if not "transfer_from_l1p" in self.l2def:
return
# Don't spam the log
try:
verbose = self.l2def.transfer_from_l1p.options.get("verbose")
except:
verbose = False
# Get and loop over data groups
data_groups, vardefs = td_branches(self.l2def.transfer_from_l1p)
for data_group, varlist in zip(data_groups, vardefs):
# Get and loop over variables per data group
var_names, vardefs = td_branches(varlist)
for var_name, vardef in zip(var_names, vardefs):
# Get variable via standard getter method
# NOTE: Will return None if not found -> create an empty array
var = l1b.get_parameter_by_name(data_group, var_name)
if var is None:
var = np.full((l2.n_records), np.nan)
# Add variable to l2 object as auxiliary variable
l2.set_auxiliary_parameter(vardef.aux_id, vardef.aux_name, var, None)
if verbose:
self.log.info("- Transfered l1p variable: %s.%s" % (data_group, var_name))
def _l2proc_summary_to_file(self):
output_ids, output_defs = td_branches(self.l2def.output)
for output_id, output_def in zip(output_ids, output_defs):
output = get_output_class(output_def.pyclass)
output.set_options(**output_def.options)
output.set_base_export_path(output_def.path)
time_range = self.report.time_range
export_folder = output.get_full_export_path(time_range.start)
self.report.write_to_file(output_id, export_folder)
def _apply_l1b_prefilter(self, l1b):
""" Apply filtering of l1b variables """
# Backward compatibility with older l2 setting files
if "l1b_pre_filtering" not in self.l2def:
return
# Apply filters
names, filters = td_branches(self.l2def.l1b_pre_filtering)
for name, filter_def in zip(names, filters):
self.log.info("- Apply l1b pre-filter: %s" % filter_def.pyclass)
l1bfilter = get_filter(filter_def.pyclass)
l1bfilter.set_options(**filter_def.options)
l1bfilter.apply_filter(l1b)
def _apply_freeboard_filter(self, l2):
""" Apply freeboard filters as defined in the level-2 settings file
under `root.filter.freeboard`
Filtering means:
- setting the freeboard value to nan
- setting the surface type classification to invalid
"""
# Extract filters from settings structure
freeboard_filters = self.l2def.filter.freeboard
names, filters = td_branches(freeboard_filters)
# Loop over freeboard filters
for name, filter_def in zip(names, filters):
# Get corresponding class name in pysiral.filter and transfer options
# XXX: This should be rewritten as (e.g.)
# `frbfilter = VariableFilter(filter_def.pyclass, **filter_def.options)`
frbfilter = get_filter(filter_def.pyclass)
frbfilter.set_options(**filter_def.options)
# XXX: This is a temporary fix of an error in the algorithm
#
# Explanation: The filter target was wrongly set to radar freeboard,
# meaning that whether a freeboard value was filtered was determined on
# the wrong parameter. Both values differ by the geometric snow propagation
# correction (22% of snow depth). While the impact on the high freeboard end
# is negligible, at the lower (negative) end more freeboard where filtered
# than necessary since radar freeboard is always lower.
#
# The `afrb` filter target was hard coded, thus an option is added to replace
# the filter target (`root.filter.freeboard.frb_valid_range.filter_target`).
# The default option is the wrong one only for consistency reasons.
filter_target = "afrb"
if filter_def.options.has_key("filter_target"):
filter_target = filter_def.options.filter_target
# Check if action is required
frbfilter.apply_filter(l2, filter_target)
if frbfilter.flag.num == 0:
continue
# Logging
self.log.info("- Filter message: %s has flagged %g waveforms" % (
filter_def.pyclass, frbfilter.flag.num))
# Set surface type flag (contains invalid)
l2.surface_type.add_flag(frbfilter.flag.flag, "invalid")
# Remove invalid elevations / freeboards
l2.frb.set_nan_indices(frbfilter.flag.indices)
def _waveform_retracking(self, l1b, l2):
""" Retracking: Obtain surface elevation from l1b waveforms """
# loop over retrackers for each surface type
surface_types, retracker_def = td_branches(self.l2def.retracker)
for i, surface_type in enumerate(surface_types):
# Check if any waveforms need to be retracked for given
# surface type
surface_type_flag = l2.surface_type.get_by_name(surface_type)
if surface_type_flag.num == 0:
self.log.info("- no waveforms of type %s" % surface_type)
continue
# Benchmark retracker performance
# XXX: is currently the bottleneck of level2 processing
timestamp = time.time()
# Retrieve the retracker assiciated with surface type
# from the l2 settings
retracker = get_retracker_class(retracker_def[i].pyclass)
# Set options (if any)
if retracker_def[i].options is not None:
retracker.set_options(**retracker_def[i].options)
# set subset of waveforms
retracker.set_indices(surface_type_flag.indices)
# Add classifier data (some retracker need that)
retracker.set_classifier(l1b.classifier)
# Start the retracking
retracker.retrack(l1b, l2)
# Retrieve the range after retracking
l2.update_retracked_range(retracker)
# XXX: Let the retracker return other parameters?
l2.set_radar_mode(l1b.waveform.radar_mode)
# retrieve potential error status and update surface type flag
if retracker.error_flag.num > 0:
l2.surface_type.add_flag(retracker.error_flag.flag, "invalid")
self.log.info("- Retrack class %s with %s in %.3f seconds" % (
surface_type, retracker_def[i].pyclass,
time.time()-timestamp))
# Error handling not yet implemented, return dummy values
return False, None
def _apply_thickness_filter(self, l2):
thickness_filters = self.l2def.filter.thickness
names, filters = td_branches(thickness_filters)
for name, filter_def in zip(names, filters):
sitfilter = get_filter(filter_def.pyclass)
sitfilter.set_options(**filter_def.options)
sitfilter.apply_filter(l2, "sit")
if sitfilter.flag.num == 0:
continue
self.log.info("- Filter message: %s has flagged %g waveforms" % (
filter_def.pyclass, sitfilter.flag.num))
# Set surface type flag (contains invalid)
l2.surface_type.add_flag(sitfilter.flag.flag, "invalid")
# Remove invalid thickness values
l2.sit.set_nan_indices(sitfilter.flag.indices)
def _validate_surface_types(self, l2):
""" Loop over stack of surface type validators """
surface_type_validators = self.l2def.validator.surface_type
names, validators = td_branches(surface_type_validators)
error_codes = ["l2proc_surface_type_discarded"]
error_states = []
error_messages = []
for name, validator_def in zip(names, validators):
validator = get_validator(validator_def.pyclass)
validator.set_options(**validator_def.options)
state, message = validator.validate(l2)
error_states.append(state)
error_messages.append(message)
if state:
self.log.info("- Validator message: "+message)
error_status = True in error_states
return error_status, error_codes
