def pysiral_l1preproc(job):
"""
Workflow script of the pysiral l1b preprocessor.
:param job: A pysiral.l1preproc.Level1PreProcJobDef instance
:return: None
"""
# Take the time
job.stopwatch.start()
# 1. Get the input handler
input_handler_def = job.l1pprocdef.input_handler
input_handler_cls = get_cls(input_handler_def.module_name,
input_handler_def.class_name,
relaxed=False)
input_handler = input_handler_cls(input_handler_def.options)
# 2. Get the adapter class that transfers
adapter_def = job.l1pprocdef.input_adapter
input_adapter_cls = get_cls(adapter_def.module_name,
adapter_def.class_name,
relaxed=False)
input_adapter = input_adapter_cls(adapter_def.options)
# 3. Get the output handler
output_handler_def = job.l1pprocdef.output_handler
output_handler = Level1POutputHandler(output_handler_def.options)
output_handler.cfg.update(**job.output_handler_cfg)
# 4. Get the pre-processor
preproc_def = job.l1pprocdef.level1_preprocessor
l1preproc = get_preproc(preproc_def.type, input_adapter, output_handler,
preproc_def.options)
# 5. Loop over monthly periods
for period in job.period_segments:
# 5.1 Get input files
file_list = input_handler.get_file_for_period(period)
if len(file_list) == 0:
job.log.warning("No input files found for period: %s, skipping" %
period.date_label)
# 5.2 Output management
# Note: This is only relevant, if the --remove-old keyword is set
output_handler.remove_old_if_applicable(period)
# 5.3 Run the pre-processor
l1preproc.process_input_files(file_list)
# Report processing time
job.stopwatch.stop()
job.info("Level-1 PreProcessor finished in %s" %
job.stopwatch.get_duration())
def l1_post_processing(self, l1_segments):
"""
Apply the post-processing procedures defined in the l1p processor definition file.
:param l1_segments: A list of Level-1 data objects
:return: None, the l1_segments are changed in place
"""
# Get the post processing options
pre_processing_items = self.cfg.get("pre_processing_items", None)
if pre_processing_items is None:
logger.info("No pre processing items defined")
return
# Measure time for the different post processors
timer = StopWatch()
# Get the list of post-processing items
for pp_item in pre_processing_items:
timer.start()
pp_class = get_cls(pp_item["module_name"],
pp_item["class_name"],
relaxed=False)
post_processor = pp_class(**pp_item["options"])
for l1 in l1_segments:
post_processor.apply(l1)
timer.stop()
msg = "- L1 pre-processing item `%s` applied in %.3f seconds" % (
pp_item["label"], timer.get_seconds())
logger.info(msg)
def get_procstep_classes(self):
"""
Retrieves the required classes from the processor definition files and stores them in a list
without initializing them. This way a freshly initialized version can be supplied to each
l2 data object without risk of interference of class properties
:return:
"""
# Loop
for procstep_def in self.cfg:
# Get the module & pyclass
module = procstep_def["module"]
full_module_name = "pysiral.{}".format(module)
obj = get_cls(full_module_name, procstep_def["pyclass"])
# This object should not be None
if obj is None:
msg = "Invalid L2 processing step class: {}.{}".format(
full_module_name, procstep_def["pyclass"])
self.error.add_error("missing-class", msg)
self.error.raise_on_error()
# Append the class
logger.info("Added L2 processor step: {}.{}".format(
full_module_name, procstep_def["pyclass"]))
self._classes.append(obj)
def _post_processing_items(self, l2):
"""
:param l2:
:return:
"""
# Get the post processing options
post_processing_items = self._l2def.get("post_processing", None)
if post_processing_items is None:
self.log.info("No post-processing items defined")
return
# Get the list of post-processing items
for pp_item in post_processing_items:
pp_class = get_cls(pp_item["module_name"], pp_item["class_name"], relaxed=False)
post_processor = pp_class(**pp_item["options"])
post_processor.apply(l2)
msg = "- Level-2 post-processing item `%s` applied" % (pp_item["label"])
self.log.info(msg)
def get_pyclass(self, auxdata_class, auxdata_id, l2_procdef_opt):
"""
Returns a class for handling auxiliary data files, that is initialized
with auxdata settings in `config/auxdata_def.yaml` and with the
directory specified in `local_machine_def.yaml`
Args:
auxdata_class (str): Auxdata class (e.g. mss, sic, sitype, snow)
auxdata_id (str): Auxdata class identifier (e.g. osisaf)
Returns:
class: The initialized auxdata handler class
"""
# Clear errors
self.error.reset()
# Initialize the class with information from auxdata_def.yaml
auxdata_def = self.get_auxdata_def(auxdata_class, auxdata_id)
if auxdata_def is None:
error_id = "auxdata_missing_definition"
error_message = PYSIRAL_ERROR_CODES[error_id] % (auxdata_class,
auxdata_id)
self.error.add_error(error_id, error_message)
self.error.raise_on_error()
# Set the auxdata config
cfg = AuxClassConfig()
# connect to repository on local machine
if "local_repository" in auxdata_def:
local_repository_id = auxdata_def.local_repository
local_repo = self.get_local_repository(auxdata_class,
local_repository_id)
if local_repo is None and local_repository_id is not None:
error_id = "auxdata_missing_localrepo_def"
error_message = PYSIRAL_ERROR_CODES[error_id] % (auxdata_class,
auxdata_id)
self.error.add_error(error_id, error_message)
self.error.raise_on_error()
cfg.set_local_repository(local_repo)
# set doc str (should be mandatory for all auxdata handlers)
if "long_name" in auxdata_def:
cfg.set_long_name(auxdata_def.long_name)
# set filename (e.g. for mss)
if "filename" in auxdata_def:
local_repository_id = auxdata_def.local_repository
local_repo = self.get_local_repository(auxdata_class,
local_repository_id)
filename = os.path.join(local_repo, auxdata_def.filename)
cfg.set_filename(filename)
# set filenaming (e.g. for sic, sitype, snow)
if "filenaming" in auxdata_def:
cfg.set_filenaming(auxdata_def.filenaming)
# set subfolders (e.g. for sic, sitype, snow)
if "subfolders" in auxdata_def:
cfg.set_subfolder(auxdata_def.subfolders)
# Set the default options from the auxiliary definition file
if "options" in auxdata_def:
options = auxdata_def.get("options", None)
if options is not None:
cfg.set_options(**options)
# Override option with definition from the l2 processor settings
if l2_procdef_opt is not None:
cfg.set_options(**l2_procdef_opt)
# Get the auxiliary data class
module_name, class_name = "pysiral.auxdata.%s" % (
auxdata_class), auxdata_def["pyclass"]
auxclass = get_cls(module_name, class_name)
if auxclass is None:
error_id = "auxdata_invalid_class_name"
msg = "Invalid Auxdata class: %s.%s" % (module_name, class_name)
self.error.add_error(PYSIRAL_ERROR_CODES[error_id], msg)
self.error.raise_on_error()
# Init the auxiliary class
# Note: This will trigger any action defined in the subclasses, such as reading static background files
auxdata_handler = auxclass(cfg)
# All done, return
return auxdata_handler
