def __init__(self, output_def):
super(OutputHandlerBase, self).__init__(self.__class__.__name__)
self.pysiral_config = ConfigInfo()
self.error = ErrorStatus()
self._basedir = "n/a"
self._init_from_output_def(output_def)
self.output_def_filename = output_def
def get_l1bdata_files(mission_id, hemisphere, year, month, config=None,
version="default"):
import glob
if config is None:
config = ConfigInfo()
l1b_repo = config.local_machine.l1b_repository[mission_id][version].l1bdata
directory = os.path.join(
l1b_repo, hemisphere, "%04g" % year, "%02g" % month)
l1bdata_files = sorted(glob.glob(os.path.join(directory, "*.nc")))
return l1bdata_files
def __init__(self, cfg):
cls_name = self.__class__.__name__
super(Level1POutputHandler, self).__init__(cls_name)
self.error = ErrorStatus(caller_id=cls_name)
self.cfg = cfg
self.pysiral_cfg = ConfigInfo()
# Init class properties
self._path = None
self._filename = None
def __init__(self,
l1p_settings_id_or_file,
tcs,
tce,
exclude_month=[],
hemisphere="global",
platform=None,
output_handler_cfg={},
source_repo_id=None):
"""
The settings for the Level-1 pre-processor job
:param l1p_settings_id_or_file: An id of an proc/l1 processor config file (filename excluding the .yaml
extension) or an full filepath to a yaml config file
:param tcs: [int list] Time coverage start (YYYY MM [DD])
:param tce: [int list] Time coverage end (YYYY MM [DD]) [int list]
:param exclude_month: [int list] A list of month that will be ignored
:param hemisphere: [str] The target hemisphere (`north`, `south`, `global`:default).
:param platform: [str] The target platform (pysiral id). Required if l1p settings files is valid for
multiple platforms (e.g. ERS-1/2, ...)
:param output_handler_cfg: [dict] An optional dictionary with options of the output handler
(`overwrite_protection`: [True, False], `remove_old`: [True, False])
:param source_repo_id: [str] The tag in local_machine_def.yaml (l1b_repository.<platform>.<source_repo_id>)
-> Overwrites the default source repo in the l1p settings
(input_handler.options.local_machine_def_tag &
output_handler.options.local_machine_def_tag)
"""
super(Level1PreProcJobDef, self).__init__(self.__class__.__name__)
self.error = ErrorStatus()
# Get pysiral configuration
# TODO: Move to global
self._cfg = ConfigInfo()
# Store command line options
self._hemisphere = hemisphere
self._platform = platform
self._source_repo_id = source_repo_id
# Parse the l1p settings file
self.set_l1p_processor_def(l1p_settings_id_or_file)
# Get full requested time range
self._time_range = TimeRangeRequest(tcs,
tce,
exclude_month=exclude_month)
self.log.info("Requested time range is %s" % self.time_range.label)
# Store the data handler options
self._output_handler_cfg = output_handler_cfg
# Measure execution time
self.stopwatch = StopWatch()
def get_local_l1bdata_files(mission_id, time_range, hemisphere, config=None,
version="default", allow_multiple_baselines=True):
"""
Returns a list of l1bdata files for a given mission, hemisphere, version
and time range
XXX: Note: this function will slowly replace `get_l1bdata_files`, which
is limited to full month
"""
# parse config data (if not provided)
if config is None or not isinstance(config, ConfigInfo):
config = ConfigInfo()
# Validate time_range (needs to be of type TimeRangeIteration)
try:
time_range_is_correct_object = time_range.base_period == "monthly"
except:
time_range_is_correct_object = False
if not time_range_is_correct_object:
error = ErrorStatus()
msg = "Invalid type of time_range, required: %s, was %s" % (
type(time_range), type(TimeRangeIteration))
error.add_error("invalid-timerange-type", msg)
error.raise_on_error()
# 1) get list of all files for monthly folders
yyyy, mm = "%04g" % time_range.start.year, "%02g" % time_range.start.month
l1b_repo = config.local_machine.l1b_repository[mission_id][version].l1bdata
directory = os.path.join(l1b_repo, hemisphere, yyyy, mm)
all_l1bdata_files = sorted(glob.glob(os.path.join(directory, "*.nc")))
# 2) First filtering step: Check if different algorithm baseline values
# exist in the list of l1bdata files
algorithm_baselines = [l1bdata_get_baseline(f) for f in all_l1bdata_files]
baselines = np.unique(np.array(algorithm_baselines))
n_baselines = len(baselines)
if not allow_multiple_baselines and n_baselines > 1:
error = ErrorStatus()
baseline_str_list = ", ".join(baselines)
msg = "Multiple l1bdata baselines (%g) [%s] found in directory: %s" % (
n_baselines, baseline_str_list, directory)
error.add_error("multiple-l1b-baselines", msg)
error.raise_on_error()
# 3) Check if files are in requested time range
# This serves two purporses: a) filter out files with timestamps that do
# not belong in the directory. b) get a subset if required
l1bdata_files_checked = [l1bdata_file for l1bdata_file in all_l1bdata_files
if l1bdata_in_trange(l1bdata_file, time_range)]
# Done return list (empty or not)
return l1bdata_files_checked, directory
def __init__(self, config=None, load_config=True):
self.error = ErrorStatus()
self.data_level = None
self.path = None
self.version = "default"
self.mission_id = None
self.year = None
self.month = None
if not load_config:
return
if config is None or not isinstance(config, ConfigInfo):
self.config = ConfigInfo()
else:
self.config = config
def _init_product_directory(self):
""" Get main product directory from local_machine_def, add mandatory
runtag subdirectory, optional second subdirectory for overwrite
protection and product level id subfolder"""
pysiral_config = ConfigInfo()
basedir = pysiral_config.local_machine.product_repository
if not isinstance(self.subdirectory, list):
basedir = os.path.join(basedir, self.subdirectory)
else:
basedir = os.path.join(basedir, *self.subdirectory)
if self.overwrite_protection:
basedir = os.path.join(basedir, self.now_directory)
basedir = os.path.join(basedir, self.product_level_subfolder)
self._set_basedir(basedir)
def __init__(self):
super(L1bPreProcJob, self).__init__(self.__class__.__name__)
# Save pointer to pysiral configuration
self.pysiral_config = ConfigInfo()
# Initialize the time range and set to monthly per default
self.time_range = None
# Error Status
self.error = ErrorStatus()
# Initialize job parameter
self.options = L1bPreProcJobOptions()
# List for iterations (currently only month-wise)
self.iterations = []
def __init__(self, product_def, auxclass_handler=None):
""" Setup of the Level-2 Processor """
super(Level2Processor, self).__init__(self.__class__.__name__)
# Error Status Handler
self.error = ErrorStatus(caller_id=self.__class__.__name__)
# Level-2 Algorithm Definition
# NOTE: This object should ony be called through the property self.l2def
self._l2def = product_def.l2def
# Auxiliary Data Handler
# NOTE: retrieves and initializes the auxdata classes based on the l2 processor definition config file
if auxclass_handler is None:
auxclass_handler = DefaultAuxdataClassHandler()
self._auxclass_handler = auxclass_handler
# This variable will contain a list with the auxiliary data handlers
self._registered_auxdata_handlers = []
self._auxhandlers = {}
# Output_handler (can be one or many)
self._output_handler = product_def.output_handler
# List of Level-2 (processed) orbit segments
self._orbit = deque()
# List of Level-1b input files
self._l1b_files = []
# pysiral config
self._config = ConfigInfo()
# Processor Initialization Flag
self._initialized = False
# Processor summary report
self.report = L2ProcessorReport()
# Initialize the class
self._initialize_processor()
def __init__(self, name):
# Enable logging capability (self.log)
super(L1bPreProc, self).__init__(name)
# Error handler
self.error = ErrorStatus()
# Job definition ( class L1bPreProcJob)
self._jobdef = None
# Mission Options
self._mdef = None
# List of l1b input files
# Needs to be filled by the mission specific classes
self._l1b_file_list = []
# pysiral configuration
self._pysiral_config = ConfigInfo()
def MaskSourceFile(mask_name, mask_cfg):
""" Wrapper method for different mask source file classes """
error = ErrorStatus(caller_id="MaskSourceFile")
# Get the full mask filename
pysiral_cfg = ConfigInfo()
try:
mask_dir = pysiral_cfg.local_machine.auxdata_repository.mask[mask_name]
except KeyError:
msg = "path to mask %s not in local_machine_def.yaml" % mask_name
error.add_error("missing-lmd-def", msg)
error.raise_on_error()
# Return the Dataset class
try:
return globals()[mask_cfg.pyclass_name](mask_dir, mask_name, mask_cfg)
except KeyError:
msg = "pysiral.mask.%s not implemented" % str(mask_cfg.pyclass_name)
error.add_error("missing-mask-class", msg)
error.raise_on_error()
def mask_filepath(self):
# Get config info
pysiral_cfg = ConfigInfo()
# Get the path to the mask file
# (needs to be in local_machine_def.yaml)
mask_dir = pysiral_cfg.local_machine.auxdata_repository.mask
try:
mask_dir = mask_dir[self.mask_name]
except KeyError:
msg = "cannot find mask entry [%s] in local_machine_def.yaml"
self.error.add_error("lmd-error", msg % self.mask_name)
return None
mask_filename = "%s_%s.nc" % (self.mask_name, self.grid_id)
filepath = os.path.join(mask_dir, mask_filename)
if not os.path.isfile(filepath):
msg = "cannot find mask file: %s" % filepath
self.error.add_error("io-error", msg)
return None
return filepath
def __init__(self):
super(Level3ProcArgParser, self).__init__(self.__class__.__name__)
self.error = ErrorStatus()
self.pysiral_config = ConfigInfo()
self._args = None
class Level3ProcArgParser(DefaultLoggingClass):
def __init__(self):
super(Level3ProcArgParser, self).__init__(self.__class__.__name__)
self.error = ErrorStatus()
self.pysiral_config = ConfigInfo()
self._args = None
def parse_command_line_arguments(self):
# use python module argparse to parse the command line arguments
# (first validation of required options and data types)
self._args = self.parser.parse_args()
# Add addtional check to make sure either `l1b-files` or
# `start ` and `stop` are set
# l1b_file_preset_is_set = self._args.l1b_files_preset is not None
# start_and_stop_is_set = self._args.start_date is not None and \
# self._args.stop_date is not None
#
# if l1b_file_preset_is_set and start_and_stop_is_set:
# self.parser.error("-start & -stop and -l1b-files are exclusive")
#
# if not l1b_file_preset_is_set and not start_and_stop_is_set:
# self.parser.error("either -start & -stop or -l1b-files required")
def critical_prompt_confirmation(self):
# Any confirmation prompts can be overriden by --no-critical-prompt
no_prompt = self._args.no_critical_prompt
# if --remove_old is set, all previous l1bdata files will be
# erased for all month
if self._args.remove_old and not no_prompt:
message = "You have selected to remove all previous " + \
"l3 files for the requested period\n" + \
"(Note: use --no-critical-prompt to skip confirmation)\n" + \
"Enter \"YES\" to confirm and continue: "
result = raw_input(message)
if result != "YES":
sys.exit(1)
@property
def parser(self):
# XXX: Move back to caller
# Take the command line options from default settings
# -> see config module for data types, destination variables, etc.
clargs = DefaultCommandLineArguments()
# List of command line option required for pre-processor
# (argname, argtype (see config module), destination, required flag)
options = [("-l2i-product-dir", "l2i-product-dir",
"l2i_basedir", True),
("-l3-settings", "l3-settings", "l3_settings", False),
("-l3-griddef", "l3-griddef", "l3_griddef", True),
("-l3-output", "l3-output", "l3_output", True),
("-start", "date", "start_date", True),
("-stop", "date", "stop_date", True),
("-period", "period", "period", False),
("-doi", "doi", "doi", False),
("-data-record-type", "data_record_type",
"data_record_type", False),
("--remove-old", "remove-old", "remove_old", False),
("--no-critical-prompt", "no-critical-prompt",
"no_critical_prompt", False)]
# create the parser
parser = argparse.ArgumentParser()
for option in options:
argname, argtype, destination, required = option
argparse_dict = clargs.get_argparse_dict(argtype, destination,
required)
parser.add_argument(argname, **argparse_dict)
return parser
@property
def arg_dict(self):
""" Return the arguments as dictionary """
return self._args.__dict__
@property
def start(self):
return self._args.start_date
@property
def stop(self):
return self._args.stop_date
@property
def period(self):
return self._args.period
@property
def doi(self):
return self._args.doi
@property
def data_record_type(self):
return self._args.data_record_type
@property
def l2i_product_directory(self):
return os.path.join(self.l3_product_basedir, "l2i")
@property
def l3_settings_file(self):
l3_settings = self._args.l3_settings
filename = self.pysiral_config.get_settings_file(
"proc", "l3", l3_settings)
if filename is None:
msg = "Invalid l3 settings filename or id: %s\n" % l3_settings
msg = msg + " \nRecognized Level-3 processor setting ids:\n"
for l3_settings_id in self.pysiral_config.get_setting_ids(
"proc", "l3"):
msg = msg + " " + l3_settings_id + "\n"
self.error.add_error("invalid-l3-settings", msg)
self.error.raise_on_error()
else:
return filename
@property
def l3_griddef(self):
l3_griddef = self._args.l3_griddef
filename = self.pysiral_config.get_settings_file(
"grid", None, l3_griddef)
if filename is None:
msg = "Invalid griddef filename or id: %s\n" % l3_griddef
msg = msg + " Recognized grid definition ids:\n"
for griddef_id in self.pysiral_config.get_setting_ids("griddef"):
msg = msg + " - " + griddef_id + "\n"
self.error.add_error("invalid-griddef", msg)
self.error.raise_on_error()
else:
return filename
@property
def l3_output_file(self):
l3_output = self._args.l3_output
filename = self.pysiral_config.get_settings_file(
"output", "l3", l3_output)
if filename is None:
msg = "Invalid output definition filename or id: %s\n" % l3_output
msg = msg + " Recognized output definition ids:\n"
for output_id in self.pysiral_config.get_setting_ids(
"output", "l3"):
msg = msg + " - " + output_id + "\n"
self.error.add_error("invalid-outputdef", msg)
self.error.raise_on_error()
else:
return filename
@property
def l3_product_basedir(self):
""" Returns the base directory (one level below l2i) """
# 1. Clean up the path
product_basedir = os.path.abspath(self._args.l2i_basedir)
dirs = os.path.split(product_basedir)
if dirs[1] == "l2i":
return dirs[0]
else:
return product_basedir
@property
def remove_old(self):
return self._args.remove_old and not self._args.overwrite_protection
def __init__(self):
super(DefaultAuxdataClassHandler,
self).__init__(self.__class__.__name__)
self.pysiral_config = ConfigInfo()
self.error = ErrorStatus(caller_id=self.__class__.__name__)
class Level2ProcArgParser(DefaultLoggingClass):
def __init__(self):
super(Level2ProcArgParser, self).__init__(self.__class__.__name__)
self.error = ErrorStatus()
self.pysiral_config = ConfigInfo()
self._args = None
def parse_command_line_arguments(self):
# use python module argparse to parse the command line arguments
# (first validation of required options and data types)
self._args = self.parser.parse_args()
# Add additional check to make sure either `l1b-files` or
# `start ` and `stop` are set
l1b_file_preset_is_set = self._args.l1b_files_preset is not None
start_and_stop_is_set = self._args.start_date is not None and \
self._args.stop_date is not None
if l1b_file_preset_is_set and start_and_stop_is_set:
self.parser.error("-start & -stop and -l1b-files are exclusive")
if not l1b_file_preset_is_set and not start_and_stop_is_set:
self.parser.error("either -start & -stop or -l1b-files required")
def critical_prompt_confirmation(self):
# Any confirmation prompts can be overridden by --no-critical-prompt
no_prompt = self._args.no_critical_prompt
# if --remove_old is set, all previous l1bdata files will be
# erased for all month
if self._args.remove_old and not no_prompt:
message = "You have selected to remove all previous " + \
"l2 files for the requested period\n" + \
"(Note: use --no-critical-prompt to skip confirmation)\n" + \
"Enter \"YES\" to confirm and continue: "
result = raw_input(message)
if result != "YES":
sys.exit(1)
@property
def parser(self):
# XXX: Move back to caller
# Take the command line options from default settings
# -> see config module for data types, destination variables, etc.
clargs = DefaultCommandLineArguments()
# List of command line option required for pre-processor
# (argname, argtype (see config module), destination, required flag)
options = [("-l2-settings", "l2-settings", "l2_settings", True),
("-run-tag", "run-tag", "run_tag", False),
("-start", "date", "start_date", False),
("-stop", "date", "stop_date", False),
("-l1b-files", "l1b_files", "l1b_files_preset", False),
("-exclude-month", "exclude-month", "exclude_month", False),
("-input-version", "input-version", "input_version", False),
("-l2-output", "l2-output", "l2_output", False),
("--remove-old", "remove-old", "remove_old", False),
("--no-critical-prompt", "no-critical-prompt",
"no_critical_prompt", False),
("--no-overwrite-protection", "no-overwrite-protection",
"overwrite_protection", False),
("--overwrite-protection", "overwrite-protection",
"overwrite_protection", False)]
# create the parser
parser = argparse.ArgumentParser()
for option in options:
argname, argtype, destination, required = option
argparse_dict = clargs.get_argparse_dict(argtype, destination,
required)
parser.add_argument(argname, **argparse_dict)
parser.set_defaults(overwrite_protection=True)
return parser
@property
def arg_dict(self):
""" Return the arguments as dictionary """
return self._args.__dict__
@property
def start(self):
return self._args.start_date
@property
def stop(self):
return self._args.stop_date
@property
def run_tag(self):
""" run_tag is a str or relative path that determines the output directory for
the Level-2 processor. If the -run-tag option is not specified, the output
directory will be the `product_repository` specification in `local_machine_def`
with the l2 settings file basename as subfolder.
One can however specify a custom string, or a relative path, with subfolders
defined by `\` or `/`, e.g.
Examples:
-run-tag cs2awi_v2p0_nrt
-run-tag c3s/cdr/cryosat2/v1p0/nh
"""
# Get from command line arguments (default: None)
run_tag = self._args.run_tag
# If argument is empty use the basename of the l2 settings file
if run_tag is None:
run_tag = self._args.l2_settings
# Settings file may be specified as full path and not just the id
if os.path.isfile(run_tag):
run_tag = file_basename(run_tag)
# split the run-tag on potential path separators
run_tag = re.split(r'[\\|/]', run_tag)
return run_tag
@property
def exclude_month(self):
return self._args.exclude_month
@property
def overwrite_protection(self):
return self._args.overwrite_protection
@property
def l2_settings_file(self):
l2_settings = self._args.l2_settings
filename = self.pysiral_config.get_settings_file(
"proc", "l2", l2_settings)
if filename is None:
msg = "Invalid l2 settings filename or id: %s\n" % l2_settings
msg = msg + " \nRecognized Level-2 processor setting ids:\n"
for l2_settings_id in self.pysiral_config.get_setting_ids(
"proc", "l2"):
msg = msg + " " + l2_settings_id + "\n"
self.error.add_error("invalid-l2-settings", msg)
self.error.raise_on_error()
else:
return filename
@property
def l1b_version(self):
return self._args.input_version
@property
def l1b_predef_files(self):
l1b_files = glob.glob(self._args.l1b_files_preset)
return l1b_files
@property
def l2_output(self):
l2_output = self._args.l2_output
filename = self.pysiral_config.get_settings_file(
"output", "l2i", l2_output)
if filename is None:
msg = "Invalid l2 outputdef filename or id: %s\n" % l2_output
msg = msg + " \nRecognized Level-2 output definitions ids:\n"
l2_output_ids = self.pysiral_config.get_setting_ids(
"output", "l2i")
for l2_output_id in l2_output_ids:
msg = msg + " - " + l2_output_id + "\n"
self.error.add_error("invalid-l2-outputdef", msg)
self.error.raise_on_error()
else:
return filename
@property
def is_time_range_request(self):
return self._args.l1b_files_preset is None
@property
def remove_old(self):
return self._args.remove_old and not self._args.overwrite_protection
def default_output_def_filename(self):
pysiral_config = ConfigInfo()
local_settings_path = pysiral_config.pysiral_local_path
return os.path.join(local_settings_path, *self.default_file_location)
class Level2PreProcArgParser(DefaultLoggingClass):
def __init__(self):
super(Level2PreProcArgParser, self).__init__(self.__class__.__name__)
self.error = ErrorStatus()
self.pysiral_config = ConfigInfo()
self._args = None
def parse_command_line_arguments(self):
# use python module argparse to parse the command line arguments
# (first validation of required options and data types)
self._args = self.parser.parse_args()
def critical_prompt_confirmation(self):
# Any confirmation prompts can be overriden by --no-critical-prompt
no_prompt = self._args.no_critical_prompt
# if --remove_old is set, all previous l1bdata files will be
# erased for all month
if self._args.remove_old and not no_prompt:
message = "You have selected to remove all previous " + \
"l2p files for the requested period\n" + \
"(Note: use --no-critical-prompt to skip confirmation)\n" + \
"Enter \"YES\" to confirm and continue: "
result = raw_input(message)
if result != "YES":
sys.exit(1)
@property
def parser(self):
# XXX: Move back to caller
# Take the command line options from default settings
# -> see config module for data types, destination variables, etc.
clargs = DefaultCommandLineArguments()
# List of command line option required for pre-processor
# (argname, argtype (see config module), destination, required flag)
options = [
("-start", "date", "start_date", False),
("-stop", "date", "stop_date", False),
("-l2i-product-dir", "l2i-product-dir", "l2i_product_dir", True),
("-l2p-output", "l2p-output", "l2p_output", False),
("-exclude-month", "exclude-month", "exclude_month", False),
("-doi", "doi", "doi", False),
("--remove-old", "remove-old", "remove_old", False),
("--no-critical-prompt", "no-critical-prompt",
"no_critical_prompt", False),
("--no-overwrite-protection", "no-overwrite-protection",
"overwrite_protection", False),
("--overwrite-protection", "overwrite-protection",
"overwrite_protection", False)]
# create the parser
parser = argparse.ArgumentParser()
for option in options:
argname, argtype, destination, required = option
argparse_dict = clargs.get_argparse_dict(
argtype, destination, required)
parser.add_argument(argname, **argparse_dict)
parser.set_defaults(overwrite_protection=True)
return parser
@property
def arg_dict(self):
""" Return the arguments as dictionary """
return self._args.__dict__
@property
def start(self):
return self._args.start_date
@property
def stop(self):
return self._args.stop_date
@property
def exclude_month(self):
return self._args.exclude_month
@property
def doi(self):
return self._args.doi
@property
def overwrite_protection(self):
return self._args.overwrite_protection
@property
def l2i_product_dir(self):
l2i_product_dir = self._args.l2i_product_dir
if os.path.isdir(l2i_product_dir):
return os.path.normpath(l2i_product_dir)
else:
msg = "Invalid l2i product dir: %s" % str(l2i_product_dir)
self.error.add_error("invalid-l2i-product-dir", msg)
self.error.raise_on_error()
@property
def l2p_output(self):
l2p_output = self._args.l2p_output
filename = self.pysiral_config.get_settings_file("output", "l2p", l2p_output)
if filename is None:
msg = "Invalid l2p outputdef filename or id: %s\n" % l2p_output
msg = msg + " \nRecognized Level-2 output definitions ids:\n"
l2p_output_ids = self.pysiral_config.get_setting_ids("output", "l2p")
for l2p_output_id in l2p_output_ids:
msg = msg + " - " + l2p_output_id+"\n"
self.error.add_error("invalid-l2p-outputdef", msg)
self.error.raise_on_error()
else:
return filename
@property
def remove_old(self):
return self._args.remove_old and not self._args.overwrite_protection
