def __init__(self, output_def): super(OutputHandlerBase, self).__init__(self.__class__.__name__) self.pysiral_config = ConfigInfo() self.error = ErrorStatus() self._basedir = "n/a" self._init_from_output_def(output_def) self.output_def_filename = output_def
def get_l1bdata_files(mission_id, hemisphere, year, month, config=None, version="default"): import glob if config is None: config = ConfigInfo() l1b_repo = config.local_machine.l1b_repository[mission_id][version].l1bdata directory = os.path.join( l1b_repo, hemisphere, "%04g" % year, "%02g" % month) l1bdata_files = sorted(glob.glob(os.path.join(directory, "*.nc"))) return l1bdata_files
def __init__(self, cfg): cls_name = self.__class__.__name__ super(Level1POutputHandler, self).__init__(cls_name) self.error = ErrorStatus(caller_id=cls_name) self.cfg = cfg self.pysiral_cfg = ConfigInfo() # Init class properties self._path = None self._filename = None
def __init__(self, l1p_settings_id_or_file, tcs, tce, exclude_month=[], hemisphere="global", platform=None, output_handler_cfg={}, source_repo_id=None): """ The settings for the Level-1 pre-processor job :param l1p_settings_id_or_file: An id of an proc/l1 processor config file (filename excluding the .yaml extension) or an full filepath to a yaml config file :param tcs: [int list] Time coverage start (YYYY MM [DD]) :param tce: [int list] Time coverage end (YYYY MM [DD]) [int list] :param exclude_month: [int list] A list of month that will be ignored :param hemisphere: [str] The target hemisphere (`north`, `south`, `global`:default). :param platform: [str] The target platform (pysiral id). Required if l1p settings files is valid for multiple platforms (e.g. ERS-1/2, ...) :param output_handler_cfg: [dict] An optional dictionary with options of the output handler (`overwrite_protection`: [True, False], `remove_old`: [True, False]) :param source_repo_id: [str] The tag in local_machine_def.yaml (l1b_repository.<platform>.<source_repo_id>) -> Overwrites the default source repo in the l1p settings (input_handler.options.local_machine_def_tag & output_handler.options.local_machine_def_tag) """ super(Level1PreProcJobDef, self).__init__(self.__class__.__name__) self.error = ErrorStatus() # Get pysiral configuration # TODO: Move to global self._cfg = ConfigInfo() # Store command line options self._hemisphere = hemisphere self._platform = platform self._source_repo_id = source_repo_id # Parse the l1p settings file self.set_l1p_processor_def(l1p_settings_id_or_file) # Get full requested time range self._time_range = TimeRangeRequest(tcs, tce, exclude_month=exclude_month) self.log.info("Requested time range is %s" % self.time_range.label) # Store the data handler options self._output_handler_cfg = output_handler_cfg # Measure execution time self.stopwatch = StopWatch()
def get_local_l1bdata_files(mission_id, time_range, hemisphere, config=None, version="default", allow_multiple_baselines=True): """ Returns a list of l1bdata files for a given mission, hemisphere, version and time range XXX: Note: this function will slowly replace `get_l1bdata_files`, which is limited to full month """ # parse config data (if not provided) if config is None or not isinstance(config, ConfigInfo): config = ConfigInfo() # Validate time_range (needs to be of type TimeRangeIteration) try: time_range_is_correct_object = time_range.base_period == "monthly" except: time_range_is_correct_object = False if not time_range_is_correct_object: error = ErrorStatus() msg = "Invalid type of time_range, required: %s, was %s" % ( type(time_range), type(TimeRangeIteration)) error.add_error("invalid-timerange-type", msg) error.raise_on_error() # 1) get list of all files for monthly folders yyyy, mm = "%04g" % time_range.start.year, "%02g" % time_range.start.month l1b_repo = config.local_machine.l1b_repository[mission_id][version].l1bdata directory = os.path.join(l1b_repo, hemisphere, yyyy, mm) all_l1bdata_files = sorted(glob.glob(os.path.join(directory, "*.nc"))) # 2) First filtering step: Check if different algorithm baseline values # exist in the list of l1bdata files algorithm_baselines = [l1bdata_get_baseline(f) for f in all_l1bdata_files] baselines = np.unique(np.array(algorithm_baselines)) n_baselines = len(baselines) if not allow_multiple_baselines and n_baselines > 1: error = ErrorStatus() baseline_str_list = ", ".join(baselines) msg = "Multiple l1bdata baselines (%g) [%s] found in directory: %s" % ( n_baselines, baseline_str_list, directory) error.add_error("multiple-l1b-baselines", msg) error.raise_on_error() # 3) Check if files are in requested time range # This serves two purporses: a) filter out files with timestamps that do # not belong in the directory. b) get a subset if required l1bdata_files_checked = [l1bdata_file for l1bdata_file in all_l1bdata_files if l1bdata_in_trange(l1bdata_file, time_range)] # Done return list (empty or not) return l1bdata_files_checked, directory
def __init__(self, config=None, load_config=True): self.error = ErrorStatus() self.data_level = None self.path = None self.version = "default" self.mission_id = None self.year = None self.month = None if not load_config: return if config is None or not isinstance(config, ConfigInfo): self.config = ConfigInfo() else: self.config = config
def _init_product_directory(self): """ Get main product directory from local_machine_def, add mandatory runtag subdirectory, optional second subdirectory for overwrite protection and product level id subfolder""" pysiral_config = ConfigInfo() basedir = pysiral_config.local_machine.product_repository if not isinstance(self.subdirectory, list): basedir = os.path.join(basedir, self.subdirectory) else: basedir = os.path.join(basedir, *self.subdirectory) if self.overwrite_protection: basedir = os.path.join(basedir, self.now_directory) basedir = os.path.join(basedir, self.product_level_subfolder) self._set_basedir(basedir)
def __init__(self): super(L1bPreProcJob, self).__init__(self.__class__.__name__) # Save pointer to pysiral configuration self.pysiral_config = ConfigInfo() # Initialize the time range and set to monthly per default self.time_range = None # Error Status self.error = ErrorStatus() # Initialize job parameter self.options = L1bPreProcJobOptions() # List for iterations (currently only month-wise) self.iterations = []
def __init__(self, product_def, auxclass_handler=None): """ Setup of the Level-2 Processor """ super(Level2Processor, self).__init__(self.__class__.__name__) # Error Status Handler self.error = ErrorStatus(caller_id=self.__class__.__name__) # Level-2 Algorithm Definition # NOTE: This object should ony be called through the property self.l2def self._l2def = product_def.l2def # Auxiliary Data Handler # NOTE: retrieves and initializes the auxdata classes based on the l2 processor definition config file if auxclass_handler is None: auxclass_handler = DefaultAuxdataClassHandler() self._auxclass_handler = auxclass_handler # This variable will contain a list with the auxiliary data handlers self._registered_auxdata_handlers = [] self._auxhandlers = {} # Output_handler (can be one or many) self._output_handler = product_def.output_handler # List of Level-2 (processed) orbit segments self._orbit = deque() # List of Level-1b input files self._l1b_files = [] # pysiral config self._config = ConfigInfo() # Processor Initialization Flag self._initialized = False # Processor summary report self.report = L2ProcessorReport() # Initialize the class self._initialize_processor()
def __init__(self, name): # Enable logging capability (self.log) super(L1bPreProc, self).__init__(name) # Error handler self.error = ErrorStatus() # Job definition ( class L1bPreProcJob) self._jobdef = None # Mission Options self._mdef = None # List of l1b input files # Needs to be filled by the mission specific classes self._l1b_file_list = [] # pysiral configuration self._pysiral_config = ConfigInfo()
def MaskSourceFile(mask_name, mask_cfg): """ Wrapper method for different mask source file classes """ error = ErrorStatus(caller_id="MaskSourceFile") # Get the full mask filename pysiral_cfg = ConfigInfo() try: mask_dir = pysiral_cfg.local_machine.auxdata_repository.mask[mask_name] except KeyError: msg = "path to mask %s not in local_machine_def.yaml" % mask_name error.add_error("missing-lmd-def", msg) error.raise_on_error() # Return the Dataset class try: return globals()[mask_cfg.pyclass_name](mask_dir, mask_name, mask_cfg) except KeyError: msg = "pysiral.mask.%s not implemented" % str(mask_cfg.pyclass_name) error.add_error("missing-mask-class", msg) error.raise_on_error()
def mask_filepath(self): # Get config info pysiral_cfg = ConfigInfo() # Get the path to the mask file # (needs to be in local_machine_def.yaml) mask_dir = pysiral_cfg.local_machine.auxdata_repository.mask try: mask_dir = mask_dir[self.mask_name] except KeyError: msg = "cannot find mask entry [%s] in local_machine_def.yaml" self.error.add_error("lmd-error", msg % self.mask_name) return None mask_filename = "%s_%s.nc" % (self.mask_name, self.grid_id) filepath = os.path.join(mask_dir, mask_filename) if not os.path.isfile(filepath): msg = "cannot find mask file: %s" % filepath self.error.add_error("io-error", msg) return None return filepath
def __init__(self): super(Level3ProcArgParser, self).__init__(self.__class__.__name__) self.error = ErrorStatus() self.pysiral_config = ConfigInfo() self._args = None
class Level3ProcArgParser(DefaultLoggingClass): def __init__(self): super(Level3ProcArgParser, self).__init__(self.__class__.__name__) self.error = ErrorStatus() self.pysiral_config = ConfigInfo() self._args = None def parse_command_line_arguments(self): # use python module argparse to parse the command line arguments # (first validation of required options and data types) self._args = self.parser.parse_args() # Add addtional check to make sure either `l1b-files` or # `start ` and `stop` are set # l1b_file_preset_is_set = self._args.l1b_files_preset is not None # start_and_stop_is_set = self._args.start_date is not None and \ # self._args.stop_date is not None # # if l1b_file_preset_is_set and start_and_stop_is_set: # self.parser.error("-start & -stop and -l1b-files are exclusive") # # if not l1b_file_preset_is_set and not start_and_stop_is_set: # self.parser.error("either -start & -stop or -l1b-files required") def critical_prompt_confirmation(self): # Any confirmation prompts can be overriden by --no-critical-prompt no_prompt = self._args.no_critical_prompt # if --remove_old is set, all previous l1bdata files will be # erased for all month if self._args.remove_old and not no_prompt: message = "You have selected to remove all previous " + \ "l3 files for the requested period\n" + \ "(Note: use --no-critical-prompt to skip confirmation)\n" + \ "Enter \"YES\" to confirm and continue: " result = raw_input(message) if result != "YES": sys.exit(1) @property def parser(self): # XXX: Move back to caller # Take the command line options from default settings # -> see config module for data types, destination variables, etc. clargs = DefaultCommandLineArguments() # List of command line option required for pre-processor # (argname, argtype (see config module), destination, required flag) options = [("-l2i-product-dir", "l2i-product-dir", "l2i_basedir", True), ("-l3-settings", "l3-settings", "l3_settings", False), ("-l3-griddef", "l3-griddef", "l3_griddef", True), ("-l3-output", "l3-output", "l3_output", True), ("-start", "date", "start_date", True), ("-stop", "date", "stop_date", True), ("-period", "period", "period", False), ("-doi", "doi", "doi", False), ("-data-record-type", "data_record_type", "data_record_type", False), ("--remove-old", "remove-old", "remove_old", False), ("--no-critical-prompt", "no-critical-prompt", "no_critical_prompt", False)] # create the parser parser = argparse.ArgumentParser() for option in options: argname, argtype, destination, required = option argparse_dict = clargs.get_argparse_dict(argtype, destination, required) parser.add_argument(argname, **argparse_dict) return parser @property def arg_dict(self): """ Return the arguments as dictionary """ return self._args.__dict__ @property def start(self): return self._args.start_date @property def stop(self): return self._args.stop_date @property def period(self): return self._args.period @property def doi(self): return self._args.doi @property def data_record_type(self): return self._args.data_record_type @property def l2i_product_directory(self): return os.path.join(self.l3_product_basedir, "l2i") @property def l3_settings_file(self): l3_settings = self._args.l3_settings filename = self.pysiral_config.get_settings_file( "proc", "l3", l3_settings) if filename is None: msg = "Invalid l3 settings filename or id: %s\n" % l3_settings msg = msg + " \nRecognized Level-3 processor setting ids:\n" for l3_settings_id in self.pysiral_config.get_setting_ids( "proc", "l3"): msg = msg + " " + l3_settings_id + "\n" self.error.add_error("invalid-l3-settings", msg) self.error.raise_on_error() else: return filename @property def l3_griddef(self): l3_griddef = self._args.l3_griddef filename = self.pysiral_config.get_settings_file( "grid", None, l3_griddef) if filename is None: msg = "Invalid griddef filename or id: %s\n" % l3_griddef msg = msg + " Recognized grid definition ids:\n" for griddef_id in self.pysiral_config.get_setting_ids("griddef"): msg = msg + " - " + griddef_id + "\n" self.error.add_error("invalid-griddef", msg) self.error.raise_on_error() else: return filename @property def l3_output_file(self): l3_output = self._args.l3_output filename = self.pysiral_config.get_settings_file( "output", "l3", l3_output) if filename is None: msg = "Invalid output definition filename or id: %s\n" % l3_output msg = msg + " Recognized output definition ids:\n" for output_id in self.pysiral_config.get_setting_ids( "output", "l3"): msg = msg + " - " + output_id + "\n" self.error.add_error("invalid-outputdef", msg) self.error.raise_on_error() else: return filename @property def l3_product_basedir(self): """ Returns the base directory (one level below l2i) """ # 1. Clean up the path product_basedir = os.path.abspath(self._args.l2i_basedir) dirs = os.path.split(product_basedir) if dirs[1] == "l2i": return dirs[0] else: return product_basedir @property def remove_old(self): return self._args.remove_old and not self._args.overwrite_protection
def __init__(self): super(DefaultAuxdataClassHandler, self).__init__(self.__class__.__name__) self.pysiral_config = ConfigInfo() self.error = ErrorStatus(caller_id=self.__class__.__name__)
class Level2ProcArgParser(DefaultLoggingClass): def __init__(self): super(Level2ProcArgParser, self).__init__(self.__class__.__name__) self.error = ErrorStatus() self.pysiral_config = ConfigInfo() self._args = None def parse_command_line_arguments(self): # use python module argparse to parse the command line arguments # (first validation of required options and data types) self._args = self.parser.parse_args() # Add additional check to make sure either `l1b-files` or # `start ` and `stop` are set l1b_file_preset_is_set = self._args.l1b_files_preset is not None start_and_stop_is_set = self._args.start_date is not None and \ self._args.stop_date is not None if l1b_file_preset_is_set and start_and_stop_is_set: self.parser.error("-start & -stop and -l1b-files are exclusive") if not l1b_file_preset_is_set and not start_and_stop_is_set: self.parser.error("either -start & -stop or -l1b-files required") def critical_prompt_confirmation(self): # Any confirmation prompts can be overridden by --no-critical-prompt no_prompt = self._args.no_critical_prompt # if --remove_old is set, all previous l1bdata files will be # erased for all month if self._args.remove_old and not no_prompt: message = "You have selected to remove all previous " + \ "l2 files for the requested period\n" + \ "(Note: use --no-critical-prompt to skip confirmation)\n" + \ "Enter \"YES\" to confirm and continue: " result = raw_input(message) if result != "YES": sys.exit(1) @property def parser(self): # XXX: Move back to caller # Take the command line options from default settings # -> see config module for data types, destination variables, etc. clargs = DefaultCommandLineArguments() # List of command line option required for pre-processor # (argname, argtype (see config module), destination, required flag) options = [("-l2-settings", "l2-settings", "l2_settings", True), ("-run-tag", "run-tag", "run_tag", False), ("-start", "date", "start_date", False), ("-stop", "date", "stop_date", False), ("-l1b-files", "l1b_files", "l1b_files_preset", False), ("-exclude-month", "exclude-month", "exclude_month", False), ("-input-version", "input-version", "input_version", False), ("-l2-output", "l2-output", "l2_output", False), ("--remove-old", "remove-old", "remove_old", False), ("--no-critical-prompt", "no-critical-prompt", "no_critical_prompt", False), ("--no-overwrite-protection", "no-overwrite-protection", "overwrite_protection", False), ("--overwrite-protection", "overwrite-protection", "overwrite_protection", False)] # create the parser parser = argparse.ArgumentParser() for option in options: argname, argtype, destination, required = option argparse_dict = clargs.get_argparse_dict(argtype, destination, required) parser.add_argument(argname, **argparse_dict) parser.set_defaults(overwrite_protection=True) return parser @property def arg_dict(self): """ Return the arguments as dictionary """ return self._args.__dict__ @property def start(self): return self._args.start_date @property def stop(self): return self._args.stop_date @property def run_tag(self): """ run_tag is a str or relative path that determines the output directory for the Level-2 processor. If the -run-tag option is not specified, the output directory will be the `product_repository` specification in `local_machine_def` with the l2 settings file basename as subfolder. One can however specify a custom string, or a relative path, with subfolders defined by `\` or `/`, e.g. Examples: -run-tag cs2awi_v2p0_nrt -run-tag c3s/cdr/cryosat2/v1p0/nh """ # Get from command line arguments (default: None) run_tag = self._args.run_tag # If argument is empty use the basename of the l2 settings file if run_tag is None: run_tag = self._args.l2_settings # Settings file may be specified as full path and not just the id if os.path.isfile(run_tag): run_tag = file_basename(run_tag) # split the run-tag on potential path separators run_tag = re.split(r'[\\|/]', run_tag) return run_tag @property def exclude_month(self): return self._args.exclude_month @property def overwrite_protection(self): return self._args.overwrite_protection @property def l2_settings_file(self): l2_settings = self._args.l2_settings filename = self.pysiral_config.get_settings_file( "proc", "l2", l2_settings) if filename is None: msg = "Invalid l2 settings filename or id: %s\n" % l2_settings msg = msg + " \nRecognized Level-2 processor setting ids:\n" for l2_settings_id in self.pysiral_config.get_setting_ids( "proc", "l2"): msg = msg + " " + l2_settings_id + "\n" self.error.add_error("invalid-l2-settings", msg) self.error.raise_on_error() else: return filename @property def l1b_version(self): return self._args.input_version @property def l1b_predef_files(self): l1b_files = glob.glob(self._args.l1b_files_preset) return l1b_files @property def l2_output(self): l2_output = self._args.l2_output filename = self.pysiral_config.get_settings_file( "output", "l2i", l2_output) if filename is None: msg = "Invalid l2 outputdef filename or id: %s\n" % l2_output msg = msg + " \nRecognized Level-2 output definitions ids:\n" l2_output_ids = self.pysiral_config.get_setting_ids( "output", "l2i") for l2_output_id in l2_output_ids: msg = msg + " - " + l2_output_id + "\n" self.error.add_error("invalid-l2-outputdef", msg) self.error.raise_on_error() else: return filename @property def is_time_range_request(self): return self._args.l1b_files_preset is None @property def remove_old(self): return self._args.remove_old and not self._args.overwrite_protection
def default_output_def_filename(self): pysiral_config = ConfigInfo() local_settings_path = pysiral_config.pysiral_local_path return os.path.join(local_settings_path, *self.default_file_location)
class Level2PreProcArgParser(DefaultLoggingClass): def __init__(self): super(Level2PreProcArgParser, self).__init__(self.__class__.__name__) self.error = ErrorStatus() self.pysiral_config = ConfigInfo() self._args = None def parse_command_line_arguments(self): # use python module argparse to parse the command line arguments # (first validation of required options and data types) self._args = self.parser.parse_args() def critical_prompt_confirmation(self): # Any confirmation prompts can be overriden by --no-critical-prompt no_prompt = self._args.no_critical_prompt # if --remove_old is set, all previous l1bdata files will be # erased for all month if self._args.remove_old and not no_prompt: message = "You have selected to remove all previous " + \ "l2p files for the requested period\n" + \ "(Note: use --no-critical-prompt to skip confirmation)\n" + \ "Enter \"YES\" to confirm and continue: " result = raw_input(message) if result != "YES": sys.exit(1) @property def parser(self): # XXX: Move back to caller # Take the command line options from default settings # -> see config module for data types, destination variables, etc. clargs = DefaultCommandLineArguments() # List of command line option required for pre-processor # (argname, argtype (see config module), destination, required flag) options = [ ("-start", "date", "start_date", False), ("-stop", "date", "stop_date", False), ("-l2i-product-dir", "l2i-product-dir", "l2i_product_dir", True), ("-l2p-output", "l2p-output", "l2p_output", False), ("-exclude-month", "exclude-month", "exclude_month", False), ("-doi", "doi", "doi", False), ("--remove-old", "remove-old", "remove_old", False), ("--no-critical-prompt", "no-critical-prompt", "no_critical_prompt", False), ("--no-overwrite-protection", "no-overwrite-protection", "overwrite_protection", False), ("--overwrite-protection", "overwrite-protection", "overwrite_protection", False)] # create the parser parser = argparse.ArgumentParser() for option in options: argname, argtype, destination, required = option argparse_dict = clargs.get_argparse_dict( argtype, destination, required) parser.add_argument(argname, **argparse_dict) parser.set_defaults(overwrite_protection=True) return parser @property def arg_dict(self): """ Return the arguments as dictionary """ return self._args.__dict__ @property def start(self): return self._args.start_date @property def stop(self): return self._args.stop_date @property def exclude_month(self): return self._args.exclude_month @property def doi(self): return self._args.doi @property def overwrite_protection(self): return self._args.overwrite_protection @property def l2i_product_dir(self): l2i_product_dir = self._args.l2i_product_dir if os.path.isdir(l2i_product_dir): return os.path.normpath(l2i_product_dir) else: msg = "Invalid l2i product dir: %s" % str(l2i_product_dir) self.error.add_error("invalid-l2i-product-dir", msg) self.error.raise_on_error() @property def l2p_output(self): l2p_output = self._args.l2p_output filename = self.pysiral_config.get_settings_file("output", "l2p", l2p_output) if filename is None: msg = "Invalid l2p outputdef filename or id: %s\n" % l2p_output msg = msg + " \nRecognized Level-2 output definitions ids:\n" l2p_output_ids = self.pysiral_config.get_setting_ids("output", "l2p") for l2p_output_id in l2p_output_ids: msg = msg + " - " + l2p_output_id+"\n" self.error.add_error("invalid-l2p-outputdef", msg) self.error.raise_on_error() else: return filename @property def remove_old(self): return self._args.remove_old and not self._args.overwrite_protection