def get_files_from_time_range(self, time_range: DatePeriod) -> List[str]: """ Query l1p files for a a given time range. :param time_range: a dateperiods.DatePeriod instance :return: """ # Validate time_range (needs to be of type DatePeriod) if not isinstance(time_range, DatePeriod): error = ErrorStatus() msg = "Invalid type of time_range, required: dateperiods.DatePeriod, was %s" % (type(time_range)) error.add_error("invalid-timerange-type", msg) error.raise_on_error() # 1) get list of all files for monthly folders yyyy, mm = "%04g" % time_range.tcs.year, "%02g" % time_range.tcs.month directory = Path(self.l1p_base_dir) if self._file_version is not None: directory = directory / self._file_version directory = directory / self._hemisphere / yyyy / mm all_l1p_files = sorted(list(directory.rglob("*.nc"))) # 3) Check if files are in requested time range # This serves two purposes: a) filter out files with timestamps that do # not belong in the directory. b) get a subset if required l1p_filepaths = [l1p_file for l1p_file in all_l1p_files if self.l1p_in_trange(l1p_file, time_range)] # Save last search directory self._last_directory = directory # Done return l1p_filepaths
def get_local_l1bdata_files(mission_id, time_range, hemisphere, config=None, version="default", allow_multiple_baselines=True): """ Returns a list of l1bdata files for a given mission, hemisphere, version and time range XXX: Note: this function will slowly replace `get_l1bdata_files`, which is limited to full month """ # parse config data (if not provided) if config is None or not isinstance(config, psrlcfg): config = psrlcfg # Validate time_range (needs to be of type dateperiods.DatePeriod) if not isinstance(time_range, DatePeriod): error = ErrorStatus() msg = "Invalid type of time_range, required: dateperiods.DatePeriod, was %s" % ( type(time_range)) error.add_error("invalid-timerange-type", msg) error.raise_on_error() # 1) get list of all files for monthly folders yyyy, mm = "%04g" % time_range.tcs.year, "%02g" % time_range.tcs.month repo_branch = config.local_machine.l1b_repository[mission_id][version] directory = Path(repo_branch["l1p"]) / hemisphere / yyyy / mm all_l1bdata_files = sorted(directory.glob("*.nc")) # 3) Check if files are in requested time range # This serves two purporses: a) filter out files with timestamps that do # not belong in the directory. b) get a subset if required l1bdata_files_checked = [ l1bdata_file for l1bdata_file in all_l1bdata_files if l1bdata_in_trange(l1bdata_file, time_range) ] # Done return list (empty or not) return l1bdata_files_checked, directory
def MaskSourceFile(mask_name, mask_cfg): """ Wrapper method for different mask source file classes """ error = ErrorStatus(caller_id="MaskSourceFile") try: mask_dir = psrlcfg.local_machine.auxdata_repository.mask[mask_name] except KeyError: mask_dir = None msg = "path to mask %s not in local_machine_def.yaml" % mask_name error.add_error("missing-lmd-def", msg) error.raise_on_error() # Return the Dataset class try: return globals()[mask_cfg.pyclass_name](mask_dir, mask_name, mask_cfg) except KeyError: msg = "pysiral.mask.%s not implemented" % str(mask_cfg.pyclass_name) error.add_error("missing-mask-class", msg) error.raise_on_error()
class MaskSourceBase(DefaultLoggingClass): """ Parent class for various source masks. Main functionality is to create gridded mask netCDF for for level-3 grid definitions """ def __init__(self, mask_dir, mask_name, cfg): super(MaskSourceBase, self).__init__(self.__class__.__name__) self.error = ErrorStatus() self._cfg = cfg self._mask_dir = mask_dir self._mask_name = mask_name self._mask = None self._area_def = None self._post_flipud = False def set_mask(self, mask, area_def): """ Set grid definition for the mask source grid using pyresample. The argument area_def needs to have the attributes needed as arguments for pyresample.geometry.AreaDefinition or be of the pyresampe types (geometry.AreaDefinition, geometry.GridDefinition) """ # Set the Mask self._mask = mask # Set the area definition pyresample_instances = (geometry.AreaDefinition, geometry.GridDefinition) if isinstance(area_def, pyresample_instances): self._area_def = area_def else: self._area_def = geometry.AreaDefinition( area_def.area_id, area_def.name, area_def.proj_id, dict(area_def.proj_dict), area_def.x_size, area_def.y_size, area_def.area_extent) def export_l3_mask(self, griddef, nc_filepath=None): """ Create a gridded mask product in pysiral compliant filenaming. The argument griddef is needs to be a pysiral.grid.GridDefinition instance """ # Get the area definition for the grid if not isinstance(griddef, GridDefinition): msg = "griddef needs to be of type pysiral.grid.GridDefinition" self.error.add_error("value-error", msg) # Resample the mask if self.cfg.pyresample_method == "ImageContainerNearest": resample = image.ImageContainerNearest(self.source_mask, self.source_area_def, **self.cfg.pyresample_keyw) resample_result = resample.resample(griddef.pyresample_area_def) target_mask = resample_result.image_data elif self.cfg.pyresample_method == "resample_gauss": result, stddev, count = kd_tree.resample_gauss( self.source_area_def, self.source_mask, griddef.pyresample_area_def, with_uncert=True, **self.cfg.pyresample_keyw) target_mask = result else: msg = "Unrecognized opt pyresample_method: %s need to be %s" % ( str(self.cfg.pyresample_method), "(ImageContainerNearest, resample_gauss)") self.error.add_error("invalid-pr-method", msg) self.error.add_error() # pyresample may use masked arrays -> set nan's to missing data try: target_mask[np.where(target_mask.mask)] = np.nan except AttributeError: pass if "post_processing" in self.cfg: pp_method = getattr(self, self.cfg.post_processing) target_mask = pp_method(target_mask, griddef) # Write the mask to a netCDF file # (the filename will be automatically generated if not specifically # passed to this method if nc_filepath is None: nc_filename = "%s_%s.nc" % (self.mask_name, griddef.grid_id) nc_filepath = os.path.join(self.mask_dir, nc_filename) self.log.info("Export mask file: %s" % nc_filepath) self._write_netcdf(nc_filepath, griddef, target_mask) def _write_netcdf(self, nc_filepath, griddef, mask): """ Write a netCDF file with the mask in the target grid projections""" # Get metadata shape = np.shape(mask) dimdict = OrderedDict([("x", shape[0]), ("y", shape[1])]) # Get longitude/latitude from grid definition lons, lats = griddef.get_grid_coordinates() # Open the file try: rootgrp = Dataset(nc_filepath, "w") except RuntimeError: msg = "Unable to create netCDF file: %s" % nc_filepath self.error.add_error("nc-runtime", msg) self.error.raise_on_error() # Write Global Attributes rootgrp.setncattr("title", "Mask file for pysiral Level3 Processor") rootgrp.setncattr("mask_id", self.mask_name) rootgrp.setncattr("comment", self.cfg.comment) rootgrp.setncattr("description", self.cfg.label) rootgrp.setncattr("grid_id", griddef.grid_id) # Write dimensions dims = dimdict.keys() for key in dims: rootgrp.createDimension(key, dimdict[key]) # Write Variables dim = tuple(dims[0:len(mask.shape)]) dtype_str = mask.dtype.str varmask = rootgrp.createVariable("mask", dtype_str, dim, zlib=True) varmask[:] = mask dtype_str = lons.dtype.str varlon = rootgrp.createVariable("longitude", dtype_str, dim, zlib=True) setattr(varlon, "long_name", "longitude of grid cell center") setattr(varlon, "standard_name", "longitude") setattr(varlon, "units", "degrees") setattr(varlon, "scale_factor", 1.0) setattr(varlon, "add_offset", 0.0) varlon[:] = lons varlat = rootgrp.createVariable("latitude", dtype_str, dim, zlib=True) setattr(varlat, "long_name", "latitude of grid cell center") setattr(varlat, "standard_name", "latitude") setattr(varlat, "units", "degrees") setattr(varlat, "scale_factor", 1.0) setattr(varlat, "add_offset", 0.0) varlat[:] = lats # Close the file rootgrp.close() @property def cfg(self): return self._cfg @property def mask_name(self): return str(self._mask_name) @property def mask_dir(self): return str(self._mask_dir) @property def source_mask(self): return self._mask @property def source_area_def(self): return self._area_def
class Level2PContainer(DefaultLoggingClass): def __init__(self, period): super(Level2PContainer, self).__init__(self.__class__.__name__) self.error = ErrorStatus() self._period = period self._l2i_stack = [] def append_l2i(self, l2i): self._l2i_stack.append(l2i) def get_merged_l2(self): """ Returns a Level2Data object with data from all l2i objects """ # Merge the parameter data = self._get_merged_data(valid_mask="sea_ice_freeboard") # There are rare occasion, where no valid freeboard data is found for an entire day if len(data["longitude"]) == 0: return None # Set up a timeorbit group timeorbit = Level2iTimeOrbit() timeorbit.from_l2i_stack(data) # Use the first l2i object in stack to retrieve metadata l2i = self._l2i_stack[0] # Set up a metadata container metadata = Level2iMetadata() metadata.set_attribute("n_records", len(timeorbit.time)) metadata.set_attribute("start_time", timeorbit.time[0]) metadata.set_attribute("stop_time", timeorbit.time[-1]) # XXX: Very ugly, but required due to a non-standard use of # region_subset_set (originally idea to crop regions in # Level-2 Processor) region_name = "north" if np.nanmean(data["latitude"]) > 0 else "south" metadata.subset_region_name = region_name # Retrieve the following constant attributes from the first # l2i object in the stack info = self.l2i_stack[0].info # Old notation (for backward compatibility) # TODO: This will soon be obsolete mission_id = None if hasattr(info, "mission_id"): mission_id = info.mission_id metadata.source_auxdata_sic = l2i.info.source_sic metadata.source_auxdata_snow = l2i.info.source_snow metadata.source_auxdata_sitype = l2i.info.source_sitype metadata.source_auxdata_mss = l2i.info.source_mss # New (fall 2017) pysiral product notation if hasattr(info, "source_mission_id"): mission_id = info.source_mission_id # Transfer auxdata information metadata.source_auxdata_sic = l2i.info.source_auxdata_sic metadata.source_auxdata_snow = l2i.info.source_auxdata_snow metadata.source_auxdata_sitype = l2i.info.source_auxdata_sitype metadata.source_auxdata_mss = l2i.info.source_auxdata_mss # Conversion of l2i to CF/ACDD conventions (Fall 2021) if hasattr(info, "platform"): mission_id = psrlcfg.platforms.get_platform_id(info.platform) if mission_id is None: self.error.add_error( "unknown-platform", "Cannot determine platform name from source l2i stack") self.error.raise_on_error() if hasattr(l2i.info, "source_timeliness"): metadata.timeliness = l2i.info.source_timeliness if hasattr(l2i.info, "data_record_type"): metadata.timeliness = l2i.info.data_record_type metadata.set_attribute("mission", mission_id) mission_sensor = psrlcfg.platforms.get_sensor(mission_id) metadata.set_attribute("mission_sensor", mission_sensor) # Construct level-2 object l2 = Level2Data(metadata, timeorbit, period=self._period) # Transfer the level-2 data items # 1. Get the list of parameters # (assuming all l2i files share the same) parameter_list_all = l2i.parameter_list # 2. Exclude variables that end with `_uncertainty` parameter_list = [ p for p in parameter_list_all if not re.search("_uncertainty", p) ] # 3. Remove parameters from the timeorbit group, surface type & # orbit id. This will be added to level 2 object by other means # or do not make sense (surface type for valid freeboard will # always be sea ice) for parameter_name in [ "timestamp", "time", "longitude", "latitude", "surface_type" ]: try: parameter_list.remove(parameter_name) except ValueError: pass # 4. Set parameters for parameter_name in parameter_list: # Get the parameter value = data[parameter_name] # Test if uncertainty exists uncertainty_name = parameter_name + "_uncertainty" if uncertainty_name in parameter_list_all: uncertainty = data[uncertainty_name] else: uncertainty = None # Add to l2 object l2.set_parameter(parameter_name, value, uncertainty=uncertainty) return l2 def _get_merged_data(self, valid_mask=None): """ Returns a dict with merged data groups for all parameters in the l2i file (assumed to be identical for all files in the stack """ parameter_list = self.l2i_stack[0].parameter_list data = self._get_empty_data_group(parameter_list) for l2i in self.l2i_stack: if valid_mask is not None: valid_mask_parameter = getattr(l2i, valid_mask) is_valid = np.where(np.isfinite(valid_mask_parameter))[0] else: is_valid = np.arange(l2i.n_records) for parameter in parameter_list: stack_data = getattr(l2i, parameter) stack_data = stack_data[is_valid] data[parameter] = np.append(data[parameter], stack_data) return data @staticmethod def _get_empty_data_group(parameter_list): data = {} for parameter_name in parameter_list: data[parameter_name] = np.array([], dtype=np.float32) return data @property def l2i_stack(self): return self._l2i_stack @property def n_l2i_objects(self): return len(self.l2i_stack) @property def period(self): return self._period
class L1bPreProcJob(DefaultLoggingClass): """ Container for the definition and handling of a pre-processor job """ def __init__(self): super(L1bPreProcJob, self).__init__(self.__class__.__name__) # Save pointer to pysiral configuration self.pysiral_config = ConfigInfo() # Initialize the time range and set to monthly per default self.time_range = None # Error Status self.error = ErrorStatus() # Initialize job parameter self.options = L1bPreProcJobOptions() # List for iterations (currently only month-wise) self.iterations = [] def generate_preprocessor_iterations(self): """ Break the requested time range into monthly iterations """ # The input data is organized in folder per month, therefore # the processing period is set accordingly self.time_range.set_period("monthly") self.log.info("Pre-processor Period is monthly") self.time_range.set_exclude_month(self.options.exclude_month) self.log.info("Excluding month: %s" % str(self.options.exclude_month)) self.iterations = self.time_range.iterations self.log.info("Number of iterations: %g" % len(self.iterations)) def process_requested_time_range(self): """ Verify time range with mission data availability and create datetime objects """ config = self.pysiral_config mission_info = config.get_mission_info(self.options.mission_id) # Set and validate the time range start_date, stop_date = self.options.start_date, self.options.stop_date self.time_range = TimeRangeRequest(start_date, stop_date) # Check if any errors in definitions self.time_range.error.raise_on_error() self.log.info("Requested time range is: %s" % self.time_range.label) # Clip time range to mission time range is_clipped = self.time_range.clip_to_range( mission_info.data_period.start, mission_info.data_period.stop) if is_clipped: self.log.info("Clipped to mission time range: %s till %s" % ( mission_info.data_period.start, mission_info.data_period.stop)) # Check if range is still valid self.time_range.raise_if_empty() def get_mission_preprocessor(self): """ Return the mission specific pre-processor class """ from pysiral.cryosat2.preproc import CryoSat2PreProc from pysiral.envisat.preproc import EnvisatPreProc from pysiral.ers.preproc import ERSPreProc from pysiral.sentinel3.preproc import Sentinel3PreProc from pysiral.icesat.preproc import ICESatPreProc if self.mission_id == "cryosat2": return CryoSat2PreProc elif self.mission_id == "envisat": return EnvisatPreProc elif self.mission_id == "ers2": return ERSPreProc elif self.mission_id == "sentinel3a": return Sentinel3PreProc elif self.mission_id == "icesat": return ICESatPreProc else: error_code = self.__class__.__name__+" (01)" error_message = "Invalid mission_id: %s" % self.mission_id self.error.add_error(error_code, error_message) self.error.raise_on_error() @property def mission_id(self): return self.options.mission_id @property def hemisphere(self): return self.options.hemisphere @property def input_version(self): return self.options.input_version @property def remove_old(self): return self.options.remove_old