def get_files_from_time_range(self, time_range: DatePeriod) -> List[str]:
"""
Query l1p files for a a given time range.
:param time_range: a dateperiods.DatePeriod instance
:return:
"""
# Validate time_range (needs to be of type DatePeriod)
if not isinstance(time_range, DatePeriod):
error = ErrorStatus()
msg = "Invalid type of time_range, required: dateperiods.DatePeriod, was %s" % (type(time_range))
error.add_error("invalid-timerange-type", msg)
error.raise_on_error()
# 1) get list of all files for monthly folders
yyyy, mm = "%04g" % time_range.tcs.year, "%02g" % time_range.tcs.month
directory = Path(self.l1p_base_dir)
if self._file_version is not None:
directory = directory / self._file_version
directory = directory / self._hemisphere / yyyy / mm
all_l1p_files = sorted(list(directory.rglob("*.nc")))
# 3) Check if files are in requested time range
# This serves two purposes: a) filter out files with timestamps that do
# not belong in the directory. b) get a subset if required
l1p_filepaths = [l1p_file for l1p_file in all_l1p_files if self.l1p_in_trange(l1p_file, time_range)]
# Save last search directory
self._last_directory = directory
# Done
return l1p_filepaths
def get_local_l1bdata_files(mission_id,
time_range,
hemisphere,
config=None,
version="default",
allow_multiple_baselines=True):
"""
Returns a list of l1bdata files for a given mission, hemisphere, version
and time range
XXX: Note: this function will slowly replace `get_l1bdata_files`, which
is limited to full month
"""
# parse config data (if not provided)
if config is None or not isinstance(config, psrlcfg):
config = psrlcfg
# Validate time_range (needs to be of type dateperiods.DatePeriod)
if not isinstance(time_range, DatePeriod):
error = ErrorStatus()
msg = "Invalid type of time_range, required: dateperiods.DatePeriod, was %s" % (
type(time_range))
error.add_error("invalid-timerange-type", msg)
error.raise_on_error()
# 1) get list of all files for monthly folders
yyyy, mm = "%04g" % time_range.tcs.year, "%02g" % time_range.tcs.month
repo_branch = config.local_machine.l1b_repository[mission_id][version]
directory = Path(repo_branch["l1p"]) / hemisphere / yyyy / mm
all_l1bdata_files = sorted(directory.glob("*.nc"))
# 3) Check if files are in requested time range
# This serves two purporses: a) filter out files with timestamps that do
# not belong in the directory. b) get a subset if required
l1bdata_files_checked = [
l1bdata_file for l1bdata_file in all_l1bdata_files
if l1bdata_in_trange(l1bdata_file, time_range)
]
# Done return list (empty or not)
return l1bdata_files_checked, directory
def MaskSourceFile(mask_name, mask_cfg):
""" Wrapper method for different mask source file classes """
error = ErrorStatus(caller_id="MaskSourceFile")
try:
mask_dir = psrlcfg.local_machine.auxdata_repository.mask[mask_name]
except KeyError:
mask_dir = None
msg = "path to mask %s not in local_machine_def.yaml" % mask_name
error.add_error("missing-lmd-def", msg)
error.raise_on_error()
# Return the Dataset class
try:
return globals()[mask_cfg.pyclass_name](mask_dir, mask_name, mask_cfg)
except KeyError:
msg = "pysiral.mask.%s not implemented" % str(mask_cfg.pyclass_name)
error.add_error("missing-mask-class", msg)
error.raise_on_error()
class MaskSourceBase(DefaultLoggingClass):
""" Parent class for various source masks. Main functionality is to
create gridded mask netCDF for for level-3 grid definitions """
def __init__(self, mask_dir, mask_name, cfg):
super(MaskSourceBase, self).__init__(self.__class__.__name__)
self.error = ErrorStatus()
self._cfg = cfg
self._mask_dir = mask_dir
self._mask_name = mask_name
self._mask = None
self._area_def = None
self._post_flipud = False
def set_mask(self, mask, area_def):
""" Set grid definition for the mask source grid using pyresample.
The argument area_def needs to have the attributes needed as arguments
for pyresample.geometry.AreaDefinition or be of the pyresampe types
(geometry.AreaDefinition, geometry.GridDefinition) """
# Set the Mask
self._mask = mask
# Set the area definition
pyresample_instances = (geometry.AreaDefinition,
geometry.GridDefinition)
if isinstance(area_def, pyresample_instances):
self._area_def = area_def
else:
self._area_def = geometry.AreaDefinition(
area_def.area_id, area_def.name, area_def.proj_id,
dict(area_def.proj_dict), area_def.x_size, area_def.y_size,
area_def.area_extent)
def export_l3_mask(self, griddef, nc_filepath=None):
""" Create a gridded mask product in pysiral compliant filenaming.
The argument griddef is needs to be a pysiral.grid.GridDefinition
instance """
# Get the area definition for the grid
if not isinstance(griddef, GridDefinition):
msg = "griddef needs to be of type pysiral.grid.GridDefinition"
self.error.add_error("value-error", msg)
# Resample the mask
if self.cfg.pyresample_method == "ImageContainerNearest":
resample = image.ImageContainerNearest(self.source_mask,
self.source_area_def,
**self.cfg.pyresample_keyw)
resample_result = resample.resample(griddef.pyresample_area_def)
target_mask = resample_result.image_data
elif self.cfg.pyresample_method == "resample_gauss":
result, stddev, count = kd_tree.resample_gauss(
self.source_area_def,
self.source_mask,
griddef.pyresample_area_def,
with_uncert=True,
**self.cfg.pyresample_keyw)
target_mask = result
else:
msg = "Unrecognized opt pyresample_method: %s need to be %s" % (
str(self.cfg.pyresample_method),
"(ImageContainerNearest, resample_gauss)")
self.error.add_error("invalid-pr-method", msg)
self.error.add_error()
# pyresample may use masked arrays -> set nan's to missing data
try:
target_mask[np.where(target_mask.mask)] = np.nan
except AttributeError:
pass
if "post_processing" in self.cfg:
pp_method = getattr(self, self.cfg.post_processing)
target_mask = pp_method(target_mask, griddef)
# Write the mask to a netCDF file
# (the filename will be automatically generated if not specifically
# passed to this method
if nc_filepath is None:
nc_filename = "%s_%s.nc" % (self.mask_name, griddef.grid_id)
nc_filepath = os.path.join(self.mask_dir, nc_filename)
self.log.info("Export mask file: %s" % nc_filepath)
self._write_netcdf(nc_filepath, griddef, target_mask)
def _write_netcdf(self, nc_filepath, griddef, mask):
""" Write a netCDF file with the mask in the target
grid projections"""
# Get metadata
shape = np.shape(mask)
dimdict = OrderedDict([("x", shape[0]), ("y", shape[1])])
# Get longitude/latitude from grid definition
lons, lats = griddef.get_grid_coordinates()
# Open the file
try:
rootgrp = Dataset(nc_filepath, "w")
except RuntimeError:
msg = "Unable to create netCDF file: %s" % nc_filepath
self.error.add_error("nc-runtime", msg)
self.error.raise_on_error()
# Write Global Attributes
rootgrp.setncattr("title", "Mask file for pysiral Level3 Processor")
rootgrp.setncattr("mask_id", self.mask_name)
rootgrp.setncattr("comment", self.cfg.comment)
rootgrp.setncattr("description", self.cfg.label)
rootgrp.setncattr("grid_id", griddef.grid_id)
# Write dimensions
dims = dimdict.keys()
for key in dims:
rootgrp.createDimension(key, dimdict[key])
# Write Variables
dim = tuple(dims[0:len(mask.shape)])
dtype_str = mask.dtype.str
varmask = rootgrp.createVariable("mask", dtype_str, dim, zlib=True)
varmask[:] = mask
dtype_str = lons.dtype.str
varlon = rootgrp.createVariable("longitude", dtype_str, dim, zlib=True)
setattr(varlon, "long_name", "longitude of grid cell center")
setattr(varlon, "standard_name", "longitude")
setattr(varlon, "units", "degrees")
setattr(varlon, "scale_factor", 1.0)
setattr(varlon, "add_offset", 0.0)
varlon[:] = lons
varlat = rootgrp.createVariable("latitude", dtype_str, dim, zlib=True)
setattr(varlat, "long_name", "latitude of grid cell center")
setattr(varlat, "standard_name", "latitude")
setattr(varlat, "units", "degrees")
setattr(varlat, "scale_factor", 1.0)
setattr(varlat, "add_offset", 0.0)
varlat[:] = lats
# Close the file
rootgrp.close()
@property
def cfg(self):
return self._cfg
@property
def mask_name(self):
return str(self._mask_name)
@property
def mask_dir(self):
return str(self._mask_dir)
@property
def source_mask(self):
return self._mask
@property
def source_area_def(self):
return self._area_def
class Level2PContainer(DefaultLoggingClass):
def __init__(self, period):
super(Level2PContainer, self).__init__(self.__class__.__name__)
self.error = ErrorStatus()
self._period = period
self._l2i_stack = []
def append_l2i(self, l2i):
self._l2i_stack.append(l2i)
def get_merged_l2(self):
""" Returns a Level2Data object with data from all l2i objects """
# Merge the parameter
data = self._get_merged_data(valid_mask="sea_ice_freeboard")
# There are rare occasion, where no valid freeboard data is found for an entire day
if len(data["longitude"]) == 0:
return None
# Set up a timeorbit group
timeorbit = Level2iTimeOrbit()
timeorbit.from_l2i_stack(data)
# Use the first l2i object in stack to retrieve metadata
l2i = self._l2i_stack[0]
# Set up a metadata container
metadata = Level2iMetadata()
metadata.set_attribute("n_records", len(timeorbit.time))
metadata.set_attribute("start_time", timeorbit.time[0])
metadata.set_attribute("stop_time", timeorbit.time[-1])
# XXX: Very ugly, but required due to a non-standard use of
# region_subset_set (originally idea to crop regions in
# Level-2 Processor)
region_name = "north" if np.nanmean(data["latitude"]) > 0 else "south"
metadata.subset_region_name = region_name
# Retrieve the following constant attributes from the first
# l2i object in the stack
info = self.l2i_stack[0].info
# Old notation (for backward compatibility)
# TODO: This will soon be obsolete
mission_id = None
if hasattr(info, "mission_id"):
mission_id = info.mission_id
metadata.source_auxdata_sic = l2i.info.source_sic
metadata.source_auxdata_snow = l2i.info.source_snow
metadata.source_auxdata_sitype = l2i.info.source_sitype
metadata.source_auxdata_mss = l2i.info.source_mss
# New (fall 2017) pysiral product notation
if hasattr(info, "source_mission_id"):
mission_id = info.source_mission_id
# Transfer auxdata information
metadata.source_auxdata_sic = l2i.info.source_auxdata_sic
metadata.source_auxdata_snow = l2i.info.source_auxdata_snow
metadata.source_auxdata_sitype = l2i.info.source_auxdata_sitype
metadata.source_auxdata_mss = l2i.info.source_auxdata_mss
# Conversion of l2i to CF/ACDD conventions (Fall 2021)
if hasattr(info, "platform"):
mission_id = psrlcfg.platforms.get_platform_id(info.platform)
if mission_id is None:
self.error.add_error(
"unknown-platform",
"Cannot determine platform name from source l2i stack")
self.error.raise_on_error()
if hasattr(l2i.info, "source_timeliness"):
metadata.timeliness = l2i.info.source_timeliness
if hasattr(l2i.info, "data_record_type"):
metadata.timeliness = l2i.info.data_record_type
metadata.set_attribute("mission", mission_id)
mission_sensor = psrlcfg.platforms.get_sensor(mission_id)
metadata.set_attribute("mission_sensor", mission_sensor)
# Construct level-2 object
l2 = Level2Data(metadata, timeorbit, period=self._period)
# Transfer the level-2 data items
# 1. Get the list of parameters
# (assuming all l2i files share the same)
parameter_list_all = l2i.parameter_list
# 2. Exclude variables that end with `_uncertainty`
parameter_list = [
p for p in parameter_list_all if not re.search("_uncertainty", p)
]
# 3. Remove parameters from the timeorbit group, surface type &
# orbit id. This will be added to level 2 object by other means
# or do not make sense (surface type for valid freeboard will
# always be sea ice)
for parameter_name in [
"timestamp", "time", "longitude", "latitude", "surface_type"
]:
try:
parameter_list.remove(parameter_name)
except ValueError:
pass
# 4. Set parameters
for parameter_name in parameter_list:
# Get the parameter
value = data[parameter_name]
# Test if uncertainty exists
uncertainty_name = parameter_name + "_uncertainty"
if uncertainty_name in parameter_list_all:
uncertainty = data[uncertainty_name]
else:
uncertainty = None
# Add to l2 object
l2.set_parameter(parameter_name, value, uncertainty=uncertainty)
return l2
def _get_merged_data(self, valid_mask=None):
""" Returns a dict with merged data groups for all parameters
in the l2i file (assumed to be identical for all files in the stack
"""
parameter_list = self.l2i_stack[0].parameter_list
data = self._get_empty_data_group(parameter_list)
for l2i in self.l2i_stack:
if valid_mask is not None:
valid_mask_parameter = getattr(l2i, valid_mask)
is_valid = np.where(np.isfinite(valid_mask_parameter))[0]
else:
is_valid = np.arange(l2i.n_records)
for parameter in parameter_list:
stack_data = getattr(l2i, parameter)
stack_data = stack_data[is_valid]
data[parameter] = np.append(data[parameter], stack_data)
return data
@staticmethod
def _get_empty_data_group(parameter_list):
data = {}
for parameter_name in parameter_list:
data[parameter_name] = np.array([], dtype=np.float32)
return data
@property
def l2i_stack(self):
return self._l2i_stack
@property
def n_l2i_objects(self):
return len(self.l2i_stack)
@property
def period(self):
return self._period
class L1bPreProcJob(DefaultLoggingClass):
""" Container for the definition and handling of a pre-processor job """
def __init__(self):
super(L1bPreProcJob, self).__init__(self.__class__.__name__)
# Save pointer to pysiral configuration
self.pysiral_config = ConfigInfo()
# Initialize the time range and set to monthly per default
self.time_range = None
# Error Status
self.error = ErrorStatus()
# Initialize job parameter
self.options = L1bPreProcJobOptions()
# List for iterations (currently only month-wise)
self.iterations = []
def generate_preprocessor_iterations(self):
""" Break the requested time range into monthly iterations """
# The input data is organized in folder per month, therefore
# the processing period is set accordingly
self.time_range.set_period("monthly")
self.log.info("Pre-processor Period is monthly")
self.time_range.set_exclude_month(self.options.exclude_month)
self.log.info("Excluding month: %s" % str(self.options.exclude_month))
self.iterations = self.time_range.iterations
self.log.info("Number of iterations: %g" % len(self.iterations))
def process_requested_time_range(self):
"""
Verify time range with mission data availability and create
datetime objects
"""
config = self.pysiral_config
mission_info = config.get_mission_info(self.options.mission_id)
# Set and validate the time range
start_date, stop_date = self.options.start_date, self.options.stop_date
self.time_range = TimeRangeRequest(start_date, stop_date)
# Check if any errors in definitions
self.time_range.error.raise_on_error()
self.log.info("Requested time range is: %s" % self.time_range.label)
# Clip time range to mission time range
is_clipped = self.time_range.clip_to_range(
mission_info.data_period.start, mission_info.data_period.stop)
if is_clipped:
self.log.info("Clipped to mission time range: %s till %s" % (
mission_info.data_period.start, mission_info.data_period.stop))
# Check if range is still valid
self.time_range.raise_if_empty()
def get_mission_preprocessor(self):
""" Return the mission specific pre-processor class """
from pysiral.cryosat2.preproc import CryoSat2PreProc
from pysiral.envisat.preproc import EnvisatPreProc
from pysiral.ers.preproc import ERSPreProc
from pysiral.sentinel3.preproc import Sentinel3PreProc
from pysiral.icesat.preproc import ICESatPreProc
if self.mission_id == "cryosat2":
return CryoSat2PreProc
elif self.mission_id == "envisat":
return EnvisatPreProc
elif self.mission_id == "ers2":
return ERSPreProc
elif self.mission_id == "sentinel3a":
return Sentinel3PreProc
elif self.mission_id == "icesat":
return ICESatPreProc
else:
error_code = self.__class__.__name__+" (01)"
error_message = "Invalid mission_id: %s" % self.mission_id
self.error.add_error(error_code, error_message)
self.error.raise_on_error()
@property
def mission_id(self):
return self.options.mission_id
@property
def hemisphere(self):
return self.options.hemisphere
@property
def input_version(self):
return self.options.input_version
@property
def remove_old(self):
return self.options.remove_old