def load_requested_auxdata(self):
""" Required subclass method: Load the data file necessary to satisfy condition for requested date"""
# Retrieve the file path for the requested date from a property of the auxdata parent class
path = self.requested_filepath
# --- Validation ---
if not os.path.isfile(path):
msg = self.pyclass+": File not found: %s " % path
self.add_handler_message(msg)
self.error.add_error("auxdata_missing_sic", msg)
return
# --- Read the data ---
self._data = ReadNC(path)
# --- Pre-process the data ---
# Remove time dimension
self._data.ice_conc = self._data.ice_conc[0, :, :]
# No negative ice concentrations
flagged = np.where(self._data.ice_conc < 0)
self._data.ice_conc[flagged] = np.nan
self.add_handler_message("OsiSafSIC: Loaded SIC file: %s" % path)
def __init__(self, *args, **kwargs):
super(DTU1MinGrid, self).__init__(*args, **kwargs)
# Read as standard netcdf
dtu_grid = ReadNC(self.cfg.filename)
# Cut to ROI regions (latitude only)
# -> no need for world mss
lat_range = self.cfg.options.latitude_range
# Get the indices for the latitude subset
latitude_indices = np.where(
np.logical_and(dtu_grid.lat >= lat_range[0],
dtu_grid.lat <= lat_range[1]))[0]
# Crop data to subset
self.elevation = dtu_grid.mss[latitude_indices, :]
self.longitude = dtu_grid.lon
self.latitude = dtu_grid.lat[latitude_indices]
# Convert elevations to WGS84
delta_h1 = egm2top_delta_h(self.latitude)
delta_h = egm2wgs_delta_h(self.latitude)
for i in np.arange(len(self.latitude)):
self.elevation[i, :] += (delta_h[i] - delta_h1[i])
def _parse_measurement_nc(self):
from pysiral.iotools import ReadNC
self._validate()
# Read the L2 netCDF file
self.nc = ReadNC(self.filename, nan_fill_value=True)
def _parse(self):
from cftime import num2pydate
content = ReadNC(self.filename)
for attribute_name in content.attributes:
self.attribute_list.append(attribute_name)
self.info.set_attribute(attribute_name,
getattr(content, attribute_name))
for parameter_name in content.parameters:
self.parameter_list.append(parameter_name)
setattr(self, parameter_name, getattr(content, parameter_name))
self._n_records = len(self.longitude)
# Get timestamp (can be either time or timestamp in l2i files)
if hasattr(self, "time"):
time = self.time
time_parameter_name = "time"
else:
time = self.time
time_parameter_name = "timestamp"
self._time_parameter_name = time_parameter_name
dt = num2pydate(time, self.time_def.units, self.time_def.calendar)
setattr(self, "time", dt)
self.time = self.time
def __init__(self, *args, **kwargs):
super(IfremerSIC, self).__init__(*args, **kwargs)
self._data = None
# XXX: This is a dirty hack, but needed for getting SIC lon/lat grid
self._grid = {}
for hemisphere in ["north", "south"]:
grid_file = os.path.join(self.cfg.local_repository, "grid_%s_12km.nc" % hemisphere)
self._grid[hemisphere] = ReadNC(grid_file)
def __init__(self, mask_dir, mask_name, cfg):
super(MaskW99Valid, self).__init__(mask_dir, mask_name, cfg)
# Read the data and transfer the ice_mask (1: valid, 0: invalid)
content = ReadNC(self.mask_filepath)
mask = content.ice_mask
mask = np.flipud(mask)
# Set the mask (pyresample area definition from config file)
self.set_mask(mask, self.cfg.area_def)
def parse(self):
self._log("Parsing cs2awi grid file: %s" %
filename_from_path(self._filename))
nc = ReadNC(self._filename)
for parameter in nc.parameters:
data = np.ma.array(getattr(nc, parameter))
data.mask = np.isnan(data)
setattr(self, parameter, data)
self._register_parameter(parameter)
self._log("- found parameter: %s dims:%s" %
(parameter, str(np.shape(data))))
def __init__(self,
variable_name,
repo_dir=None,
ctlg=None,
squeeze_empty_dims=True,
auxiliary_vars=[]):
super(L3ParameterCollection, self).__init__(self.__class__.__name__)
# Name of the parameter from the netCDF files
self.variable_name = variable_name
self.auxiliary_vars = auxiliary_vars
self.squeeze_empty_dims = squeeze_empty_dims
self._product = {}
self._mask = {}
self.error = ErrorStatus()
# Simple consistency check
if repo_dir is None and ctlg is None:
msg = "Either repo_dir or ctlg must be specified"
self.error.add_error("invalid-args", msg)
self.error.raise_on_error()
if repo_dir is not None and ctlg is not None:
msg = "Both repo_dir or ctlg are specified, using ctlg"
self.error.add_error("invalid-args", msg)
# Construct L3 product catalog
if repo_dir is not None:
self.ctlg = L3CProductCatalog(repo_dir)
# use existing catalog
if ctlg is not None:
self.ctlg = ctlg
# parse the files
for product in self.ctlg.product_list:
nc = ReadNC(product.path)
var = getattr(nc, self.variable_name)
if self.squeeze_empty_dims:
var = np.squeeze(var)
l3par = L3Parameter(self.variable_name, var, product)
for auxiliary_var_name in self.auxiliary_vars:
auxvar = getattr(nc, auxiliary_var_name)
if self.squeeze_empty_dims:
auxvar = np.squeeze(auxvar)
l3par.set_auxiliary_var(auxiliary_var_name, auxvar)
self._product[product.id] = l3par
# self.log.debug("Add product: %s" % product.id)
self.log.info("Added %g product(s)" % len(self._product))
def load_requested_auxdata(self):
""" Loads file from local repository only if needed """
# Retrieve the file path for the requested date from a property of the auxdata parent class
path = self.requested_filepath
# Validation
if not os.path.isfile(path):
msg = "%s: File not found: %s " % (self.__class__.__name__, path)
self.add_handler_message(msg)
self.error.add_error("auxdata_missing_sitype", msg)
return
# Read and prepare input data
self._data = ReadNC(path)
def load_requested_auxdata(self):
""" Loads file from local repository only if needed """
# Retrieve the file path for the requested date from a property of the auxdata parent class
path = Path(self.requested_filepath)
# Validation
if not path.is_file():
msg = "%s: File not found: %s " % (self.__class__.__name__, path)
self.add_handler_message(msg)
self.error.add_error("auxdata_missing_snow", msg)
return
# Store the netCDF data object
self._data = ReadNC(path)
def parse(self):
self._log("Parsing cs2awi grid file: %s" %
filename_from_path(self._filename))
nc = ReadNC(self._filename)
for parameter in nc.parameters:
data = np.ma.array(getattr(nc, parameter))
data.mask = np.isnan(data)
setattr(self, parameter, data)
self._register_parameter(parameter)
self._log("- found parameter: %s dims:%s" %
(parameter, str(np.shape(data))))
# XXX: dirty hack
nan_mask = np.isnan(self.sea_surface_height_anomaly)
self.radar_freeboard.mask = nan_mask
self.sea_ice_freeboard = self.radar_freeboard
def parse(self):
from pysiral.iotools import ReadNC
nc = ReadNC(self.filename)
self.parameters = nc.parameters
for parameter in nc.parameters:
data = np.ma.array(getattr(nc, parameter))
if self.squeeze:
data = np.squeeze(data)
data.mask = np.isnan(data)
setattr(self, parameter, data)
self.attributes = nc.attributes
for attribute in nc.attributes:
setattr(self, attribute, getattr(nc, attribute))
def load_requested_auxdata(self):
""" Required subclass method: Load the data file necessary to satisfy condition for requested date"""
# Retrieve the file path for the requested date from a property of the auxdata parent class
path = self.requested_filepath
# Validation
if not os.path.isfile(path):
msg = "OsiSafSIType: File not found: %s " % path
self.add_handler_message(msg)
self.error.add_error("auxdata_missing_sitype", msg)
return
# --- Read the data ---
self._data = ReadNC(path)
# Report
self.add_handler_message("OsiSafSIType: Loaded SIType file: %s" % path)
def load_requested_auxdata(self):
""" Required subclass method: Load the data file necessary to satisfy condition for requested date"""
# Retrieve the file path for the requested date from a property of the auxdata parent class
path = Path(self.requested_filepath)
# Validation
if not path.is_file():
msg = self.pyclass + ": File not found: %s " % path
self.add_handler_message(msg)
self.error.add_error("auxdata_missing_snow", msg)
return
# Read the data
self._data = ReadNC(path)
# This step is important for calculation of image coordinates
self.add_handler_message(self.__class__.__name__ +
": Loaded snow file: %s" % path)
def _get_data(self, l2):
""" Loads file from local repository only if needed """
opt = self.cfg.options
# Check if file is already loaded
if self._requested_date == self._current_date:
# Data already loaded, nothing to do
self.add_handler_message(
"ICDCNasaTeam: Daily grid already present")
return
# construct filename
path = Path(self._get_local_repository_filename(l2))
# Check if the file exists, add an error if not
# (error is not raised at this point)
if not path.isfile():
msg = "ICDCNasaTeam: File not found: %s " % path
self.add_handler_message(msg)
self.error.add_error("auxdata_missing_sitype", msg)
return
# Bulk read the netcdf file
self._data = ReadNC(path)
# There are multiple myi concentrations fields in the product
# The one used here is defined in the auxdata definition file
# in the pysiral config folder (`auxdata_def.yaml`)
# -> root.sitype.icdc_nasateam.options.variable_name
myi_fraction = getattr(self._data, opt.variable_name)
self._data.ice_type = myi_fraction[0, :, :]
# Same for the uncertainty variable
# (see description directly above for how to access variable namde
# definition)
myi_fraction_unc = getattr(self._data, opt.uncertainty_variable_name)
self._data.ice_type_uncertainty = myi_fraction_unc[0, :, :]
# Report and save current data period
self.add_handler_message("ICDCNasaTeam: Loaded SIType file: %s" % path)
self._current_date = self._requested_date
def _get_data(self, l2):
""" Loads file from local repository only if needed """
if self._requested_date == self._current_date:
# Data already loaded, nothing to do
return
path = self._get_local_repository_filename(l2)
# Validation
if not os.path.isfile(path):
msg ="IfremerSIC: File not found: %s " % path
self.add_handler_message(msg)
self.error.add_error("auxdata_missing_sic", msg)
return
self._data = ReadNC(path)
self._data.ice_conc = self._data.concentration[0, :, :]
flagged = np.where(np.logical_or(self._data.ice_conc < 0, self._data.ice_conc > 100))
self._data.ice_conc[flagged] = 0
# This step is important for calculation of image coordinates
self._data.ice_conc = np.flipud(self._data.ice_conc)
self.add_handler_message("IfremerSIC: Loaded SIC file: %s" % path)
self._current_date = self._requested_date
def _get_data(self, l2):
""" Loads file from local repository only if needed """
if self._requested_date == self._current_date:
# Data already loaded, nothing to do
return
# construct filename
path = self._get_local_repository_filename(l2)
# Validation
if not os.path.isfile(path):
msg = "OsiSafSIType: File not found: %s " % path
self.error.add_error("auxdata_missing_sitype", msg)
return
# Read and prepare input data
self._data = ReadNC(path)
self._data.ice_type = np.flipud(self._data.ice_type[0, :, :])
self._data.uncertainty = np.flipud(self._data.uncertainty[0, :, :])
# Logging
self.add_handler_message("OsiSafSIType: Loaded SIType file: %s" % path)
self._current_date = self._requested_date
def _read_sgdr(self):
""" Read the L1b file and create a ERS native L1b object """
self.sgdr = ReadNC(self.filepath, nan_fill_value=True)
def parse(self):
self._validate()
self.nc = ReadNC(self.filename, nan_fill_value=True)
def _read_mask_netcdf(self):
""" Read the mask """
if self.mask_filepath is not None:
self._nc = ReadNC(self.mask_filepath)
def parse(self):
from pysiral.iotools import ReadNC
self._validate()
self.nc = ReadNC(self.filename, nan_fill_value=True)
def __init__(self, *args, **kwargs):
super(NSIDCRegionMask, self).__init__(*args, **kwargs)
# The region mask is static, parse the file during init
self._data = ReadNC(self.cfg.filename)