def __parse_netcdf_bands_metadata(self, user_bands):
"""
Parse the netCDF file to extract the bands' metadata for the coverage
:return: dict:
"""
netcdf_files = self.session.get_files()
file_path = netcdf_files[0].filepath
# NOTE: all files should have same bands's metadata for each file
netCDF4 = import_netcdf4()
dataset = netCDF4.Dataset(file_path, 'r')
bands_metadata = {}
for user_band in user_bands:
band_id = user_band.identifier
attrs_list = dataset.variables[band_id].ncattrs()
bands_metadata[band_id] = {}
for attr in attrs_list:
try:
bands_metadata[band_id][attr] = str(getattr(dataset.variables[band_id], attr))
except:
log.warn("Attribute '" + attr + "' of band '" + band_id + "' cannot be parsed as string, ignored.")
return bands_metadata
def parse_netcdf_bands_metadata(file_path, user_bands):
"""
Parse the netCDF file to extract the bands' metadata for the coverage's global metadata
str file_path: path to the first input netCDF file
list[UserBand] user_bands: list of configured bands in ingredient file
:return: dict:
"""
# NOTE: all files should have same bands's metadata for each file
netCDF4 = import_netcdf4()
dataset = netCDF4.Dataset(file_path, 'r')
bands_metadata = {}
for user_band in user_bands:
band_id = user_band.identifier
attrs_list = dataset.variables[band_id].ncattrs()
bands_metadata[band_id] = {}
for attr in attrs_list:
try:
bands_metadata[band_id][attr] = str(getattr(dataset.variables[band_id], attr))
except:
log.warn("Attribute '" + attr + "' of band '" + band_id + "' cannot be parsed as string, ignored.")
return bands_metadata
def _file_band_nil_values(self, index):
"""
This is used to get the null values (Only 1) from the given band index if one exists when nilValue was not defined
in ingredient file
:param integer index: the current band index to get the nilValues
:rtype: List[RangeTypeNilValue] with only 1 element
"""
if len(self.files) < 1:
raise RuntimeException("No netcdf files given for import!")
if len(self.default_null_values) > 0:
return self.default_null_values
netCDF4 = import_netcdf4()
# NOTE: all files should have same bands's metadata for each file
nci = netCDF4.Dataset(self.files[0].get_filepath(), 'r')
try:
nil_value = nci.variables[self.bands[index].identifier].missing_value
except AttributeError:
# if file has no missing_value attribute of variable, then try with _FillValue
try:
nil_value = nci.variables[self.bands[index].identifier]._FillValue
except AttributeError:
# so variable does not have any null property
nil_value = None
if nil_value is None:
return None
else:
return [nil_value]
def get_dataset(self):
"""
Returns the dataset ofthe file
:rtype: netCDF4.Dataset
"""
netCDF4 = import_netcdf4()
return netCDF4.Dataset(self.get_file().get_filepath(), "r")
def get_dataset(self):
"""
Returns the dataset ofthe file
:rtype: netCDF4.Dataset
"""
netCDF4 = import_netcdf4()
if self.dataset is None:
file_path = self.get_file().get_filepath()
self.dataset = netCDF4.Dataset(file_path, "r")
return self.dataset
def _data_type(self):
"""
Returns the data type for this netcdf dataset
:rtype: str
"""
if len(self.files) < 1:
raise RuntimeException("No files to import were specified.")
netCDF4 = import_netcdf4()
nci = netCDF4.Dataset(self.files[0].get_filepath(), 'r')
netcdf_data_type = nci.variables[self.bands[0].identifier].dtype.name
return GDALGmlUtil.data_type_to_gdal_type(netcdf_data_type)
def parse_netcdf_axes_metadata(file_path, crs_axes_configured_dict):
"""
Parse the netCDF file to extract the axes metadata for the coverage's global metadata
str file_path: path to the first input netCDF file
dict crs_axes_configured_dict: dictionary of crs axis labels and themselves configuration in ingredient file
under "slicer"/"axes" section.
:return: dict:
"""
netCDF4 = import_netcdf4()
# NOTE: all files should have same axes's metadata for each file
dataset = netCDF4.Dataset(file_path, 'r')
axes_metadata = {}
# Iterate all slicer/axes configured in ingredient file
for crs_axis_label, axis_configured_dict in crs_axes_configured_dict.items(
):
min = axis_configured_dict["min"]
# Get the axis variable name in netCDF file from the min configuration
# e.g: "Long": { "min": "${netcdf:variable:lon:min}" } -> axis variable name is "lon"
variable_axis_label = None
# Find the variable axis label from netCDF expression for this axis
for key, value in axis_configured_dict.items():
tmp = re.search("variable:(.*):.*}", str(value))
if tmp is not None:
variable_axis_label = tmp.group(1)
break
if variable_axis_label is not None:
if variable_axis_label in dataset.variables:
axes_metadata[crs_axis_label] = {}
attrs_list = dataset.variables[
variable_axis_label].ncattrs()
for attr in attrs_list:
try:
# crs axis (e.g: Long) -> variable axis (e.g: lon)
axes_metadata[crs_axis_label][attr] = str(
getattr(dataset.variables[variable_axis_label],
attr))
except:
log.warn("Attribute '" + attr + "' of axis '" +
variable_axis_label +
"' cannot be parsed as string, ignored.")
return axes_metadata
def parse_netcdf_global_metadata(file_path):
"""
Parse the first file of importing netCDF files to extract the global metadata for the coverage
str file_path: path to first netCDF input file
:return: dict: global_metadata
"""
# NOTE: all files should have same global metadata for each file
netCDF4 = import_netcdf4()
dataset = netCDF4.Dataset(file_path, 'r')
global_metadata = {}
for attr in dataset.ncattrs():
try:
global_metadata[attr] = str(getattr(dataset, attr))
except:
log.warn("Attribute '" + attr + "' of global metadata cannot be parsed as string, ignored.")
return global_metadata
def _get_netcdf_coverage(self, recipe_type):
"""
Returns a coverage that uses the netcdf slicer
:param: string recipe_type the type of netcdf
:rtype: master.importer.coverage.Coverage
"""
crs = self._resolve_crs(self.options['coverage']['crs'])
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
# by default pixelIsPoint is not set to true in ingredient file
pixel_is_point = False
if 'pixelIsPoint' in self.options['coverage']['slicer'] and self.options['coverage']['slicer']['pixelIsPoint']:
pixel_is_point = True
input_files = self.session.get_files()
user_axes = self._read_axes(crs)
bands = self._read_bands()
first_band = bands[0]
first_band_variable_identifier = first_band.identifier
netCDF4 = import_netcdf4()
# Get the number of dimensions of band variable to validate with number of axes specified in the ingredients file
number_of_dimensions = len(netCDF4.Dataset(input_files[0], 'r').variables[first_band_variable_identifier].dimensions)
self.__validate_data_bound_axes(user_axes, number_of_dimensions)
coverage = NetcdfToCoverageConverter(self.resumer, self.session.default_null_values,
recipe_type, sentence_evaluator, self.session.get_coverage_id(),
bands,
input_files, crs, user_axes,
self.options['tiling'], self._global_metadata_fields(),
self._local_metadata_fields(),
self._netcdf_bands_metadata_fields(),
self._netcdf_axes_metadata_fields(),
self._metadata_type(),
self.options['coverage']['grid_coverage'], pixel_is_point,
self.options['import_order'],
self.session).to_coverages()
return coverage