def check_timeseries_id(self, dataset):
'''
Checks that if a variable exists for the time series id it has the appropriate attributes
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(BaseCheck.HIGH, 'Required variable for time series identifier')
recommended_ctx = TestCtx(BaseCheck.MEDIUM, 'Recommended attributes for the timeSeries variable')
# A variable with cf_role="timeseries_id" MUST exist for this to be a valid timeseries incomplete
timeseries_ids = dataset.get_variables_by_attributes(cf_role='timeseries_id')
required_ctx.assert_true(timeseries_ids, 'a unique variable must define attribute cf_role="timeseries_id"')
results.append(required_ctx.to_result())
if not timeseries_ids:
return results
timevar = util.get_time_variable(dataset)
nc_timevar = dataset.variables[timevar]
time_dimensions = nc_timevar.dimensions
timeseries_variable = timeseries_ids[0]
dims = timeseries_variable.dimensions
required_ctx.assert_true(
time_dimensions and time_dimensions[0] == dims[0],
'{} must have a dimension and that dimension must be shared by the time variable'.format(timeseries_variable.name)
)
recommended_ctx.assert_true(
getattr(timeseries_variable, 'long_name', '') != "",
"long_name attribute should exist and not be empty"
)
results.append(recommended_ctx.to_result())
return results
def check_bounds_variables(self, dataset):
'''
Checks the grid boundary variables.
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
recommended_ctx = TestCtx(BaseCheck.MEDIUM, 'Recommended variables to describe grid boundaries')
bounds_map = {
'lat_bounds': {
'units': 'degrees_north',
'comment': 'latitude values at the north and south bounds of each pixel.'
},
'lon_bounds': {
'units': 'degrees_east',
'comment': 'longitude values at the west and east bounds of each pixel.'
},
'z_bounds': {
'comment': 'z bounds for each z value',
},
'time_bounds': {
'comment': 'time bounds for each time value'
}
}
bounds_variables = [v.bounds for v in dataset.get_variables_by_attributes(bounds=lambda x: x is not None)]
for variable in bounds_variables:
ncvar = dataset.variables.get(variable, {})
recommended_ctx.assert_true(ncvar != {}, 'a variable {} should exist as indicated by a bounds attribute'.format(variable))
if ncvar == {}:
continue
units = getattr(ncvar, 'units', '')
if variable in bounds_map and 'units' in bounds_map[variable]:
recommended_ctx.assert_true(
units == bounds_map[variable]['units'],
'variable {} should have units {}'.format(variable, bounds_map[variable]['units'])
)
else:
recommended_ctx.assert_true(
units != '',
'variable {} should have a units attribute that is not empty'.format(variable)
)
comment = getattr(ncvar, 'comment', '')
recommended_ctx.assert_true(
comment != '',
'variable {} should have a comment and not be empty'
)
return recommended_ctx.to_result()
def check_dimensions(self, dataset):
'''
Checks that the feature types of this dataset are consitent with a time series orthogonal dataset
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
required_ctx = TestCtx(BaseCheck.HIGH, 'All geophysical variables are time-series orthogonal feature types')
message = '{} must be a valid timeseries feature type. It must have dimensions of (timeSeries, time) or (time).'
message += ' And x, y and z coordinates must have dimensions (timeSeries) or be dimensionless'
for variable in util.get_geophysical_variables(dataset):
is_valid = util.is_timeseries(dataset, variable) or util.is_multi_timeseries_orthogonal(dataset, variable)
required_ctx.assert_true(
is_valid,
message.format(variable)
)
return required_ctx.to_result()
def check_timeseries_id(self, dataset):
'''
Checks that if a variable exists for the time series id it has the appropriate attributes
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
timeseries_ids = dataset.get_variables_by_attributes(cf_role='timeseries_id')
# No need to check
if not timeseries_ids:
return
test_ctx = TestCtx(BaseCheck.MEDIUM, 'Recommended attributes for the timeSeries variable')
timeseries_variable = timeseries_ids[0]
test_ctx.assert_true(
getattr(timeseries_variable, 'long_name', '') != "",
"long_name attribute should exist and not be empty"
)
return test_ctx.to_result()
def check_dimensions(self, dataset):
'''
Checks that the feature types of this dataset are consistent with a profile-orthogonal dataset.
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(BaseCheck.HIGH, 'All geophysical variables are profile-orthogonal feature types')
message = '{} must be a valid profile-orthogonal feature type. It must have dimensions of (profile, depth).'
message += ' x and y should have dimensions of (profile), z should have dimension of (depth) and t should have dimension (profile)'
for variable in util.get_geophysical_variables(dataset):
is_valid = util.is_profile_orthogonal(dataset, variable)
required_ctx.assert_true(
is_valid,
message.format(variable)
)
results.append(required_ctx.to_result())
return results
def check_dimensions(self, dataset):
'''
Checks that the feature types of this dataset are consitent with a trajectory profile incomplete dataset
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(
BaseCheck.HIGH,
'All geophysical variables are trajectory profile incomplete feature types'
)
message = '{} must be a valid trajectory profile incomplete feature type. It and z must have dimensions of (trajectory, obs, nzMax).'
message += ' Also, x, y, and t must have dimensions (trajectory, obs).'
for variable in util.get_geophysical_variables(dataset):
is_valid = util.is_trajectory_profile_incomplete(dataset, variable)
required_ctx.assert_true(is_valid, message.format(variable))
results.append(required_ctx.to_result())
return results
def check_dimensions(self, dataset):
'''
Checks that the feature types of this dataset are consitent with a trajectory profile orthogonal dataset
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(BaseCheck.HIGH, 'All geophysical variables are trajectory profile orthogonal feature types')
message = '{} must be a valid trajectory profile orthogonal feature type. It must have dimensions of (trajectory, obs, z).'
message += ' Also, x, y, and t must have dimensions (trajectory, obs). z must be a coordinate variable with dimensions (z).'
for variable in util.get_geophysical_variables(dataset):
is_valid = util.is_trajectory_profile_orthogonal(dataset, variable)
required_ctx.assert_true(
is_valid,
message.format(variable)
)
results.append(required_ctx.to_result())
return results
def check_dimensions(self, dataset):
'''
Checks that the feature types of this dataset are consitent with a trajectory dataset
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(BaseCheck.HIGH, 'All geophysical variables are trajectory feature types')
message = ("{} must be a valid trajectory feature type. It must have dimensions of (trajectoryID, time)."
" And all coordinates must have dimensions (trajectoryID, time)")
for variable in util.get_geophysical_variables(dataset):
is_valid = util.is_cf_trajectory(dataset, variable)
is_valid = is_valid or util.is_single_trajectory(dataset, variable)
required_ctx.assert_true(
is_valid,
message.format(variable)
)
results.append(required_ctx.to_result())
return results
def check_dimensions(self, dataset):
'''
Checks that the feature types of this dataset are consistent with a timeseries-profile-incomplete dataset.
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(BaseCheck.HIGH, 'All geophysical variables are timeseries-profile-incomplete feature types')
message = '{} must be a valid timeseries-profile-incomplete feature type.'
message += ' it must have dimensions (station, nTimeMax, zMax). x and y must have dimensions (station).'
message += ' time must have dimensions (station, nTimeMax). And z must have dimensions (station, nTimeMax, zMax).'
for variable in util.get_geophysical_variables(dataset):
is_valid = util.is_timeseries_profile_incomplete(dataset, variable)
required_ctx.assert_true(
is_valid,
message.format(variable)
)
results.append(required_ctx.to_result())
return results
def check_dimensions(self, dataset):
'''
Checks that the feature types of this dataset are consistent with a regular gridded dataset
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(
BaseCheck.HIGH,
'All geophysical variables are regular gridded feature types')
message = '{} must be a valid regular gridded feature type. It must have dimensions (t, z, y, x)'
message += ' and each dimension must be a coordinate variable with a dimension with the same name'
message += ' as the variable. z is optional.'
for variable in util.get_geophysical_variables(dataset):
is_valid = util.is_2d_regular_grid(dataset, variable)
is_valid = is_valid or util.is_3d_regular_grid(dataset, variable)
required_ctx.assert_true(is_valid, message.format(variable))
results.append(required_ctx.to_result())
return results
def check_dimensions(self, dataset):
'''
Checks that the feature types of this dataset are consistent with a timeseries-profile-incomplete dataset.
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(
BaseCheck.HIGH,
'All geophysical variables are timeseries-profile-incomplete feature types'
)
message = '{} must be a valid timeseries-profile-incomplete feature type.'
message += ' it must have dimensions (station, nTimeMax, zMax). x and y must have dimensions (station).'
message += ' time must have dimensions (station, nTimeMax). And z must have dimensions (station, nTimeMax, zMax).'
for variable in util.get_geophysical_variables(dataset):
is_valid = util.is_timeseries_profile_incomplete(dataset, variable)
required_ctx.assert_true(is_valid, message.format(variable))
results.append(required_ctx.to_result())
return results
def check_dimensions(self, dataset):
'''
Checks that the feature types of this dataset are consistent with a regular gridded dataset
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(BaseCheck.HIGH, 'All geophysical variables are regular gridded feature types')
message = '{} must be a valid regular gridded feature type. It must have dimensions (t, z, y, x)'
message += ' and each dimension must be a coordinate variable with a dimension with the same name'
message += ' as the variable. z is optional.'
for variable in util.get_geophysical_variables(dataset):
is_valid = util.is_2d_regular_grid(dataset, variable)
is_valid = is_valid or util.is_3d_regular_grid(dataset, variable)
required_ctx.assert_true(
is_valid,
message.format(variable)
)
results.append(required_ctx.to_result())
return results
def check_dimensions(self, dataset):
'''
Checks that the feature types of this dataset are consistent with a timeseries-profile-orthogonal dataset.
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(BaseCheck.HIGH, 'All geophysical variables are timeseries-profile-orthogonal feature types')
message = '{} must be a valid profile-orthogonal feature type. It must have dimensions of (station, time, z).'
message += ' If it\'s a single station, it must have dimensions (time, z). x and y dimensions must be scalar or have'
message += ' dimensions (station). time must be a coordinate variable with dimension (time) and z must be a'
message += ' coordinate variabel with dimension (z).'
for variable in util.get_geophysical_variables(dataset):
is_valid = util.is_timeseries_profile_single_station(dataset, variable)
is_valid = is_valid or util.is_timeseries_profile_multi_station(dataset, variable)
required_ctx.assert_true(
is_valid,
message.format(variable)
)
results.append(required_ctx.to_result())
return results
def check_required_attributes(self, dataset):
'''
Verifies that the dataset contains the NCEI required and highly recommended global attributes
'''
results = []
required_ctx = TestCtx(BaseCheck.HIGH,
'Required Global Attributes for Timeseries')
required_ctx.assert_true(
getattr(dataset, 'nodc_template_version',
'').lower() == self.valid_templates[0].lower(),
'nodc_template_version attribute must be {}'.format(
self.valid_templates[0]))
required_ctx.assert_true(
getattr(dataset, 'cdm_data_type', '') == 'Point',
'cdm_data_type attribute must be set to Point')
required_ctx.assert_true(
getattr(dataset, 'featureType', '') == 'point',
'featureType attribute must be set to point')
results.append(required_ctx.to_result())
return results
def check_trajectory_id(self, dataset):
'''
Checks that if a variable exists for the trajectory id it has the appropriate attributes
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
exists_ctx = TestCtx(BaseCheck.MEDIUM, 'Variable defining "trajectory_id" exists')
trajectory_ids = dataset.get_variables_by_attributes(cf_role='trajectory_id')
# No need to check
exists_ctx.assert_true(trajectory_ids, 'variable defining cf_role="trajectory_id" exists')
if not trajectory_ids:
return exists_ctx.to_result()
results.append(exists_ctx.to_result())
test_ctx = TestCtx(BaseCheck.MEDIUM, 'Recommended attributes for the {} variable'.format(trajectory_ids[0].name))
test_ctx.assert_true(
getattr(trajectory_ids[0], 'long_name', '') != "",
"long_name attribute should exist and not be empty"
)
results.append(test_ctx.to_result())
return results
def check_required_attributes(self, dataset):
"""
Verifies that the dataset contains the NCEI required and highly recommended global attributes
"""
results = []
required_ctx = TestCtx(BaseCheck.HIGH, "Required Global Attributes for Timeseries")
required_ctx.assert_true(
getattr(dataset, "nodc_template_version", "").lower() == self.valid_templates[0].lower(),
"nodc_template_version attribute must be {}".format(self.valid_templates[0]),
)
required_ctx.assert_true(
getattr(dataset, "cdm_data_type", "") == "Point", "cdm_data_type attribute must be set to Point"
)
required_ctx.assert_true(
getattr(dataset, "featureType", "") == "point", "featureType attribute must be set to point"
)
results.append(required_ctx.to_result())
return results
def check_required_attributes(self, dataset):
'''
Feature type specific check of global required and highly recommended attributes.
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(BaseCheck.HIGH,
'Required Global Attributes for Grid')
required_ctx.assert_true(
getattr(dataset, 'ncei_template_version',
'').lower() == self.valid_templates[0].lower(),
'ncei_template_version attribute must be {}'.format(
self.valid_templates[0]))
required_ctx.assert_true(
getattr(dataset, 'cdm_data_type', '') == 'Grid',
'cdm_data_type attribute must be set to Grid')
required_ctx.assert_true(
getattr(dataset, 'featureType', '') == 'grid',
'featureType attribute must be set to grid')
results.append(required_ctx.to_result())
return results
def check_required_attributes(self, dataset):
'''
Feature type specific check of global required and highly recommended attributes.
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(
BaseCheck.HIGH,
'Required Global Attributes for Timeseries Profile Incomplete Time and Depth'
)
required_ctx.assert_true(
getattr(dataset, 'ncei_template_version',
'') == self.valid_templates[0],
'ncei_template_version attribute must be {}'.format(
self.valid_templates[0]))
required_ctx.assert_true(
getattr(dataset, 'cdm_data_type', '') == 'Station',
'cdm_data_type attribute must be set to Station')
required_ctx.assert_true(
getattr(dataset, 'featureType', '') == 'timeSeriesProfile',
'featureType attribute must be set to timeSeriesProfile')
results.append(required_ctx.to_result())
return results
def check_required_attributes(self, dataset):
'''
Feature type specific check of global required and highly recommended attributes.
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(BaseCheck.HIGH, 'Required Global Attributes for Timeseries Profile Incomplete Time and Depth')
required_ctx.assert_true(
getattr(dataset, 'ncei_template_version', '') == self.valid_templates[0],
'ncei_template_version attribute must be {}'.format(self.valid_templates[0])
)
required_ctx.assert_true(
getattr(dataset, 'cdm_data_type', '') == 'Station',
'cdm_data_type attribute must be set to Station'
)
required_ctx.assert_true(
getattr(dataset, 'featureType', '') == 'timeSeriesProfile',
'featureType attribute must be set to timeSeriesProfile'
)
results.append(required_ctx.to_result())
return results
def check_required_attributes(self, dataset):
'''
Feature type specific check of global required and highly recommended attributes.
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(BaseCheck.HIGH, 'Required Global Attributes for Trajectory dataset')
required_ctx.assert_true(
getattr(dataset, 'nodc_template_version', '').lower() == self.valid_templates[0].lower(),
'nodc_template_version attribute must be {}'.format(self.valid_templates[0])
)
required_ctx.assert_true(
getattr(dataset, 'cdm_data_type', '') == 'Trajectory',
'cdm_data_type attribute must be set to Trajectory'
)
required_ctx.assert_true(
getattr(dataset, 'featureType', '') == 'trajectory',
'featureType attribute must be set to trajectory'
)
results.append(required_ctx.to_result())
return results
def check_dimensions(self, dataset):
"""
Checks that the feature types of this dataset are consitent with a point dataset
"""
required_ctx = TestCtx(BaseCheck.HIGH, "All geophysical variables are point feature types")
t = util.get_time_variable(dataset)
# Exit prematurely
if not t:
required_ctx.assert_true(False, "A dimension representing time is required for point feature types")
return required_ctx.to_result()
t_dims = dataset.variables[t].dimensions
o = None or (t_dims and t_dims[0])
message = "{} must be a valid timeseries feature type. It must have dimensions of ({}), and all coordinates must have dimensions of ({})"
for variable in util.get_geophysical_variables(dataset):
is_valid = util.is_point(dataset, variable)
required_ctx.assert_true(is_valid, message.format(variable, o, o))
return required_ctx.to_result()
def check_dimensions(self, dataset):
'''
Checks that the feature types of this dataset are consitent with a point dataset
'''
required_ctx = TestCtx(
BaseCheck.HIGH,
'All geophysical variables are point feature types')
t = util.get_time_variable(dataset)
# Exit prematurely
if not t:
required_ctx.assert_true(
False,
'A dimension representing time is required for point feature types'
)
return required_ctx.to_result()
t_dims = dataset.variables[t].dimensions
o = None or (t_dims and t_dims[0])
message = '{} must be a valid timeseries feature type. It must have dimensions of ({}), and all coordinates must have dimensions of ({})'
for variable in util.get_geophysical_variables(dataset):
is_valid = util.is_point(dataset, variable)
required_ctx.assert_true(is_valid, message.format(variable, o, o))
return required_ctx.to_result()
def check_recommended_attributes(self, dataset):
'''
Feature type specific check of global recommended attributes.
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
recommended_ctx = TestCtx(BaseCheck.MEDIUM, 'Recommended global attributes')
# Check time_coverage_duration and resolution
for attr in ['time_coverage_duration', 'time_coverage_resolution']:
attr_value = getattr(dataset, attr, '')
try:
parse_duration(attr_value)
recommended_ctx.assert_true(True, '') # Score it True!
except Exception:
recommended_ctx.assert_true(False, '{} should exist and be ISO-8601 format (example: PT1M30S), currently: {}'.format(attr, attr_value))
results.append(recommended_ctx.to_result())
return results
def check_recommended_attributes(self, dataset):
'''
Feature type specific check of global recommended attributes.
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
recommended_ctx = TestCtx(BaseCheck.MEDIUM,
'Recommended global attributes')
# Check time_coverage_duration and resolution
for attr in ['time_coverage_duration', 'time_coverage_resolution']:
attr_value = getattr(dataset, attr, '')
try:
parse_duration(attr_value)
recommended_ctx.assert_true(True, '') # Score it True!
except Exception:
recommended_ctx.assert_true(
False,
'{} should exist and be ISO-8601 format (example: PT1M30S), currently: {}'
.format(attr, attr_value))
results.append(recommended_ctx.to_result())
return results
def check_trajectory_id(self, dataset):
'''
Checks that if a variable exists for the trajectory id it has the appropriate attributes
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
exists_ctx = TestCtx(BaseCheck.MEDIUM,
'Variable defining "trajectory_id" exists')
trajectory_ids = dataset.get_variables_by_attributes(
cf_role='trajectory_id')
# No need to check
exists_ctx.assert_true(
trajectory_ids, 'variable defining cf_role="trajectory_id" exists')
if not trajectory_ids:
return exists_ctx.to_result()
results.append(exists_ctx.to_result())
test_ctx = TestCtx(
BaseCheck.MEDIUM,
'Recommended attributes for the {} variable'.format(
trajectory_ids[0].name))
test_ctx.assert_true(
getattr(trajectory_ids[0], 'long_name', '') != "",
"long_name attribute should exist and not be empty")
results.append(test_ctx.to_result())
return results
def check_timeseries_id(self, dataset):
'''
Checks that if a variable exists for the time series id it has the appropriate attributes
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
results = []
required_ctx = TestCtx(BaseCheck.HIGH,
'Required variable for time series identifier')
recommended_ctx = TestCtx(
BaseCheck.MEDIUM,
'Recommended attributes for the timeSeries variable')
# A variable with cf_role="timeseries_id" MUST exist for this to be a valid timeseries incomplete
timeseries_ids = dataset.get_variables_by_attributes(
cf_role='timeseries_id')
required_ctx.assert_true(
timeseries_ids,
'a unique variable must define attribute cf_role="timeseries_id"')
results.append(required_ctx.to_result())
if not timeseries_ids:
return results
timevar = util.get_time_variable(dataset)
nc_timevar = dataset.variables[timevar]
time_dimensions = nc_timevar.dimensions
timeseries_variable = timeseries_ids[0]
dims = timeseries_variable.dimensions
required_ctx.assert_true(
time_dimensions and time_dimensions[0] == dims[0],
'{} must have a dimension and that dimension must be shared by the time variable'
.format(timeseries_variable.name))
recommended_ctx.assert_true(
getattr(timeseries_variable, 'long_name', '') != "",
"long_name attribute should exist and not be empty")
results.append(recommended_ctx.to_result())
return results
def check_bounds_variables(self, dataset):
'''
Checks the grid boundary variables.
:param netCDF4.Dataset dataset: An open netCDF dataset
'''
recommended_ctx = TestCtx(
BaseCheck.MEDIUM,
'Recommended variables to describe grid boundaries')
bounds_map = {
'lat_bounds': {
'units':
'degrees_north',
'comment':
'latitude values at the north and south bounds of each pixel.'
},
'lon_bounds': {
'units':
'degrees_east',
'comment':
'longitude values at the west and east bounds of each pixel.'
},
'z_bounds': {
'comment': 'z bounds for each z value',
},
'time_bounds': {
'comment': 'time bounds for each time value'
}
}
bounds_variables = [
v.bounds for v in dataset.get_variables_by_attributes(
bounds=lambda x: x is not None)
]
for variable in bounds_variables:
ncvar = dataset.variables.get(variable, {})
recommended_ctx.assert_true(
ncvar != {},
'a variable {} should exist as indicated by a bounds attribute'
.format(variable))
if ncvar == {}:
continue
units = getattr(ncvar, 'units', '')
if variable in bounds_map and 'units' in bounds_map[variable]:
recommended_ctx.assert_true(
units == bounds_map[variable]['units'],
'variable {} should have units {}'.format(
variable, bounds_map[variable]['units']))
else:
recommended_ctx.assert_true(
units != '',
'variable {} should have a units attribute that is not empty'
.format(variable))
comment = getattr(ncvar, 'comment', '')
recommended_ctx.assert_true(
comment != '',
'variable {} should have a comment and not be empty')
return recommended_ctx.to_result()