def test_check_child_attr_data_types(self):
"""
Tests check_child_attr_data_types() to ensure the attributes specified in Section 2.5.1
have a matching data type to their parent variables."""
# create dataset using MockDataset (default constructor gives it time dimension)
ds = MockTimeSeries()
ds.createVariable("temp", np.float64, dimensions=("time")) # add variable "temp" with dimension "time"
# check where no special data attrs are present, should result good
result = self.cf.check_child_attr_data_types(ds) # checks all special attrs for all variables
self.assert_result_is_good(result)
# give temp _FillValue as a float, expect good result
ds.variables['temp'].setncattr("_FillValue", np.float(99999999999999999999.))
result = self.cf.check_child_attr_data_types(ds)
self.assert_result_is_good(result)
# give temp valid_range as an array of floats, all should check out
ds.variables['temp'].setncattr("valid_range", np.array([35., 38.]))
result = self.cf.check_child_attr_data_types(ds)
self.assert_result_is_good(result)
# now give invalid integer for valid_min; above two should still check out, this one should fail
ds.variables['temp'].setncattr("valid_min", 45)
result = self.cf.check_child_attr_data_types(ds)
self.assert_result_is_bad(result)
# now give invalid string for valid_max
ds.variables['temp'].setncattr("valid_max", "eighty")
result = self.cf.check_child_attr_data_types(ds)
self.assert_result_is_bad(result)
def test_compare_string_var_dtype(self):
"""
Checks that the special dtype comparison case for variable-length
strings is checked properly. Calling var.dtype on a variable-length
string returns Python `str` type instead of a NumPy dtype
"""
ts = MockTimeSeries()
str_var = ts.createVariable('str_var', str, ('time',))
self.assertTrue(util.compare_dtype(str_var.dtype, str))
def test_time_monotonically_increasing(self):
"""Checks that the time variable is monotonically increasing"""
ts = MockTimeSeries()
# first check failure case
ts.variables['time'][:] = np.zeros(500)
result = self.check.check_monotonically_increasing_time(ts)
self.assertLess(result.value[0], result.value[1])
# now make a monotonically increasing time variable
ts.variables['time'][:] = np.linspace(1, 500, 500)
result = self.check.check_monotonically_increasing_time(ts)
self.assertEqual(result.value[0], result.value[1])
def test_depth_diff(self):
"""
Checks that the sum of the first order difference over the start to
the end time is non-negligible
"""
ts = MockTimeSeries()
ts.variables['depth'][:] = np.zeros(500)
result = self.check.check_depth_array(ts)
self.assertLess(result.value[0], result.value[1])
ts.variables['depth'][:] = np.linspace(1, 500, 500)
result = self.check.check_depth_array(ts)
self.assertEqual(result.value[0], result.value[1])
def test_bad_standard_name_table(self):
"""
Test that failure in case a bad standard name table is passed.
"""
# would this ever actually be reached by the code?
with pytest.raises(IOError):
StandardNameTable('dummy_non_existent_file.ext')
nc_obj = MockTimeSeries()
nc_obj.standard_name_table = 'dummy_non_existent_file.ext'
self.assertFalse(self.cf._find_cf_standard_name_table(nc_obj))
nc_obj.standard_name_table = np.array([], np.float64)
self.assertFalse(self.cf._find_cf_standard_name_table(nc_obj))
def test_time_depth_non_nan(self):
"""
Check that the cartesian product of time and depth coordinate variables
have at least two non-NaN combinations
"""
ts = MockTimeSeries()
ts.variables['time'][0] = 0
ts.variables['depth'][0] = 5
# cartesian product should only contain one element and fail
result = self.check.check_dim_no_data(ts)
self.assertLess(result.value[0], result.value[1])
# adding one more coordinate variable should make the number of passing
# combinations equal to two, which should pass this check
ts.variables['time'][1] = 1
result = self.check.check_dim_no_data(ts)
self.assertEqual(result.value[0], result.value[1])
def test_check_lon_extents(self):
"""Test the check_lon_extents() method behaves expectedly"""
# create dataset using MockDataset, give it lat/lon dimensions
ds = MockTimeSeries()
ds.variables["lon"][:] = np.linspace(65., 67., num=500)
# test no values, expect failure
#result = self.acdd.check_lon_extents(ds)
#self.assert_result_is_bad(result)
# give integer geospatial_lon_max/min, test
ds.setncattr("geospatial_lon_min", 65)
ds.setncattr("geospatial_lon_max", 67)
result = self.acdd.check_lon_extents(ds)
self.assert_result_is_good(result)
# give float geospatial_lon_min/max, test
ds.setncattr("geospatial_lon_min", 65.)
ds.setncattr("geospatial_lon_max", 67.)
result = self.acdd.check_lon_extents(ds)
self.assert_result_is_good(result)
# give string (in number-form), test
ds.setncattr("geospatial_lon_min", "65.")
ds.setncattr("geospatial_lon_max", "67.")
result = self.acdd.check_lon_extents(ds)
self.assert_result_is_good(result)
# give garbage string -- expect failure
ds.setncattr("geospatial_lon_min", "bad")
ds.setncattr("geospatial_lon_max", "val")
result = self.acdd.check_lon_extents(ds)
self.assert_result_is_bad(result)
def test_check_lat_extents(self):
"""Test the check_lat_extents() method behaves expectedly"""
# create dataset using MockDataset, give it lat/lon dimensions
ds = MockTimeSeries()
ds.variables["lat"][:] = np.linspace(-135., -130., num=500) # arbitrary, but matches time dim size
# test no values, expect failure
result = self.acdd.check_lat_extents(ds)
self.assert_result_is_bad(result)
# give integer geospatial_lat_max/min, test
ds.setncattr("geospatial_lat_min", -135)
ds.setncattr("geospatial_lat_max", -130)
result = self.acdd.check_lat_extents(ds)
self.assert_result_is_good(result)
# give float geospatial_lat_min/max, test
ds.setncattr("geospatial_lat_min", -135.)
ds.setncattr("geospatial_lat_max", -130.)
result = self.acdd.check_lat_extents(ds)
self.assert_result_is_good(result)
# give string (in number-form), test
ds.setncattr("geospatial_lat_min", "-135.")
ds.setncattr("geospatial_lat_max", "-130.")
result = self.acdd.check_lat_extents(ds)
self.assert_result_is_good(result)
# give garbage string -- expect failure
ds.setncattr("geospatial_lat_min", "bad")
ds.setncattr("geospatial_lat_max", "val")
result = self.acdd.check_lat_extents(ds)
self.assert_result_is_bad(result)
def test_check_geospatial_vertical_max(self):
ds = MockTimeSeries()
ds.variables["depth"][:] = np.linspace(0., 30., num=500)
# give integer geospatial_vertical_max/min, test
ds.setncattr("geospatial_vertical_min", 0)
ds.setncattr("geospatial_vertical_max", 30)
result = self.acdd.check_vertical_extents(ds)
self.assert_result_is_good(result)
# give float geospatial_vertical_min/max, test
ds.setncattr("geospatial_vertical_min", 0.)
ds.setncattr("geospatial_vertical_max", 30.)
result = self.acdd.check_vertical_extents(ds)
self.assert_result_is_good(result)
# give string (in number-form), test
ds.setncattr("geospatial_vertical_min", "0.")
ds.setncattr("geospatial_vertical_max", "30.")
result = self.acdd.check_vertical_extents(ds)
self.assert_result_is_good(result)
# give garbage string -- expect failure
ds.setncattr("geospatial_vertical_min", "bad")
ds.setncattr("geospatial_vertical_max", "val")
result = self.acdd.check_vertical_extents(ds)
self.assert_result_is_bad(result)
# all masked values mean that there are no valid array elements to get
# the min/max of
ds.setncattr("geospatial_vertical_min", 0.)
ds.setncattr("geospatial_vertical_max", 30.)
ds.variables["depth"][:] = np.ma.masked_all(ds.variables["depth"]
.shape)
result = self.acdd.check_vertical_extents(ds)
self.assert_result_is_bad(result)
def test_ncei_compliance(self):
"""Tests that the NCEI compliance suite works"""
ncei_base_table_url = 'https://gliders.ioos.us/ncei_authority_tables/'
# this is only a small subset
institutions = """MARACOOS
University of Delaware
Woods Hole Oceanographic Institution"""
projects = """MARACOOS"""
platforms = """Test123"""
instrument_makes = """Seabird GCTD
Sea-Bird 41CP
Sea-Bird GCTD
Seabird GPCTD"""
mock_nc_file = MockTimeSeries()
mock_nc_file.project = 'MARACOOS'
mock_nc_file.institution = 'MARACOOS'
instrument_var = mock_nc_file.createVariable('instrument', 'i', ())
instrument_var.make_model = 'Sea-Bird GCTD'
mock_nc_file.variables['depth'].instrument = 'instrument'
mock_nc_file.variables['depth'].platform = 'platform'
platform_var = mock_nc_file.createVariable('platform', 'i', ())
platform_var.id = "Test123"
with requests_mock.Mocker() as mock:
mock.get(urljoin(ncei_base_table_url, 'institutions.txt'),
text=institutions)
mock.get(urljoin(ncei_base_table_url, 'projects.txt'),
text=projects)
mock.get(urljoin(ncei_base_table_url, 'platforms.txt'),
text=platforms)
mock.get(urljoin(ncei_base_table_url, 'instruments.txt'),
text=instrument_makes)
result = self.check.check_ncei_tables(mock_nc_file)
# everything should pass here
self.assertEqual(result.value[0], result.value[1])
# now change to values that should fail
mock_nc_file.project = 'N/A'
mock_nc_file.institution = 'N/A'
# set instrument_var make_model to something not contained in the
# list
instrument_var.make_model = 'Unknown'
platform_var.id = 'No platform'
# create a dummy variable which points to an instrument that doesn't
# exist
dummy_var = mock_nc_file.createVariable('dummy', 'i', ())
dummy_var.instrument = 'nonexistent_var_name'
dummy_var.platform = 'nonexistent_var_name_2'
result_fail = self.check.check_ncei_tables(mock_nc_file)
expected_msg_set = {
"Global attribute project value 'N/A' not contained in https://gliders.ioos.us/ncei_authority_tables/projects.txt",
"Global attribute institution value 'N/A' not contained in https://gliders.ioos.us/ncei_authority_tables/institutions.txt",
"Attribute make_model 'Unknown' for variable instrument not contained in https://gliders.ioos.us/ncei_authority_tables/instruments.txt",
"Referenced instrument variable nonexistent_var_name does not exist",
"Attribute id 'No platform' for variable platform not contained in https://gliders.ioos.us/ncei_authority_tables/platforms.txt",
"Referenced platform variable nonexistent_var_name_2 does not exist"
}
self.assertSetEqual(expected_msg_set, set(result_fail.msgs))
# remove attributes which need to be checked to see if properly
# detected
del (instrument_var.make_model, platform_var.id,
mock_nc_file.project, mock_nc_file.institution)
missing_attr_results = self.check.check_ncei_tables(mock_nc_file)
expected_missing_msgs = {
"Attribute project not in dataset",
"Attribute institution not in dataset",
"Attribute make_model should exist in variable instrument",
"Attribute id should exist in variable platform"
}
# check that all the missing attribute messages are contained in the
# test results
self.assertTrue(expected_missing_msgs <=
set(missing_attr_results.msgs))
def test_check_ioos_ingest(self):
ds = MockTimeSeries()
# no value, pass
res = self.ioos.check_ioos_ingest(ds)
self.assertTrue(res.value)
self.assertEqual(res.msgs, [])
# value false
ds.setncattr("ioos_ingest", "false")
self.assertTrue(self.ioos.check_ioos_ingest(ds).value)
# value true
ds.setncattr("ioos_ingest", "true")
self.assertTrue(self.ioos.check_ioos_ingest(ds).value)
# case insensitive
ds.setncattr("ioos_ingest", "True")
self.assertTrue(self.ioos.check_ioos_ingest(ds).value)
ds.setncattr("ioos_ingest", "False")
self.assertTrue(self.ioos.check_ioos_ingest(ds).value)
# anything else fails
ds.setncattr("ioos_ingest", "badval")
self.assertFalse(self.ioos.check_ioos_ingest(ds).value)
ds.setncattr("ioos_ingest", 0)
self.assertFalse(self.ioos.check_ioos_ingest(ds).value)
def test_check_feattype_timeseries_cf_role(self):
### featureType: timeseries and timeseries - msingle station require same tests ###
# for ftype in ("timeseries", "timeseries - single station", "timeseries - multiple station"):
ftype = "timeseries"
ds = MockTimeSeries() # time, lat, lon, depth
ds.setncattr("featureType", ftype)
temp = ds.createVariable("temp", "d", ("time"))
# no platform variables or geophys vars with cf_role=timeseries_id, fail
result = self.ioos._check_feattype_timeseries_cf_role(ds)
self.assertFalse(result.value)
# create dimensionless platform variable, set bad cf_role
plat = ds.createVariable("station", "|S1", ())
plat.setncattr("cf_role", "badbadbad")
ds.variables["temp"].setncattr("platform", "station")
# should still fail as no vars with cf_role=timeseries_id exist
result = self.ioos._check_feattype_timeseries_cf_role(ds)
self.assertFalse(result.value)
# set correct cf_role on platform var, should fail as dim == 0
plat.setncattr("cf_role", "timeseries_id")
result = self.ioos._check_feattype_timeseries_cf_role(ds)
self.assertFalse(result.value)
# remove platform variable, put cf_role on variable "station", should fail as dim = 0
ds = MockTimeSeries() # time, lat, lon, depth
ds.createDimension("timeseries_dim", 0)
ds.setncattr("featureType", ftype)
plat = ds.createVariable("station",
"|S1",
dimensions=("timeseries_dim"))
plat.setncattr("cf_role", "timeseries_id")
result = self.ioos._check_feattype_timeseries_cf_role(ds)
self.assertFalse(result.value)
# remove cf_role from station variable, put it on another with dim = 1
ds = MockTimeSeries() # time, lat, lon, depth
ds.createDimension("station_dim", 1)
ds.setncattr("featureType", ftype)
plat = ds.createVariable("station", "|S1", dimensions=("station_dim"))
plat.setncattr("cf_role", "timeseries_id")
result = self.ioos._check_feattype_timeseries_cf_role(ds)
self.assertTrue(result.value)
# featureType = timeseries - multiple station can have cf_role be on a var with dim >= 1
# but still fails the check so as to display message
ds = MockTimeSeries() # time, lat, lon, depth
ds.createDimension("station_dim", 21)
ds.setncattr("featureType", "timeseries")
temp = ds.createVariable("temp", "d", ("time"))
plat = ds.createVariable("station", "|S1", dimensions=("station_dim"))
plat.setncattr("cf_role", "timeseries_id")
result = self.ioos._check_feattype_timeseries_cf_role(ds)
self.assertFalse(result.value)
def test_check_feattype_timeseriesprof_cf_role(self):
ftype = "timeseriesprofile"
ds = MockTimeSeries() # time, lat, lon, depth
ds.setncattr("featureType", ftype)
ds.createDimension("station_dim", 1)
temp = ds.createVariable("temp", "d", ("time"))
# no platform variables or geophys vars with cf_role=timeseries_id, fail
result = self.ioos._check_feattype_timeseriesprof_cf_role(ds)
self.assertFalse(result.value)
# create dimensionless platform variable, set bad cf_role
plat = ds.createVariable("station", "|S1", ("station_dim"))
plat.setncattr("cf_role", "badbadbad")
ds.variables["temp"].setncattr("platform", "station")
# should still fail as no vars with cf_role=timeseries_id or profile_id exist
result = self.ioos._check_feattype_timeseriesprof_cf_role(ds)
self.assertFalse(result.value)
# set correct cf_role on platform var, should still fail, no variable with profile_id
plat.setncattr("cf_role", "timeseries_id")
result = self.ioos._check_feattype_timeseriesprof_cf_role(ds)
self.assertFalse(result.value)
# add profile_id var with bad dim, fail
ds.createDimension("profile_dim", 0)
pf = ds.createVariable("profile", "|S1", ("profile_dim"))
pf.setncattr("cf_role", "profile_id")
result = self.ioos._check_feattype_timeseriesprof_cf_role(ds)
self.assertFalse(result.value)
# make the profile dim 1, no platform variable though
ds = MockTimeSeries() # time, lat, lon, depth
ds.setncattr("featureType", ftype)
ds.createDimension("station_dim", 1)
ds.createDimension("profile_dim", 1)
temp = ds.createVariable("temp", "d", ("time"))
plat = ds.createVariable("station", "|S1", ("station_dim"))
plat.setncattr("cf_role", "timeseries_id")
pf = ds.createVariable("profile", "|S1", ("profile_dim"))
pf.setncattr("cf_role", "profile_id")
result = self.ioos._check_feattype_timeseriesprof_cf_role(ds)
self.assertTrue(result.value)
# profile dim > 1, pass
ds = MockTimeSeries() # time, lat, lon, depth
ds.setncattr("featureType", ftype)
ds.createDimension("station_dim", 1)
ds.createDimension("profile_dim", 21)
temp = ds.createVariable("temp", "d", ("time"))
plat = ds.createVariable("station", "|S1", ("station_dim"))
plat.setncattr("cf_role", "timeseries_id")
pf = ds.createVariable("profile", "|S1", ("profile_dim"))
pf.setncattr("cf_role", "profile_id")
result = self.ioos._check_feattype_timeseriesprof_cf_role(ds)
self.assertTrue(result.value)
def test_check_cf_dsg(self):
ds = MockTimeSeries() # time, lat, lon, depth
ds.setncattr("platform", "single_string")
# correct cf_role & featureType, pass
ds.setncattr("featureType", "profile")
ds.createDimension("profile", 1)
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
cf_role_var = ds.createVariable("cf_role_var", np.byte, ("profile",))
cf_role_var.setncattr("cf_role", "timeseries_id")
results = self.ioos.check_cf_dsg(ds)
self.assertTrue(all(r.value for r in results))
self.assertTrue(all(r.msgs == [] for r in results))
# correct featureType, incorrect cf_role var dimension
ds = MockTimeSeries() # time, lat, lon, depth
ds.setncattr("featureType", "trajectoryprofile")
ds.createDimension("trajectory", 2) # should only be 1
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectory",))
cf_role_var.setncattr("cf_role", "trajectory_id")
results = self.ioos.check_cf_dsg(ds)
self.assertFalse(results[0].value)
# featureType==timeSeries, cf_role=timeseries_id
ds = MockTimeSeries()
ds.setncattr("featureType", "timeSeries")
ds.createDimension("station", 1)
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
cf_role_var = ds.createVariable("cf_role_var", np.byte, ("station",))
cf_role_var.setncattr("cf_role", "timeseries_id")
results = self.ioos.check_cf_dsg(ds)
# check should pass with no results
self.assertEqual(results, [])
# featureType==timeSeriesProfile, cf_role==timeseries_id, dim 1, pass
ds = MockTimeSeries()
ds.setncattr("featureType", "timeSeriesProfile")
ds.createDimension("station", 1)
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
cf_role_var = ds.createVariable("cf_role_var", np.byte, ("station",))
cf_role_var.setncattr("cf_role", "timeseries_id")
results = self.ioos.check_cf_dsg(ds)
self.assertEqual(results, [])
# featureType==timeSeriesProfile, cf_role==timeseries_id, dim 2, fail
ds = MockTimeSeries()
ds.setncattr("platform", "platform")
ds.setncattr("featureType", "timeSeriesProfile")
ds.createDimension("station", 2)
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
cf_role_var = ds.createVariable("cf_role_var", np.byte, ("station",))
cf_role_var.setncattr("cf_role", "timeseries_id")
results = self.ioos.check_cf_dsg(ds)
self.assertFalse(results[0].value)
# featureType==trajectory, cf_role==trajectory_id, dim 1, pass
ds = MockTimeSeries()
ds.setncattr("featureType", "trajectory")
ds.createDimension("trajectory", 1)
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectory",))
cf_role_var.setncattr("cf_role", "trajectory_id")
results = self.ioos.check_cf_dsg(ds)
self.assertEqual(results, [])
# featureType==trajectory, cf_role==trajectory, dim 2, fail
ds = MockTimeSeries()
ds.setncattr("featureType", "trajectory")
ds.createDimension("trajectory", 2)
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectory",))
cf_role_var.setncattr("cf_role", "trajectory_id")
results = self.ioos.check_cf_dsg(ds)
self.assertFalse(results[0].value)
# featureType==trajectoryProfile, cf_role==trajectory_id, dim 1, pass
ds = MockTimeSeries()
ds.setncattr("featureType", "trajectoryProfile")
ds.createDimension("trajectoryprof", 1)
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectoryprof",))
cf_role_var.setncattr("cf_role", "trajectory_id")
results = self.ioos.check_cf_dsg(ds)
self.assertEqual(results, [])
# featureType==trajectoryProfile, cf_role==trajectory_id, dim 2, fail
ds = MockTimeSeries()
ds.setncattr("featureType", "trajectoryProfile")
ds.createDimension("trajectoryprof", 2)
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
cf_role_var = ds.createVariable("cf_role_var", np.byte, ("trajectoryprof",))
cf_role_var.setncattr("cf_role", "trajectory_id")
results = self.ioos.check_cf_dsg(ds)
self.assertFalse(results[0].value)
# featureType==profile, cf_role==profile_id, dim 1, pass
ds = MockTimeSeries()
ds.setncattr("featureType", "profile")
ds.createDimension("prof", 1)
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
cf_role_var = ds.createVariable("cf_role_var", np.byte, ("prof",))
cf_role_var.setncattr("cf_role", "profile_id")
results = self.ioos.check_cf_dsg(ds)
self.assertEqual(results, [])
# featureType==profile, cf_role==profile_id, dim 2, fail
ds = MockTimeSeries()
ds.setncattr("featureType", "profile")
ds.createDimension("prof", 2)
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
cf_role_var = ds.createVariable("cf_role_var", np.byte, ("prof",))
cf_role_var.setncattr("cf_role", "profile_id")
results = self.ioos.check_cf_dsg(ds)
self.assertFalse(results[0].value)
# featureType==point -- do nothing
ds = MockTimeSeries()
ds.setncattr("featureType", "point")
ds.createDimension("blah", 2)
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
cf_role_var = ds.createVariable("cf_role_var", np.byte, ("blah",))
cf_role_var.setncattr("cf_role", "profile_id")
results = self.ioos.check_cf_dsg(ds)
self.assertEqual(results, [])
def test_check_single_platform(self):
ds = MockTimeSeries() # time, lat, lon, depth
# no global attr but also no platform variables, should pass
result = self.ioos.check_single_platform(ds)
self.assertTrue(result.value)
self.assertEqual(result.msgs, [])
# give platform global, no variables, fail
ds.setncattr("platform", "buoy")
result = self.ioos.check_single_platform(ds)
self.assertFalse(result.value)
# global platform, one platform variable, pass
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
result = self.ioos.check_single_platform(ds)
self.assertTrue(result.value)
self.assertEqual(result.msgs, [])
# two platform variables, fail
temp2 = ds.createVariable("temp2", "d", ("time"))
temp2.setncattr("platform", "platform_var2")
plat = ds.createVariable("platform_var2", np.byte)
result = self.ioos.check_single_platform(ds)
self.assertFalse(result.value)
# no global attr, one variable, fail
ds = MockTimeSeries() # time, lat, lon, depth
temp = ds.createVariable("temp", "d", ("time"))
temp.setncattr("platform", "platform_var")
plat = ds.createVariable("platform_var", np.byte)
result = self.ioos.check_single_platform(ds)
self.assertFalse(result.value)
def test_check_wmo_platform_code(self):
ds = MockTimeSeries() # time, lat, lon, depth
# no wmo_platform_code, pass
result = self.ioos.check_wmo_platform_code(ds)
self.assertTrue(result.value)
self.assertEqual(result.msgs, [])
# valid code
ds.setncattr("wmo_platform_code", "12345")
result = self.ioos.check_wmo_platform_code(ds)
self.assertTrue(result.value)
# valid code
ds.setncattr("wmo_platform_code", "7654321")
result = self.ioos.check_wmo_platform_code(ds)
self.assertTrue(result.value)
# alphanumeric, valid
ds.setncattr("wmo_platform_code", "abcd1")
result = self.ioos.check_wmo_platform_code(ds)
self.assertTrue(result.value)
# invalid length, fail
ds.setncattr("wmo_platform_code", "123")
result = self.ioos.check_wmo_platform_code(ds)
self.assertFalse(result.value)
# alphanumeric len 7, fail
ds.setncattr("wmo_platform_code", "1a2b3c7")
result = self.ioos.check_wmo_platform_code(ds)
self.assertFalse(result.value)
def test_check_gts_ingest_requirements(self):
ds = MockTimeSeries() # time, lat, lon, depth
# NOTE: this check will always have a "failing" result; see
# https://github.com/ioos/compliance-checker/issues/759#issuecomment-625356938
# and subsequent discussion
# no gts_ingest_requirements, should pass
result = self.ioos.check_gts_ingest_requirements(ds)
self.assertFalse(result.value)
# flag for ingest, no variables flagged - default pass
ds.setncattr("gts_ingest", "true")
result = self.ioos.check_gts_ingest_requirements(ds)
self.assertFalse(result.value)
# give one variable the gts_ingest attribute
# no standard_name or ancillary vars, should fail
ds.variables["time"].setncattr("gts_ingest", "true")
result = self.ioos.check_gts_ingest_requirements(ds)
self.assertFalse(result.value)
# no ancillary vars, should fail
ds.variables["time"].setncattr("gts_ingest", "true")
ds.variables["time"].setncattr("standard_name", "time")
result = self.ioos.check_gts_ingest_requirements(ds)
self.assertFalse(result.value)
self.assertIn(
"The following variables did not qualify for NDBC/GTS Ingest: time\n",
result.msgs,
)
# set ancillary var with bad standard name
tmp = ds.createVariable("tmp", np.byte, ("time", ))
tmp.setncattr("standard_name", "bad")
ds.variables["time"].setncattr("ancillary_variables", "tmp")
result = self.ioos.check_gts_ingest_requirements(ds)
self.assertFalse(result.value)
self.assertIn(
"The following variables did not qualify for NDBC/GTS Ingest: time\n",
result.msgs,
)
# good ancillary var standard name, time units are bad
tmp.setncattr("standard_name", "aggregate_quality_flag")
ds.variables["time"].setncattr("units", "bad since bad")
result = self.ioos.check_gts_ingest_requirements(ds)
self.assertFalse(result.value)
self.assertIn(
"The following variables did not qualify for NDBC/GTS Ingest: time\n",
result.msgs,
)
# good ancillary var stdname, good units, pass
tmp.setncattr("standard_name", "aggregate_quality_flag")
ds.variables["time"].setncattr("units",
"seconds since 1970-01-01T00:00:00Z")
result = self.ioos.check_gts_ingest_requirements(ds)
self.assertFalse(result.value)
self.assertIn(
"The following variables qualified for NDBC/GTS Ingest: time\n",
result.msgs)
def test_check_cell_methods(self):
"""Load a dataset (climatology.nc) and check the cell methods.
This dataset has variable "temperature" which has valid cell_methods
format, cell_methods attribute, and valid names within the
cell_methods attribute."""
dataset = self.load_dataset(STATIC_FILES['climatology'])
results = self.cf.check_cell_methods(dataset)
scored, out_of, messages = self.get_results(results)
# use itertools.chain() to unpack the lists of messages
results_list = list(chain(*(r.msgs for r in results if r.msgs)))
# check the results only have expected headers
assert set([r.name for r in results]).issubset(set([u'§7.1 Cell Boundaries', u'§7.3 Cell Methods']))
# check that all the expected variables have been hit
assert all("temperature" in msg for msg in results_list)
# check that all the results have come back passing
assert all(r.value[0] == r.value[1] for r in results)
# create a temporary variable and test this only
nc_obj = MockTimeSeries()
nc_obj.createVariable('temperature', 'd', ('time',))
temp = nc_obj.variables['temperature']
temp.cell_methods = 'lat: lon: mean depth: mean (interval: 20 meters)'
results = self.cf.check_cell_methods(nc_obj)
# invalid components lat, lon, and depth -- expect score == (6, 9)
scored, out_of, messages = self.get_results(results)
assert scored != out_of
temp.cell_methods = 'lat: lon: mean depth: mean (interval: x whizbangs)'
results = self.cf.check_cell_methods(nc_obj)
scored, out_of, messages = self.get_results(results)
# check non-standard comments are gauged correctly
temp.cell_methods = 'lat: lon: mean depth: mean (comment: should not go here interval: 2.5 m)'
results = self.cf.check_cell_methods(nc_obj)
scored, out_of, messages = self.get_results(results)
self.assertTrue(u'§7.3.3 The non-standard "comment:" element must come after any standard elements in cell_methods for variable temperature' in messages)
# standalone comments require no keyword
temp.cell_methods = 'lon: mean (This is a standalone comment)'
results = self.cf.check_cell_methods(nc_obj)
scored, out_of, messages = self.get_results(results)
assert "standalone" not in messages
# check that invalid keywords dealt with
temp.cell_methods = 'lat: lon: mean depth: mean (invalid_keyword: this is invalid)'
results = self.cf.check_cell_methods(nc_obj)
scored, out_of, messages = self.get_results(results)
self.assertTrue(u'§7.3.3 Invalid cell_methods keyword "invalid_keyword:" for variable temperature. Must be one of [interval, comment]' in messages)
# check that "parenthetical elements" are well-formed (they should not be)
temp.cell_methods = 'lat: lon: mean depth: mean (interval 0.2 m interval: 0.01 degrees)'
results = self.cf.check_cell_methods(nc_obj)
scored, out_of, messages = self.get_results(results)
assert u'§7.3.3 Parenthetical content inside temperature:cell_methods is not well formed: interval 0.2 m interval: 0.01 degrees' in messages
def test_check_geospatial_vertical_max(self):
ds = MockTimeSeries()
ds.variables["depth"][:] = np.linspace(0., 30., num=500)
# give integer geospatial_vertical_max/min, test
ds.setncattr("geospatial_vertical_min", 0)
ds.setncattr("geospatial_vertical_max", 30)
result = self.acdd.check_vertical_extents(ds)
self.assert_result_is_good(result)
# give float geospatial_vertical_min/max, test
ds.setncattr("geospatial_vertical_min", 0.)
ds.setncattr("geospatial_vertical_max", 30.)
result = self.acdd.check_vertical_extents(ds)
self.assert_result_is_good(result)
# give string (in number-form), test
ds.setncattr("geospatial_vertical_min", "0.")
ds.setncattr("geospatial_vertical_max", "30.")
result = self.acdd.check_vertical_extents(ds)
self.assert_result_is_good(result)
# give garbage string -- expect failure
ds.setncattr("geospatial_vertical_min", "bad")
ds.setncattr("geospatial_vertical_max", "val")
result = self.acdd.check_vertical_extents(ds)
self.assert_result_is_bad(result)
# all masked values mean that there are no valid array elements to get
# the min/max of
ds.setncattr("geospatial_vertical_min", 0.)
ds.setncattr("geospatial_vertical_max", 30.)
ds.variables["depth"][:] = np.ma.masked_all(
ds.variables["depth"].shape)
result = self.acdd.check_vertical_extents(ds)
self.assert_result_is_bad(result)
def test_check_lon_extents(self):
"""Test the check_lon_extents() method behaves expectedly"""
# create dataset using MockDataset, give it lat/lon dimensions
ds = MockTimeSeries()
ds.variables["lon"][:] = np.linspace(65., 67., num=500)
# test no values, expect failure
#result = self.acdd.check_lon_extents(ds)
#self.assert_result_is_bad(result)
# give integer geospatial_lon_max/min, test
ds.setncattr("geospatial_lon_min", 65)
ds.setncattr("geospatial_lon_max", 67)
result = self.acdd.check_lon_extents(ds)
self.assert_result_is_good(result)
# give float geospatial_lon_min/max, test
ds.setncattr("geospatial_lon_min", 65.)
ds.setncattr("geospatial_lon_max", 67.)
result = self.acdd.check_lon_extents(ds)
self.assert_result_is_good(result)
# give string (in number-form), test
ds.setncattr("geospatial_lon_min", "65.")
ds.setncattr("geospatial_lon_max", "67.")
result = self.acdd.check_lon_extents(ds)
self.assert_result_is_good(result)
# give garbage string -- expect failure
ds.setncattr("geospatial_lon_min", "bad")
ds.setncattr("geospatial_lon_max", "val")
result = self.acdd.check_lon_extents(ds)
self.assert_result_is_bad(result)
def test_check_lat_extents(self):
"""Test the check_lat_extents() method behaves expectedly"""
# create dataset using MockDataset, give it lat/lon dimensions
ds = MockTimeSeries()
ds.variables["lat"][:] = np.linspace(
-135., -130., num=500) # arbitrary, but matches time dim size
# test no values, expect failure
result = self.acdd.check_lat_extents(ds)
self.assert_result_is_bad(result)
# give integer geospatial_lat_max/min, test
ds.setncattr("geospatial_lat_min", -135)
ds.setncattr("geospatial_lat_max", -130)
result = self.acdd.check_lat_extents(ds)
self.assert_result_is_good(result)
# give float geospatial_lat_min/max, test
ds.setncattr("geospatial_lat_min", -135.)
ds.setncattr("geospatial_lat_max", -130.)
result = self.acdd.check_lat_extents(ds)
self.assert_result_is_good(result)
# give string (in number-form), test
ds.setncattr("geospatial_lat_min", "-135.")
ds.setncattr("geospatial_lat_max", "-130.")
result = self.acdd.check_lat_extents(ds)
self.assert_result_is_good(result)
# give garbage string -- expect failure
ds.setncattr("geospatial_lat_min", "bad")
ds.setncattr("geospatial_lat_max", "val")
result = self.acdd.check_lat_extents(ds)
self.assert_result_is_bad(result)
def test_check_contributor_role_and_vocabulary(self):
ds = MockTimeSeries() # time, lat, lon, depth
# no contributor_role or vocab, fail both
results = self.ioos.check_contributor_role_and_vocabulary(ds)
self.assertFalse(all(r.value for r in results))
# bad contributor_role and vocab
ds.setncattr("contributor_role", "bad")
ds.setncattr("contributor_role_vocabulary", "bad")
results = self.ioos.check_contributor_role_and_vocabulary(ds)
self.assertFalse(all(r.value for r in results))
# good role, bad vocab
ds.setncattr("contributor_role", "contributor")
results = self.ioos.check_contributor_role_and_vocabulary(ds)
self.assertTrue(results[0].value)
self.assertEqual(results[0].msgs, [])
self.assertFalse(results[1].value)
# bad role, good vocab
ds.setncattr("contributor_role", "bad")
ds.setncattr(
"contributor_role_vocabulary",
"http://vocab.nerc.ac.uk/collection/G04/current/",
)
results = self.ioos.check_contributor_role_and_vocabulary(ds)
self.assertFalse(results[0].value)
self.assertTrue(results[1].value)
self.assertEqual(results[1].msgs, [])
# good role, good vocab
ds.setncattr("contributor_role", "contributor")
ds.setncattr(
"contributor_role_vocabulary",
"http://vocab.nerc.ac.uk/collection/G04/current/",
)
results = self.ioos.check_contributor_role_and_vocabulary(ds)
self.assertTrue(results[0].value)
self.assertEqual(results[0].msgs, [])
self.assertTrue(results[1].value)
self.assertEqual(results[1].msgs, [])
ds.setncattr("contributor_role", "resourceProvider")
ds.setncattr(
"contributor_role_vocabulary",
"https://www.ngdc.noaa.gov/wiki/index.php?title=ISO_19115_and_19115-2_CodeList_Dictionaries#CI_RoleCode",
)
results = self.ioos.check_contributor_role_and_vocabulary(ds)
self.assertTrue(results[0].value)
self.assertEqual(results[0].msgs, [])
self.assertTrue(results[1].value)
self.assertEqual(results[1].msgs, [])
def test_check_feattype_profile_cf_role(self):
ds = MockTimeSeries() # time, lat, lon, depth
ds.setncattr("featureType", "profile")
# no platform variables or geophys vars with cf_role=profile_id, fail
result = self.ioos._check_feattype_profile_cf_role(ds)
self.assertFalse(result.value)
# create dimensionless platform variable, set bad cf_role
ds.createDimension("profile_dim", 0)
prf = ds.createVariable("profile", "|S1", ("profile_dim"))
prf.setncattr("cf_role", "badbadbad")
result = self.ioos._check_feattype_profile_cf_role(ds)
self.assertFalse(result.value)
# set correct cf_role, should still fail as profile is not 1
prf.setncattr("cf_role", "profile_id")
result = self.ioos._check_feattype_profile_cf_role(ds)
self.assertFalse(result.value)
# set correct dim size
ds = MockTimeSeries() # time, lat, lon, depth
ds.setncattr("featureType", "profile")
ds.createDimension("profile_dim", 1)
prf = ds.createVariable("profile", "|S1", ("profile_dim"))
prf.setncattr("cf_role", "profile_id")
result = self.ioos._check_feattype_profile_cf_role(ds)
self.assertTrue(result.value)
