def test():
profile = DummyData()
pqc = ProfileQC(profile, saveauxiliary=False)
pqc = ProfileQC(profile, saveauxiliary=True)
keys = ['PRES', 'TEMP', 'PSAL', 'flag']
for v in profile.keys():
assert v in pqc.keys()
assert np.allclose(profile[v], pqc[v])
for a in profile.attributes:
assert a in pqc.attributes
assert profile.attributes[a] == pqc.attributes[a]
assert hasattr(pqc, 'flags')
assert type(pqc.flags) is dict
vs = list(pqc.flags.keys())
vs.remove('common')
for v in vs:
for f in pqc.flags[v]:
assert pqc.flags[v][f].dtype == 'i1'
assert hasattr(pqc, 'features')
assert type(pqc.features) is dict
def test_LocationAtSea_track():
"""Test standard with multiple locations
lat & lon defined in the dataset. This would be the case for a TSG
where each measurement is associated with a location.
Locking etopo resolution, since it can change the values.
Note that there is no restriction in the number of locations. In this
example there are multiple depths but only 3 positions. It's not the
LocationAtSea job to make sense of that. Should it match with which
variable? It can't be done here, but should be done once the tests
are combined.
"""
data = DummyData()
data.data["LATITUDE"] = [15, 12, 8]
data.data["LONGITUDE"] = [-38, 222, 0]
y = LocationAtSea(data, cfg={"resolution": "5min"})
assert hasattr(y, "features")
assert "bathymetry" in y.features
assert ma.allclose(y.features["bathymetry"], [5036, 4995, -122])
assert hasattr(y, "flags")
assert "location_at_sea" in y.flags
assert ma.allclose(y.flags["location_at_sea"], [1, 1, 4])
def test():
profile = DummyData()
profile.data['PRES'] = ma.masked_array([1.0, 100, 200, 300, 500, 5000])
profile.data['TEMP'] = ma.masked_array(
[27.44, 14.55, 11.96, 11.02, 7.65, 2.12])
profile.data['PSAL'] = ma.masked_array(
[35.71, 35.50, 35.13, 35.02, 34.72, 35.03])
features = {
'tukey53H':
ma.masked_array([0, 0, 0.3525000000000009, 0.35249999999999915, 0, 0],
mask=[True, True, False, False, True, True]),
'tukey53H_norm':
ma.masked_array([0, 0, 0.07388721803621254, 0.07388721803621218, 0, 0],
mask=[True, True, False, False, True, True])
}
flags = {'tukey53H_norm': np.array([0, 0, 1, 1, 0, 0], dtype='i1')}
cfg = {'l': 5, 'threshold': 6, 'flag_good': 1, 'flag_bad': 4}
y = Tukey53H(profile, 'TEMP', cfg)
y.test()
assert type(y.features) is dict
for f in y.features:
assert ma.allclose(y.features[f], features[f])
for f in y.flags:
assert ma.allclose(y.flags[f], flags[f])
def test_nolatlon():
"""If missing lat or lon, return flag 0 for everything
"""
cfg = {
'name': 'PIRATA',
'region': 'POLYGON ((-40 10, -40 20, -30 20, -30 10, -40 10))',
'minval': 2.0,
'maxval': 40
}
profile = DummyData()
del (profile.attrs['LATITUDE'])
y = RegionalRange(profile, 'TEMP', cfg)
assert 'regional_range' in y.flags
flag = y.flags['regional_range']
assert profile['TEMP'].shape == flag.shape
assert (flag == 0).all()
profile = DummyData()
del (profile.attrs['LONGITUDE'])
y = RegionalRange(profile, 'TEMP', cfg)
assert 'regional_range' in y.flags
flag = y.flags['regional_range']
assert profile['TEMP'].shape == flag.shape
assert (flag == 0).all()
def test_single_negative_depth():
"""Evaluate a profile with a single measurement
WOD has some profiles with a single measurement. Something certainly went
wrong on those profiles, despite that, CoTeDe should be able to do the
best assessement possible. Some tests can't be applied, like spike which
requires neighbor measurements, but those should return flag 0.
"""
profile = DummyData()
profile.attrs = {
"id": 609483,
"LATITUDE": 6.977,
"LONGITUDE": 79.873,
"datetime": datetime(2009, 8, 14, 1, 18, 36),
"date": date(2009, 8, 14),
"schema": "pfl",
}
profile.data = {
"id": ma.masked_array(data=[51190527], mask=[False], dtype="i"),
"PRES": ma.masked_array(data=[-1.0], mask=[False], dtype="f"),
"TEMP": ma.masked_array(data=[25.81], mask=[False], dtype="f"),
"PSAL": ma.masked_array(data=[0.01], mask=[False], dtype="f"),
}
ProfileQC(profile, saveauxiliary=False)
ProfileQC(profile, saveauxiliary=True)
def test_attribute():
data = DummyData()
coords = [[10, -30, 1], [10, 330, 1]]
for lat, lon, flag in coords:
data.attrs["LATITUDE"] = lat
data.attrs["LONGITUDE"] = lon
assert location_at_sea(data) == flag
def test_attribute_inland():
data = DummyData()
coords = [[-10, -60, 3], [-10, 300, 3]]
for lat, lon, flag in coords:
data.attrs["LATITUDE"] = lat
data.attrs["LONGITUDE"] = lon
assert location_at_sea(data) == flag
def test_badlocation():
data = DummyData()
coords = [[91, -30, 3], [-91, -30, 3], [10, -361, 3], [10, 1000, 3]]
for lat, lon, flag in coords:
data.attrs["LATITUDE"] = lat
data.attrs["LONGITUDE"] = lon
assert location_at_sea(data) == flag
def test_default():
"""Test Regional Range on the default DummyData
Must recognize flags 1, 4, and 9.
"""
profile = DummyData()
cfg = {
'name': 'PIRATA',
'region': 'POLYGON ((-40 10, -40 20, -30 20, -30 10, -40 10))',
'minval': 15.0,
'maxval': 40
}
y = RegionalRange(profile, 'TEMP', cfg)
assert 'regional_range' in y.flags
assert profile['TEMP'].shape == y.flags['regional_range'].shape
idx = ma.getmaskarray(profile['TEMP'])
assert idx.any(), "Redefine cfg to have at least one masked value"
try:
import shapely
assert np.all(y.flags['regional_range'][idx] == 9)
except ImportError:
# If Shapely is not available, return 0, i.e. no evaluation.
assert np.all(y.flags['regional_range'][idx] == 0)
return
x = profile['TEMP'][y.flags['regional_range'] == 1]
idx = (x >= cfg['minval']) & (x <= cfg['maxval'])
assert idx.any(), "Redefine cfg to have at least one valid value"
assert idx.all()
x = profile['TEMP'][y.flags['regional_range'] == 4]
idx = (x < cfg['minval']) | (x > cfg['maxval'])
assert idx.any(), "Redefine cfg to have at least one non-valid value"
assert idx.all()
def test_nocoverage():
"""Test when Regional Range does not cover the data position.
Returns flag 9 for invalid measurements and 0 for everything else.
"""
profile = DummyData()
cfg = {
'name': 'red_sea',
'region': 'POLYGON ((10 40, 20 50, 30 30, 10 40))',
'minval': 21.7,
'maxval': 40
}
y = RegionalRange(profile, 'TEMP', cfg)
assert 'regional_range' in y.flags
assert profile['TEMP'].shape == y.flags['regional_range'].shape
idx = ma.getmaskarray(profile['TEMP'])
assert idx.any(), "Redefine cfg to have at least one masked value"
try:
import shapely
assert np.all(y.flags['regional_range'][idx] == 9)
except ImportError:
# If Shapely is not available, return 0, i.e. no evaluation.
assert np.all(y.flags['regional_range'][idx] == 0)
return
assert np.all(y.flags['regional_range'][~idx] == 0)
def test_ProfileQC():
if nogsw:
print("GSW package not available. Can't run density_inversion test.")
return
profile = DummyData()
profile["TEMP"][4] = profile["TEMP"][3] + 5
cfg = {
"TEMP": {
"density_inversion": {
"threshold": -0.03,
"flag_good": 1,
"flag_bad": 4
}
}
}
pqc = ProfileQC(profile, cfg)
assert type(pqc.features) is dict
assert "densitystep" in pqc.features["TEMP"]
assert type(pqc.flags) is dict
assert "density_inversion" in pqc.flags["TEMP"]
assert pqc.flags["TEMP"]["density_inversion"][4] == 4
def test_standard_error():
"""I need to improve this!!
"""
profile = DummyData()
profile.attrs['datetime'] = datetime(2016,6,4)
profile.attrs['LATITUDE'] = -30.0
profile.attrs['LONGITUDE'] = 15
profile.data['PRES'] = np.array([2.0, 5.0, 6.0, 21.0, 44.0, 79.0, 1000, 5000])
profile.data['TEMP'] = np.array([16.0, 15.6, 15.9, 15.7, 15.2, 14.1, 8.6, 2.0])
cfg = {"TEMP": {"woa_normbias": {"threshold": 10}}}
pqc = ProfileQC(profile, cfg=cfg)
assert 'woa_normbias' in pqc.flags['TEMP']
assert pqc.flags['TEMP']['woa_normbias'].shape == profile.data['TEMP'].shape
assert (pqc.flags['TEMP']['woa_normbias'] == [1, 1, 1, 1, 1, 1, 4, 0]).all()
cfg = {"TEMP": {"woa_normbias": {
"threshold": 10, "use_standard_error": False}}}
pqc_noSE = ProfileQC(profile, cfg=cfg)
assert 'woa_normbias' in pqc.flags['TEMP']
assert pqc.flags['TEMP']['woa_normbias'].shape == profile.data['TEMP'].shape
assert (pqc.flags['TEMP']['woa_normbias'] == [1, 1, 1, 1, 1, 1, 4, 0]).all()
cfg = {"TEMP": {"woa_normbias": {"threshold": 10, "use_standard_error": True}}}
pqc_SE = ProfileQC(profile, cfg=cfg)
assert 'woa_normbias' in pqc.flags['TEMP']
assert pqc.flags['TEMP']['woa_normbias'].shape == profile.data['TEMP'].shape
assert (pqc.flags['TEMP']['woa_normbias'] == [1, 1, 1, 1, 1, 1, 4, 0]).all()
def test():
if nogsw:
print("GSW package not available. Can't run density_inversion test.")
return
profile = DummyData()
profile.data["PRES"] = ma.masked_array([1.0, 100, 200, 300, 500, 5000])
profile.data["TEMP"] = ma.masked_array(
[27.44, 14.55, 11.96, 11.02, 7.65, 2.12])
profile.data["PSAL"] = ma.masked_array(
[35.71, 35.50, 35.13, 35.02, 34.72, 35.03])
cfg = {"threshold": -0.03, "flag_good": 1, "flag_bad": 4}
y = DensityInversion(profile, cfg)
assert type(y.features) is dict
assert "densitystep" in y.features
def test_default():
profile = DummyData()
cfg = {'flag_good': 1, 'flag_bad': 4}
y = StuckValue(profile, 'TEMP', cfg)
assert type(y.features) is dict
idx = ma.getmaskarray(profile['TEMP'])
assert (y.flags['stuck_value'][idx] == 9).all()
assert (y.flags['stuck_value'][~idx] == 1).all()
def test():
profile = DummyData()
cfg = [["> 0", "<= 25", -2, 37], ["> 25", "<= 50", -2, 36]]
flags = profile_envelop(profile, cfg, "TEMP")
# Check for BUG #9.
assert 9 in flags
def test_attribute():
profile = DummyData()
cfg = {"TEMP": {"woa_normbias": {"threshold": 3}}}
pqc = ProfileQC(profile, cfg=cfg)
assert 'woa_normbias' in pqc.flags['TEMP']
assert pqc.flags['TEMP']['woa_normbias'].shape == profile.data['TEMP'].shape
assert np.unique(pqc.features['TEMP']['woa_mean']).size > 1
assert (pqc.flags['TEMP']['woa_normbias'] ==
[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 9]).all()
def test():
try:
import gsw
except:
print('GSW package not available. Can\'t run density_inversion test.')
return
profile = DummyData()
profile.data['PRES'] = ma.masked_array([1.0, 100, 200, 300, 500, 5000])
profile.data['TEMP'] = ma.masked_array(
[27.44, 14.55, 11.96, 11.02, 7.65, 2.12])
profile.data['PSAL'] = ma.masked_array(
[35.71, 35.50, 35.13, 35.02, 34.72, 35.03])
cfg = {'threshold': -0.03, 'flag_good': 1, 'flag_bad': 4}
y = DensityInversion(profile, cfg)
assert type(y.features) is dict
assert 'densitystep' in y.features
def test_basic():
"""
"""
profile = DummyData()
cfg = {"TEMP": {"woa_normbias": {"threshold": 3, "flag_bad": 3}},
"PSAL": {"woa_normbias": {"threshold": 3, "flag_bad": 3}}}
pqc = ProfileQC(profile, cfg=cfg)
assert 'woa_normbias' in pqc.flags['TEMP']
assert sorted(np.unique(pqc.flags['TEMP']['woa_normbias'])) == [1, 3, 9]
assert sorted(np.unique(pqc.flags['PSAL']['woa_normbias'])) == [1, 9]
def test_neglible_difference():
profile = DummyData()
profile['TEMP'][:] = profile['TEMP'] * 0 + 3.14
profile['TEMP'][:] += np.random.randn(profile['TEMP'].size) * 1e-10
cfg = {'flag_good': 1, 'flag_bad': 4}
y = StuckValue(profile, 'TEMP', cfg)
assert type(y.features) is dict
idx = ma.getmaskarray(profile['TEMP'])
assert (y.flags['stuck_value'][idx] == 9).all()
assert (y.flags['stuck_value'][~idx] == 4).all()
def test():
"""
"""
profile = DummyData()
cfg = {"TEMP": {"cars_normbias": {"threshold": 10}},
"PSAL": {"cars_normbias": {"threshold": 10}}}
pqc = ProfileQC(profile, cfg=cfg)
assert 'cars_normbias' in pqc.flags['TEMP']
assert sorted(np.unique(pqc.flags['TEMP']['cars_normbias'])) == [1, 9]
#assert sorted(np.unique(pqc.flags['TEMP2']['cars_normbias'])) == [1]
assert sorted(np.unique(pqc.flags['PSAL']['cars_normbias'])) == [1, 9]
def test():
"""
"""
profile = DummyData()
pqc = ProfileQC(profile, cfg='morello2014')
assert 'morello2014' in pqc.flags['TEMP']
assert 'morello2014' in pqc.flags['PSAL']
assert profile['TEMP'].shape == pqc.flags['TEMP']['morello2014'].shape
assert profile['PSAL'].shape == pqc.flags['PSAL']['morello2014'].shape
def test():
profile = DummyData()
feature = ma.masked_array([0, 1.25, 5.875, 0],
mask=[True, False, False, True])
cfg = {'threshold': 6, 'flag_good': 1, 'flag_bad': 4}
y = Gradient(profile, 'TEMP', cfg)
y.test()
assert type(y.features) is dict
def test():
"""
"""
profile = DummyData()
pqc = ProfileQC(profile, cfg='fuzzylogic')
assert 'fuzzylogic' in pqc.flags['TEMP']
assert 'fuzzylogic' in pqc.flags['PSAL']
assert profile['TEMP'].shape == pqc.flags['TEMP']['fuzzylogic'].shape
assert profile['PSAL'].shape == pqc.flags['PSAL']['fuzzylogic'].shape
def test_attribute():
profile = DummyData()
cfg = {"TEMP": {"woa_normbias": {"threshold": 3}}}
pqc = ProfileQC(profile, cfg=cfg)
assert "woa_normbias" in pqc.flags["TEMP"]
assert pqc.flags["TEMP"]["woa_normbias"].shape == profile.data[
"TEMP"].shape
assert np.unique(pqc.features["TEMP"]["woa_mean"]).size > 1
assert (pqc.flags["TEMP"]["woa_normbias"] == [
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 9
]).all()
def test():
""" Only test if run. Must improve this.
"""
profile = DummyData()
pqc = ProfileQC(profile, cfg='anomaly_detection')
assert 'anomaly_detection' in pqc.flags['TEMP']
assert 'anomaly_detection' in pqc.flags['PSAL']
assert profile['TEMP'].shape == pqc.flags['TEMP'][
'anomaly_detection'].shape
assert profile['PSAL'].shape == pqc.flags['PSAL'][
'anomaly_detection'].shape
def test_attribute():
profile = DummyData()
profile.attrs['datetime'] = datetime(2016, 6, 4)
profile.attrs['LATITUDE'] = -30.0
profile.attrs['LONGITUDE'] = 15
profile.data['PRES'] = np.array([2.0, 5.0, 6.0, 21.0, 44.0, 79.0])
profile.data['TEMP'] = np.array([16.0, 15.6, 15.9, 5.7, 15.2, 14.1])
cfg = {"TEMP": {"cars_normbias": {"threshold": 6}}}
pqc = ProfileQC(profile, cfg=cfg)
assert 'cars_normbias' in pqc.flags['TEMP']
assert pqc.flags['TEMP']['cars_normbias'].shape == profile.data['TEMP'].shape
assert (pqc.flags['TEMP']['cars_normbias'] == [1, 1, 1, 3, 1, 1]).all()
def test_all_valid_no_9():
""" If all measurements are valid it can't return flag 9
This is to test a special condition when all values are valid, .mask
return False, instead of an array on the same size with False.
This test input all valid values, and check if there is no flag 9.
"""
profile = DummyData()
pqc = ProfileQC(profile)
assert pqc['TEMP'].mask.all() == False
assert np.allclose(
combined_flag(pqc.flags['TEMP']) == 9, profile['TEMP'].mask)
def test_common_flags():
profile = DummyData()
cfg = {
"main": {
"valid_datetime": None,
"valid_geolocation": None
},
"TEMP": {}
}
pqc = ProfileQC(profile, cfg=cfg)
assert 'common' in pqc.flags
assert 'valid_datetime' in pqc.flags['TEMP']
assert pqc.flags['TEMP']['valid_datetime'].shape == profile['TEMP'].shape
def test_LocationAtSea_attrs():
"""Test standard with single location
Lat & Lon defined in the attrs
Locking etopo resolution, since it can change the values.
"""
data = DummyData()
y = LocationAtSea(data, cfg={"resolution": "5min"})
assert hasattr(y, "features")
assert "bathymetry" in y.features
assert ma.allclose(y.features["bathymetry"], 5036)
assert hasattr(y, "flags")
assert "location_at_sea" in y.flags
assert ma.allclose(y.flags["location_at_sea"], 1)
def test_LocationAtSea_attrs():
"""Test standard with single location
Lat & Lon defined in the attrs
Locking etopo resolution, since it can change the values.
"""
data = DummyData()
y = LocationAtSea(data, cfg={'resolution': '5min'})
assert hasattr(y, 'features')
assert 'bathymetry' in y.features
assert ma.allclose(y.features['bathymetry'], 5036)
assert hasattr(y, 'flags')
assert 'location_at_sea' in y.flags
assert ma.allclose(y.flags['location_at_sea'], 1)
