def test_nolatlon(): """If missing lat or lon, return flag 0 for everything """ cfg = { 'name': 'PIRATA', 'region': 'POLYGON ((-40 10, -40 20, -30 20, -30 10, -40 10))', 'minval': 2.0, 'maxval': 40 } profile = DummyData() del (profile.attrs['LATITUDE']) y = RegionalRange(profile, 'TEMP', cfg) assert 'regional_range' in y.flags flag = y.flags['regional_range'] assert profile['TEMP'].shape == flag.shape assert (flag == 0).all() profile = DummyData() del (profile.attrs['LONGITUDE']) y = RegionalRange(profile, 'TEMP', cfg) assert 'regional_range' in y.flags flag = y.flags['regional_range'] assert profile['TEMP'].shape == flag.shape assert (flag == 0).all()
def test(): profile = DummyData() profile.data['PRES'] = ma.masked_array([1.0, 100, 200, 300, 500, 5000]) profile.data['TEMP'] = ma.masked_array( [27.44, 14.55, 11.96, 11.02, 7.65, 2.12]) profile.data['PSAL'] = ma.masked_array( [35.71, 35.50, 35.13, 35.02, 34.72, 35.03]) features = { 'tukey53H': ma.masked_array([0, 0, 0.3525000000000009, 0.35249999999999915, 0, 0], mask=[True, True, False, False, True, True]), 'tukey53H_norm': ma.masked_array([0, 0, 0.07388721803621254, 0.07388721803621218, 0, 0], mask=[True, True, False, False, True, True]) } flags = {'tukey53H_norm': np.array([0, 0, 1, 1, 0, 0], dtype='i1')} cfg = {'l': 5, 'threshold': 6, 'flag_good': 1, 'flag_bad': 4} y = Tukey53H(profile, 'TEMP', cfg) y.test() assert type(y.features) is dict for f in y.features: assert ma.allclose(y.features[f], features[f]) for f in y.flags: assert ma.allclose(y.flags[f], flags[f])
def test_default(): """Test Regional Range on the default DummyData Must recognize flags 1, 4, and 9. """ profile = DummyData() cfg = { 'name': 'PIRATA', 'region': 'POLYGON ((-40 10, -40 20, -30 20, -30 10, -40 10))', 'minval': 15.0, 'maxval': 40 } y = RegionalRange(profile, 'TEMP', cfg) assert 'regional_range' in y.flags assert profile['TEMP'].shape == y.flags['regional_range'].shape idx = ma.getmaskarray(profile['TEMP']) assert idx.any(), "Redefine cfg to have at least one masked value" try: import shapely assert np.all(y.flags['regional_range'][idx] == 9) except ImportError: # If Shapely is not available, return 0, i.e. no evaluation. assert np.all(y.flags['regional_range'][idx] == 0) return x = profile['TEMP'][y.flags['regional_range'] == 1] idx = (x >= cfg['minval']) & (x <= cfg['maxval']) assert idx.any(), "Redefine cfg to have at least one valid value" assert idx.all() x = profile['TEMP'][y.flags['regional_range'] == 4] idx = (x < cfg['minval']) | (x > cfg['maxval']) assert idx.any(), "Redefine cfg to have at least one non-valid value" assert idx.all()
def test_standard_error(): """I need to improve this!! """ profile = DummyData() profile.attrs['datetime'] = datetime(2016,6,4) profile.attrs['LATITUDE'] = -30.0 profile.attrs['LONGITUDE'] = 15 profile.data['PRES'] = np.array([2.0, 5.0, 6.0, 21.0, 44.0, 79.0, 1000, 5000]) profile.data['TEMP'] = np.array([16.0, 15.6, 15.9, 15.7, 15.2, 14.1, 8.6, 2.0]) cfg = {"TEMP": {"woa_normbias": {"threshold": 10}}} pqc = ProfileQC(profile, cfg=cfg) assert 'woa_normbias' in pqc.flags['TEMP'] assert pqc.flags['TEMP']['woa_normbias'].shape == profile.data['TEMP'].shape assert (pqc.flags['TEMP']['woa_normbias'] == [1, 1, 1, 1, 1, 1, 4, 0]).all() cfg = {"TEMP": {"woa_normbias": { "threshold": 10, "use_standard_error": False}}} pqc_noSE = ProfileQC(profile, cfg=cfg) assert 'woa_normbias' in pqc.flags['TEMP'] assert pqc.flags['TEMP']['woa_normbias'].shape == profile.data['TEMP'].shape assert (pqc.flags['TEMP']['woa_normbias'] == [1, 1, 1, 1, 1, 1, 4, 0]).all() cfg = {"TEMP": {"woa_normbias": {"threshold": 10, "use_standard_error": True}}} pqc_SE = ProfileQC(profile, cfg=cfg) assert 'woa_normbias' in pqc.flags['TEMP'] assert pqc.flags['TEMP']['woa_normbias'].shape == profile.data['TEMP'].shape assert (pqc.flags['TEMP']['woa_normbias'] == [1, 1, 1, 1, 1, 1, 4, 0]).all()
def test_nocoverage(): """Test when Regional Range does not cover the data position. Returns flag 9 for invalid measurements and 0 for everything else. """ profile = DummyData() cfg = { 'name': 'red_sea', 'region': 'POLYGON ((10 40, 20 50, 30 30, 10 40))', 'minval': 21.7, 'maxval': 40 } y = RegionalRange(profile, 'TEMP', cfg) assert 'regional_range' in y.flags assert profile['TEMP'].shape == y.flags['regional_range'].shape idx = ma.getmaskarray(profile['TEMP']) assert idx.any(), "Redefine cfg to have at least one masked value" try: import shapely assert np.all(y.flags['regional_range'][idx] == 9) except ImportError: # If Shapely is not available, return 0, i.e. no evaluation. assert np.all(y.flags['regional_range'][idx] == 0) return assert np.all(y.flags['regional_range'][~idx] == 0)
def test(): profile = DummyData() pqc = ProfileQC(profile, saveauxiliary=False) pqc = ProfileQC(profile, saveauxiliary=True) keys = ['PRES', 'TEMP', 'PSAL', 'flag'] for v in profile.keys(): assert v in pqc.keys() assert np.allclose(profile[v], pqc[v]) for a in profile.attributes: assert a in pqc.attributes assert profile.attributes[a] == pqc.attributes[a] assert hasattr(pqc, 'flags') assert type(pqc.flags) is dict vs = list(pqc.flags.keys()) vs.remove('common') for v in vs: for f in pqc.flags[v]: assert pqc.flags[v][f].dtype == 'i1' assert hasattr(pqc, 'features') assert type(pqc.features) is dict
def test_LocationAtSea_track(): """Test standard with multiple locations lat & lon defined in the dataset. This would be the case for a TSG where each measurement is associated with a location. Locking etopo resolution, since it can change the values. Note that there is no restriction in the number of locations. In this example there are multiple depths but only 3 positions. It's not the LocationAtSea job to make sense of that. Should it match with which variable? It can't be done here, but should be done once the tests are combined. """ data = DummyData() data.data["LATITUDE"] = [15, 12, 8] data.data["LONGITUDE"] = [-38, 222, 0] y = LocationAtSea(data, cfg={"resolution": "5min"}) assert hasattr(y, "features") assert "bathymetry" in y.features assert ma.allclose(y.features["bathymetry"], [5036, 4995, -122]) assert hasattr(y, "flags") assert "location_at_sea" in y.flags assert ma.allclose(y.flags["location_at_sea"], [1, 1, 4])
def test_single_negative_depth(): """Evaluate a profile with a single measurement WOD has some profiles with a single measurement. Something certainly went wrong on those profiles, despite that, CoTeDe should be able to do the best assessement possible. Some tests can't be applied, like spike which requires neighbor measurements, but those should return flag 0. """ profile = DummyData() profile.attrs = { "id": 609483, "LATITUDE": 6.977, "LONGITUDE": 79.873, "datetime": datetime(2009, 8, 14, 1, 18, 36), "date": date(2009, 8, 14), "schema": "pfl", } profile.data = { "id": ma.masked_array(data=[51190527], mask=[False], dtype="i"), "PRES": ma.masked_array(data=[-1.0], mask=[False], dtype="f"), "TEMP": ma.masked_array(data=[25.81], mask=[False], dtype="f"), "PSAL": ma.masked_array(data=[0.01], mask=[False], dtype="f"), } ProfileQC(profile, saveauxiliary=False) ProfileQC(profile, saveauxiliary=True)
def test_ProfileQC(): if nogsw: print("GSW package not available. Can't run density_inversion test.") return profile = DummyData() profile["TEMP"][4] = profile["TEMP"][3] + 5 cfg = { "TEMP": { "density_inversion": { "threshold": -0.03, "flag_good": 1, "flag_bad": 4 } } } pqc = ProfileQC(profile, cfg) assert type(pqc.features) is dict assert "densitystep" in pqc.features["TEMP"] assert type(pqc.flags) is dict assert "density_inversion" in pqc.flags["TEMP"] assert pqc.flags["TEMP"]["density_inversion"][4] == 4
def test_badlocation(): data = DummyData() coords = [[91, -30, 3], [-91, -30, 3], [10, -361, 3], [10, 1000, 3]] for lat, lon, flag in coords: data.attrs["LATITUDE"] = lat data.attrs["LONGITUDE"] = lon assert location_at_sea(data) == flag
def test_attribute(): data = DummyData() coords = [[10, -30, 1], [10, 330, 1]] for lat, lon, flag in coords: data.attrs["LATITUDE"] = lat data.attrs["LONGITUDE"] = lon assert location_at_sea(data) == flag
def test_attribute_inland(): data = DummyData() coords = [[-10, -60, 3], [-10, 300, 3]] for lat, lon, flag in coords: data.attrs["LATITUDE"] = lat data.attrs["LONGITUDE"] = lon assert location_at_sea(data) == flag
def test_default(): profile = DummyData() cfg = {'flag_good': 1, 'flag_bad': 4} y = StuckValue(profile, 'TEMP', cfg) assert type(y.features) is dict idx = ma.getmaskarray(profile['TEMP']) assert (y.flags['stuck_value'][idx] == 9).all() assert (y.flags['stuck_value'][~idx] == 1).all()
def test(): profile = DummyData() cfg = [["> 0", "<= 25", -2, 37], ["> 25", "<= 50", -2, 36]] flags = profile_envelop(profile, cfg, "TEMP") # Check for BUG #9. assert 9 in flags
def test_attribute(): profile = DummyData() cfg = {"TEMP": {"woa_normbias": {"threshold": 3}}} pqc = ProfileQC(profile, cfg=cfg) assert 'woa_normbias' in pqc.flags['TEMP'] assert pqc.flags['TEMP']['woa_normbias'].shape == profile.data['TEMP'].shape assert np.unique(pqc.features['TEMP']['woa_mean']).size > 1 assert (pqc.flags['TEMP']['woa_normbias'] == [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 9]).all()
def test_basic(): """ """ profile = DummyData() cfg = {"TEMP": {"woa_normbias": {"threshold": 3, "flag_bad": 3}}, "PSAL": {"woa_normbias": {"threshold": 3, "flag_bad": 3}}} pqc = ProfileQC(profile, cfg=cfg) assert 'woa_normbias' in pqc.flags['TEMP'] assert sorted(np.unique(pqc.flags['TEMP']['woa_normbias'])) == [1, 3, 9] assert sorted(np.unique(pqc.flags['PSAL']['woa_normbias'])) == [1, 9]
def test_neglible_difference(): profile = DummyData() profile['TEMP'][:] = profile['TEMP'] * 0 + 3.14 profile['TEMP'][:] += np.random.randn(profile['TEMP'].size) * 1e-10 cfg = {'flag_good': 1, 'flag_bad': 4} y = StuckValue(profile, 'TEMP', cfg) assert type(y.features) is dict idx = ma.getmaskarray(profile['TEMP']) assert (y.flags['stuck_value'][idx] == 9).all() assert (y.flags['stuck_value'][~idx] == 4).all()
def test_data(): data = DummyData() data.data["LATITUDE"] = 10 data.data["LONGITUDE"] = -30 flag = location_at_sea(data) assert flag == 1 data.data["LATITUDE"] = 10 data.data["LONGITUDE"] = 330 flag = location_at_sea(data) assert flag == 1
def test(): """ """ profile = DummyData() pqc = ProfileQC(profile, cfg='morello2014') assert 'morello2014' in pqc.flags['TEMP'] assert 'morello2014' in pqc.flags['PSAL'] assert profile['TEMP'].shape == pqc.flags['TEMP']['morello2014'].shape assert profile['PSAL'].shape == pqc.flags['PSAL']['morello2014'].shape
def test(): """ """ profile = DummyData() pqc = ProfileQC(profile, cfg='fuzzylogic') assert 'fuzzylogic' in pqc.flags['TEMP'] assert 'fuzzylogic' in pqc.flags['PSAL'] assert profile['TEMP'].shape == pqc.flags['TEMP']['fuzzylogic'].shape assert profile['PSAL'].shape == pqc.flags['PSAL']['fuzzylogic'].shape
def notready_test_data(): data = DummyData() data.data['LATITUDE'] = 10 data.data['LONGITUDE'] = -30 flag = location_at_sea(data) assert flag == 1 data.data['LATITUDE'] = 10 data.data['LONGITUDE'] = 330 flag = location_at_sea(data) assert flag == 1
def test_attribute(): profile = DummyData() cfg = {"TEMP": {"woa_normbias": {"threshold": 3}}} pqc = ProfileQC(profile, cfg=cfg) assert "woa_normbias" in pqc.flags["TEMP"] assert pqc.flags["TEMP"]["woa_normbias"].shape == profile.data[ "TEMP"].shape assert np.unique(pqc.features["TEMP"]["woa_mean"]).size > 1 assert (pqc.flags["TEMP"]["woa_normbias"] == [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 9 ]).all()
def test(): """ """ profile = DummyData() cfg = {"TEMP": {"cars_normbias": {"threshold": 10}}, "PSAL": {"cars_normbias": {"threshold": 10}}} pqc = ProfileQC(profile, cfg=cfg) assert 'cars_normbias' in pqc.flags['TEMP'] assert sorted(np.unique(pqc.flags['TEMP']['cars_normbias'])) == [1, 9] #assert sorted(np.unique(pqc.flags['TEMP2']['cars_normbias'])) == [1] assert sorted(np.unique(pqc.flags['PSAL']['cars_normbias'])) == [1, 9]
def test(): profile = DummyData() feature = ma.masked_array([0, 1.25, 5.875, 0], mask=[True, False, False, True]) cfg = {'threshold': 6, 'flag_good': 1, 'flag_bad': 4} y = Gradient(profile, 'TEMP', cfg) y.test() assert type(y.features) is dict
def test_attribute(): profile = DummyData() profile.attrs['datetime'] = datetime(2016, 6, 4) profile.attrs['LATITUDE'] = -30.0 profile.attrs['LONGITUDE'] = 15 profile.data['PRES'] = np.array([2.0, 5.0, 6.0, 21.0, 44.0, 79.0]) profile.data['TEMP'] = np.array([16.0, 15.6, 15.9, 5.7, 15.2, 14.1]) cfg = {"TEMP": {"cars_normbias": {"threshold": 6}}} pqc = ProfileQC(profile, cfg=cfg) assert 'cars_normbias' in pqc.flags['TEMP'] assert pqc.flags['TEMP']['cars_normbias'].shape == profile.data['TEMP'].shape assert (pqc.flags['TEMP']['cars_normbias'] == [1, 1, 1, 3, 1, 1]).all()
def test(): """ Only test if run. Must improve this. """ profile = DummyData() pqc = ProfileQC(profile, cfg='anomaly_detection') assert 'anomaly_detection' in pqc.flags['TEMP'] assert 'anomaly_detection' in pqc.flags['PSAL'] assert profile['TEMP'].shape == pqc.flags['TEMP'][ 'anomaly_detection'].shape assert profile['PSAL'].shape == pqc.flags['PSAL'][ 'anomaly_detection'].shape
def test_common_flags(): profile = DummyData() cfg = { "main": { "valid_datetime": None, "valid_geolocation": None }, "TEMP": {} } pqc = ProfileQC(profile, cfg=cfg) assert 'common' in pqc.flags assert 'valid_datetime' in pqc.flags['TEMP'] assert pqc.flags['TEMP']['valid_datetime'].shape == profile['TEMP'].shape
def test_all_valid_no_9(): """ If all measurements are valid it can't return flag 9 This is to test a special condition when all values are valid, .mask return False, instead of an array on the same size with False. This test input all valid values, and check if there is no flag 9. """ profile = DummyData() pqc = ProfileQC(profile) assert pqc['TEMP'].mask.all() == False assert np.allclose( combined_flag(pqc.flags['TEMP']) == 9, profile['TEMP'].mask)
def test_LocationAtSea_attrs(): """Test standard with single location Lat & Lon defined in the attrs Locking etopo resolution, since it can change the values. """ data = DummyData() y = LocationAtSea(data, cfg={"resolution": "5min"}) assert hasattr(y, "features") assert "bathymetry" in y.features assert ma.allclose(y.features["bathymetry"], 5036) assert hasattr(y, "flags") assert "location_at_sea" in y.flags assert ma.allclose(y.flags["location_at_sea"], 1)
def test_nonelocation(): data = DummyData() coords = [[None, 1, 0], [1, None, 0]] for lat, lon, flag in coords: data.attrs["LATITUDE"] = lat data.attrs["LONGITUDE"] = lon assert location_at_sea(data) == flag del data.attrs["LATITUDE"] data.attrs["LONGITUDE"] = 1 assert location_at_sea(data) == 0 del data.attrs["LONGITUDE"] data.attrs["LATITUDE"] = 1 assert location_at_sea(data) == 0