def test():
""" Only test if run. Must improve this.
"""
datafile = download_testdata("dPIRX010.cnv")
pqc = fProfileQC(datafile, cfg='anomaly_detection')
# While anomaly detection is limited to spike, gradient and tukey tests it
# will always return 0 for the first and last.
assert sorted(np.unique(pqc.flags['TEMP']['anomaly_detection'])) == [0,1,4]
def test():
""" Only test if run. Must improve this.
"""
datafile = download_testdata("dPIRX010.cnv")
pqc = fProfileQC(datafile, cfg='anomaly_detection')
# While anomaly detection is limited to spike, gradient and tukey tests it
# will always return 0 for the first and last.
assert sorted(np.unique(
pqc.flags['TEMP']['anomaly_detection'])) == [0, 1, 4]
def test_answer():
datafile = download_testdata("dPIRX010.cnv")
ped = func(datafile=datafile)
keys = [
'timeS', 'PRES', 'TEMP', 'TEMP2', 'CNDC', 'CNDC2', 'potemperature',
'potemperature2', 'PSAL', 'PSAL2', 'flag'
]
assert ped.keys() == keys
assert len(ped.attributes) == 13
def test():
"""
"""
datafile = download_testdata("dPIRX010.cnv")
pqc = fProfileQC(datafile, cfg='morello2014')
assert sorted(np.unique(pqc.flags['TEMP']['morello2014'])) == [1, 2, 3, 4]
assert sorted(np.unique(pqc.flags['TEMP2']['morello2014'])) == [1]
assert sorted(np.unique(pqc.flags['PSAL']['morello2014'])) == [1, 2, 4]
assert sorted(np.unique(pqc.flags['PSAL2']['morello2014'])) == [1]
def test():
"""
"""
datafile = download_testdata("dPIRX010.cnv")
pqc = fProfileQC(datafile, cfg='fuzzylogic')
assert 'fuzzylogic' in pqc.flags['TEMP']
assert sorted(np.unique(pqc.flags['TEMP']['fuzzylogic'])) == [1, 2, 4]
assert sorted(np.unique(pqc.flags['TEMP2']['fuzzylogic'])) == [1]
assert sorted(np.unique(pqc.flags['PSAL']['fuzzylogic'])) == [1, 2, 4]
assert sorted(np.unique(pqc.flags['PSAL2']['fuzzylogic'])) == [1]
def test_tsg():
""" I should think about a way to test if the output make sense.
"""
datafile = download_testdata("TSG_PIR_001.cnv")
data = cnv.fCNV(datafile)
# Fails with configuration cotede. It's not ready to handle missing variable, like missing PRES.
#pqc = cotede.qc.ProfileQC(data)
pqc = cotede.qc.ProfileQC(data, cfg='tsg')
assert len(pqc.flags) > 0
def test():
"""
"""
datafile = download_testdata("dPIRX010.cnv")
pqc = fProfileQC(datafile, cfg='fuzzylogic')
assert 'fuzzylogic' in pqc.flags['TEMP']
assert sorted(np.unique(pqc.flags['TEMP']['fuzzylogic'])) == [1, 2, 4]
assert sorted(np.unique(pqc.flags['TEMP2']['fuzzylogic'])) == [1]
assert sorted(np.unique(pqc.flags['PSAL']['fuzzylogic'])) == [1, 2, 4]
assert sorted(np.unique(pqc.flags['PSAL2']['fuzzylogic'])) == [1]
def test():
""" Only test if run. Must improve this.
"""
datafile = download_testdata("dPIRX010.cnv")
pqc = fProfileQC(datafile, cfg='fuzzy', saveauxiliary=True)
# While anomaly detection is limited to spike, gradient and tukey tests it
# will always return 0 for the first and last.
assert sorted(np.unique(pqc.flags['TEMP']['fuzzylogic'])) == [1,4]
assert sorted(np.unique(pqc.flags['TEMP2']['fuzzylogic'])) == [1]
assert sorted(np.unique(pqc.flags['PSAL']['fuzzylogic'])) == [1,4]
assert sorted(np.unique(pqc.flags['PSAL2']['fuzzylogic'])) == [1]
def test_all_valid_no_9():
""" If all measurements are valid it can't return flag 9
This is to test a special condition when all values are valid, .mask
return False, instead of an array on the same size with False.
This test input all valid values, and check if there is no flag 9.
"""
datafile = download_testdata("dPIRX010.cnv")
pqc = cotede.qc.fProfileQC(datafile)
assert pqc['TEMP'].mask == False
assert ~(combined_flag(pqc.flags['TEMP']) == 9).any()
def test_all_valid_no_9():
""" If all measurements are valid it can't return flag 9
This is to test a special condition when all values are valid, .mask
return False, instead of an array on the same size with False.
This test input all valid values, and check if there is no flag 9.
"""
datafile = download_testdata("dPIRX010.cnv")
pqc = cotede.qc.fProfileQC(datafile)
assert pqc['TEMP'].mask == False
assert ~(combined_flag(pqc.flags['TEMP']) == 9).any()
def test_multiple_cfg():
""" I should think about a way to test if the output make sense.
"""
datafile = download_testdata("dPIRX010.cnv")
data = cnv.fCNV(datafile)
for cfg in [None, 'cotede', 'gtspp', 'eurogoos']:
pqc = cotede.qc.ProfileQC(data, cfg=cfg)
assert sorted(pqc.flags.keys()) == \
['PSAL', 'PSAL2', 'TEMP', 'TEMP2', 'common'], \
"Incomplete flagging for %s: %s" % (cfg, pqc.flags.keys())
# Manually defined
pqc = cotede.qc.ProfileQC(data, cfg={'TEMP': {"spike": 6.0,}})
assert len(pqc.flags) > 0
def test_multiple_cfg():
""" I should think about a way to test if the output make sense.
"""
datafile = download_testdata("dPIRX010.cnv")
data = cnv.fCNV(datafile)
for cfg in [None, 'cotede', 'gtspp', 'eurogoos']:
pqc = cotede.qc.ProfileQC(data, cfg=cfg)
assert sorted(pqc.flags.keys()) == \
['PSAL', 'PSAL2', 'TEMP', 'TEMP2', 'common'], \
"Incomplete flagging for %s: %s" % (cfg, pqc.flags.keys())
# Manually defined
pqc = cotede.qc.ProfileQC(data, cfg={'TEMP': {
"spike": 6.0,
}})
assert len(pqc.flags) > 0
def test_answer():
datafile = download_testdata("dPIRX010.cnv")
pqc = func(datafile=datafile)
assert type(pqc.keys()) == list
assert type(pqc.attributes) == dict
assert hasattr(pqc, 'input')
assert hasattr(pqc, 'flags')
assert hasattr(pqc, 'auxiliary')
assert type(pqc.flags) == dict
for k in pqc.flags.keys():
assert type(pqc.flags[k]) == dict
for kk in pqc.flags[k].keys():
assert (type(pqc.flags[k][kk]) == np.ndarray) or \
(type(pqc.flags[k][kk]) == int)
if (type(pqc.flags[k][kk]) == np.ndarray):
assert pqc.flags[k][kk].dtype == 'int8'
def test_argo():
datafile = download_testdata("20150127_prof.nc")
profile = argo.profile_from_nc(datafile)[0]
pqc = ProfileQC(profile, cfg='argo')
assert hasattr(pqc, 'flags')
for v in ['TEMP', 'PSAL']:
assert v in pqc.keys()
assert len(pqc[v]) == 1034
assert v in pqc.flags
for f in pqc.flags[v]:
assert len(pqc.flags[v][f]) == 1034
for a in ['datetime', 'LATITUDE', 'LONGITUDE']:
assert a in pqc.attributes
assert type(pqc.attributes['datetime']) == datetime
def test_argo():
datafile = download_testdata("20150127_prof.nc")
profile = argo.profile_from_nc(datafile)[0]
pqc = ProfileQC(profile, cfg='argo')
print pqc.keys()
print dir(pqc.input)
print pqc.flags
assert hasattr(pqc, 'flags')
for v in ['TEMP', 'PSAL']:
assert v in pqc.keys()
assert len(pqc[v]) == 1034
assert v in pqc.flags
for f in pqc.flags[v]:
assert len(pqc.flags[v][f]) == 1034
for a in ['datetime', 'LATITUDE', 'LONGITUDE']:
assert a in pqc.attributes
assert type(pqc.attributes['datetime']) == datetime
def test_answer():
datafile = download_testdata("dPIRX010.cnv")
pqc = func(datafile=datafile, saveauxiliary=False)
pqc = func(datafile=datafile, saveauxiliary=True)
keys = ['timeS', 'PRES', 'TEMP', 'TEMP2', 'CNDC', 'CNDC2',
'potemperature', 'potemperature2', 'PSAL',
'PSAL2', 'flag']
assert pqc.keys() == keys
assert len(pqc.attributes) == 13
assert hasattr(pqc, 'flags')
assert type(pqc.flags) is dict
vs = pqc.flags.keys()
vs.remove('common')
for v in vs:
for f in pqc.flags[v]:
assert pqc.flags[v][f].dtype == 'i1'
assert hasattr(pqc, 'auxiliary')
assert type(pqc.auxiliary) is dict
def test_answer():
datafile = download_testdata("dPIRX010.cnv")
pqc = func(datafile=datafile, saveauxiliary=False)
pqc = func(datafile=datafile, saveauxiliary=True)
keys = [
'timeS', 'PRES', 'TEMP', 'TEMP2', 'CNDC', 'CNDC2', 'potemperature',
'potemperature2', 'PSAL', 'PSAL2', 'flag'
]
assert pqc.keys() == keys
assert len(pqc.attributes) == 13
assert hasattr(pqc, 'flags')
assert type(pqc.flags) is dict
vs = pqc.flags.keys()
vs.remove('common')
for v in vs:
for f in pqc.flags[v]:
assert pqc.flags[v][f].dtype == 'i1'
assert hasattr(pqc, 'auxiliary')
assert type(pqc.auxiliary) is dict
import shutil
import numpy as np
from numpy import ma
from cotede.utils import ProfilesQCPandasCollection
from cotede.utils.supportdata import download_testdata
from cotede.anomaly_detection import split_data_groups
from cotede.anomaly_detection import rank_files
from cotede.anomaly_detection import i2b_flags
from cotede.anomaly_detection import estimate_p_optimal
from cotede.anomaly_detection import calibrate_anomaly_detection
from cotede.anomaly_detection import calibrate4flags
datalist = ["dPIRX010.cnv", "PIRA001.cnv", "dPIRX003.cnv"]
INPUTFILES = [download_testdata(f) for f in datalist]
def test_i2b_flags(n=100):
flag = ma.concatenate(
[np.random.randint(0, 5, n),
ma.masked_all(2, dtype='int8')])
binflags = i2b_flags(flag)
assert type(binflags) == ma.MaskedArray
assert binflags.dtype == 'bool'
assert binflags.shape == (n + 2, )
assert binflags.mask[flag.mask].all(), \
"All masked flags records should be also masked at binflags"
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import shutil
import tempfile
from cotede.utils.supportdata import download_testdata
#from cotede.utils import ProfilesQCCollection
from cotede.utils import ProfilesQCPandasCollection
datalist = ["dPIRX010.cnv", "PIRA001.cnv", "dPIRX003.cnv"]
INPUTFILES = [download_testdata(f) for f in datalist]
def check_profiles(profiles):
#assert len(INPUTFILES) == len(profiles)
for p in profiles:
assert hasattr(p, 'keys')
assert len(p.keys()) > 0
assert hasattr(p, 'flags')
assert len(p.flags.keys()) > 0
def test_ProfilesQCPandasCollection():
try:
tmpdir = tempfile.mkdtemp()
print tmpdir
for f in INPUTFILES:
shutil.copy(f, tmpdir)