def test_inheritance_priority():
"""Test priority when inheriting
When inheritance is a list, the first item has priority over
the last one.
"""
def walk_and_check(cfg, cfg2):
for c in cfg:
assert c in cfg2, "Missing %s in inherited cfg2" % c
if not isinstance(cfg[c], dict):
assert cfg[c] == cfg2[c], \
"Missing %s in cfg2" % c
else:
walk_and_check(cfg[c], cfg2[c])
cfg = load_cfg('cotede')
# If is a list, the last is the lowest priority
cfg2 = load_cfg({'inherit': ['cotede', 'gtspp']})
walk_and_check(cfg, cfg2)
try:
cfg2 = load_cfg({'inherit': ['gtspp', 'cotede']})
walk_and_check(cfg, cfg2)
failed = False
except:
failed = True
assert failed, "It should fail in inverse priority"
def test_inheritance():
"""Test inheritance
"""
cfg = load_cfg('cotede')
cfg2 = load_cfg({'inherit': 'cotede'})
for c in cfg:
assert c in cfg
assert cfg[c] == cfg2[c]
def __init__(self, input, cfg=None, saveauxiliary=True, verbose=True,
attributes=None, logger=None):
"""
Input: dictionary with data.
- pressure[\d]:
- temperature[\d]:
- salinity[\d]:
cfg: Check cotede.utils.load_cfg() for the possible input formats
for cfg.
=======================
- Must have a log system
- Probably accept incomplete cfg. If some threshold is
not defined, take the default value.
- Is the best return another dictionary?
"""
# self.logger = logging.getLogger(logger or 'cotede.ProfileQC')
try:
self.name = input.filename
except:
self.name = None
self.verbose = verbose
if attributes is None:
assert (hasattr(input, 'attributes'))
assert (hasattr(input, 'keys')) and (len(input.keys()) > 0)
self.cfg = load_cfg(cfg)
module_logger.debug("Using cfg: {}".format(self.cfg))
self.input = input
if attributes is None:
self.attrs = input.attributes
else:
self.attrs = attributes
self.flags = {}
self.saveauxiliary = saveauxiliary
if saveauxiliary:
#self.auxiliary = {}
# build_auxiliary is not exactly the best way to do it.
self.build_features()
if 'common' in self.cfg:
self.evaluate_common(self.cfg)
for v in self.input.keys():
if v == 'TEMP':
vv = 'sea_water_temperature'
elif v == 'PSAL':
vv = 'sea_water_salinity'
else:
vv = v
for c in self.cfg['variables']:
if re.match("(%s)2?$" % c, vv):
module_logger.debug(" %s - evaluating: %s, as type: %s" %
(self.name, v, c))
self.evaluate(v, self.cfg['variables'][c])
break
def test_dict_input():
"""Test a dictionary input, i.e. manually defined config
It should return the same dictionary
"""
cfg = {'temperature': {'global_range': 'test'}}
cfg_out = load_cfg(cfg)
assert cfg_out['variables']['temperature'] == cfg['temperature']
def test_factory_cfgs():
for cfg in CFG:
print("Loading %s" % cfg)
try:
cfg_out = load_cfg(cfg)
except:
assert False, "Couldn't load: %s" % cfg
assert type(cfg_out) is dict
assert len(cfg_out) > 0
def __init__(self, input, cfg=None, saveauxiliary=True, verbose=True,
attributes=None, logger=None):
"""
Input: dictionary with data.
- pressure[\d]:
- temperature[\d]:
- salinity[\d]:
cfg: Check cotede.utils.load_cfg() for the possible input formats
for cfg.
=======================
- Must have a log system
- Probably accept incomplete cfg. If some threshold is
not defined, take the default value.
- Is the best return another dictionary?
"""
#self.logger = logger or logging.getLogger(__name__)
logging.getLogger(logger or __name__)
try:
self.name = input.filename
except:
self.name = None
self.verbose = verbose
if attributes is None:
assert (hasattr(input, 'attributes'))
assert (hasattr(input, 'keys')) and (len(input.keys()) > 0)
self.cfg = load_cfg(cfg)
self.input = input
if attributes is None:
self.attributes = input.attributes
else:
self.attributes = attributes
self.flags = {}
self.saveauxiliary = saveauxiliary
if saveauxiliary:
#self.auxiliary = {}
# build_auxiliary is not exactly the best way to do it.
self.build_auxiliary()
# I should use common or main, but must be consistent
# between defaults and flags.keys()
# Think about it
self.evaluate_common(self.cfg)
for v in self.input.keys():
for c in self.cfg.keys():
if re.match("(%s)2?$" % c, v):
logging.debug(" %s - evaluating: %s, as type: %s" %
(self.name, v, c))
self.evaluate(v, self.cfg[c])
break
def __init__(self, input, cfg=None, saveauxiliary=True, verbose=True,
attributes=None, logger=None):
"""
Input: dictionary with data.
- pressure[\d]:
- temperature[\d]:
- salinity[\d]:
cfg: Check cotede.utils.load_cfg() for the possible input formats
for cfg.
=======================
- Must have a log system
- Probably accept incomplete cfg. If some threshold is
not defined, take the default value.
- Is the best return another dictionary?
"""
#self.logger = logger or logging.getLogger(__name__)
logging.getLogger(logger or __name__)
try:
self.name = input.filename
except:
self.name = None
self.verbose = verbose
if attributes is None:
assert (hasattr(input, 'attributes'))
assert (hasattr(input, 'keys')) and (len(input.keys()) > 0)
self.cfg = load_cfg(cfg)
self.input = input
if attributes is None:
self.attributes = input.attributes
else:
self.attributes = attributes
self.flags = {}
self.saveauxiliary = saveauxiliary
if saveauxiliary:
#self.auxiliary = {}
# build_auxiliary is not exactly the best way to do it.
self.build_auxiliary()
# I should use common or main, but must be consistent
# between defaults and flags.keys()
# Think about it
self.evaluate_common(self.cfg)
for v in self.input.keys():
for c in self.cfg.keys():
if re.match("%s\d?$" % c, v):
logging.debug(" %s - evaluating: %s, as type: %s" %
(self.name, v, c))
self.evaluate(v, self.cfg[c])
break
def test_dict_input():
"""Test a dictionary input, i.e. manually defined config
The output configuration can't miss anything given in the input but
can have extra content.
"""
# Scalar argument
cfg = {'temperature': {'spike': 1234}}
cfg_out = load_cfg(cfg)
assert cfg_out['variables']['temperature']['spike']['threshold'] == 1234
# Dictionary argument
cfg = {'temperature': {'global_range': {'minvalue': 0, 'maxvalue': 60}}}
cfg_out = load_cfg(cfg)
assert 'global_range' in cfg_out['variables']['temperature']
tmp = cfg_out['variables']['temperature']['global_range']
for v in cfg['temperature']['global_range']:
assert v in tmp
assert cfg['temperature']['global_range'][v] == tmp[v]
def test_dict():
"""Test a user dict config
It is possible to define a full config instead of choosing one of the
builtins. This is done by giving a dictionary with the correct
structure.
"""
cfg = {'common': {'valid_datetime': None}}
cfg_out = load_cfg(cfg)
assert 'common' in cfg_out, "Missing 'common' in load_cfg output"
assert cfg_out['common'] == cfg['common']
def test_factory_cfgs():
"""Load all available configs, one at a time
CoTeDe comes with builtin config. This test checks if can
load all those available configs.
"""
for cfg in CFG:
print("Loading %s" % cfg)
try:
cfg_out = load_cfg(cfg)
except:
assert False, "Couldn't load: %s" % cfg
assert isinstance(cfg_out, dict)
assert len(cfg_out) > 0
def test_default():
cfg_out = load_cfg()
assert type(cfg_out) is dict
assert len(cfg_out) > 0
def test_default():
cfg_out = load_cfg()
assert isinstance(cfg_out, dict)
assert len(cfg_out) > 0
def test_inout():
""" load_cfg shouldn't modify input variable cfg
"""
cfg = 'cotede'
out = load_cfg(cfg)
assert out != cfg
def test_dict():
cfg = {'main': {'valid_datetime': None}}
cfg_out = load_cfg(cfg)
assert cfg_out == cfg
def test_default():
cfg_out = load_cfg()
assert type(cfg_out) is dict
assert len(cfg_out) > 0
def get_qc(p, config, test):
'''Wrapper for running and returning results of CoTeDe tests.
Inputs are:
p is a wodpy profile object.
config is the suite of tests that test comes from e.g. gtspp.
test is the specific test to get the results from.
'''
global cotede_results
# Disable logging messages from CoTeDe unless they are more
# severe than a warning.
logging.disable(logging.WARNING)
# Create a dummy results variable if this is the first call.
try:
cotede_results
except NameError:
cotede_results = [-1, '', {}, None]
var = 'TEMP'
# Check if we need to perform the quality control.
if (p.uid() != cotede_results[0] or config != cotede_results[1]
or test not in cotede_results[2] or p.uid() is None):
inputs = Wod4CoTeDe(p)
dt = inputs.attributes['datetime']
if dt.year < 1900:
inputs.attributes['datetime'] = dt.replace(year=1900)
# If config is a dictionary, use it.
if type(config) is not dict:
try:
# Load config from CoTeDe
cfg = load_cfg(config)
if test == config:
# AutoQC runs only on TEMP, so clean the rest.
for v in list(cfg):
if v not in ['main', var]:
del (cfg[v])
# If is a specific test,
elif test != config:
# Load from TEMP,
try:
cfg = {var: {test: cfg[var][test]}}
# otherwise load it from main.
except:
# The dummy configuration ensures that the results from
# 'main' is copied into the results for var.
cfg = {
'main': {
test: cfg['main'][test]
},
var: {
'dummy': None
}
}
except:
with open('cotede_qc/qc_cfg/' + config + '.json') as f:
cfg = json.load(f)
pqc = ProfileQC(inputs, cfg=cfg)
cotede_results = [p.uid(), config, pqc.flags[var].keys(), pqc]
# Get the QC results, which use the IOC conventions.
qc_returned = cotede_results[3].flags[var][test]
# It looks like CoTeDe never returns a QC decision
# of 2. If it ever does, we need to decide whether
# this counts as a pass or reject.
# Gui: Yes, some tests can return 2. My suggestions is to flag as good.
qc = np.ma.zeros(p.n_levels(), dtype=bool)
qc[np.logical_or(qc_returned == 3, qc_returned == 4)] = True
return qc
def test_dict():
cfg = {'main': {'valid_datetime': None}}
cfg_out = load_cfg(cfg)
assert cfg_out == cfg
def test_inout():
""" load_cfg shouldn't overwrite input variable cfg
"""
cfg = 'cotede'
out = load_cfg(cfg)
assert out != cfg
def __init__(self, input, cfg=None, saveauxiliary=True, verbose=True,
attributes=None):
"""A procedure to QC a hydrographic profile
Parameters
----------
input: dict-like
An object with the data to be evaluated that responds like a
dictionary. For instance, a variable pressure should be acessible
as input['pressure'], or temperature as input['temperature'].
This input object could have attrs, with global attributes for
the whole dataset. For instance, input.attrs['lat'] would give the
nominal latitude of the dataset input.
cfg: dict-like or str
The QC configuration to be used in the current profile. If a
string, it should be the name of a JSON QC configuration. Check
the manual for the available options.
saveauxiliary: bool
Save features as .features
verbose: bool
Show extra information
attributes: dict-like, optional
If given, append/overwirte the input.attrs
Methods
-------
keys(self): List of input contents
"""
# self.logger = logging.getLogger(logger or 'cotede.ProfileQC')
try:
self.name = input.filename
except:
self.name = None
self.verbose = verbose
assert (hasattr(input, 'keys')) and (len(input.keys()) > 0)
self.cfg = load_cfg(cfg)
module_logger.debug("Using cfg: {}".format(self.cfg))
self.input = deepcopy(input)
self._set_attrs(attributes)
self.flags = {}
self.saveauxiliary = saveauxiliary
if saveauxiliary:
#self.auxiliary = {}
# build_auxiliary is not exactly the best way to do it.
self.build_features()
if 'common' in self.cfg:
self.evaluate_common(self.cfg)
for v in self.input.keys():
if v == 'TEMP':
vv = 'sea_water_temperature'
elif v == 'PSAL':
vv = 'sea_water_salinity'
else:
vv = v
for c in self.cfg['variables']:
if re.match("(%s)2?$" % c, vv):
module_logger.debug(" %s - evaluating: %s, as type: %s" %
(self.name, v, c))
self.evaluate(v, self.cfg['variables'][c])
break
def test_factory_cfgs():
for cfg in CFG:
print("Loading %s" % cfg)
cfg_out = load_cfg(cfg)
assert type(cfg_out) is dict
assert len(cfg_out) > 0
def test_factory_cfgs():
for cfg in CFG:
print("Loading %s" % cfg)
cfg_out = load_cfg(cfg)
assert type(cfg_out) is dict
assert len(cfg_out) > 0
