def test_inheritance_priority(): """Test priority when inheriting When inheritance is a list, the first item has priority over the last one. """ def walk_and_check(cfg, cfg2): for c in cfg: assert c in cfg2, "Missing %s in inherited cfg2" % c if not isinstance(cfg[c], dict): assert cfg[c] == cfg2[c], \ "Missing %s in cfg2" % c else: walk_and_check(cfg[c], cfg2[c]) cfg = load_cfg('cotede') # If is a list, the last is the lowest priority cfg2 = load_cfg({'inherit': ['cotede', 'gtspp']}) walk_and_check(cfg, cfg2) try: cfg2 = load_cfg({'inherit': ['gtspp', 'cotede']}) walk_and_check(cfg, cfg2) failed = False except: failed = True assert failed, "It should fail in inverse priority"
def test_inheritance(): """Test inheritance """ cfg = load_cfg('cotede') cfg2 = load_cfg({'inherit': 'cotede'}) for c in cfg: assert c in cfg assert cfg[c] == cfg2[c]
def __init__(self, input, cfg=None, saveauxiliary=True, verbose=True, attributes=None, logger=None): """ Input: dictionary with data. - pressure[\d]: - temperature[\d]: - salinity[\d]: cfg: Check cotede.utils.load_cfg() for the possible input formats for cfg. ======================= - Must have a log system - Probably accept incomplete cfg. If some threshold is not defined, take the default value. - Is the best return another dictionary? """ # self.logger = logging.getLogger(logger or 'cotede.ProfileQC') try: self.name = input.filename except: self.name = None self.verbose = verbose if attributes is None: assert (hasattr(input, 'attributes')) assert (hasattr(input, 'keys')) and (len(input.keys()) > 0) self.cfg = load_cfg(cfg) module_logger.debug("Using cfg: {}".format(self.cfg)) self.input = input if attributes is None: self.attrs = input.attributes else: self.attrs = attributes self.flags = {} self.saveauxiliary = saveauxiliary if saveauxiliary: #self.auxiliary = {} # build_auxiliary is not exactly the best way to do it. self.build_features() if 'common' in self.cfg: self.evaluate_common(self.cfg) for v in self.input.keys(): if v == 'TEMP': vv = 'sea_water_temperature' elif v == 'PSAL': vv = 'sea_water_salinity' else: vv = v for c in self.cfg['variables']: if re.match("(%s)2?$" % c, vv): module_logger.debug(" %s - evaluating: %s, as type: %s" % (self.name, v, c)) self.evaluate(v, self.cfg['variables'][c]) break
def test_dict_input(): """Test a dictionary input, i.e. manually defined config It should return the same dictionary """ cfg = {'temperature': {'global_range': 'test'}} cfg_out = load_cfg(cfg) assert cfg_out['variables']['temperature'] == cfg['temperature']
def test_factory_cfgs(): for cfg in CFG: print("Loading %s" % cfg) try: cfg_out = load_cfg(cfg) except: assert False, "Couldn't load: %s" % cfg assert type(cfg_out) is dict assert len(cfg_out) > 0
def __init__(self, input, cfg=None, saveauxiliary=True, verbose=True, attributes=None, logger=None): """ Input: dictionary with data. - pressure[\d]: - temperature[\d]: - salinity[\d]: cfg: Check cotede.utils.load_cfg() for the possible input formats for cfg. ======================= - Must have a log system - Probably accept incomplete cfg. If some threshold is not defined, take the default value. - Is the best return another dictionary? """ #self.logger = logger or logging.getLogger(__name__) logging.getLogger(logger or __name__) try: self.name = input.filename except: self.name = None self.verbose = verbose if attributes is None: assert (hasattr(input, 'attributes')) assert (hasattr(input, 'keys')) and (len(input.keys()) > 0) self.cfg = load_cfg(cfg) self.input = input if attributes is None: self.attributes = input.attributes else: self.attributes = attributes self.flags = {} self.saveauxiliary = saveauxiliary if saveauxiliary: #self.auxiliary = {} # build_auxiliary is not exactly the best way to do it. self.build_auxiliary() # I should use common or main, but must be consistent # between defaults and flags.keys() # Think about it self.evaluate_common(self.cfg) for v in self.input.keys(): for c in self.cfg.keys(): if re.match("(%s)2?$" % c, v): logging.debug(" %s - evaluating: %s, as type: %s" % (self.name, v, c)) self.evaluate(v, self.cfg[c]) break
def __init__(self, input, cfg=None, saveauxiliary=True, verbose=True, attributes=None, logger=None): """ Input: dictionary with data. - pressure[\d]: - temperature[\d]: - salinity[\d]: cfg: Check cotede.utils.load_cfg() for the possible input formats for cfg. ======================= - Must have a log system - Probably accept incomplete cfg. If some threshold is not defined, take the default value. - Is the best return another dictionary? """ #self.logger = logger or logging.getLogger(__name__) logging.getLogger(logger or __name__) try: self.name = input.filename except: self.name = None self.verbose = verbose if attributes is None: assert (hasattr(input, 'attributes')) assert (hasattr(input, 'keys')) and (len(input.keys()) > 0) self.cfg = load_cfg(cfg) self.input = input if attributes is None: self.attributes = input.attributes else: self.attributes = attributes self.flags = {} self.saveauxiliary = saveauxiliary if saveauxiliary: #self.auxiliary = {} # build_auxiliary is not exactly the best way to do it. self.build_auxiliary() # I should use common or main, but must be consistent # between defaults and flags.keys() # Think about it self.evaluate_common(self.cfg) for v in self.input.keys(): for c in self.cfg.keys(): if re.match("%s\d?$" % c, v): logging.debug(" %s - evaluating: %s, as type: %s" % (self.name, v, c)) self.evaluate(v, self.cfg[c]) break
def test_dict_input(): """Test a dictionary input, i.e. manually defined config The output configuration can't miss anything given in the input but can have extra content. """ # Scalar argument cfg = {'temperature': {'spike': 1234}} cfg_out = load_cfg(cfg) assert cfg_out['variables']['temperature']['spike']['threshold'] == 1234 # Dictionary argument cfg = {'temperature': {'global_range': {'minvalue': 0, 'maxvalue': 60}}} cfg_out = load_cfg(cfg) assert 'global_range' in cfg_out['variables']['temperature'] tmp = cfg_out['variables']['temperature']['global_range'] for v in cfg['temperature']['global_range']: assert v in tmp assert cfg['temperature']['global_range'][v] == tmp[v]
def test_dict(): """Test a user dict config It is possible to define a full config instead of choosing one of the builtins. This is done by giving a dictionary with the correct structure. """ cfg = {'common': {'valid_datetime': None}} cfg_out = load_cfg(cfg) assert 'common' in cfg_out, "Missing 'common' in load_cfg output" assert cfg_out['common'] == cfg['common']
def test_factory_cfgs(): """Load all available configs, one at a time CoTeDe comes with builtin config. This test checks if can load all those available configs. """ for cfg in CFG: print("Loading %s" % cfg) try: cfg_out = load_cfg(cfg) except: assert False, "Couldn't load: %s" % cfg assert isinstance(cfg_out, dict) assert len(cfg_out) > 0
def test_default(): cfg_out = load_cfg() assert type(cfg_out) is dict assert len(cfg_out) > 0
def test_default(): cfg_out = load_cfg() assert isinstance(cfg_out, dict) assert len(cfg_out) > 0
def test_inout(): """ load_cfg shouldn't modify input variable cfg """ cfg = 'cotede' out = load_cfg(cfg) assert out != cfg
def test_dict(): cfg = {'main': {'valid_datetime': None}} cfg_out = load_cfg(cfg) assert cfg_out == cfg
def test_default(): cfg_out = load_cfg() assert type(cfg_out) is dict assert len(cfg_out) > 0
def get_qc(p, config, test): '''Wrapper for running and returning results of CoTeDe tests. Inputs are: p is a wodpy profile object. config is the suite of tests that test comes from e.g. gtspp. test is the specific test to get the results from. ''' global cotede_results # Disable logging messages from CoTeDe unless they are more # severe than a warning. logging.disable(logging.WARNING) # Create a dummy results variable if this is the first call. try: cotede_results except NameError: cotede_results = [-1, '', {}, None] var = 'TEMP' # Check if we need to perform the quality control. if (p.uid() != cotede_results[0] or config != cotede_results[1] or test not in cotede_results[2] or p.uid() is None): inputs = Wod4CoTeDe(p) dt = inputs.attributes['datetime'] if dt.year < 1900: inputs.attributes['datetime'] = dt.replace(year=1900) # If config is a dictionary, use it. if type(config) is not dict: try: # Load config from CoTeDe cfg = load_cfg(config) if test == config: # AutoQC runs only on TEMP, so clean the rest. for v in list(cfg): if v not in ['main', var]: del (cfg[v]) # If is a specific test, elif test != config: # Load from TEMP, try: cfg = {var: {test: cfg[var][test]}} # otherwise load it from main. except: # The dummy configuration ensures that the results from # 'main' is copied into the results for var. cfg = { 'main': { test: cfg['main'][test] }, var: { 'dummy': None } } except: with open('cotede_qc/qc_cfg/' + config + '.json') as f: cfg = json.load(f) pqc = ProfileQC(inputs, cfg=cfg) cotede_results = [p.uid(), config, pqc.flags[var].keys(), pqc] # Get the QC results, which use the IOC conventions. qc_returned = cotede_results[3].flags[var][test] # It looks like CoTeDe never returns a QC decision # of 2. If it ever does, we need to decide whether # this counts as a pass or reject. # Gui: Yes, some tests can return 2. My suggestions is to flag as good. qc = np.ma.zeros(p.n_levels(), dtype=bool) qc[np.logical_or(qc_returned == 3, qc_returned == 4)] = True return qc
def test_dict(): cfg = {'main': {'valid_datetime': None}} cfg_out = load_cfg(cfg) assert cfg_out == cfg
def test_inout(): """ load_cfg shouldn't overwrite input variable cfg """ cfg = 'cotede' out = load_cfg(cfg) assert out != cfg
def __init__(self, input, cfg=None, saveauxiliary=True, verbose=True, attributes=None): """A procedure to QC a hydrographic profile Parameters ---------- input: dict-like An object with the data to be evaluated that responds like a dictionary. For instance, a variable pressure should be acessible as input['pressure'], or temperature as input['temperature']. This input object could have attrs, with global attributes for the whole dataset. For instance, input.attrs['lat'] would give the nominal latitude of the dataset input. cfg: dict-like or str The QC configuration to be used in the current profile. If a string, it should be the name of a JSON QC configuration. Check the manual for the available options. saveauxiliary: bool Save features as .features verbose: bool Show extra information attributes: dict-like, optional If given, append/overwirte the input.attrs Methods ------- keys(self): List of input contents """ # self.logger = logging.getLogger(logger or 'cotede.ProfileQC') try: self.name = input.filename except: self.name = None self.verbose = verbose assert (hasattr(input, 'keys')) and (len(input.keys()) > 0) self.cfg = load_cfg(cfg) module_logger.debug("Using cfg: {}".format(self.cfg)) self.input = deepcopy(input) self._set_attrs(attributes) self.flags = {} self.saveauxiliary = saveauxiliary if saveauxiliary: #self.auxiliary = {} # build_auxiliary is not exactly the best way to do it. self.build_features() if 'common' in self.cfg: self.evaluate_common(self.cfg) for v in self.input.keys(): if v == 'TEMP': vv = 'sea_water_temperature' elif v == 'PSAL': vv = 'sea_water_salinity' else: vv = v for c in self.cfg['variables']: if re.match("(%s)2?$" % c, vv): module_logger.debug(" %s - evaluating: %s, as type: %s" % (self.name, v, c)) self.evaluate(v, self.cfg['variables'][c]) break
def test_factory_cfgs(): for cfg in CFG: print("Loading %s" % cfg) cfg_out = load_cfg(cfg) assert type(cfg_out) is dict assert len(cfg_out) > 0
def test_factory_cfgs(): for cfg in CFG: print("Loading %s" % cfg) cfg_out = load_cfg(cfg) assert type(cfg_out) is dict assert len(cfg_out) > 0