def test_items(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d['len'] = '89' for _, v in list(d.items()): self.assertTrue(v in ['Birds', '89'])
def m_override(target, sparse): """Override items in a target pdict. Target keys must already exist unless there is a "__MANY__" placeholder in the right position. """ if not sparse: return stack = [(sparse, target, [], OrderedDictWithDefaults())] defaults_list = [] while stack: source, dest, keylist, many_defaults = stack.pop(0) if many_defaults: defaults_list.append((dest, many_defaults)) for key, val in source.items(): if isinstance(val, dict): if key in many_defaults: child_many_defaults = many_defaults[key] else: child_many_defaults = OrderedDictWithDefaults() if key not in dest: if '__MANY__' in dest: dest[key] = OrderedDictWithDefaults() child_many_defaults = dest['__MANY__'] elif '__MANY__' in many_defaults: # A 'sub-many' dict - would it ever exist in real life? dest[key] = OrderedDictWithDefaults() child_many_defaults = many_defaults['__MANY__'] elif key in many_defaults: dest[key] = OrderedDictWithDefaults() else: # TODO - validation prevents this, but handle properly # for completeness. raise Exception( "parsec dict override: no __MANY__ placeholder" + "%s" % (keylist + [key])) stack.append( (val, dest[key], keylist + [key], child_many_defaults)) else: if key not in dest: if ('__MANY__' in dest or key in many_defaults or '__MANY__' in many_defaults): if isinstance(val, list): dest[key] = val[:] else: dest[key] = val else: # TODO - validation prevents this, but handle properly # for completeness. raise Exception( "parsec dict override: no __MANY__ placeholder" + "%s" % (keylist + [key])) if isinstance(val, list): dest[key] = val[:] else: dest[key] = val for dest_dict, defaults in defaults_list: dest_dict.defaults_ = defaults
def test_values(self): d = OrderedDictWithDefaults() d['name'] = 'Paul' d['color'] = 'Green' values = d.values() self.assertTrue(len(values) == 2) self.assertTrue('Paul' in values) self.assertTrue('Green' in values)
def test_contains(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d.defaults_ = {'value': '10'} self.assertTrue('key' in d) self.assertTrue('value' in d) self.assertFalse('test' in d)
def test_values(self): d = OrderedDictWithDefaults() d['name'] = 'Paul' d['color'] = 'Green' values = list(d.values()) self.assertTrue(len(values) == 2) self.assertTrue('Paul' in values) self.assertTrue('Green' in values)
def test_getitem(self): d = OrderedDictWithDefaults() d['name'] = 'Joseph' d.defaults_ = { 'surname': 'Wyndham' } self.assertEqual('Joseph', d['name']) self.assertEqual('Wyndham', d['surname'])
def get_parsec_validator_invalid_values(): """ Data provider or invalid values for parsec validator. All values must not be null (covered elsewhere), and not dict's. Possible invalid scenarios must include: - cfg[key] is a list AND a value is not in list of the possible values - OR - cfg[key] is not a list AND cfg[key] not in the list of possible values :return: a list with sets of tuples for the test parameters :rtype: list """ values = [] # variables reused throughout spec = None msg = None # set 1 (t, f, f, t) spec = { 'value': [VDR.V_INTEGER_LIST, 1, 2, 3, 4] } cfg = OrderedDictWithDefaults() cfg['value'] = "1, 2, 3" msg = None values.append((spec, cfg, msg)) # set 2 (t, t, f, t) spec = { 'value': [VDR.V_INTEGER_LIST, 1, 2, 3, 4] } cfg = OrderedDictWithDefaults() cfg['value'] = "1, 2, 5" msg = 'Illegal option value: value = [1, 2, 5]' values.append((spec, cfg, msg)) # set 3 (f, f, t, f) spec = { 'value': [VDR.V_INTEGER, 1, 2, 3, 4] } cfg = OrderedDictWithDefaults() cfg['value'] = "2" msg = None values.append((spec, cfg, msg)) # set 4 (f, f, t, t) spec = { 'value': [VDR.V_INTEGER, 1, 2, 3, 4] } cfg = OrderedDictWithDefaults() cfg['value'] = "5" msg = 'Illegal option value: value = 5' values.append((spec, cfg, msg)) return values
def __init__(self, spec, upgrader=None, output_fname=None, tvars=None): self.sparse = OrderedDictWithDefaults() self.dense = OrderedDictWithDefaults() self.upgrader = upgrader self.tvars = tvars self.output_fname = output_fname self.checkspec(spec) self.spec = spec
def test_parsec_validator_invalid_key(self): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section22'] = 'abc' with self.assertRaises(IllegalItemError): parsec_validator.validate(cfg, SAMPLE_SPEC_1)
def test_keys(self): d = OrderedDictWithDefaults() d['name'] = 'Andrew' d['surname'] = 'Gray' d.defaults_ = {'address': 'N/A'} keys = list(d.keys()) self.assertTrue(len(keys) == 3) self.assertTrue('name' in keys) self.assertTrue('surname' in keys) self.assertTrue('address' in keys)
def test_parsec_validator_invalid_key_with_many_1(self): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section3000000'] = OrderedDictWithDefaults() parsec_validator.validate(cfg, SAMPLE_SPEC_1) # TBD assertIsNotNone when 2.6+ self.assertTrue(parsec_validator is not None)
def test_contains(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d.defaults_ = { 'value': '10' } self.assertTrue('key' in d) self.assertTrue('value' in d) self.assertFalse('test' in d)
def test_itervalues(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d['len'] = '89' d.defaults_ = {'surname': 'Wyndham'} count = 0 for k in d.values(): self.assertTrue(k in ['Birds', '89', 'Wyndham']) count += 1 self.assertEqual(3, count)
def __init__(self, spec, upgrader=None, output_fname=None, tvars=None, validator=None): self.sparse = OrderedDictWithDefaults() self.dense = OrderedDictWithDefaults() self.upgrader = upgrader self.tvars = tvars self.output_fname = output_fname self.spec = spec if validator is None: validator = parsec_validate self.validator = validator
def test_parsec_validator_invalid_key_with_many_spaces(self): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section 3000000'] = 'test' with self.assertRaises(IllegalItemError) as cm: parsec_validator.validate(cfg, SAMPLE_SPEC_1) self.assertEqual("Illegal item (consecutive spaces): " "section 3000000", cm.exception.msg)
def test_parsec_validator(self): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section3'] = OrderedDictWithDefaults() cfg['section3']['title'] = None parsec_validator.validate(cfg, SAMPLE_SPEC_1) # TBD assertIsNotNone when 2.6+ self.assertTrue(parsec_validator is not None)
def test_parsec_validator_invalid_key_no_spec(self): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section22'] = 'abc' spec = SAMPLE_SPEC_1.copy() del (spec['__MANY__']) with self.assertRaises(IllegalItemError): parsec_validator.validate(cfg, spec)
def test_parsec_validator_invalid_key_with_many_2(self): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section3'] = OrderedDictWithDefaults() cfg['section3']['title'] = '1' cfg['section3']['entries'] = OrderedDictWithDefaults() cfg['section3']['entries']['key'] = 'name' cfg['section3']['entries']['value'] = "1, 2, 3, 4" parsec_validator.validate(cfg, SAMPLE_SPEC_1) # TBD assertIsNotNone when 2.6+ self.assertTrue(parsec_validator is not None)
def __init__( self, spec, upgrader=None, write_proc=False, tvars=[], tvars_file=None ): self.sparse = OrderedDictWithDefaults() self.dense = OrderedDictWithDefaults() self.upgrader = upgrader self.tvars = tvars self.tvars_file = tvars_file self.write_proc = write_proc self.checkspec( spec ) self.spec = spec
def test_keys(self): d = OrderedDictWithDefaults() d['name'] = 'Andrew' d['surname'] = 'Gray' d.defaults_ = { 'address': 'N/A' } keys = list(d.keys()) self.assertTrue(len(keys) == 3) self.assertTrue('name' in keys) self.assertTrue('surname' in keys) self.assertTrue('address' in keys)
def test_itervalues(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d['len'] = '89' d.defaults_ = { 'surname': 'Wyndham' } count = 0 for k in d.values(): self.assertTrue(k in ['Birds', '89', 'Wyndham']) count += 1 self.assertEqual(3, count)
def poverride(target, sparse, prepend=False): """Override or add items in a target pdict. Target sub-dicts must already exist. For keys that already exist in the target, the value is overridden in-place. New keys can be prepended in the target (Cylc use case: broadcast environment variables should be defined first in the user environment section, to allow use in subsequent variable definitions. """ if not sparse: target = OrderedDictWithDefaults() return for key, val in sparse.items(): if isinstance(val, dict): poverride(target[key], val, prepend) else: if prepend and (key not in target): # Prepend new items in the target ordered dict. setitem = target.prepend else: # Override in-place in the target ordered dict. setitem = target.__setitem__ if isinstance(val, list): setitem(key, val[:]) else: setitem(key, val)
def _expand_graph(self, line, all_params, param_list, line_set, values=None): """Expand line into line_set for any number of parameters. line is a graph string line as described above in the calling method. param_list is a list of tuples (name, max-val) for each parameter. results is a set to hold each expanded line. """ if values is None: values = {} if not param_list: # Inner loop. for p_group in set(REC_P_GROUP.findall(line)): # Parameters must be expanded in the order found. param_values = OrderedDictWithDefaults() tmpl = '' for item in p_group.split(','): pname, offs = REC_P_OFFS.match(item).groups() if offs is None: param_values[pname] = values[pname] elif offs.startswith('='): # Specific value. try: # Template may require an integer param_values[pname] = int(offs[1:]) except ValueError: param_values[pname] = offs[1:] else: # Index offset. plist = all_params[pname] cur_idx = plist.index(values[pname]) off_idx = cur_idx + int(offs) if 0 <= off_idx < len(plist): offval = plist[off_idx] else: offval = self._REMOVE param_values[pname] = offval for pname in param_values: tmpl += self.param_tmpl_cfg[pname] try: repl = tmpl % param_values except KeyError as exc: raise ParamExpandError('ERROR: parameter %s is not ' 'defined.' % str(exc.args[0])) line = line.replace('<' + p_group + '>', repl) # Remove out-of-range nodes line = self._REMOVE_REC.sub('', line) if line: line_set.add(line) else: # Recurse through index ranges. for param_val in param_list[0][1]: values[param_list[0][0]] = param_val self._expand_graph(line, all_params, param_list[1:], line_set, values)
def get_checkspec_params(): """Data provider for test_checkspec""" spec1 = OrderedDictWithDefaults() spec1['key'] = OrderedDictWithDefaults() spec1['key']['values'] = ['a', 'b', 'c'] parents1 = [] spec2 = OrderedDictWithDefaults() spec2['key'] = OrderedDictWithDefaults() spec2['key']['values'] = ['a', 'b', 'c'] spec2['entry'] = "not a dict, not a list" parents2 = [] return [(spec1, parents1, None), (spec2, parents2, parsec.config.ParsecError)]
def _populate_spec_defaults(defs, spec): """Populate a nested dict with default values from a spec.""" for key, val in spec.items(): if isinstance(val, dict): if key not in defs: defs[key] = OrderedDictWithDefaults() _populate_spec_defaults(defs[key], spec[key]) else: defs[key] = spec[key].args['default']
def test_prepend(self): d = OrderedDictWithDefaults() d['key'] = 'Birds' d.prepend('year', 1980) d.prepend('key', 2000) iterator = iter(d.keys()) self.assertEqual('key', next(iterator)) self.assertEqual('year', next(iterator)) d = OrderedDictWithDefaults() d['key'] = 'Birds' d.prepend('year', 1980) iterator = iter(d.keys()) self.assertEqual('year', next(iterator)) self.assertEqual('key', next(iterator))
def addsect(cfig, sname, parents): """Add a new section to a nested dict.""" for p in parents: # drop down the parent list cfig = cfig[p] if sname in cfig: # this doesn't warrant a warning unless contained items are repeated LOG.debug('Section already encountered: %s', itemstr(parents + [sname])) else: cfig[sname] = OrderedDictWithDefaults()
def replicate(target, source): """Replicate source *into* target. Source elements need not exist in target already, so source overrides common elements in target and otherwise adds elements to it. """ if not source: return if hasattr(source, "defaults_"): target.defaults_ = pdeepcopy(source.defaults_) for key, val in source.items(): if isinstance(val, dict): if key not in target: target[key] = OrderedDictWithDefaults() if hasattr(val, 'defaults_'): target[key].defaults_ = pdeepcopy(val.defaults_) replicate(target[key], val) elif isinstance(val, list): target[key] = val[:] else: target[key] = val
def poverride( target, sparse ): """Override items in a target pdict, target sub-dicts must already exist.""" if not sparse: target = OrderedDictWithDefaults() return for key,val in sparse.items(): if isinstance( val, dict ): poverride( target[key], val ) elif isinstance( val, list ): target[key] = val[:] else: target[key] = val
def test_validate(self): """ An interesting aspect of the ParsecConfig.validate, is that if you have a sparse dict produced by this class, and you call the validate on that dict again, you may have TypeErrors. That occurs because the values like 'True' are validated against the spec and converted from Strings with quotes, to bool types. So the next type you run the validation if expects Strings... :return: """ spec = { 'section': { 'name': [VDR.V_STRING], 'address': [VDR.V_STRING], } } parsec_config = parsec.config.ParsecConfig( spec=spec, upgrader=None, # new spec output_fname=None, # not going to call the loadcfg tvars=None, validator=None # use default ) sparse = OrderedDictWithDefaults() parsec_config.validate(sparse) # empty dict is OK with self.assertRaises(parsec.validate.IllegalItemError): sparse = OrderedDictWithDefaults() sparse['name'] = 'True' parsec_config.validate(sparse) # name is not valid sparse = OrderedDictWithDefaults() sparse['section'] = OrderedDictWithDefaults() sparse['section']['name'] = 'Wind' sparse['section']['address'] = 'Corner' parsec_config.validate(sparse)
def parse(fpath, write_proc=False, template_vars=[], template_vars_file=None): "Parse file items line-by-line into a corresponding nested dict." # read and process the file (jinja2, include-files, line continuation) flines = read_and_proc(fpath, template_vars, template_vars_file) # write the processed for suite.rc if it lives in a writable directory if write_proc and \ os.access(os.path.dirname(fpath), os.W_OK): fpath_processed = fpath + '.processed' if cylc.flags.verbose: print "Writing file " + fpath_processed f = open(fpath_processed, 'w') f.write('\n'.join(flines) + '\n') f.close() nesting_level = 0 config = OrderedDictWithDefaults() sect_name = None parents = [] maxline = len(flines) - 1 index = -1 while index < maxline: index += 1 line = flines[index] if re.match(_LINECOMMENT, line): # skip full-line comments continue if re.match(_BLANKLINE, line): # skip blank lines continue m = re.match(_HEADING, line) if m: # matched a section heading indent, s_open, sect_name, s_close, comment = m.groups() nb = len(s_open) if nb != len(s_close): raise FileParseError('bracket mismatch', index, line) elif nb == nesting_level: # sibling section parents = parents[:-1] + [sect_name] elif nb == nesting_level + 1: # child section parents = parents + [sect_name] elif nb < nesting_level: # back up one or more levels ndif = nesting_level - nb parents = parents[:-ndif - 1] + [sect_name] else: raise FileParseError('Error line ' + str(index + 1) + ': ' + line) nesting_level = nb addsect(config, sect_name, parents[:-1]) else: m = re.match(_KEY_VALUE, line) if m: # matched a key=value item indent, key, val = m.groups() if val.startswith('"""') or val.startswith("'''"): # triple quoted - may be a multiline value val, index = multiline(flines, val, index, maxline) addict(config, key, val, parents, index) else: # no match raise FileParseError('Invalid line ' + str(index + 1) + ': ' + line) return config
def test_nonzero(self): d = OrderedDictWithDefaults() self.assertFalse(d) d['value'] = 10 self.assertTrue(d)
def parse(fpath, output_fname=None, template_vars=None): "Parse file items line-by-line into a corresponding nested dict." # read and process the file (jinja2, include-files, line continuation) flines = read_and_proc(fpath, template_vars) if output_fname: with open(output_fname, 'wb') as handle: handle.write('\n'.join(flines) + '\n') LOG.debug('Processed configuration dumped: %s', output_fname) nesting_level = 0 config = OrderedDictWithDefaults() parents = [] maxline = len(flines) - 1 index = -1 while index < maxline: index += 1 line = flines[index] if re.match(_LINECOMMENT, line): # skip full-line comments continue if re.match(_BLANKLINE, line): # skip blank lines continue m = re.match(_HEADING, line) if m: # matched a section heading s_open, sect_name, s_close = m.groups()[1:-1] nb = len(s_open) if nb != len(s_close): raise FileParseError('bracket mismatch', index, line) elif nb == nesting_level: # sibling section parents = parents[:-1] + [sect_name] elif nb == nesting_level + 1: # child section parents = parents + [sect_name] elif nb < nesting_level: # back up one or more levels ndif = nesting_level - nb parents = parents[:-ndif - 1] + [sect_name] else: raise FileParseError('Error line ' + str(index + 1) + ': ' + line) nesting_level = nb addsect(config, sect_name, parents[:-1]) else: m = re.match(_KEY_VALUE, line) if m: # matched a key=value item key, _, val = m.groups()[1:] if val.startswith('"""') or val.startswith("'''"): # triple quoted - may be a multiline value val, index = multiline(flines, val, index, maxline) addict(config, key, val, parents, index) else: # no match raise FileParseError('Invalid line ' + str(index + 1) + ': ' + line) return config
def pdeepcopy(source): """Make a deep copy of a pdict source""" target = OrderedDictWithDefaults() replicate(target, source) return target
def test_prepend(self): d = OrderedDictWithDefaults() d['key'] = 'Birds' d.prepend('year', 1980) d.prepend('key', 2000) iterator = d.iterkeys() self.assertEqual('key', iterator.next()) self.assertEqual('year', iterator.next()) d = OrderedDictWithDefaults() d['key'] = 'Birds' d.prepend('year', 1980) iterator = d.iterkeys() self.assertEqual('year', iterator.next()) self.assertEqual('key', iterator.next())
def test_setitem(self): d = OrderedDictWithDefaults() d['name'] = 'Matthews' self.assertEqual('Matthews', d['name']) d['name'] = 'Zaccharias' self.assertEqual('Zaccharias', d['name'])