def expand(self): """Flesh out undefined items with defaults, if any, from the spec.""" if not self.dense: dense = OrderedDictWithDefaults() # Populate dict with default values from the spec stack = [[dense, self.spec]] while stack: defs, spec = stack.pop() for key, val in spec.items(): if isinstance(val, dict): if key not in defs: defs[key] = OrderedDictWithDefaults() stack.append((defs[key], spec[key])) else: try: defs[key] = spec[key][1] except IndexError: if spec[key][0].endswith('_LIST'): defs[key] = [] else: defs[key] = None # override defaults with sparse values m_override(dense, self.sparse) un_many(dense) self.dense = dense
def test_items(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d['len'] = '89' for _, v in list(d.items()): self.assertTrue(v in ['Birds', '89'])
def expand(self): """Flesh out undefined items with defaults, if any, from the spec.""" if not self.dense: dense = OrderedDictWithDefaults() # Populate dict with default values from the spec stack = [[dense, self.spec]] while stack: defs, spec = stack.pop() for node in spec: if not node.is_leaf(): if node.name not in defs: defs[node.name] = OrderedDictWithDefaults() stack.append((defs[node.name], node)) else: if node.default is ConfigNode.UNSET: if node.vdr and node.vdr.endswith('_LIST'): defs[node.name] = [] else: defs[node.name] = None else: defs[node.name] = node.default # override defaults with sparse values m_override(dense, self.sparse) un_many(dense) self.dense = dense
def test_items(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d['len'] = '89' for _, v in list(d.items()): self.assertTrue(v in ['Birds', '89'])
def test_parse_with_sections(): with tempfile.NamedTemporaryFile() as of: with tempfile.NamedTemporaryFile() as tf: fpath = tf.name template_vars = {'name': 'Cylc'} tf.write(("#!jinja2\n[section1]\n" "a={{ name }}\n# comment!\n" "[[subsection1]]\n" "[[subsection2]]\n" "[section2]").encode()) tf.flush() r = parse(fpath=fpath, output_fname=of.name, template_vars=template_vars) expected = OrderedDictWithDefaults() expected['section1'] = OrderedDictWithDefaults() expected['section1']['a'] = 'Cylc' expected['section1']['subsection1'] = OrderedDictWithDefaults() expected['section1']['subsection2'] = OrderedDictWithDefaults() expected['section2'] = OrderedDictWithDefaults() assert r == expected of.flush() output_file_contents = of.read().decode() assert output_file_contents == ('[section1]\na=Cylc\n# comment!\n' '[[subsection1]]\n' '[[subsection2]]\n' '[section2]\n')
def __init__(self, spec, upgrader=None, output_fname=None, tvars=None, validator=None, options=None): """Instatiate a parsec config object. Args: spec: Specification for the config. upgrader: An upgrader function, which converts old config items to new ones, or returns errors for obselete items. output_fname: Filename to dump parsed config to. tvars: Template variables. validator: Function checkin that config is valid; defaults to ``parsec_validate``. options: Command line options. """ self.sparse = OrderedDictWithDefaults() self.dense = OrderedDictWithDefaults() self.upgrader = upgrader self.tvars = tvars self.output_fname = output_fname self.spec = spec if validator is None: validator = parsec_validate self.validator = validator # Get a list of config items which have a private name ``__MANY__``: self.manyparents = self._get_namespace_parents() self.options = options
def m_override(target, sparse): """Override items in a target pdict. Target keys must already exist unless there is a "__MANY__" placeholder in the right position. """ if not sparse: return stack = [(sparse, target, [], OrderedDictWithDefaults())] defaults_list = [] while stack: source, dest, keylist, many_defaults = stack.pop(0) if many_defaults: defaults_list.append((dest, many_defaults)) for key, val in source.items(): if isinstance(val, dict): if key in many_defaults: child_many_defaults = many_defaults[key] else: child_many_defaults = OrderedDictWithDefaults() if key not in dest: if '__MANY__' in dest: dest[key] = OrderedDictWithDefaults() child_many_defaults = dest['__MANY__'] elif '__MANY__' in many_defaults: # A 'sub-many' dict - would it ever exist in real life? dest[key] = OrderedDictWithDefaults() child_many_defaults = many_defaults['__MANY__'] elif key in many_defaults: dest[key] = OrderedDictWithDefaults() else: # TODO - validation prevents this, but handle properly # for completeness. raise Exception( "parsec dict override: no __MANY__ placeholder" + "%s" % (keylist + [key])) stack.append( (val, dest[key], keylist + [key], child_many_defaults)) else: if key not in dest: if ('__MANY__' in dest or key in many_defaults or '__MANY__' in many_defaults): if isinstance(val, list): dest[key] = val[:] else: dest[key] = val else: # TODO - validation prevents this, but handle properly # for completeness. raise Exception( "parsec dict override: no __MANY__ placeholder" + "%s" % (keylist + [key])) if isinstance(val, list): dest[key] = val[:] else: dest[key] = val for dest_dict, defaults in defaults_list: dest_dict.defaults_ = defaults
def test_values(self): d = OrderedDictWithDefaults() d['name'] = 'Paul' d['color'] = 'Green' values = list(d.values()) self.assertTrue(len(values) == 2) self.assertTrue('Paul' in values) self.assertTrue('Green' in values)
def test_values(self): d = OrderedDictWithDefaults() d['name'] = 'Paul' d['color'] = 'Green' values = list(d.values()) self.assertTrue(len(values) == 2) self.assertTrue('Paul' in values) self.assertTrue('Green' in values)
def test_getitem(self): d = OrderedDictWithDefaults() d['name'] = 'Joseph' d.defaults_ = { 'surname': 'Wyndham' } self.assertEqual('Joseph', d['name']) self.assertEqual('Wyndham', d['surname'])
def test_iter_defaults(): """It doesn't yield keys from defaults.""" this = ODD(a=1) that = ODD(b=2) this.defaults = dict(c=3) that.defaults = dict(d=4) a = DictTree(this, that) assert list(sorted(a)) == ['a', 'b']
def test_contains(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d.defaults_ = {'value': '10'} self.assertTrue('key' in d) self.assertTrue('value' in d) self.assertFalse('test' in d)
def get_parsec_validator_invalid_values(): """ Data provider or invalid values for parsec validator. All values must not be null (covered elsewhere), and not dict's. Possible invalid scenarios must include: - cfg[key] is a list AND a value is not in list of the possible values - OR - cfg[key] is not a list AND cfg[key] not in the list of possible values :return: a list with sets of tuples for the test parameters :rtype: list """ values = [] # variables reused throughout spec = None msg = None # set 1 (t, f, f, t) spec = { 'value': [VDR.V_INTEGER_LIST, 1, 2, 3, 4] } cfg = OrderedDictWithDefaults() cfg['value'] = "1, 2, 3" msg = None values.append((spec, cfg, msg)) # set 2 (t, t, f, t) spec = { 'value': [VDR.V_INTEGER_LIST, 1, 2, 3, 4] } cfg = OrderedDictWithDefaults() cfg['value'] = "1, 2, 5" msg = '(type=option) value = [1, 2, 5]' values.append((spec, cfg, msg)) # set 3 (f, f, t, f) spec = { 'value': [VDR.V_INTEGER, 1, 2, 3, 4] } cfg = OrderedDictWithDefaults() cfg['value'] = "2" msg = None values.append((spec, cfg, msg)) # set 4 (f, f, t, t) spec = { 'value': [VDR.V_INTEGER, 1, 2, 3, 4] } cfg = OrderedDictWithDefaults() cfg['value'] = "5" msg = '(type=option) value = 5' values.append((spec, cfg, msg)) return values
def test_parsec_validator_invalid_key(sample_spec): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section22'] = 'abc' with pytest.raises(IllegalItemError): parsec_validator.validate(cfg, sample_spec)
def test_parsec_validator_invalid_key(self): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section22'] = 'abc' with self.assertRaises(IllegalItemError): parsec_validator.validate(cfg, SAMPLE_SPEC_1)
def test_keys(self): d = OrderedDictWithDefaults() d['name'] = 'Andrew' d['surname'] = 'Gray' d.defaults_ = {'address': 'N/A'} keys = list(d.keys()) self.assertTrue(len(keys) == 3) self.assertTrue('name' in keys) self.assertTrue('surname' in keys) self.assertTrue('address' in keys)
def test_contains(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d.defaults_ = { 'value': '10' } self.assertTrue('key' in d) self.assertTrue('value' in d) self.assertFalse('test' in d)
def test_parsec_validator_invalid_key_with_many_1(sample_spec): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section3000000'] = OrderedDictWithDefaults() parsec_validator.validate(cfg, sample_spec) # TBD assertIsNotNone when 2.6+ assert parsec_validator is not None
def test_parsec_validator_invalid_key_with_many_spaces(sample_spec): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section 3000000'] = 'test' with pytest.raises(IllegalItemError) as cm: parsec_validator.validate(cfg, sample_spec) assert "section 3000000 - (consecutive spaces)" == str(cm.exception)
def test_parsec_validator_invalid_key_with_many_1(self): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section3000000'] = OrderedDictWithDefaults() parsec_validator.validate(cfg, SAMPLE_SPEC_1) # TBD assertIsNotNone when 2.6+ self.assertTrue(parsec_validator is not None)
def test_parsec_validator_invalid_key_with_many_spaces(self): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section 3000000'] = 'test' with self.assertRaises(IllegalItemError) as cm: parsec_validator.validate(cfg, SAMPLE_SPEC_1) self.assertEqual("section 3000000 - (consecutive spaces)", str(cm.exception))
def test_pdeepcopy(self): """This is tested entirely by the tests in replicate as well""" source = OrderedDictWithDefaults() source["name"] = OrderedDictWithDefaults() source["name"]["value"] = "oil" source["name"]["key"] = 1 source["name"].defaults_ = {"value": 1} target = pdeepcopy(source) self.assertEqual(source, target)
def test_printcfg_list_values(self): cfg = OrderedDictWithDefaults() cfg['root'] = OrderedDictWithDefaults() cfg['root']['special'] = ['a', 'b', 'c', None] cfg['root']['normal'] = 0 myhandle = StringIO() printcfg(cfg, handle=myhandle, none_str='d') expected = "[root]\n special = a, b, c, d\n normal = 0\n" actual = myhandle.getvalue() self.assertEqual(expected, actual)
def test_parsec_validator(sample_spec): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section3'] = OrderedDictWithDefaults() cfg['section3']['title'] = None parsec_validator.validate(cfg, sample_spec) # TBD assertIsNotNone when 2.6+ assert parsec_validator is not None
def __init__(self, spec, upgrader=None, output_fname=None, tvars=None, validator=None): self.sparse = OrderedDictWithDefaults() self.dense = OrderedDictWithDefaults() self.upgrader = upgrader self.tvars = tvars self.output_fname = output_fname self.spec = spec if validator is None: validator = parsec_validate self.validator = validator
def test_parsec_validator_invalid_key_with_many_2(self): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section3'] = OrderedDictWithDefaults() cfg['section3']['title'] = '1' cfg['section3']['entries'] = OrderedDictWithDefaults() cfg['section3']['entries']['key'] = 'name' cfg['section3']['entries']['value'] = "1, 2, 3, 4" parsec_validator.validate(cfg, SAMPLE_SPEC_1) # TBD assertIsNotNone when 2.6+ self.assertTrue(parsec_validator is not None)
def test_parsec_validator_invalid_key_with_many_2(sample_spec): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section3'] = OrderedDictWithDefaults() cfg['section3']['title'] = '1' cfg['section3']['entries'] = OrderedDictWithDefaults() cfg['section3']['entries']['key'] = 'name' cfg['section3']['entries']['value'] = "1, 2, 3, 4" parsec_validator.validate(cfg, sample_spec) # TBD assertIsNotNone when 2.6+ assert parsec_validator is not None
def test_parsec_validator(self): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section3'] = OrderedDictWithDefaults() cfg['section3']['title'] = None parsec_validator.validate(cfg, SAMPLE_SPEC_1) # TBD assertIsNotNone when 2.6+ self.assertTrue(parsec_validator is not None)
def test_parsec_validator_invalid_key_no_spec(self): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section22'] = 'abc' spec = SAMPLE_SPEC_1.copy() del (spec['__MANY__']) with self.assertRaises(IllegalItemError): parsec_validator.validate(cfg, spec)
def test_itervalues(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d['len'] = '89' d.defaults_ = {'surname': 'Wyndham'} count = 0 for k in d.values(): self.assertTrue(k in ['Birds', '89', 'Wyndham']) count += 1 self.assertEqual(3, count)
def test_keys(self): d = OrderedDictWithDefaults() d['name'] = 'Andrew' d['surname'] = 'Gray' d.defaults_ = { 'address': 'N/A' } keys = list(d.keys()) self.assertTrue(len(keys) == 3) self.assertTrue('name' in keys) self.assertTrue('surname' in keys) self.assertTrue('address' in keys)
def test_m_override_without_many_1(self): source = OrderedDictWithDefaults() source["name"] = OrderedDictWithDefaults() source["name"]["value"] = "oil" source["name"]["key"] = [1, 2, 3, 4] target = OrderedDictWithDefaults() target["name"] = OrderedDictWithDefaults() target["name"]["index"] = 0 with self.assertRaises(Exception): m_override(target, source)
def test_printcfg_break_lines(self): cfg = OrderedDictWithDefaults() cfg['root'] = OrderedDictWithDefaults() cfg['root']['special'] = "\nthis is\nvalid" cfg['root']['normal'] = 0 myhandle = StringIO() printcfg(cfg, handle=myhandle) expected = "[root]\n special = \"\"\"\n \n " \ " this is\n valid\n \"\"\"\n normal = 0\n" actual = myhandle.getvalue() self.assertEqual(expected, actual)
def test_itervalues(self): d = OrderedDictWithDefaults() self.assertEqual([], list(d.items())) d['key'] = 'Birds' d['len'] = '89' d.defaults_ = { 'surname': 'Wyndham' } count = 0 for k in d.values(): self.assertTrue(k in ['Birds', '89', 'Wyndham']) count += 1 self.assertEqual(3, count)
def test_printcfg_none_str_is_none(self): cfg = OrderedDictWithDefaults() cfg['root'] = OrderedDictWithDefaults() cfg['root']['special'] = 1 cfg['root']['normal'] = 0 cfg['root'][None] = None cfg[None] = None myhandle = StringIO() printcfg(cfg, handle=myhandle, none_str=None) expected = "[root]\n special = 1\n normal = 0\n" actual = myhandle.getvalue() self.assertEqual(expected, actual)
def test_parsec_validator_invalid_key_no_spec(sample_spec): parsec_validator = ParsecValidator() cfg = OrderedDictWithDefaults() cfg['section1'] = OrderedDictWithDefaults() cfg['section1']['value1'] = '1' cfg['section1']['value2'] = '2' cfg['section22'] = 'abc' # remove the user-defined section from the spec sample_spec._children = { key: value for key, value in sample_spec._children.items() if key != '__MANY__' } with pytest.raises(IllegalItemError): parsec_validator.validate(cfg, sample_spec)
def test_prepend(self): d = OrderedDictWithDefaults() d['key'] = 'Birds' d.prepend('year', 1980) d.prepend('key', 2000) iterator = iter(d.keys()) self.assertEqual('key', next(iterator)) self.assertEqual('year', next(iterator)) d = OrderedDictWithDefaults() d['key'] = 'Birds' d.prepend('year', 1980) iterator = iter(d.keys()) self.assertEqual('year', next(iterator)) self.assertEqual('key', next(iterator))
def replicate(target, source): """Replicate source *into* target. Source elements need not exist in target already, so source overrides common elements in target and otherwise adds elements to it. """ if not source: return if hasattr(source, "defaults_"): target.defaults_ = pdeepcopy(source.defaults_) for key, val in source.items(): if isinstance(val, dict): if key not in target: target[key] = OrderedDictWithDefaults() if hasattr(val, 'defaults_'): target[key].defaults_ = pdeepcopy(val.defaults_) replicate(target[key], val) elif isinstance(val, list): target[key] = val[:] else: target[key] = val