def expand(self):
"""Flesh out undefined items with defaults, if any, from the spec."""
if not self.dense:
dense = OrderedDictWithDefaults()
# Populate dict with default values from the spec
stack = [[dense, self.spec]]
while stack:
defs, spec = stack.pop()
for key, val in spec.items():
if isinstance(val, dict):
if key not in defs:
defs[key] = OrderedDictWithDefaults()
stack.append((defs[key], spec[key]))
else:
try:
defs[key] = spec[key][1]
except IndexError:
if spec[key][0].endswith('_LIST'):
defs[key] = []
else:
defs[key] = None
# override defaults with sparse values
m_override(dense, self.sparse)
un_many(dense)
self.dense = dense
def test_items(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d['len'] = '89'
for _, v in list(d.items()):
self.assertTrue(v in ['Birds', '89'])
def expand(self):
"""Flesh out undefined items with defaults, if any, from the spec."""
if not self.dense:
dense = OrderedDictWithDefaults()
# Populate dict with default values from the spec
stack = [[dense, self.spec]]
while stack:
defs, spec = stack.pop()
for node in spec:
if not node.is_leaf():
if node.name not in defs:
defs[node.name] = OrderedDictWithDefaults()
stack.append((defs[node.name], node))
else:
if node.default is ConfigNode.UNSET:
if node.vdr and node.vdr.endswith('_LIST'):
defs[node.name] = []
else:
defs[node.name] = None
else:
defs[node.name] = node.default
# override defaults with sparse values
m_override(dense, self.sparse)
un_many(dense)
self.dense = dense
def test_items(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d['len'] = '89'
for _, v in list(d.items()):
self.assertTrue(v in ['Birds', '89'])
def test_parse_with_sections():
with tempfile.NamedTemporaryFile() as of:
with tempfile.NamedTemporaryFile() as tf:
fpath = tf.name
template_vars = {'name': 'Cylc'}
tf.write(("#!jinja2\n[section1]\n"
"a={{ name }}\n# comment!\n"
"[[subsection1]]\n"
"[[subsection2]]\n"
"[section2]").encode())
tf.flush()
r = parse(fpath=fpath,
output_fname=of.name,
template_vars=template_vars)
expected = OrderedDictWithDefaults()
expected['section1'] = OrderedDictWithDefaults()
expected['section1']['a'] = 'Cylc'
expected['section1']['subsection1'] = OrderedDictWithDefaults()
expected['section1']['subsection2'] = OrderedDictWithDefaults()
expected['section2'] = OrderedDictWithDefaults()
assert r == expected
of.flush()
output_file_contents = of.read().decode()
assert output_file_contents == ('[section1]\na=Cylc\n# comment!\n'
'[[subsection1]]\n'
'[[subsection2]]\n'
'[section2]\n')
def __init__(self,
spec,
upgrader=None,
output_fname=None,
tvars=None,
validator=None,
options=None):
"""Instatiate a parsec config object.
Args:
spec: Specification for the config.
upgrader: An upgrader function, which converts old config items
to new ones, or returns errors for obselete items.
output_fname: Filename to dump parsed config to.
tvars: Template variables.
validator: Function checkin that config is valid; defaults to
``parsec_validate``.
options: Command line options.
"""
self.sparse = OrderedDictWithDefaults()
self.dense = OrderedDictWithDefaults()
self.upgrader = upgrader
self.tvars = tvars
self.output_fname = output_fname
self.spec = spec
if validator is None:
validator = parsec_validate
self.validator = validator
# Get a list of config items which have a private name ``__MANY__``:
self.manyparents = self._get_namespace_parents()
self.options = options
def m_override(target, sparse):
"""Override items in a target pdict.
Target keys must already exist unless there is a "__MANY__" placeholder in
the right position.
"""
if not sparse:
return
stack = [(sparse, target, [], OrderedDictWithDefaults())]
defaults_list = []
while stack:
source, dest, keylist, many_defaults = stack.pop(0)
if many_defaults:
defaults_list.append((dest, many_defaults))
for key, val in source.items():
if isinstance(val, dict):
if key in many_defaults:
child_many_defaults = many_defaults[key]
else:
child_many_defaults = OrderedDictWithDefaults()
if key not in dest:
if '__MANY__' in dest:
dest[key] = OrderedDictWithDefaults()
child_many_defaults = dest['__MANY__']
elif '__MANY__' in many_defaults:
# A 'sub-many' dict - would it ever exist in real life?
dest[key] = OrderedDictWithDefaults()
child_many_defaults = many_defaults['__MANY__']
elif key in many_defaults:
dest[key] = OrderedDictWithDefaults()
else:
# TODO - validation prevents this, but handle properly
# for completeness.
raise Exception(
"parsec dict override: no __MANY__ placeholder" +
"%s" % (keylist + [key]))
stack.append(
(val, dest[key], keylist + [key], child_many_defaults))
else:
if key not in dest:
if ('__MANY__' in dest or key in many_defaults
or '__MANY__' in many_defaults):
if isinstance(val, list):
dest[key] = val[:]
else:
dest[key] = val
else:
# TODO - validation prevents this, but handle properly
# for completeness.
raise Exception(
"parsec dict override: no __MANY__ placeholder" +
"%s" % (keylist + [key]))
if isinstance(val, list):
dest[key] = val[:]
else:
dest[key] = val
for dest_dict, defaults in defaults_list:
dest_dict.defaults_ = defaults
def test_values(self):
d = OrderedDictWithDefaults()
d['name'] = 'Paul'
d['color'] = 'Green'
values = list(d.values())
self.assertTrue(len(values) == 2)
self.assertTrue('Paul' in values)
self.assertTrue('Green' in values)
def test_values(self):
d = OrderedDictWithDefaults()
d['name'] = 'Paul'
d['color'] = 'Green'
values = list(d.values())
self.assertTrue(len(values) == 2)
self.assertTrue('Paul' in values)
self.assertTrue('Green' in values)
def test_getitem(self):
d = OrderedDictWithDefaults()
d['name'] = 'Joseph'
d.defaults_ = {
'surname': 'Wyndham'
}
self.assertEqual('Joseph', d['name'])
self.assertEqual('Wyndham', d['surname'])
def test_iter_defaults():
"""It doesn't yield keys from defaults."""
this = ODD(a=1)
that = ODD(b=2)
this.defaults = dict(c=3)
that.defaults = dict(d=4)
a = DictTree(this, that)
assert list(sorted(a)) == ['a', 'b']
def test_contains(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d.defaults_ = {'value': '10'}
self.assertTrue('key' in d)
self.assertTrue('value' in d)
self.assertFalse('test' in d)
def get_parsec_validator_invalid_values():
"""
Data provider or invalid values for parsec validator. All values must not
be null (covered elsewhere), and not dict's.
Possible invalid scenarios must include:
- cfg[key] is a list AND a value is not in list of the possible values
- OR
- cfg[key] is not a list AND cfg[key] not in the list of possible values
:return: a list with sets of tuples for the test parameters
:rtype: list
"""
values = []
# variables reused throughout
spec = None
msg = None
# set 1 (t, f, f, t)
spec = {
'value': [VDR.V_INTEGER_LIST, 1, 2, 3, 4]
}
cfg = OrderedDictWithDefaults()
cfg['value'] = "1, 2, 3"
msg = None
values.append((spec, cfg, msg))
# set 2 (t, t, f, t)
spec = {
'value': [VDR.V_INTEGER_LIST, 1, 2, 3, 4]
}
cfg = OrderedDictWithDefaults()
cfg['value'] = "1, 2, 5"
msg = '(type=option) value = [1, 2, 5]'
values.append((spec, cfg, msg))
# set 3 (f, f, t, f)
spec = {
'value': [VDR.V_INTEGER, 1, 2, 3, 4]
}
cfg = OrderedDictWithDefaults()
cfg['value'] = "2"
msg = None
values.append((spec, cfg, msg))
# set 4 (f, f, t, t)
spec = {
'value': [VDR.V_INTEGER, 1, 2, 3, 4]
}
cfg = OrderedDictWithDefaults()
cfg['value'] = "5"
msg = '(type=option) value = 5'
values.append((spec, cfg, msg))
return values
def test_parsec_validator_invalid_key(sample_spec):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section22'] = 'abc'
with pytest.raises(IllegalItemError):
parsec_validator.validate(cfg, sample_spec)
def test_parsec_validator_invalid_key(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section22'] = 'abc'
with self.assertRaises(IllegalItemError):
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
def test_keys(self):
d = OrderedDictWithDefaults()
d['name'] = 'Andrew'
d['surname'] = 'Gray'
d.defaults_ = {'address': 'N/A'}
keys = list(d.keys())
self.assertTrue(len(keys) == 3)
self.assertTrue('name' in keys)
self.assertTrue('surname' in keys)
self.assertTrue('address' in keys)
def test_contains(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d.defaults_ = {
'value': '10'
}
self.assertTrue('key' in d)
self.assertTrue('value' in d)
self.assertFalse('test' in d)
def test_parsec_validator_invalid_key_with_many_1(sample_spec):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section3000000'] = OrderedDictWithDefaults()
parsec_validator.validate(cfg, sample_spec)
# TBD assertIsNotNone when 2.6+
assert parsec_validator is not None
def test_parsec_validator_invalid_key_with_many_spaces(sample_spec):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section 3000000'] = 'test'
with pytest.raises(IllegalItemError) as cm:
parsec_validator.validate(cfg, sample_spec)
assert "section 3000000 - (consecutive spaces)" == str(cm.exception)
def test_parsec_validator_invalid_key_with_many_1(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section3000000'] = OrderedDictWithDefaults()
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
# TBD assertIsNotNone when 2.6+
self.assertTrue(parsec_validator is not None)
def test_parsec_validator_invalid_key_with_many_spaces(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section 3000000'] = 'test'
with self.assertRaises(IllegalItemError) as cm:
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
self.assertEqual("section 3000000 - (consecutive spaces)",
str(cm.exception))
def test_pdeepcopy(self):
"""This is tested entirely by the tests in replicate as well"""
source = OrderedDictWithDefaults()
source["name"] = OrderedDictWithDefaults()
source["name"]["value"] = "oil"
source["name"]["key"] = 1
source["name"].defaults_ = {"value": 1}
target = pdeepcopy(source)
self.assertEqual(source, target)
def test_printcfg_list_values(self):
cfg = OrderedDictWithDefaults()
cfg['root'] = OrderedDictWithDefaults()
cfg['root']['special'] = ['a', 'b', 'c', None]
cfg['root']['normal'] = 0
myhandle = StringIO()
printcfg(cfg, handle=myhandle, none_str='d')
expected = "[root]\n special = a, b, c, d\n normal = 0\n"
actual = myhandle.getvalue()
self.assertEqual(expected, actual)
def test_parsec_validator(sample_spec):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section3'] = OrderedDictWithDefaults()
cfg['section3']['title'] = None
parsec_validator.validate(cfg, sample_spec)
# TBD assertIsNotNone when 2.6+
assert parsec_validator is not None
def __init__(self, spec, upgrader=None, output_fname=None, tvars=None,
validator=None):
self.sparse = OrderedDictWithDefaults()
self.dense = OrderedDictWithDefaults()
self.upgrader = upgrader
self.tvars = tvars
self.output_fname = output_fname
self.spec = spec
if validator is None:
validator = parsec_validate
self.validator = validator
def test_parsec_validator_invalid_key_with_many_2(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section3'] = OrderedDictWithDefaults()
cfg['section3']['title'] = '1'
cfg['section3']['entries'] = OrderedDictWithDefaults()
cfg['section3']['entries']['key'] = 'name'
cfg['section3']['entries']['value'] = "1, 2, 3, 4"
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
# TBD assertIsNotNone when 2.6+
self.assertTrue(parsec_validator is not None)
def test_parsec_validator_invalid_key_with_many_2(sample_spec):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section3'] = OrderedDictWithDefaults()
cfg['section3']['title'] = '1'
cfg['section3']['entries'] = OrderedDictWithDefaults()
cfg['section3']['entries']['key'] = 'name'
cfg['section3']['entries']['value'] = "1, 2, 3, 4"
parsec_validator.validate(cfg, sample_spec)
# TBD assertIsNotNone when 2.6+
assert parsec_validator is not None
def test_parsec_validator(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section3'] = OrderedDictWithDefaults()
cfg['section3']['title'] = None
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
# TBD assertIsNotNone when 2.6+
self.assertTrue(parsec_validator is not None)
def test_parsec_validator_invalid_key_no_spec(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section22'] = 'abc'
spec = SAMPLE_SPEC_1.copy()
del (spec['__MANY__'])
with self.assertRaises(IllegalItemError):
parsec_validator.validate(cfg, spec)
def test_itervalues(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d['len'] = '89'
d.defaults_ = {'surname': 'Wyndham'}
count = 0
for k in d.values():
self.assertTrue(k in ['Birds', '89', 'Wyndham'])
count += 1
self.assertEqual(3, count)
def test_keys(self):
d = OrderedDictWithDefaults()
d['name'] = 'Andrew'
d['surname'] = 'Gray'
d.defaults_ = {
'address': 'N/A'
}
keys = list(d.keys())
self.assertTrue(len(keys) == 3)
self.assertTrue('name' in keys)
self.assertTrue('surname' in keys)
self.assertTrue('address' in keys)
def test_m_override_without_many_1(self):
source = OrderedDictWithDefaults()
source["name"] = OrderedDictWithDefaults()
source["name"]["value"] = "oil"
source["name"]["key"] = [1, 2, 3, 4]
target = OrderedDictWithDefaults()
target["name"] = OrderedDictWithDefaults()
target["name"]["index"] = 0
with self.assertRaises(Exception):
m_override(target, source)
def test_printcfg_break_lines(self):
cfg = OrderedDictWithDefaults()
cfg['root'] = OrderedDictWithDefaults()
cfg['root']['special'] = "\nthis is\nvalid"
cfg['root']['normal'] = 0
myhandle = StringIO()
printcfg(cfg, handle=myhandle)
expected = "[root]\n special = \"\"\"\n \n " \
" this is\n valid\n \"\"\"\n normal = 0\n"
actual = myhandle.getvalue()
self.assertEqual(expected, actual)
def test_itervalues(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d['len'] = '89'
d.defaults_ = {
'surname': 'Wyndham'
}
count = 0
for k in d.values():
self.assertTrue(k in ['Birds', '89', 'Wyndham'])
count += 1
self.assertEqual(3, count)
def test_printcfg_none_str_is_none(self):
cfg = OrderedDictWithDefaults()
cfg['root'] = OrderedDictWithDefaults()
cfg['root']['special'] = 1
cfg['root']['normal'] = 0
cfg['root'][None] = None
cfg[None] = None
myhandle = StringIO()
printcfg(cfg, handle=myhandle, none_str=None)
expected = "[root]\n special = 1\n normal = 0\n"
actual = myhandle.getvalue()
self.assertEqual(expected, actual)
def test_parsec_validator_invalid_key_no_spec(sample_spec):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section22'] = 'abc'
# remove the user-defined section from the spec
sample_spec._children = {
key: value
for key, value in sample_spec._children.items() if key != '__MANY__'
}
with pytest.raises(IllegalItemError):
parsec_validator.validate(cfg, sample_spec)
def test_prepend(self):
d = OrderedDictWithDefaults()
d['key'] = 'Birds'
d.prepend('year', 1980)
d.prepend('key', 2000)
iterator = iter(d.keys())
self.assertEqual('key', next(iterator))
self.assertEqual('year', next(iterator))
d = OrderedDictWithDefaults()
d['key'] = 'Birds'
d.prepend('year', 1980)
iterator = iter(d.keys())
self.assertEqual('year', next(iterator))
self.assertEqual('key', next(iterator))
def replicate(target, source):
"""Replicate source *into* target.
Source elements need not exist in target already, so source overrides
common elements in target and otherwise adds elements to it.
"""
if not source:
return
if hasattr(source, "defaults_"):
target.defaults_ = pdeepcopy(source.defaults_)
for key, val in source.items():
if isinstance(val, dict):
if key not in target:
target[key] = OrderedDictWithDefaults()
if hasattr(val, 'defaults_'):
target[key].defaults_ = pdeepcopy(val.defaults_)
replicate(target[key], val)
elif isinstance(val, list):
target[key] = val[:]
else:
target[key] = val
