def expand(self):
"""Flesh out undefined items with defaults, if any, from the spec."""
if not self.dense:
dense = OrderedDictWithDefaults()
# Populate dict with default values from the spec
stack = [[dense, self.spec]]
while stack:
defs, spec = stack.pop()
for key, val in spec.items():
if isinstance(val, dict):
if key not in defs:
defs[key] = OrderedDictWithDefaults()
stack.append((defs[key], spec[key]))
else:
try:
defs[key] = spec[key][1]
except IndexError:
if spec[key][0].endswith('_LIST'):
defs[key] = []
else:
defs[key] = None
# override defaults with sparse values
m_override(dense, self.sparse)
un_many(dense)
self.dense = dense
def expand(self):
"""Flesh out undefined items with defaults, if any, from the spec."""
if not self.dense:
dense = OrderedDictWithDefaults()
# Populate dict with default values from the spec
stack = [[dense, self.spec]]
while stack:
defs, spec = stack.pop()
for node in spec:
if not node.is_leaf():
if node.name not in defs:
defs[node.name] = OrderedDictWithDefaults()
stack.append((defs[node.name], node))
else:
if node.default is ConfigNode.UNSET:
if node.vdr and node.vdr.endswith('_LIST'):
defs[node.name] = []
else:
defs[node.name] = None
else:
defs[node.name] = node.default
# override defaults with sparse values
m_override(dense, self.sparse)
un_many(dense)
self.dense = dense
def test_parse_with_sections():
with tempfile.NamedTemporaryFile() as of:
with tempfile.NamedTemporaryFile() as tf:
fpath = tf.name
template_vars = {'name': 'Cylc'}
tf.write(("#!jinja2\n[section1]\n"
"a={{ name }}\n# comment!\n"
"[[subsection1]]\n"
"[[subsection2]]\n"
"[section2]").encode())
tf.flush()
r = parse(fpath=fpath,
output_fname=of.name,
template_vars=template_vars)
expected = OrderedDictWithDefaults()
expected['section1'] = OrderedDictWithDefaults()
expected['section1']['a'] = 'Cylc'
expected['section1']['subsection1'] = OrderedDictWithDefaults()
expected['section1']['subsection2'] = OrderedDictWithDefaults()
expected['section2'] = OrderedDictWithDefaults()
assert r == expected
of.flush()
output_file_contents = of.read().decode()
assert output_file_contents == ('[section1]\na=Cylc\n# comment!\n'
'[[subsection1]]\n'
'[[subsection2]]\n'
'[section2]\n')
def __init__(self,
spec,
upgrader=None,
output_fname=None,
tvars=None,
validator=None,
options=None):
"""Instatiate a parsec config object.
Args:
spec: Specification for the config.
upgrader: An upgrader function, which converts old config items
to new ones, or returns errors for obselete items.
output_fname: Filename to dump parsed config to.
tvars: Template variables.
validator: Function checkin that config is valid; defaults to
``parsec_validate``.
options: Command line options.
"""
self.sparse = OrderedDictWithDefaults()
self.dense = OrderedDictWithDefaults()
self.upgrader = upgrader
self.tvars = tvars
self.output_fname = output_fname
self.spec = spec
if validator is None:
validator = parsec_validate
self.validator = validator
# Get a list of config items which have a private name ``__MANY__``:
self.manyparents = self._get_namespace_parents()
self.options = options
def m_override(target, sparse):
"""Override items in a target pdict.
Target keys must already exist unless there is a "__MANY__" placeholder in
the right position.
"""
if not sparse:
return
stack = [(sparse, target, [], OrderedDictWithDefaults())]
defaults_list = []
while stack:
source, dest, keylist, many_defaults = stack.pop(0)
if many_defaults:
defaults_list.append((dest, many_defaults))
for key, val in source.items():
if isinstance(val, dict):
if key in many_defaults:
child_many_defaults = many_defaults[key]
else:
child_many_defaults = OrderedDictWithDefaults()
if key not in dest:
if '__MANY__' in dest:
dest[key] = OrderedDictWithDefaults()
child_many_defaults = dest['__MANY__']
elif '__MANY__' in many_defaults:
# A 'sub-many' dict - would it ever exist in real life?
dest[key] = OrderedDictWithDefaults()
child_many_defaults = many_defaults['__MANY__']
elif key in many_defaults:
dest[key] = OrderedDictWithDefaults()
else:
# TODO - validation prevents this, but handle properly
# for completeness.
raise Exception(
"parsec dict override: no __MANY__ placeholder" +
"%s" % (keylist + [key]))
stack.append(
(val, dest[key], keylist + [key], child_many_defaults))
else:
if key not in dest:
if ('__MANY__' in dest or key in many_defaults
or '__MANY__' in many_defaults):
if isinstance(val, list):
dest[key] = val[:]
else:
dest[key] = val
else:
# TODO - validation prevents this, but handle properly
# for completeness.
raise Exception(
"parsec dict override: no __MANY__ placeholder" +
"%s" % (keylist + [key]))
if isinstance(val, list):
dest[key] = val[:]
else:
dest[key] = val
for dest_dict, defaults in defaults_list:
dest_dict.defaults_ = defaults
def get_parsec_validator_invalid_values():
"""
Data provider or invalid values for parsec validator. All values must not
be null (covered elsewhere), and not dict's.
Possible invalid scenarios must include:
- cfg[key] is a list AND a value is not in list of the possible values
- OR
- cfg[key] is not a list AND cfg[key] not in the list of possible values
:return: a list with sets of tuples for the test parameters
:rtype: list
"""
values = []
# variables reused throughout
spec = None
msg = None
# set 1 (t, f, f, t)
spec = {
'value': [VDR.V_INTEGER_LIST, 1, 2, 3, 4]
}
cfg = OrderedDictWithDefaults()
cfg['value'] = "1, 2, 3"
msg = None
values.append((spec, cfg, msg))
# set 2 (t, t, f, t)
spec = {
'value': [VDR.V_INTEGER_LIST, 1, 2, 3, 4]
}
cfg = OrderedDictWithDefaults()
cfg['value'] = "1, 2, 5"
msg = '(type=option) value = [1, 2, 5]'
values.append((spec, cfg, msg))
# set 3 (f, f, t, f)
spec = {
'value': [VDR.V_INTEGER, 1, 2, 3, 4]
}
cfg = OrderedDictWithDefaults()
cfg['value'] = "2"
msg = None
values.append((spec, cfg, msg))
# set 4 (f, f, t, t)
spec = {
'value': [VDR.V_INTEGER, 1, 2, 3, 4]
}
cfg = OrderedDictWithDefaults()
cfg['value'] = "5"
msg = '(type=option) value = 5'
values.append((spec, cfg, msg))
return values
def test_parsec_validator_invalid_key(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section22'] = 'abc'
with self.assertRaises(IllegalItemError):
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
def test_parsec_validator_invalid_key(sample_spec):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section22'] = 'abc'
with pytest.raises(IllegalItemError):
parsec_validator.validate(cfg, sample_spec)
def test_parsec_validator_invalid_key_with_many_1(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section3000000'] = OrderedDictWithDefaults()
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
# TBD assertIsNotNone when 2.6+
self.assertTrue(parsec_validator is not None)
def test_parsec_validator_invalid_key_with_many_spaces(sample_spec):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section 3000000'] = 'test'
with pytest.raises(IllegalItemError) as cm:
parsec_validator.validate(cfg, sample_spec)
assert "section 3000000 - (consecutive spaces)" == str(cm.exception)
def test_parsec_validator_invalid_key_with_many_1(sample_spec):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section3000000'] = OrderedDictWithDefaults()
parsec_validator.validate(cfg, sample_spec)
# TBD assertIsNotNone when 2.6+
assert parsec_validator is not None
def __init__(self, spec, upgrader=None, output_fname=None, tvars=None,
validator=None):
self.sparse = OrderedDictWithDefaults()
self.dense = OrderedDictWithDefaults()
self.upgrader = upgrader
self.tvars = tvars
self.output_fname = output_fname
self.spec = spec
if validator is None:
validator = parsec_validate
self.validator = validator
def test_printcfg_list_values(self):
cfg = OrderedDictWithDefaults()
cfg['root'] = OrderedDictWithDefaults()
cfg['root']['special'] = ['a', 'b', 'c', None]
cfg['root']['normal'] = 0
myhandle = StringIO()
printcfg(cfg, handle=myhandle, none_str='d')
expected = "[root]\n special = a, b, c, d\n normal = 0\n"
actual = myhandle.getvalue()
self.assertEqual(expected, actual)
def test_pdeepcopy(self):
"""This is tested entirely by the tests in replicate as well"""
source = OrderedDictWithDefaults()
source["name"] = OrderedDictWithDefaults()
source["name"]["value"] = "oil"
source["name"]["key"] = 1
source["name"].defaults_ = {"value": 1}
target = pdeepcopy(source)
self.assertEqual(source, target)
def test_parsec_validator_invalid_key_with_many_spaces(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section 3000000'] = 'test'
with self.assertRaises(IllegalItemError) as cm:
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
self.assertEqual("section 3000000 - (consecutive spaces)",
str(cm.exception))
def test_parsec_validator(sample_spec):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section3'] = OrderedDictWithDefaults()
cfg['section3']['title'] = None
parsec_validator.validate(cfg, sample_spec)
# TBD assertIsNotNone when 2.6+
assert parsec_validator is not None
def test_parsec_validator_invalid_key_with_many_2(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section3'] = OrderedDictWithDefaults()
cfg['section3']['title'] = '1'
cfg['section3']['entries'] = OrderedDictWithDefaults()
cfg['section3']['entries']['key'] = 'name'
cfg['section3']['entries']['value'] = "1, 2, 3, 4"
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
# TBD assertIsNotNone when 2.6+
self.assertTrue(parsec_validator is not None)
def test_parsec_validator_invalid_key_with_many_2(sample_spec):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section3'] = OrderedDictWithDefaults()
cfg['section3']['title'] = '1'
cfg['section3']['entries'] = OrderedDictWithDefaults()
cfg['section3']['entries']['key'] = 'name'
cfg['section3']['entries']['value'] = "1, 2, 3, 4"
parsec_validator.validate(cfg, sample_spec)
# TBD assertIsNotNone when 2.6+
assert parsec_validator is not None
def test_parsec_validator(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section3'] = OrderedDictWithDefaults()
cfg['section3']['title'] = None
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
# TBD assertIsNotNone when 2.6+
self.assertTrue(parsec_validator is not None)
def test_parsec_validator_invalid_key_no_spec(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section22'] = 'abc'
spec = SAMPLE_SPEC_1.copy()
del (spec['__MANY__'])
with self.assertRaises(IllegalItemError):
parsec_validator.validate(cfg, spec)
def test_m_override_without_many_1(self):
source = OrderedDictWithDefaults()
source["name"] = OrderedDictWithDefaults()
source["name"]["value"] = "oil"
source["name"]["key"] = [1, 2, 3, 4]
target = OrderedDictWithDefaults()
target["name"] = OrderedDictWithDefaults()
target["name"]["index"] = 0
with self.assertRaises(Exception):
m_override(target, source)
def test_printcfg_break_lines(self):
cfg = OrderedDictWithDefaults()
cfg['root'] = OrderedDictWithDefaults()
cfg['root']['special'] = "\nthis is\nvalid"
cfg['root']['normal'] = 0
myhandle = StringIO()
printcfg(cfg, handle=myhandle)
expected = "[root]\n special = \"\"\"\n \n " \
" this is\n valid\n \"\"\"\n normal = 0\n"
actual = myhandle.getvalue()
self.assertEqual(expected, actual)
def test_printcfg_none_str_is_none(self):
cfg = OrderedDictWithDefaults()
cfg['root'] = OrderedDictWithDefaults()
cfg['root']['special'] = 1
cfg['root']['normal'] = 0
cfg['root'][None] = None
cfg[None] = None
myhandle = StringIO()
printcfg(cfg, handle=myhandle, none_str=None)
expected = "[root]\n special = 1\n normal = 0\n"
actual = myhandle.getvalue()
self.assertEqual(expected, actual)
def test_parsec_validator_invalid_key_no_spec(sample_spec):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section22'] = 'abc'
# remove the user-defined section from the spec
sample_spec._children = {
key: value
for key, value in sample_spec._children.items() if key != '__MANY__'
}
with pytest.raises(IllegalItemError):
parsec_validator.validate(cfg, sample_spec)
def test_un_many(self):
target = OrderedDictWithDefaults()
target["name"] = OrderedDictWithDefaults()
target["name"]["index"] = 0
target['__MANY__'] = OrderedDictWithDefaults()
target['__MANY__']['name2'] = OrderedDictWithDefaults()
target['__MANY__']['subdict'] = OrderedDictWithDefaults()
target['__MANY__']['key'] = []
target['__MANY__']['text'] = "Ad infinitum"
un_many(None) # harmless, no error/exception
un_many(target)
self.assertFalse('__MANY__' in list(target))
def get_checkspec_params():
"""Data provider for test_checkspec"""
spec1 = OrderedDictWithDefaults()
spec1['key'] = OrderedDictWithDefaults()
spec1['key']['values'] = ['a', 'b', 'c']
parents1 = []
spec2 = OrderedDictWithDefaults()
spec2['key'] = OrderedDictWithDefaults()
spec2['key']['values'] = ['a', 'b', 'c']
spec2['entry'] = "not a dict, not a list"
parents2 = []
return [(spec1, parents1, None), (spec2, parents2, ParsecError)]
def test_items(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d['len'] = '89'
for _, v in list(d.items()):
self.assertTrue(v in ['Birds', '89'])
def _expand_graph(self,
line,
all_params,
param_list,
line_set,
values=None):
"""Expand line into line_set for any number of parameters.
line is a graph string line as described above in the calling method.
param_list is a list of tuples (name, max-val) for each parameter.
results is a set to hold each expanded line.
"""
if values is None:
values = {}
if not param_list:
# Inner loop.
for p_group in set(REC_P_GROUP.findall(line)):
# Parameters must be expanded in the order found.
param_values = OrderedDictWithDefaults()
tmpl = ''
for item in p_group.split(','):
pname, offs = REC_P_OFFS.match(item).groups()
if offs is None:
param_values[pname] = values[pname]
elif offs.startswith('='):
# Specific value.
try:
# Template may require an integer
param_values[pname] = int(offs[1:])
except ValueError:
param_values[pname] = offs[1:]
else:
# Index offset.
plist = all_params[pname]
cur_idx = plist.index(values[pname])
off_idx = cur_idx + int(offs)
if 0 <= off_idx < len(plist):
offval = plist[off_idx]
else:
offval = self._REMOVE
param_values[pname] = offval
for pname in param_values:
tmpl += self.param_tmpl_cfg[pname]
try:
repl = tmpl % param_values
except KeyError as exc:
raise ParamExpandError('parameter %s is not '
'defined.' % str(exc.args[0]))
line = line.replace('<' + p_group + '>', repl)
# Remove out-of-range nodes
line = self._REMOVE_REC.sub('', line)
if line:
line_set.add(line)
else:
# Recurse through index ranges.
for param_val in param_list[0][1]:
values[param_list[0][0]] = param_val
self._expand_graph(line, all_params, param_list[1:], line_set,
values)
def test_contains(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d.defaults_ = {'value': '10'}
self.assertTrue('key' in d)
self.assertTrue('value' in d)
self.assertFalse('test' in d)
def test_values(self):
d = OrderedDictWithDefaults()
d['name'] = 'Paul'
d['color'] = 'Green'
values = list(d.values())
self.assertTrue(len(values) == 2)
self.assertTrue('Paul' in values)
self.assertTrue('Green' in values)