def test_items(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d['len'] = '89'
for _, v in list(d.items()):
self.assertTrue(v in ['Birds', '89'])
def m_override(target, sparse):
"""Override items in a target pdict.
Target keys must already exist unless there is a "__MANY__" placeholder in
the right position.
"""
if not sparse:
return
stack = [(sparse, target, [], OrderedDictWithDefaults())]
defaults_list = []
while stack:
source, dest, keylist, many_defaults = stack.pop(0)
if many_defaults:
defaults_list.append((dest, many_defaults))
for key, val in source.items():
if isinstance(val, dict):
if key in many_defaults:
child_many_defaults = many_defaults[key]
else:
child_many_defaults = OrderedDictWithDefaults()
if key not in dest:
if '__MANY__' in dest:
dest[key] = OrderedDictWithDefaults()
child_many_defaults = dest['__MANY__']
elif '__MANY__' in many_defaults:
# A 'sub-many' dict - would it ever exist in real life?
dest[key] = OrderedDictWithDefaults()
child_many_defaults = many_defaults['__MANY__']
elif key in many_defaults:
dest[key] = OrderedDictWithDefaults()
else:
# TODO - validation prevents this, but handle properly
# for completeness.
raise Exception(
"parsec dict override: no __MANY__ placeholder" +
"%s" % (keylist + [key]))
stack.append(
(val, dest[key], keylist + [key], child_many_defaults))
else:
if key not in dest:
if ('__MANY__' in dest or key in many_defaults
or '__MANY__' in many_defaults):
if isinstance(val, list):
dest[key] = val[:]
else:
dest[key] = val
else:
# TODO - validation prevents this, but handle properly
# for completeness.
raise Exception(
"parsec dict override: no __MANY__ placeholder" +
"%s" % (keylist + [key]))
if isinstance(val, list):
dest[key] = val[:]
else:
dest[key] = val
for dest_dict, defaults in defaults_list:
dest_dict.defaults_ = defaults
def test_values(self):
d = OrderedDictWithDefaults()
d['name'] = 'Paul'
d['color'] = 'Green'
values = d.values()
self.assertTrue(len(values) == 2)
self.assertTrue('Paul' in values)
self.assertTrue('Green' in values)
def test_contains(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d.defaults_ = {'value': '10'}
self.assertTrue('key' in d)
self.assertTrue('value' in d)
self.assertFalse('test' in d)
def test_values(self):
d = OrderedDictWithDefaults()
d['name'] = 'Paul'
d['color'] = 'Green'
values = list(d.values())
self.assertTrue(len(values) == 2)
self.assertTrue('Paul' in values)
self.assertTrue('Green' in values)
def test_getitem(self):
d = OrderedDictWithDefaults()
d['name'] = 'Joseph'
d.defaults_ = {
'surname': 'Wyndham'
}
self.assertEqual('Joseph', d['name'])
self.assertEqual('Wyndham', d['surname'])
def test_getitem(self):
d = OrderedDictWithDefaults()
d['name'] = 'Joseph'
d.defaults_ = {
'surname': 'Wyndham'
}
self.assertEqual('Joseph', d['name'])
self.assertEqual('Wyndham', d['surname'])
def get_parsec_validator_invalid_values():
"""
Data provider or invalid values for parsec validator. All values must not
be null (covered elsewhere), and not dict's.
Possible invalid scenarios must include:
- cfg[key] is a list AND a value is not in list of the possible values
- OR
- cfg[key] is not a list AND cfg[key] not in the list of possible values
:return: a list with sets of tuples for the test parameters
:rtype: list
"""
values = []
# variables reused throughout
spec = None
msg = None
# set 1 (t, f, f, t)
spec = {
'value': [VDR.V_INTEGER_LIST, 1, 2, 3, 4]
}
cfg = OrderedDictWithDefaults()
cfg['value'] = "1, 2, 3"
msg = None
values.append((spec, cfg, msg))
# set 2 (t, t, f, t)
spec = {
'value': [VDR.V_INTEGER_LIST, 1, 2, 3, 4]
}
cfg = OrderedDictWithDefaults()
cfg['value'] = "1, 2, 5"
msg = 'Illegal option value: value = [1, 2, 5]'
values.append((spec, cfg, msg))
# set 3 (f, f, t, f)
spec = {
'value': [VDR.V_INTEGER, 1, 2, 3, 4]
}
cfg = OrderedDictWithDefaults()
cfg['value'] = "2"
msg = None
values.append((spec, cfg, msg))
# set 4 (f, f, t, t)
spec = {
'value': [VDR.V_INTEGER, 1, 2, 3, 4]
}
cfg = OrderedDictWithDefaults()
cfg['value'] = "5"
msg = 'Illegal option value: value = 5'
values.append((spec, cfg, msg))
return values
def __init__(self, spec, upgrader=None, output_fname=None, tvars=None):
self.sparse = OrderedDictWithDefaults()
self.dense = OrderedDictWithDefaults()
self.upgrader = upgrader
self.tvars = tvars
self.output_fname = output_fname
self.checkspec(spec)
self.spec = spec
def test_parsec_validator_invalid_key(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section22'] = 'abc'
with self.assertRaises(IllegalItemError):
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
def test_keys(self):
d = OrderedDictWithDefaults()
d['name'] = 'Andrew'
d['surname'] = 'Gray'
d.defaults_ = {'address': 'N/A'}
keys = list(d.keys())
self.assertTrue(len(keys) == 3)
self.assertTrue('name' in keys)
self.assertTrue('surname' in keys)
self.assertTrue('address' in keys)
def test_parsec_validator_invalid_key_with_many_1(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section3000000'] = OrderedDictWithDefaults()
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
# TBD assertIsNotNone when 2.6+
self.assertTrue(parsec_validator is not None)
def test_contains(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d.defaults_ = {
'value': '10'
}
self.assertTrue('key' in d)
self.assertTrue('value' in d)
self.assertFalse('test' in d)
def test_itervalues(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d['len'] = '89'
d.defaults_ = {'surname': 'Wyndham'}
count = 0
for k in d.values():
self.assertTrue(k in ['Birds', '89', 'Wyndham'])
count += 1
self.assertEqual(3, count)
def __init__(self, spec, upgrader=None, output_fname=None, tvars=None,
validator=None):
self.sparse = OrderedDictWithDefaults()
self.dense = OrderedDictWithDefaults()
self.upgrader = upgrader
self.tvars = tvars
self.output_fname = output_fname
self.spec = spec
if validator is None:
validator = parsec_validate
self.validator = validator
def test_parsec_validator_invalid_key_with_many_spaces(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section 3000000'] = 'test'
with self.assertRaises(IllegalItemError) as cm:
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
self.assertEqual("Illegal item (consecutive spaces): "
"section 3000000", cm.exception.msg)
def test_parsec_validator(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section3'] = OrderedDictWithDefaults()
cfg['section3']['title'] = None
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
# TBD assertIsNotNone when 2.6+
self.assertTrue(parsec_validator is not None)
def test_parsec_validator_invalid_key_no_spec(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section1'] = OrderedDictWithDefaults()
cfg['section1']['value1'] = '1'
cfg['section1']['value2'] = '2'
cfg['section22'] = 'abc'
spec = SAMPLE_SPEC_1.copy()
del (spec['__MANY__'])
with self.assertRaises(IllegalItemError):
parsec_validator.validate(cfg, spec)
def test_parsec_validator_invalid_key_with_many_2(self):
parsec_validator = ParsecValidator()
cfg = OrderedDictWithDefaults()
cfg['section3'] = OrderedDictWithDefaults()
cfg['section3']['title'] = '1'
cfg['section3']['entries'] = OrderedDictWithDefaults()
cfg['section3']['entries']['key'] = 'name'
cfg['section3']['entries']['value'] = "1, 2, 3, 4"
parsec_validator.validate(cfg, SAMPLE_SPEC_1)
# TBD assertIsNotNone when 2.6+
self.assertTrue(parsec_validator is not None)
def __init__( self, spec, upgrader=None, write_proc=False,
tvars=[], tvars_file=None ):
self.sparse = OrderedDictWithDefaults()
self.dense = OrderedDictWithDefaults()
self.upgrader = upgrader
self.tvars = tvars
self.tvars_file = tvars_file
self.write_proc = write_proc
self.checkspec( spec )
self.spec = spec
def test_keys(self):
d = OrderedDictWithDefaults()
d['name'] = 'Andrew'
d['surname'] = 'Gray'
d.defaults_ = {
'address': 'N/A'
}
keys = list(d.keys())
self.assertTrue(len(keys) == 3)
self.assertTrue('name' in keys)
self.assertTrue('surname' in keys)
self.assertTrue('address' in keys)
def test_itervalues(self):
d = OrderedDictWithDefaults()
self.assertEqual([], list(d.items()))
d['key'] = 'Birds'
d['len'] = '89'
d.defaults_ = {
'surname': 'Wyndham'
}
count = 0
for k in d.values():
self.assertTrue(k in ['Birds', '89', 'Wyndham'])
count += 1
self.assertEqual(3, count)
def poverride(target, sparse, prepend=False):
"""Override or add items in a target pdict.
Target sub-dicts must already exist. For keys that already exist in the
target, the value is overridden in-place. New keys can be prepended in the
target (Cylc use case: broadcast environment variables should be defined
first in the user environment section, to allow use in subsequent variable
definitions.
"""
if not sparse:
target = OrderedDictWithDefaults()
return
for key, val in sparse.items():
if isinstance(val, dict):
poverride(target[key], val, prepend)
else:
if prepend and (key not in target):
# Prepend new items in the target ordered dict.
setitem = target.prepend
else:
# Override in-place in the target ordered dict.
setitem = target.__setitem__
if isinstance(val, list):
setitem(key, val[:])
else:
setitem(key, val)
def _expand_graph(self,
line,
all_params,
param_list,
line_set,
values=None):
"""Expand line into line_set for any number of parameters.
line is a graph string line as described above in the calling method.
param_list is a list of tuples (name, max-val) for each parameter.
results is a set to hold each expanded line.
"""
if values is None:
values = {}
if not param_list:
# Inner loop.
for p_group in set(REC_P_GROUP.findall(line)):
# Parameters must be expanded in the order found.
param_values = OrderedDictWithDefaults()
tmpl = ''
for item in p_group.split(','):
pname, offs = REC_P_OFFS.match(item).groups()
if offs is None:
param_values[pname] = values[pname]
elif offs.startswith('='):
# Specific value.
try:
# Template may require an integer
param_values[pname] = int(offs[1:])
except ValueError:
param_values[pname] = offs[1:]
else:
# Index offset.
plist = all_params[pname]
cur_idx = plist.index(values[pname])
off_idx = cur_idx + int(offs)
if 0 <= off_idx < len(plist):
offval = plist[off_idx]
else:
offval = self._REMOVE
param_values[pname] = offval
for pname in param_values:
tmpl += self.param_tmpl_cfg[pname]
try:
repl = tmpl % param_values
except KeyError as exc:
raise ParamExpandError('ERROR: parameter %s is not '
'defined.' % str(exc.args[0]))
line = line.replace('<' + p_group + '>', repl)
# Remove out-of-range nodes
line = self._REMOVE_REC.sub('', line)
if line:
line_set.add(line)
else:
# Recurse through index ranges.
for param_val in param_list[0][1]:
values[param_list[0][0]] = param_val
self._expand_graph(line, all_params, param_list[1:], line_set,
values)
def get_checkspec_params():
"""Data provider for test_checkspec"""
spec1 = OrderedDictWithDefaults()
spec1['key'] = OrderedDictWithDefaults()
spec1['key']['values'] = ['a', 'b', 'c']
parents1 = []
spec2 = OrderedDictWithDefaults()
spec2['key'] = OrderedDictWithDefaults()
spec2['key']['values'] = ['a', 'b', 'c']
spec2['entry'] = "not a dict, not a list"
parents2 = []
return [(spec1, parents1, None),
(spec2, parents2, parsec.config.ParsecError)]
def _populate_spec_defaults(defs, spec):
"""Populate a nested dict with default values from a spec."""
for key, val in spec.items():
if isinstance(val, dict):
if key not in defs:
defs[key] = OrderedDictWithDefaults()
_populate_spec_defaults(defs[key], spec[key])
else:
defs[key] = spec[key].args['default']
def test_prepend(self):
d = OrderedDictWithDefaults()
d['key'] = 'Birds'
d.prepend('year', 1980)
d.prepend('key', 2000)
iterator = iter(d.keys())
self.assertEqual('key', next(iterator))
self.assertEqual('year', next(iterator))
d = OrderedDictWithDefaults()
d['key'] = 'Birds'
d.prepend('year', 1980)
iterator = iter(d.keys())
self.assertEqual('year', next(iterator))
self.assertEqual('key', next(iterator))
def addsect(cfig, sname, parents):
"""Add a new section to a nested dict."""
for p in parents:
# drop down the parent list
cfig = cfig[p]
if sname in cfig:
# this doesn't warrant a warning unless contained items are repeated
LOG.debug('Section already encountered: %s',
itemstr(parents + [sname]))
else:
cfig[sname] = OrderedDictWithDefaults()
def replicate(target, source):
"""Replicate source *into* target.
Source elements need not exist in target already, so source overrides
common elements in target and otherwise adds elements to it.
"""
if not source:
return
if hasattr(source, "defaults_"):
target.defaults_ = pdeepcopy(source.defaults_)
for key, val in source.items():
if isinstance(val, dict):
if key not in target:
target[key] = OrderedDictWithDefaults()
if hasattr(val, 'defaults_'):
target[key].defaults_ = pdeepcopy(val.defaults_)
replicate(target[key], val)
elif isinstance(val, list):
target[key] = val[:]
else:
target[key] = val
def poverride( target, sparse ):
"""Override items in a target pdict, target sub-dicts must already exist."""
if not sparse:
target = OrderedDictWithDefaults()
return
for key,val in sparse.items():
if isinstance( val, dict ):
poverride( target[key], val )
elif isinstance( val, list ):
target[key] = val[:]
else:
target[key] = val
def replicate(target, source):
"""Replicate source *into* target.
Source elements need not exist in target already, so source overrides
common elements in target and otherwise adds elements to it.
"""
if not source:
return
if hasattr(source, "defaults_"):
target.defaults_ = pdeepcopy(source.defaults_)
for key, val in source.items():
if isinstance(val, dict):
if key not in target:
target[key] = OrderedDictWithDefaults()
if hasattr(val, 'defaults_'):
target[key].defaults_ = pdeepcopy(val.defaults_)
replicate(target[key], val)
elif isinstance(val, list):
target[key] = val[:]
else:
target[key] = val
def test_validate(self):
"""
An interesting aspect of the ParsecConfig.validate, is that if you
have a sparse dict produced by this class, and you call the validate
on that dict again, you may have TypeErrors.
That occurs because the values like 'True' are validated against the
spec and converted from Strings with quotes, to bool types. So the
next type you run the validation if expects Strings...
:return:
"""
spec = {
'section': {
'name': [VDR.V_STRING],
'address': [VDR.V_STRING],
}
}
parsec_config = parsec.config.ParsecConfig(
spec=spec,
upgrader=None, # new spec
output_fname=None, # not going to call the loadcfg
tvars=None,
validator=None # use default
)
sparse = OrderedDictWithDefaults()
parsec_config.validate(sparse) # empty dict is OK
with self.assertRaises(parsec.validate.IllegalItemError):
sparse = OrderedDictWithDefaults()
sparse['name'] = 'True'
parsec_config.validate(sparse) # name is not valid
sparse = OrderedDictWithDefaults()
sparse['section'] = OrderedDictWithDefaults()
sparse['section']['name'] = 'Wind'
sparse['section']['address'] = 'Corner'
parsec_config.validate(sparse)
def parse(fpath, write_proc=False, template_vars=[], template_vars_file=None):
"Parse file items line-by-line into a corresponding nested dict."
# read and process the file (jinja2, include-files, line continuation)
flines = read_and_proc(fpath, template_vars, template_vars_file)
# write the processed for suite.rc if it lives in a writable directory
if write_proc and \
os.access(os.path.dirname(fpath), os.W_OK):
fpath_processed = fpath + '.processed'
if cylc.flags.verbose:
print "Writing file " + fpath_processed
f = open(fpath_processed, 'w')
f.write('\n'.join(flines) + '\n')
f.close()
nesting_level = 0
config = OrderedDictWithDefaults()
sect_name = None
parents = []
maxline = len(flines) - 1
index = -1
while index < maxline:
index += 1
line = flines[index]
if re.match(_LINECOMMENT, line):
# skip full-line comments
continue
if re.match(_BLANKLINE, line):
# skip blank lines
continue
m = re.match(_HEADING, line)
if m:
# matched a section heading
indent, s_open, sect_name, s_close, comment = m.groups()
nb = len(s_open)
if nb != len(s_close):
raise FileParseError('bracket mismatch', index, line)
elif nb == nesting_level:
# sibling section
parents = parents[:-1] + [sect_name]
elif nb == nesting_level + 1:
# child section
parents = parents + [sect_name]
elif nb < nesting_level:
# back up one or more levels
ndif = nesting_level - nb
parents = parents[:-ndif - 1] + [sect_name]
else:
raise FileParseError('Error line ' + str(index + 1) + ': ' +
line)
nesting_level = nb
addsect(config, sect_name, parents[:-1])
else:
m = re.match(_KEY_VALUE, line)
if m:
# matched a key=value item
indent, key, val = m.groups()
if val.startswith('"""') or val.startswith("'''"):
# triple quoted - may be a multiline value
val, index = multiline(flines, val, index, maxline)
addict(config, key, val, parents, index)
else:
# no match
raise FileParseError('Invalid line ' + str(index + 1) + ': ' +
line)
return config
def test_nonzero(self):
d = OrderedDictWithDefaults()
self.assertFalse(d)
d['value'] = 10
self.assertTrue(d)
def parse(fpath, output_fname=None, template_vars=None):
"Parse file items line-by-line into a corresponding nested dict."
# read and process the file (jinja2, include-files, line continuation)
flines = read_and_proc(fpath, template_vars)
if output_fname:
with open(output_fname, 'wb') as handle:
handle.write('\n'.join(flines) + '\n')
LOG.debug('Processed configuration dumped: %s', output_fname)
nesting_level = 0
config = OrderedDictWithDefaults()
parents = []
maxline = len(flines) - 1
index = -1
while index < maxline:
index += 1
line = flines[index]
if re.match(_LINECOMMENT, line):
# skip full-line comments
continue
if re.match(_BLANKLINE, line):
# skip blank lines
continue
m = re.match(_HEADING, line)
if m:
# matched a section heading
s_open, sect_name, s_close = m.groups()[1:-1]
nb = len(s_open)
if nb != len(s_close):
raise FileParseError('bracket mismatch', index, line)
elif nb == nesting_level:
# sibling section
parents = parents[:-1] + [sect_name]
elif nb == nesting_level + 1:
# child section
parents = parents + [sect_name]
elif nb < nesting_level:
# back up one or more levels
ndif = nesting_level - nb
parents = parents[:-ndif - 1] + [sect_name]
else:
raise FileParseError('Error line ' + str(index + 1) + ': ' +
line)
nesting_level = nb
addsect(config, sect_name, parents[:-1])
else:
m = re.match(_KEY_VALUE, line)
if m:
# matched a key=value item
key, _, val = m.groups()[1:]
if val.startswith('"""') or val.startswith("'''"):
# triple quoted - may be a multiline value
val, index = multiline(flines, val, index, maxline)
addict(config, key, val, parents, index)
else:
# no match
raise FileParseError('Invalid line ' + str(index + 1) + ': ' +
line)
return config
def pdeepcopy(source):
"""Make a deep copy of a pdict source"""
target = OrderedDictWithDefaults()
replicate(target, source)
return target
def test_prepend(self):
d = OrderedDictWithDefaults()
d['key'] = 'Birds'
d.prepend('year', 1980)
d.prepend('key', 2000)
iterator = d.iterkeys()
self.assertEqual('key', iterator.next())
self.assertEqual('year', iterator.next())
d = OrderedDictWithDefaults()
d['key'] = 'Birds'
d.prepend('year', 1980)
iterator = d.iterkeys()
self.assertEqual('year', iterator.next())
self.assertEqual('key', iterator.next())
def test_setitem(self):
d = OrderedDictWithDefaults()
d['name'] = 'Matthews'
self.assertEqual('Matthews', d['name'])
d['name'] = 'Zaccharias'
self.assertEqual('Zaccharias', d['name'])
