def test_merge():
src = OrderedTree()
src['a.b.c'] = 'foo'
src['a.b.d'] = 'foo'
print_yaml('src:', src)
other = OrderedTree()
other['x.y.z'] = 'foobar'
print_yaml('other_tree:', other)
src.merge(other)
print_yaml("After updating x.y.z: ", src)
assert 'a.b.c' in src
assert 'a.b.d' in src
assert 'x.y.z' in src
assert src['a.b.d'] == 'foo'
other['a.b.d.e'] = 'foobar'
print_yaml('add a.b.d.e to other:', other)
print_yaml('src:', src)
src.merge(other)
print_yaml("After merge: ", src)
assert 'a.b.c' in src
assert 'a.b.d' in src
assert 'x.y.z' in src
assert 'a.b.d.e' in src
assert src['a.b.d'] != 'foo'
def test_merge_2():
src = OrderedTree()
src['foo.bar.baz'] = 'boo boo'
other = OrderedTree()
other['x.y.z'] = 'xyz'
other['foo.bar.soo'] = 'moo moo'
# other['foo.bar.baz.too'] = 'moo moo'
src.merge(other)
print_yaml("add merge src and other", src)
src.merge(other)
print_yaml("add merge src and other", src)
def test_init_dict():
d = {'foo': 'bar'}
tree = OrderedTree('.', **d)
print_yaml("tree init using a dict", tree)
assert 'foo' in tree
assert tree['foo'] == 'bar'
d2 = {'foo': {'bar': 'baz'}}
tree = OrderedTree('.', **d2)
print_yaml("tree init using a complex dict", tree)
assert 'foo.bar' in tree
assert tree['foo.bar'] == 'baz'
import configure
x = configure.Configuration({'foo': 'bar'})
assert is_dict(x)
def test_insert():
tree = OrderedTree()
logging.debug(tree)
tree.insert('foo.bar', 'moomoo')
assert 'foo' in tree
assert 'foo.bar' in tree
assert tree['foo.bar'] == 'moomoo'
assert tree['foo']['bar'] == 'moomoo'
print_yaml("After foo.bar", tree)
tree.insert('foobar', 'moomoo')
tree.insert('foo.bar.baz', 'moomoo')
assert tree['foo.bar'] != 'moomoo'
assert tree['foo.bar.baz'] == 'moomoo'
print_yaml("foo.bar overridden ", tree)
x = tree['foo.bar']
x['x.y.z'] = 'blah blah'
print_yaml("sub-tree x", x)
print_yaml("extend sub-tree foo.bar ", tree)
assert x['x.y.z'] == 'blah blah'
assert 'foo' in tree
assert 'foo.bar' in tree
assert tree['foo.bar.x.y.z'] == 'blah blah'
def test_del():
t = OrderedTree()
t['foo.bar.baz'] = 'good'
t['foo.bar.boz'] = 'good'
t['foo.zoo.yoo'] = 'good'
print_yaml("initial tree", t)
assert 'foo' in t
assert 'foo.bar.baz' in t
assert 'foo.bar.boz' in t
assert 'foo.zoo.yoo' in t
del t['foo.bar.boz']
print_yaml("del foo.bar.boz", t)
x = t['foo']
del x['bar']
assert 'foo.bar' not in t
assert 'foo.bar.baz' not in t
print_yaml("del foo.bar", t)
del x['zoo']
assert 'foo' in t
assert 'foo.zoo' not in t
print_yaml("del foo.zoo", t)
del t['foo']
assert 'foo' not in t
print_yaml("del foo", t)
def test_merge_dict_objects():
tree = OrderedTree()
tree['foo'] = 'foo'
tree['bar'] = 'baz'
obj = {
"test": "foo",
"foo": "bar"
}
tree.merge(obj)
print_yaml("updating a simple dict", tree)
assert 'test' in tree
assert 'foo' in tree
assert isinstance(tree['foo'], OrderedTree) is False
assert tree['test'] == 'foo'
assert tree['foo'] == 'bar'
assert tree['bar'] == 'baz'
# add to existing tree
replace_bar = {
"bar": {
"baz": {
"boo": "baaaad"
}
}
}
tree.merge(replace_bar)
print_yaml("replace bar with a dict", tree)
assert 'test' in tree
assert 'foo' in tree
assert 'bar' in tree
assert isinstance(tree['foo'], OrderedTree) is False
assert isinstance(tree['bar'], OrderedTree)
assert tree['test'] == 'foo'
assert tree['foo'] == 'bar'
assert tree['bar.baz.boo'] == 'baaaad'
assert isinstance(tree['bar.baz'], OrderedTree)
# add more values to the existing dictionary
add_to_bar = {
"bar": {
"too": {
"moo": "yo yo"
}
}
}
tree.merge(add_to_bar)
print_yaml("replace bar with a dict", tree)
assert 'test' in tree
assert 'foo' in tree
assert 'bar' in tree
assert isinstance(tree['foo'], OrderedTree) is False
assert isinstance(tree['bar'], OrderedTree)
assert tree['test'] == 'foo'
assert tree['foo'] == 'bar'
assert tree['bar.baz.boo'] == 'baaaad'
assert isinstance(tree['bar.baz'], OrderedTree)
assert tree['bar.too.moo'] == 'yo yo'
assert isinstance(tree['bar.too'], OrderedTree)
def _parse(self):
# create the settings tree and preserve the order in which arguments
# are passed. Convert all args into an ordered tree so that
# --foo fooz --too moo --foo-bar baaz --foo-arg vali
# will be like the ordered tree below
# foo:
# <special-key>: fooz
# bar: ### <-- foo-bar is not foo/bar
# <special-key>: baaz
# arg: ### <-- arg comes after bar
# <special-key>: val
# too:
# <special-key>: moo
logger.debug("Parsing: %s", self.args)
logger.debug("DocString for Generate: %s", self._doc_string)
try:
self.parsed = docopt(self._doc_string,
options_first=True,
argv=self.args)
except DocoptExit:
logger.error(self._doc_string)
return False
logger.info("Parsed \n%s", self.parsed)
self._prepare_defaults()
if not self._apply_rules():
logger.error("Error while validating rules: check args %s",
' \n'.join(self.args))
return False
logger.debug("New Args: %s", self.args)
logger.info("After applying rules Parsed: \n%s", self.parsed)
self.output_file = utils.extract_value(self.parsed,
'<output-file>',
optional=False)
self.extra_vars = utils.extract_value(self.parsed, '--extra-vars')
# filter only options; [ --foo, fooz, --bar baz ] -> [--foo, --bar]
options = [x for x in self.args + self.defaults if x.startswith('--')]
settings = OrderedTree(delimiter='-')
for option in options: # iterate options to preserve order of args
option = option.split('=')[0]
value = self.parsed.get(option)
if not value:
continue
key = option[2:] + settings.delimiter + VALUES_KEY
settings[key] = value
logger.debug("%s: %s", key, value)
logger.debug(
yaml_utils.to_yaml("Directory structure from args:", settings))
self.settings = settings
return True
def test_insert_complex_objects():
tree = OrderedTree()
obj = {
"test": "foo",
"foo": "bar"
}
tree.insert('root', obj)
print_yaml("After inserting a dict", tree)
assert 'root.test' in tree
assert 'root.foo' in tree
assert isinstance(tree['root'], OrderedTree)
assert isinstance(tree['root.foo'], OrderedTree) is False
assert tree['root.test'] == 'foo'
assert tree['root.foo'] == 'bar'
# add to existing tree
another_dict = {
"bar": {
"baz": {
"boo": "baaaad"
}
}
}
tree.insert('root', another_dict)
assert 'root.test' in tree
assert 'root.foo' in tree
assert 'root.bar' in tree
assert isinstance(tree['root'], OrderedTree)
assert isinstance(tree['root.foo'], OrderedTree) is False
assert tree['root.test'] == 'foo'
assert tree['root.foo'] == 'bar'
print_yaml("inserting a complex dict", tree)
replace_foo = {
'foo': {
'bar': 'baz'
}
}
tree.insert('root', replace_foo)
assert 'root.test' in tree
assert 'root.foo' in tree
assert 'root.bar' in tree
assert isinstance(tree['root'], OrderedTree)
assert isinstance(tree['root.foo'], OrderedTree)
assert isinstance(tree['root.foo.bar'], OrderedTree) is False
assert tree['root.foo'] != 'bar'
print_yaml("replace foo", tree)
tree['root.array'] = range(4)
print_yaml("insert and array", tree)
assert 'root.array' in tree
assert isinstance(tree['root.array'], OrderedTree) is False
tree['root.array.array'] = range(4)
assert 'root.array' in tree
assert 'root.array.array' in tree
assert isinstance(tree['root.array'], OrderedTree)
def __init__(self, config_dir, settings):
self._settings = settings
self._config_dir = config_dir
self._loaded = False
self._all_settings = None
self._file_list = None
self._invalid_paths = None
self._all_settings = OrderedTree('!')
def test_in():
_enable_logging()
tree = OrderedTree()
tree['a.b.c.d'] = 'foo'
assert 'a' in tree
assert 'a.b' in tree
assert 'a.b.c' in tree
assert 'a.b.c.d' in tree
assert 'a.b.c.d.foo' not in tree
assert 'x' not in tree
assert 'x.y.z' not in tree
print_yaml("tree", tree)
def _update_extra_vars(extra_vars, loader):
if not extra_vars or len(extra_vars) == 0:
return
for var in extra_vars:
if var.startswith('@'):
loader.load_file(var[1:])
elif '=' in var:
key, val = var.split('=', 1)
tree = OrderedTree(delimiter='.')
tree[key] = val
loader.update(tree)
else:
raise KeyValueError(var, "No = found between key and value")
def _merge_extra_vars(self, loader):
if not self.extra_vars or len(self.extra_vars) == 0:
return
for var in self.extra_vars:
if var.startswith('@'):
loader.load_file(var[1:]) # remove @
elif '=' in var:
key, val = var.split('=', 1)
tree = OrderedTree(delimiter='.')
tree[key] = val
loader.merge(tree)
else:
raise KeyValueError(var, "No = found between key and value")
def _load(self):
if self._loaded:
return
self._all_settings = OrderedTree('/')
self._file_list = []
self._invalid_paths = []
self._create_file_list(self._settings, "", self._file_list)
logging.info("files to load :\n %s", '\n'.join(self._file_list))
logging.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
if self._invalid_paths:
raise OptionError(self._invalid_paths)
for f in self._file_list:
self._load_file(f)
self._loaded = True
def test_yaml_dump():
tree = OrderedTree()
tree['os'] = ['Sunil', 'Thaha']
tree['os.test'] = ['Sunil', 'Thaha']
logging.debug(tree)
print_yaml("tree:", tree)
def generate(config_dir, args):
"""
Usage:
generate [options] <output-file>
generate [--extra-vars=KEY_PAIR]... [options] <output-file>
Options:
--extra-vars=<val>... Provide extra vars {options}
"""
logging.debug("config_dir: %s, args: %s", config_dir, args)
doc_string = generate.__doc__.format(
options=docstring.Generator(config_dir).generate())
logging.debug("Parsing: %s", args)
logging.debug("DocString for Generate: %s", doc_string)
parsed = docopt(doc_string, options_first=True, argv=args)
logging.info("Parsed: \n%s", parsed)
output_file = _extract_value_for_option(parsed, '<output-file>')
extra_vars = _extract_value_for_option(parsed,
'--extra-vars',
must_exist=False)
# create the settings tree and preserve the order in which arguments
# are passed. Convert all args into an ordered tree so that
# --foo fooz --too moo --foo-bar baaz --foo-arg vali
# will be like the ordered tree below
# foo:
# <special-key>: fooz
# bar: ### <-- foo-bar is not foo/bar
# <special-key>: baaz
# arg: ### <-- arg comes after bar
# <special-key>: val
# too:
# <special-key>: moo
settings_tree = OrderedTree(delimiter='-')
# filter only options; [ --foo, fooz, --bar baz ] -> [--foo, --bar]
options = [x for x in args if x.startswith('--')]
for option in options: # iterate options to preserve order of args
option = option.split('=')[0]
value = parsed.get(option)
if not value:
continue
key = option[2:] + '-' + settings.VALUES_KEY
settings_tree[key] = value
logging.debug("%s: %s", key, value)
logging.debug(
yaml_utils.to_yaml("Directory structure from args:", settings_tree))
loader = settings.Loader(config_dir, settings_tree)
_update_extra_vars(extra_vars, loader)
all_settings = loader.settings_tree()
logging.debug("\n" +
yaml.safe_dump(all_settings, default_flow_style=False))
logging.info("Writing to file: %s", output_file)
with open(output_file, 'w') as out:
out.write(yaml.safe_dump(all_settings, default_flow_style=False))
def test_monkey_patch_merge():
"""
Monkey patch configure so that merge will
append lists instead of replacing them
"""
src_dict = {
"d1": [1, 2, 3],
"s": "foo",
"a": [1, 2, 3],
"nested_dict": {
"d1": "ok",
"d": "ok"
}
}
src = Configuration.from_dict(src_dict)
print_yaml("Src", src)
other_dict = {
"d2": [1, 3, 5],
"s": "bar",
"a": [3, 2, 8],
"nested_dict": {
"d1": "ok",
"d": "ok"
}
}
src = Configuration.from_dict(src_dict)
print_yaml("Src", src)
other = Configuration.from_dict(other_dict)
print_yaml("Other", src)
merged = deepcopy(src).merge(other)
print_yaml("src", merged)
print_yaml("Merged", src)
assert merged['d1'] == [1, 2, 3]
assert merged['d2'] == [1, 3, 5]
assert merged['a'] == [1, 2, 3, 3, 2, 8]
assert merged['s'] == 'bar'
src = Configuration.from_string("""array: [1, 2, 3] """)
print_yaml("Original config", src)
other = Configuration.from_string("""array: [2, 3, 8, 9] """)
merged = deepcopy(src).merge(other)
print_yaml("Merged", merged)
assert merged['array'] == [1, 2, 3, 2, 3, 8, 9]
# ### test overwrite
overwrite = Configuration.from_string(""" array: !overwrite [0, 0, 0] """)
print_yaml("Overwrite", overwrite)
merged = deepcopy(src).merge(overwrite)
print_yaml('Merge with overwrite', merged)
assert merged['array'] == [0, 0, 0]
another_overwrite = Configuration.from_string(
"array: !overwrite [1, 1, 1] ")
print_yaml("Another Overwrite", another_overwrite)
merged = merged.merge(another_overwrite)
print_yaml('Merge with another overwrite', merged)
assert merged['array'] == [1, 1, 1]
# extend overwritten
print_yaml("Extending src", src)
merged = merged.merge(src)
print_yaml('Merge with src', merged)
assert merged['array'] == [1, 1, 1, 1, 2, 3]
from ksgen.tree import OrderedTree
tree = OrderedTree(delimiter='.')
tree.update(merged)
print_yaml("Merged", tree)
