def test_update():
src = OrderedTree()
src['a.b.c'] = 'foo'
src['a.b.d'] = 'foo'
print_yaml('src:', src)
other = OrderedTree()
other['x.y.z'] = 'foobar'
print_yaml('other_tree:', other)
src.update(other)
print_yaml("After updating x.y.z: ", src)
assert 'a.b.c' in src
assert 'a.b.d' in src
assert 'x.y.z' in src
assert src['a.b.d'] == 'foo'
other['a.b.d.e'] = 'foobar'
print_yaml('add a.b.d.e to other:', other)
print_yaml('src:', src)
src.update(other)
print_yaml("After update: ", src)
assert 'a.b.c' in src
assert 'a.b.d' in src
assert 'x.y.z' in src
assert 'a.b.d.e' in src
assert src['a.b.d'] != 'foo'
def test_update_2():
src = OrderedTree()
src['foo.bar.baz'] = 'boo boo'
other = OrderedTree()
other['x.y.z'] = 'xyz'
other['foo.bar.soo'] = 'moo moo'
# other['foo.bar.baz.too'] = 'moo moo'
src.update(other)
print_yaml("add update src and other", src)
src.update(other)
print_yaml("add update src and other", src)
def test_update_dict_objects():
tree = OrderedTree()
tree['foo'] = 'foo'
tree['bar'] = 'baz'
obj = {
"test": "foo",
"foo": "bar"
}
tree.update(obj)
print_yaml("updating a simple dict", tree)
assert 'test' in tree
assert 'foo' in tree
assert isinstance(tree['foo'], OrderedTree) is False
assert tree['test'] == 'foo'
assert tree['foo'] == 'bar'
assert tree['bar'] == 'baz'
# add to existing tree
replace_bar = {
"bar": {
"baz": {
"boo": "baaaad"
}
}
}
tree.update(replace_bar)
print_yaml("replace bar with a dict", tree)
assert 'test' in tree
assert 'foo' in tree
assert 'bar' in tree
assert isinstance(tree['foo'], OrderedTree) is False
assert isinstance(tree['bar'], OrderedTree)
assert tree['test'] == 'foo'
assert tree['foo'] == 'bar'
assert tree['bar.baz.boo'] == 'baaaad'
assert isinstance(tree['bar.baz'], OrderedTree)
# add more values to the existing dictionary
add_to_bar = {
"bar": {
"too": {
"moo": "yo yo"
}
}
}
tree.update(add_to_bar)
print_yaml("replace bar with a dict", tree)
assert 'test' in tree
assert 'foo' in tree
assert 'bar' in tree
assert isinstance(tree['foo'], OrderedTree) is False
assert isinstance(tree['bar'], OrderedTree)
assert tree['test'] == 'foo'
assert tree['foo'] == 'bar'
assert tree['bar.baz.boo'] == 'baaaad'
assert isinstance(tree['bar.baz'], OrderedTree)
assert tree['bar.too.moo'] == 'yo yo'
assert isinstance(tree['bar.too'], OrderedTree)
class Loader(object):
def __init__(self, config_dir, settings):
self._settings = settings
self._config_dir = config_dir
self._loaded = False
self._all_settings = None
self._file_list = None
self._invalid_paths = None
def settings_tree(self):
self._load()
return self._all_settings
def load_file(self, f):
self._load()
self._load_file(f)
def update(self, tree):
self._load()
self._all_settings.update(tree)
def _load(self):
if self._loaded:
return
self._all_settings = OrderedTree('/')
self._file_list = []
self._invalid_paths = []
self._create_file_list(self._settings, "", self._file_list)
logging.info("files to load :\n %s", '\n'.join(self._file_list))
logging.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
if self._invalid_paths:
raise OptionError(self._invalid_paths)
for f in self._file_list:
self._load_file(f)
self._loaded = True
def _load_file(self, f):
logging.debug('Loading file: %s', f)
cfg = Configuration.from_file(f).configure()
self._all_settings.update(cfg)
def _create_file_list(self, settings, parent_path, file_list):
""" Appends list of files to be process to self._file_list
and list of invalid file paths to self._invalid_paths
"""
logging.debug('settings:\n %s \n parent: %s \n files: %s', settings,
parent_path, file_list)
for key, sub_tree in settings.items():
# ignore the special key value
if key == VALUES_KEY:
continue
logging.debug("key: %s, subtree: %s", key, sub_tree)
path = "%(parent_path)s%(key)s%(sep)s%(file)s" % {
'parent_path': parent_path,
'key': key,
'sep': os.sep,
'file': sub_tree[VALUES_KEY]
}
abs_file_path = os.path.abspath(self._config_dir + os.sep + path +
'.yml')
file_list.append(abs_file_path)
logging.debug('path: %s', abs_file_path)
if not os.path.exists(abs_file_path):
self._invalid_paths.append(abs_file_path)
# recurse if there are sub settings
if (isinstance(sub_tree, dict) and len(sub_tree.keys()) > 1):
logging.debug('recursing into: sub-tree: %s', sub_tree)
self._create_file_list(sub_tree, path + os.sep, file_list)
def test_monkey_patch_merge():
"""
Monkey patch configure so that merge will
append lists instead of replacing them
"""
src_dict = {
"d1": [1, 2, 3],
"s": "foo",
"a": [1, 2, 3],
"nested_dict": {
"d1": "ok",
"d": "ok"
}
}
src = Configuration.from_dict(src_dict)
print_yaml("Src", src)
other_dict = {
"d2": [1, 3, 5],
"s": "bar",
"a": [3, 2, 8],
"nested_dict": {
"d1": "ok",
"d": "ok"
}
}
src = Configuration.from_dict(src_dict)
print_yaml("Src", src)
other = Configuration.from_dict(other_dict)
print_yaml("Other", src)
merged = deepcopy(src).merge(other)
print_yaml("src", merged)
print_yaml("Merged", src)
assert merged['d1'] == [1, 2, 3]
assert merged['d2'] == [1, 3, 5]
assert merged['a'] == [1, 2, 3, 3, 2, 8]
assert merged['s'] == 'bar'
src = Configuration.from_string("""array: [1, 2, 3] """)
print_yaml("Original config", src)
other = Configuration.from_string("""array: [2, 3, 8, 9] """)
merged = deepcopy(src).merge(other)
print_yaml("Merged", merged)
assert merged['array'] == [1, 2, 3, 2, 3, 8, 9]
# ### test overwrite
overwrite = Configuration.from_string(""" array: !overwrite [0, 0, 0] """)
print_yaml("Overwrite", overwrite)
merged = deepcopy(src).merge(overwrite)
print_yaml('Merge with overwrite', merged)
assert merged['array'] == [0, 0, 0]
another_overwrite = Configuration.from_string(
"array: !overwrite [1, 1, 1] ")
print_yaml("Another Overwrite", another_overwrite)
merged = merged.merge(another_overwrite)
print_yaml('Merge with another overwrite', merged)
assert merged['array'] == [1, 1, 1]
# extend overwritten
print_yaml("Extending src", src)
merged = merged.merge(src)
print_yaml('Merge with src', merged)
assert merged['array'] == [1, 1, 1, 1, 2, 3]
from ksgen.tree import OrderedTree
tree = OrderedTree(delimiter='.')
tree.update(merged)
print_yaml("Merged", tree)
class Loader(object):
def __init__(self, config_dir, settings):
self._settings = settings
self._config_dir = config_dir
self._loaded = False
self._all_settings = None
self._file_list = None
self._invalid_paths = None
def settings_tree(self):
self._load()
return self._all_settings
def load_file(self, f):
self._load()
self._load_file(f)
def update(self, tree):
self._load()
self._all_settings.update(tree)
def _load(self):
if self._loaded:
return
self._all_settings = OrderedTree('/')
self._file_list = []
self._invalid_paths = []
self._create_file_list(self._settings, "", self._file_list)
logging.info("files to load :\n %s", '\n'.join(self._file_list))
logging.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
if self._invalid_paths:
raise OptionError(self._invalid_paths)
for f in self._file_list:
self._load_file(f)
self._loaded = True
def _load_file(self, f):
logging.debug('Loading file: %s', f)
cfg = Configuration.from_file(f).configure()
self._all_settings.update(cfg)
def _create_file_list(self, settings, parent_path, file_list):
""" Appends list of files to be process to self._file_list
and list of invalid file paths to self._invalid_paths
"""
logging.debug('settings:\n %s \n parent: %s \n files: %s',
settings, parent_path, file_list)
for key, sub_tree in settings.items():
# ignore the special key value
if key == VALUES_KEY:
continue
logging.debug("key: %s, subtree: %s", key, sub_tree)
path = "%(parent_path)s%(key)s%(sep)s%(file)s" % {
'parent_path': parent_path,
'key': key,
'sep': os.sep,
'file': sub_tree[VALUES_KEY]
}
abs_file_path = os.path.abspath(self._config_dir + os.sep
+ path + '.yml')
file_list.append(abs_file_path)
logging.debug('path: %s', abs_file_path)
if not os.path.exists(abs_file_path):
self._invalid_paths.append(abs_file_path)
# recurse if there are sub settings
if (isinstance(sub_tree, dict)
and len(sub_tree.keys()) > 1):
logging.debug('recursing into: sub-tree: %s', sub_tree)
self._create_file_list(sub_tree, path + os.sep, file_list)
def test_monkey_patch_merge():
"""
Monkey patch configure so that merge will
append lists instead of replacing them
"""
src_dict = {
"d1": [1, 2, 3],
"s": "foo",
"a": [1, 2, 3],
"nested_dict": {
"d1": "ok",
"d": "ok"
}
}
src = Configuration.from_dict(src_dict)
print_yaml("Src", src)
other_dict = {
"d2": [1, 3, 5],
"s": "bar",
"a": [3, 2, 8],
"nested_dict": {
"d1": "ok",
"d": "ok"
}
}
src = Configuration.from_dict(src_dict)
print_yaml("Src", src)
other = Configuration.from_dict(other_dict)
print_yaml("Other", src)
merged = deepcopy(src).merge(other)
print_yaml("src", merged)
print_yaml("Merged", src)
assert merged['d1'] == [1, 2, 3]
assert merged['d2'] == [1, 3, 5]
assert merged['a'] == [1, 2, 3, 3, 2, 8]
assert merged['s'] == 'bar'
src = Configuration.from_string("""array: [1, 2, 3] """)
print_yaml("Original config", src)
other = Configuration.from_string("""array: [2, 3, 8, 9] """)
merged = deepcopy(src).merge(other)
print_yaml("Merged", merged)
assert merged['array'] == [1, 2, 3, 2, 3, 8, 9]
# ### test overwrite
overwrite = Configuration.from_string(""" array: !overwrite [0, 0, 0] """)
print_yaml("Overwrite", overwrite)
merged = deepcopy(src).merge(overwrite)
print_yaml('Merge with overwrite', merged)
assert merged['array'] == [0, 0, 0]
another_overwrite = Configuration.from_string(
"array: !overwrite [1, 1, 1] ")
print_yaml("Another Overwrite", another_overwrite)
merged = merged.merge(another_overwrite)
print_yaml('Merge with another overwrite', merged)
assert merged['array'] == [1, 1, 1]
# extend overwritten
print_yaml("Extending src", src)
merged = merged.merge(src)
print_yaml('Merge with src', merged)
assert merged['array'] == [1, 1, 1, 1, 2, 3]
from ksgen.tree import OrderedTree
tree = OrderedTree(delimiter='.')
tree.update(merged)
print_yaml("Merged", tree)
