def test_merge():
src = OrderedTree()
src['a.b.c'] = 'foo'
src['a.b.d'] = 'foo'
print_yaml('src:', src)
other = OrderedTree()
other['x.y.z'] = 'foobar'
print_yaml('other_tree:', other)
src.merge(other)
print_yaml("After updating x.y.z: ", src)
assert 'a.b.c' in src
assert 'a.b.d' in src
assert 'x.y.z' in src
assert src['a.b.d'] == 'foo'
other['a.b.d.e'] = 'foobar'
print_yaml('add a.b.d.e to other:', other)
print_yaml('src:', src)
src.merge(other)
print_yaml("After merge: ", src)
assert 'a.b.c' in src
assert 'a.b.d' in src
assert 'x.y.z' in src
assert 'a.b.d.e' in src
assert src['a.b.d'] != 'foo'
def __init__(self, config_dir, settings):
self._settings = settings
self._config_dir = config_dir
self._loaded = False
self._all_settings = None
self._file_list = None
self._invalid_paths = None
self._all_settings = OrderedTree('!')
def test_merge_2():
src = OrderedTree()
src['foo.bar.baz'] = 'boo boo'
other = OrderedTree()
other['x.y.z'] = 'xyz'
other['foo.bar.soo'] = 'moo moo'
# other['foo.bar.baz.too'] = 'moo moo'
src.merge(other)
print_yaml("add merge src and other", src)
src.merge(other)
print_yaml("add merge src and other", src)
def test_insert_complex_objects():
tree = OrderedTree()
obj = {
"test": "foo",
"foo": "bar"
}
tree.insert('root', obj)
print_yaml("After inserting a dict", tree)
assert 'root.test' in tree
assert 'root.foo' in tree
assert isinstance(tree['root'], OrderedTree)
assert isinstance(tree['root.foo'], OrderedTree) is False
assert tree['root.test'] == 'foo'
assert tree['root.foo'] == 'bar'
# add to existing tree
another_dict = {
"bar": {
"baz": {
"boo": "baaaad"
}
}
}
tree.insert('root', another_dict)
assert 'root.test' in tree
assert 'root.foo' in tree
assert 'root.bar' in tree
assert isinstance(tree['root'], OrderedTree)
assert isinstance(tree['root.foo'], OrderedTree) is False
assert tree['root.test'] == 'foo'
assert tree['root.foo'] == 'bar'
print_yaml("inserting a complex dict", tree)
replace_foo = {
'foo': {
'bar': 'baz'
}
}
tree.insert('root', replace_foo)
assert 'root.test' in tree
assert 'root.foo' in tree
assert 'root.bar' in tree
assert isinstance(tree['root'], OrderedTree)
assert isinstance(tree['root.foo'], OrderedTree)
assert isinstance(tree['root.foo.bar'], OrderedTree) is False
assert tree['root.foo'] != 'bar'
print_yaml("replace foo", tree)
tree['root.array'] = range(4)
print_yaml("insert and array", tree)
assert 'root.array' in tree
assert isinstance(tree['root.array'], OrderedTree) is False
tree['root.array.array'] = range(4)
assert 'root.array' in tree
assert 'root.array.array' in tree
assert isinstance(tree['root.array'], OrderedTree)
def test_merge_dict_objects():
tree = OrderedTree()
tree['foo'] = 'foo'
tree['bar'] = 'baz'
obj = {
"test": "foo",
"foo": "bar"
}
tree.merge(obj)
print_yaml("updating a simple dict", tree)
assert 'test' in tree
assert 'foo' in tree
assert isinstance(tree['foo'], OrderedTree) is False
assert tree['test'] == 'foo'
assert tree['foo'] == 'bar'
assert tree['bar'] == 'baz'
# add to existing tree
replace_bar = {
"bar": {
"baz": {
"boo": "baaaad"
}
}
}
tree.merge(replace_bar)
print_yaml("replace bar with a dict", tree)
assert 'test' in tree
assert 'foo' in tree
assert 'bar' in tree
assert isinstance(tree['foo'], OrderedTree) is False
assert isinstance(tree['bar'], OrderedTree)
assert tree['test'] == 'foo'
assert tree['foo'] == 'bar'
assert tree['bar.baz.boo'] == 'baaaad'
assert isinstance(tree['bar.baz'], OrderedTree)
# add more values to the existing dictionary
add_to_bar = {
"bar": {
"too": {
"moo": "yo yo"
}
}
}
tree.merge(add_to_bar)
print_yaml("replace bar with a dict", tree)
assert 'test' in tree
assert 'foo' in tree
assert 'bar' in tree
assert isinstance(tree['foo'], OrderedTree) is False
assert isinstance(tree['bar'], OrderedTree)
assert tree['test'] == 'foo'
assert tree['foo'] == 'bar'
assert tree['bar.baz.boo'] == 'baaaad'
assert isinstance(tree['bar.baz'], OrderedTree)
assert tree['bar.too.moo'] == 'yo yo'
assert isinstance(tree['bar.too'], OrderedTree)
def test_insert():
tree = OrderedTree()
logging.debug(tree)
tree.insert('foo.bar', 'moomoo')
assert 'foo' in tree
assert 'foo.bar' in tree
assert tree['foo.bar'] == 'moomoo'
assert tree['foo']['bar'] == 'moomoo'
print_yaml("After foo.bar", tree)
tree.insert('foobar', 'moomoo')
tree.insert('foo.bar.baz', 'moomoo')
assert tree['foo.bar'] != 'moomoo'
assert tree['foo.bar.baz'] == 'moomoo'
print_yaml("foo.bar overridden ", tree)
x = tree['foo.bar']
x['x.y.z'] = 'blah blah'
print_yaml("sub-tree x", x)
print_yaml("extend sub-tree foo.bar ", tree)
assert x['x.y.z'] == 'blah blah'
assert 'foo' in tree
assert 'foo.bar' in tree
assert tree['foo.bar.x.y.z'] == 'blah blah'
def _parse(self):
# create the settings tree and preserve the order in which arguments
# are passed. Convert all args into an ordered tree so that
# --foo fooz --too moo --foo-bar baaz --foo-arg vali
# will be like the ordered tree below
# foo:
# <special-key>: fooz
# bar: ### <-- foo-bar is not foo/bar
# <special-key>: baaz
# arg: ### <-- arg comes after bar
# <special-key>: val
# too:
# <special-key>: moo
logger.debug("Parsing: %s", self.args)
logger.debug("DocString for Generate: %s", self._doc_string)
try:
self.parsed = docopt(self._doc_string,
options_first=True,
argv=self.args)
except DocoptExit:
logger.error(self._doc_string)
return False
logger.info("Parsed \n%s", self.parsed)
self._prepare_defaults()
if not self._apply_rules():
logger.error("Error while validating rules: check args %s",
' \n'.join(self.args))
return False
logger.debug("New Args: %s", self.args)
logger.info("After applying rules Parsed: \n%s", self.parsed)
self.output_file = utils.extract_value(self.parsed,
'<output-file>',
optional=False)
self.extra_vars = utils.extract_value(self.parsed, '--extra-vars')
# filter only options; [ --foo, fooz, --bar baz ] -> [--foo, --bar]
options = [x for x in self.args + self.defaults if x.startswith('--')]
settings = OrderedTree(delimiter='-')
for option in options: # iterate options to preserve order of args
option = option.split('=')[0]
value = self.parsed.get(option)
if not value:
continue
key = option[2:] + settings.delimiter + VALUES_KEY
settings[key] = value
logger.debug("%s: %s", key, value)
logger.debug(
yaml_utils.to_yaml("Directory structure from args:", settings))
self.settings = settings
return True
def test_del():
t = OrderedTree()
t['foo.bar.baz'] = 'good'
t['foo.bar.boz'] = 'good'
t['foo.zoo.yoo'] = 'good'
print_yaml("initial tree", t)
assert 'foo' in t
assert 'foo.bar.baz' in t
assert 'foo.bar.boz' in t
assert 'foo.zoo.yoo' in t
del t['foo.bar.boz']
print_yaml("del foo.bar.boz", t)
x = t['foo']
del x['bar']
assert 'foo.bar' not in t
assert 'foo.bar.baz' not in t
print_yaml("del foo.bar", t)
del x['zoo']
assert 'foo' in t
assert 'foo.zoo' not in t
print_yaml("del foo.zoo", t)
del t['foo']
assert 'foo' not in t
print_yaml("del foo", t)
def test_init_dict():
d = {'foo': 'bar'}
tree = OrderedTree('.', **d)
print_yaml("tree init using a dict", tree)
assert 'foo' in tree
assert tree['foo'] == 'bar'
d2 = {'foo': {'bar': 'baz'}}
tree = OrderedTree('.', **d2)
print_yaml("tree init using a complex dict", tree)
assert 'foo.bar' in tree
assert tree['foo.bar'] == 'baz'
import configure
x = configure.Configuration({'foo': 'bar'})
assert is_dict(x)
def __init__(self, config_dir, settings):
self._settings = settings
self._config_dir = config_dir
self._loaded = False
self._all_settings = None
self._file_list = None
self._invalid_paths = None
self._all_settings = OrderedTree('!')
def _load(self):
if self._loaded:
return
self._all_settings = OrderedTree('/')
self._file_list = []
self._invalid_paths = []
self._create_file_list(self._settings, "", self._file_list)
logging.info("files to load :\n %s", '\n'.join(self._file_list))
logging.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
if self._invalid_paths:
raise OptionError(self._invalid_paths)
for f in self._file_list:
self._load_file(f)
self._loaded = True
def test_insert():
tree = OrderedTree()
logging.debug(tree)
tree.insert('foo.bar', 'moomoo')
assert 'foo' in tree
assert 'foo.bar' in tree
assert tree['foo.bar'] == 'moomoo'
assert tree['foo']['bar'] == 'moomoo'
print_yaml("After foo.bar", tree)
tree.insert('foobar', 'moomoo')
tree.insert('foo.bar.baz', 'moomoo')
assert tree['foo.bar'] != 'moomoo'
assert tree['foo.bar.baz'] == 'moomoo'
print_yaml("foo.bar overridden ", tree)
x = tree['foo.bar']
x['x.y.z'] = 'blah blah'
print_yaml("sub-tree x", x)
print_yaml("extend sub-tree foo.bar ", tree)
assert x['x.y.z'] == 'blah blah'
assert 'foo' in tree
assert 'foo.bar' in tree
assert tree['foo.bar.x.y.z'] == 'blah blah'
def test_merge():
src = OrderedTree()
src['a.b.c'] = 'foo'
src['a.b.d'] = 'foo'
print_yaml('src:', src)
other = OrderedTree()
other['x.y.z'] = 'foobar'
print_yaml('other_tree:', other)
src.merge(other)
print_yaml("After updating x.y.z: ", src)
assert 'a.b.c' in src
assert 'a.b.d' in src
assert 'x.y.z' in src
assert src['a.b.d'] == 'foo'
other['a.b.d.e'] = 'foobar'
print_yaml('add a.b.d.e to other:', other)
print_yaml('src:', src)
src.merge(other)
print_yaml("After merge: ", src)
assert 'a.b.c' in src
assert 'a.b.d' in src
assert 'x.y.z' in src
assert 'a.b.d.e' in src
assert src['a.b.d'] != 'foo'
def test_in():
_enable_logging()
tree = OrderedTree()
tree['a.b.c.d'] = 'foo'
assert 'a' in tree
assert 'a.b' in tree
assert 'a.b.c' in tree
assert 'a.b.c.d' in tree
assert 'a.b.c.d.foo' not in tree
assert 'x' not in tree
assert 'x.y.z' not in tree
print_yaml("tree", tree)
def _merge_extra_vars(self, loader):
if not self.extra_vars or len(self.extra_vars) == 0:
return
for var in self.extra_vars:
if var.startswith('@'):
loader.load_file(var[1:]) # remove @
elif '=' in var:
key, val = var.split('=', 1)
tree = OrderedTree(delimiter='.')
tree[key] = val
loader.merge(tree)
else:
raise KeyValueError(var, "No = found between key and value")
def _update_extra_vars(extra_vars, loader):
if not extra_vars or len(extra_vars) == 0:
return
for var in extra_vars:
if var.startswith('@'):
loader.load_file(var[1:])
elif '=' in var:
key, val = var.split('=', 1)
tree = OrderedTree(delimiter='.')
tree[key] = val
loader.update(tree)
else:
raise KeyValueError(var, "No = found between key and value")
def test_merge_2():
src = OrderedTree()
src['foo.bar.baz'] = 'boo boo'
other = OrderedTree()
other['x.y.z'] = 'xyz'
other['foo.bar.soo'] = 'moo moo'
# other['foo.bar.baz.too'] = 'moo moo'
src.merge(other)
print_yaml("add merge src and other", src)
src.merge(other)
print_yaml("add merge src and other", src)
def _load(self):
if self._loaded:
return
self._all_settings = OrderedTree('/')
self._file_list = []
self._invalid_paths = []
self._create_file_list(self._settings, "", self._file_list)
logging.info("files to load :\n %s", '\n'.join(self._file_list))
logging.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
if self._invalid_paths:
raise OptionError(self._invalid_paths)
for f in self._file_list:
self._load_file(f)
self._loaded = True
def test_monkey_patch_merge():
"""
Monkey patch configure so that merge will
append lists instead of replacing them
"""
src_dict = {
"d1": [1, 2, 3],
"s": "foo",
"a": [1, 2, 3],
"nested_dict": {
"d1": "ok",
"d": "ok"
}
}
src = Configuration.from_dict(src_dict)
print_yaml("Src", src)
other_dict = {
"d2": [1, 3, 5],
"s": "bar",
"a": [3, 2, 8],
"nested_dict": {
"d1": "ok",
"d": "ok"
}
}
src = Configuration.from_dict(src_dict)
print_yaml("Src", src)
other = Configuration.from_dict(other_dict)
print_yaml("Other", src)
merged = deepcopy(src).merge(other)
print_yaml("src", merged)
print_yaml("Merged", src)
assert merged['d1'] == [1, 2, 3]
assert merged['d2'] == [1, 3, 5]
assert merged['a'] == [1, 2, 3, 3, 2, 8]
assert merged['s'] == 'bar'
src = Configuration.from_string("""array: [1, 2, 3] """)
print_yaml("Original config", src)
other = Configuration.from_string("""array: [2, 3, 8, 9] """)
merged = deepcopy(src).merge(other)
print_yaml("Merged", merged)
assert merged['array'] == [1, 2, 3, 2, 3, 8, 9]
# ### test overwrite
overwrite = Configuration.from_string(""" array: !overwrite [0, 0, 0] """)
print_yaml("Overwrite", overwrite)
merged = deepcopy(src).merge(overwrite)
print_yaml('Merge with overwrite', merged)
assert merged['array'] == [0, 0, 0]
another_overwrite = Configuration.from_string(
"array: !overwrite [1, 1, 1] ")
print_yaml("Another Overwrite", another_overwrite)
merged = merged.merge(another_overwrite)
print_yaml('Merge with another overwrite', merged)
assert merged['array'] == [1, 1, 1]
# extend overwritten
print_yaml("Extending src", src)
merged = merged.merge(src)
print_yaml('Merge with src', merged)
assert merged['array'] == [1, 1, 1, 1, 2, 3]
from ksgen.tree import OrderedTree
tree = OrderedTree(delimiter='.')
tree.update(merged)
print_yaml("Merged", tree)
class Loader(object):
def __init__(self, config_dir, settings):
self._settings = settings
self._config_dir = config_dir
self._loaded = False
self._all_settings = None
self._file_list = None
self._invalid_paths = None
self._all_settings = OrderedTree('!')
def settings(self):
self.load()
LookupDirective.lookup_table = self._all_settings
return self._all_settings
def load_file(self, f):
self.load()
cfg = load_configuration(f, self._config_dir)
self._all_settings.merge(cfg)
def merge(self, tree):
self._all_settings.merge(tree)
def load(self):
if self._loaded:
return
self._file_list = []
self._invalid_paths = []
self._create_file_list(self._settings, self._file_list)
logger.info(
"\nList of files to load :\n - %s", '\n - '.join(
[x[len(self._config_dir) + 1:] for x in self._file_list]))
if self._invalid_paths:
logger.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
raise OptionError(self._invalid_paths)
all_cfg = Configuration.from_dict({})
for f in self._file_list:
cfg = load_configuration(f, self._config_dir)
try:
del cfg[DEFAULTS_TAG]
except KeyError:
pass
else:
logger.debug("Successfully removed default traces from %s" % f)
all_cfg.merge(cfg)
self._all_settings.merge(all_cfg)
self._loaded = True
def _create_file_list(self, settings, file_list, parent_path=""):
""" Appends list of files to be process to self._file_list
and list of invalid file paths to self._invalid_paths
"""
logger.debug('settings:\n %s \n parent: %s \n files: %s', settings,
parent_path, file_list)
for key, sub_tree in settings.items():
# ignore the special key value
if key == VALUES_KEY:
continue
logger.debug("key: %s, subtree: %s", key, sub_tree)
path = "%(parent_path)s%(key)s%(sep)s%(file)s" % {
'parent_path': parent_path,
'key': key,
'sep': os.sep,
'file': sub_tree[VALUES_KEY]
}
abs_file_path = os.path.abspath(self._config_dir + os.sep + path +
'.yml')
file_list.append(abs_file_path)
logger.debug('path: %s', abs_file_path)
if not os.path.exists(abs_file_path):
self._invalid_paths.append(abs_file_path)
# recurse if there are sub settings
if (isinstance(sub_tree, dict) and len(sub_tree.keys()) > 1):
logger.debug('recursing into: sub-tree: %s', sub_tree)
self._create_file_list(sub_tree, file_list, path + os.sep)
def test_yaml_dump():
tree = OrderedTree()
tree['os'] = ['Sunil', 'Thaha']
tree['os.test'] = ['Sunil', 'Thaha']
logging.debug(tree)
print_yaml("tree:", tree)
def generate(config_dir, args):
"""
Usage:
generate [options] <output-file>
generate [--extra-vars=KEY_PAIR]... [options] <output-file>
Options:
--extra-vars=<val>... Provide extra vars {options}
"""
logging.debug("config_dir: %s, args: %s", config_dir, args)
doc_string = generate.__doc__.format(
options=docstring.Generator(config_dir).generate())
logging.debug("Parsing: %s", args)
logging.debug("DocString for Generate: %s", doc_string)
parsed = docopt(doc_string, options_first=True, argv=args)
logging.info("Parsed: \n%s", parsed)
output_file = _extract_value_for_option(parsed, '<output-file>')
extra_vars = _extract_value_for_option(parsed,
'--extra-vars',
must_exist=False)
# create the settings tree and preserve the order in which arguments
# are passed. Convert all args into an ordered tree so that
# --foo fooz --too moo --foo-bar baaz --foo-arg vali
# will be like the ordered tree below
# foo:
# <special-key>: fooz
# bar: ### <-- foo-bar is not foo/bar
# <special-key>: baaz
# arg: ### <-- arg comes after bar
# <special-key>: val
# too:
# <special-key>: moo
settings_tree = OrderedTree(delimiter='-')
# filter only options; [ --foo, fooz, --bar baz ] -> [--foo, --bar]
options = [x for x in args if x.startswith('--')]
for option in options: # iterate options to preserve order of args
option = option.split('=')[0]
value = parsed.get(option)
if not value:
continue
key = option[2:] + '-' + settings.VALUES_KEY
settings_tree[key] = value
logging.debug("%s: %s", key, value)
logging.debug(
yaml_utils.to_yaml("Directory structure from args:", settings_tree))
loader = settings.Loader(config_dir, settings_tree)
_update_extra_vars(extra_vars, loader)
all_settings = loader.settings_tree()
logging.debug("\n" +
yaml.safe_dump(all_settings, default_flow_style=False))
logging.info("Writing to file: %s", output_file)
with open(output_file, 'w') as out:
out.write(yaml.safe_dump(all_settings, default_flow_style=False))
class Loader(object):
def __init__(self, config_dir, settings):
self._settings = settings
self._config_dir = config_dir
self._loaded = False
self._all_settings = None
self._file_list = None
self._invalid_paths = None
def settings_tree(self):
self._load()
return self._all_settings
def load_file(self, f):
self._load()
self._load_file(f)
def update(self, tree):
self._load()
self._all_settings.update(tree)
def _load(self):
if self._loaded:
return
self._all_settings = OrderedTree('/')
self._file_list = []
self._invalid_paths = []
self._create_file_list(self._settings, "", self._file_list)
logging.info("files to load :\n %s", '\n'.join(self._file_list))
logging.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
if self._invalid_paths:
raise OptionError(self._invalid_paths)
for f in self._file_list:
self._load_file(f)
self._loaded = True
def _load_file(self, f):
logging.debug('Loading file: %s', f)
cfg = Configuration.from_file(f).configure()
self._all_settings.update(cfg)
def _create_file_list(self, settings, parent_path, file_list):
""" Appends list of files to be process to self._file_list
and list of invalid file paths to self._invalid_paths
"""
logging.debug('settings:\n %s \n parent: %s \n files: %s', settings,
parent_path, file_list)
for key, sub_tree in settings.items():
# ignore the special key value
if key == VALUES_KEY:
continue
logging.debug("key: %s, subtree: %s", key, sub_tree)
path = "%(parent_path)s%(key)s%(sep)s%(file)s" % {
'parent_path': parent_path,
'key': key,
'sep': os.sep,
'file': sub_tree[VALUES_KEY]
}
abs_file_path = os.path.abspath(self._config_dir + os.sep + path +
'.yml')
file_list.append(abs_file_path)
logging.debug('path: %s', abs_file_path)
if not os.path.exists(abs_file_path):
self._invalid_paths.append(abs_file_path)
# recurse if there are sub settings
if (isinstance(sub_tree, dict) and len(sub_tree.keys()) > 1):
logging.debug('recursing into: sub-tree: %s', sub_tree)
self._create_file_list(sub_tree, path + os.sep, file_list)
class Loader(object):
def __init__(self, config_dir, settings):
self._settings = settings
self._config_dir = config_dir
self._loaded = False
self._all_settings = None
self._file_list = None
self._invalid_paths = None
def settings_tree(self):
self._load()
return self._all_settings
def load_file(self, f):
self._load()
self._load_file(f)
def update(self, tree):
self._load()
self._all_settings.update(tree)
def _load(self):
if self._loaded:
return
self._all_settings = OrderedTree('/')
self._file_list = []
self._invalid_paths = []
self._create_file_list(self._settings, "", self._file_list)
logging.info("files to load :\n %s", '\n'.join(self._file_list))
logging.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
if self._invalid_paths:
raise OptionError(self._invalid_paths)
for f in self._file_list:
self._load_file(f)
self._loaded = True
def _load_file(self, f):
logging.debug('Loading file: %s', f)
cfg = Configuration.from_file(f).configure()
self._all_settings.update(cfg)
def _create_file_list(self, settings, parent_path, file_list):
""" Appends list of files to be process to self._file_list
and list of invalid file paths to self._invalid_paths
"""
logging.debug('settings:\n %s \n parent: %s \n files: %s',
settings, parent_path, file_list)
for key, sub_tree in settings.items():
# ignore the special key value
if key == VALUES_KEY:
continue
logging.debug("key: %s, subtree: %s", key, sub_tree)
path = "%(parent_path)s%(key)s%(sep)s%(file)s" % {
'parent_path': parent_path,
'key': key,
'sep': os.sep,
'file': sub_tree[VALUES_KEY]
}
abs_file_path = os.path.abspath(self._config_dir + os.sep
+ path + '.yml')
file_list.append(abs_file_path)
logging.debug('path: %s', abs_file_path)
if not os.path.exists(abs_file_path):
self._invalid_paths.append(abs_file_path)
# recurse if there are sub settings
if (isinstance(sub_tree, dict)
and len(sub_tree.keys()) > 1):
logging.debug('recursing into: sub-tree: %s', sub_tree)
self._create_file_list(sub_tree, path + os.sep, file_list)
class Loader(object):
def __init__(self, config_dir, settings):
self._settings = settings
self._config_dir = config_dir
self._loaded = False
self._all_settings = None
self._file_list = None
self._invalid_paths = None
self._all_settings = OrderedTree('!')
def settings(self):
self.load()
LookupDirective.lookup_table = self._all_settings
return self._all_settings
def load_file(self, f):
self.load()
cfg = load_configuration(f, self._config_dir)
self._all_settings.merge(cfg)
def merge(self, tree):
self._all_settings.merge(tree)
def load(self):
if self._loaded:
return
self._file_list = []
self._invalid_paths = []
self._create_file_list(self._settings, self._file_list)
logger.info(
"\nList of files to load :\n - %s",
'\n - '.join([
x[len(self._config_dir) + 1:] for x in self._file_list
]))
if self._invalid_paths:
logger.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
raise OptionError(self._invalid_paths)
all_cfg = Configuration.from_dict({})
for f in self._file_list:
cfg = load_configuration(f, self._config_dir)
try:
del cfg[DEFAULTS_TAG]
except KeyError:
pass
else:
logger.debug("Successfully removed default traces from %s" % f)
all_cfg.merge(cfg)
self._all_settings.merge(all_cfg)
self._loaded = True
def _create_file_list(self, settings, file_list, parent_path=""):
""" Appends list of files to be process to self._file_list
and list of invalid file paths to self._invalid_paths
"""
logger.debug('settings:\n %s \n parent: %s \n files: %s',
settings, parent_path, file_list)
for key, sub_tree in settings.items():
# ignore the special key value
if key == VALUES_KEY:
continue
logger.debug("key: %s, subtree: %s", key, sub_tree)
path = "%(parent_path)s%(key)s%(sep)s%(file)s" % {
'parent_path': parent_path,
'key': key,
'sep': os.sep,
'file': sub_tree[VALUES_KEY]
}
abs_file_path = os.path.abspath(
self._config_dir + os.sep + path + '.yml')
file_list.append(abs_file_path)
logger.debug('path: %s', abs_file_path)
if not os.path.exists(abs_file_path):
self._invalid_paths.append(abs_file_path)
# recurse if there are sub settings
if (isinstance(sub_tree, dict)
and len(sub_tree.keys()) > 1):
logger.debug('recursing into: sub-tree: %s', sub_tree)
self._create_file_list(sub_tree, file_list, path + os.sep)
def test_monkey_patch_merge():
"""
Monkey patch configure so that merge will
append lists instead of replacing them
"""
src_dict = {
"d1": [1, 2, 3],
"s": "foo",
"a": [1, 2, 3],
"nested_dict": {
"d1": "ok",
"d": "ok"
}
}
src = Configuration.from_dict(src_dict)
print_yaml("Src", src)
other_dict = {
"d2": [1, 3, 5],
"s": "bar",
"a": [3, 2, 8],
"nested_dict": {
"d1": "ok",
"d": "ok"
}
}
src = Configuration.from_dict(src_dict)
print_yaml("Src", src)
other = Configuration.from_dict(other_dict)
print_yaml("Other", src)
merged = deepcopy(src).merge(other)
print_yaml("src", merged)
print_yaml("Merged", src)
assert merged['d1'] == [1, 2, 3]
assert merged['d2'] == [1, 3, 5]
assert merged['a'] == [1, 2, 3, 3, 2, 8]
assert merged['s'] == 'bar'
src = Configuration.from_string("""array: [1, 2, 3] """)
print_yaml("Original config", src)
other = Configuration.from_string("""array: [2, 3, 8, 9] """)
merged = deepcopy(src).merge(other)
print_yaml("Merged", merged)
assert merged['array'] == [1, 2, 3, 2, 3, 8, 9]
# ### test overwrite
overwrite = Configuration.from_string(""" array: !overwrite [0, 0, 0] """)
print_yaml("Overwrite", overwrite)
merged = deepcopy(src).merge(overwrite)
print_yaml('Merge with overwrite', merged)
assert merged['array'] == [0, 0, 0]
another_overwrite = Configuration.from_string(
"array: !overwrite [1, 1, 1] ")
print_yaml("Another Overwrite", another_overwrite)
merged = merged.merge(another_overwrite)
print_yaml('Merge with another overwrite', merged)
assert merged['array'] == [1, 1, 1]
# extend overwritten
print_yaml("Extending src", src)
merged = merged.merge(src)
print_yaml('Merge with src', merged)
assert merged['array'] == [1, 1, 1, 1, 2, 3]
from ksgen.tree import OrderedTree
tree = OrderedTree(delimiter='.')
tree.update(merged)
print_yaml("Merged", tree)
