def test_merge(): src = OrderedTree() src['a.b.c'] = 'foo' src['a.b.d'] = 'foo' print_yaml('src:', src) other = OrderedTree() other['x.y.z'] = 'foobar' print_yaml('other_tree:', other) src.merge(other) print_yaml("After updating x.y.z: ", src) assert 'a.b.c' in src assert 'a.b.d' in src assert 'x.y.z' in src assert src['a.b.d'] == 'foo' other['a.b.d.e'] = 'foobar' print_yaml('add a.b.d.e to other:', other) print_yaml('src:', src) src.merge(other) print_yaml("After merge: ", src) assert 'a.b.c' in src assert 'a.b.d' in src assert 'x.y.z' in src assert 'a.b.d.e' in src assert src['a.b.d'] != 'foo'
def __init__(self, config_dir, settings): self._settings = settings self._config_dir = config_dir self._loaded = False self._all_settings = None self._file_list = None self._invalid_paths = None self._all_settings = OrderedTree('!')
def test_merge_2(): src = OrderedTree() src['foo.bar.baz'] = 'boo boo' other = OrderedTree() other['x.y.z'] = 'xyz' other['foo.bar.soo'] = 'moo moo' # other['foo.bar.baz.too'] = 'moo moo' src.merge(other) print_yaml("add merge src and other", src) src.merge(other) print_yaml("add merge src and other", src)
def test_insert_complex_objects(): tree = OrderedTree() obj = { "test": "foo", "foo": "bar" } tree.insert('root', obj) print_yaml("After inserting a dict", tree) assert 'root.test' in tree assert 'root.foo' in tree assert isinstance(tree['root'], OrderedTree) assert isinstance(tree['root.foo'], OrderedTree) is False assert tree['root.test'] == 'foo' assert tree['root.foo'] == 'bar' # add to existing tree another_dict = { "bar": { "baz": { "boo": "baaaad" } } } tree.insert('root', another_dict) assert 'root.test' in tree assert 'root.foo' in tree assert 'root.bar' in tree assert isinstance(tree['root'], OrderedTree) assert isinstance(tree['root.foo'], OrderedTree) is False assert tree['root.test'] == 'foo' assert tree['root.foo'] == 'bar' print_yaml("inserting a complex dict", tree) replace_foo = { 'foo': { 'bar': 'baz' } } tree.insert('root', replace_foo) assert 'root.test' in tree assert 'root.foo' in tree assert 'root.bar' in tree assert isinstance(tree['root'], OrderedTree) assert isinstance(tree['root.foo'], OrderedTree) assert isinstance(tree['root.foo.bar'], OrderedTree) is False assert tree['root.foo'] != 'bar' print_yaml("replace foo", tree) tree['root.array'] = range(4) print_yaml("insert and array", tree) assert 'root.array' in tree assert isinstance(tree['root.array'], OrderedTree) is False tree['root.array.array'] = range(4) assert 'root.array' in tree assert 'root.array.array' in tree assert isinstance(tree['root.array'], OrderedTree)
def test_merge_dict_objects(): tree = OrderedTree() tree['foo'] = 'foo' tree['bar'] = 'baz' obj = { "test": "foo", "foo": "bar" } tree.merge(obj) print_yaml("updating a simple dict", tree) assert 'test' in tree assert 'foo' in tree assert isinstance(tree['foo'], OrderedTree) is False assert tree['test'] == 'foo' assert tree['foo'] == 'bar' assert tree['bar'] == 'baz' # add to existing tree replace_bar = { "bar": { "baz": { "boo": "baaaad" } } } tree.merge(replace_bar) print_yaml("replace bar with a dict", tree) assert 'test' in tree assert 'foo' in tree assert 'bar' in tree assert isinstance(tree['foo'], OrderedTree) is False assert isinstance(tree['bar'], OrderedTree) assert tree['test'] == 'foo' assert tree['foo'] == 'bar' assert tree['bar.baz.boo'] == 'baaaad' assert isinstance(tree['bar.baz'], OrderedTree) # add more values to the existing dictionary add_to_bar = { "bar": { "too": { "moo": "yo yo" } } } tree.merge(add_to_bar) print_yaml("replace bar with a dict", tree) assert 'test' in tree assert 'foo' in tree assert 'bar' in tree assert isinstance(tree['foo'], OrderedTree) is False assert isinstance(tree['bar'], OrderedTree) assert tree['test'] == 'foo' assert tree['foo'] == 'bar' assert tree['bar.baz.boo'] == 'baaaad' assert isinstance(tree['bar.baz'], OrderedTree) assert tree['bar.too.moo'] == 'yo yo' assert isinstance(tree['bar.too'], OrderedTree)
def test_insert(): tree = OrderedTree() logging.debug(tree) tree.insert('foo.bar', 'moomoo') assert 'foo' in tree assert 'foo.bar' in tree assert tree['foo.bar'] == 'moomoo' assert tree['foo']['bar'] == 'moomoo' print_yaml("After foo.bar", tree) tree.insert('foobar', 'moomoo') tree.insert('foo.bar.baz', 'moomoo') assert tree['foo.bar'] != 'moomoo' assert tree['foo.bar.baz'] == 'moomoo' print_yaml("foo.bar overridden ", tree) x = tree['foo.bar'] x['x.y.z'] = 'blah blah' print_yaml("sub-tree x", x) print_yaml("extend sub-tree foo.bar ", tree) assert x['x.y.z'] == 'blah blah' assert 'foo' in tree assert 'foo.bar' in tree assert tree['foo.bar.x.y.z'] == 'blah blah'
def _parse(self): # create the settings tree and preserve the order in which arguments # are passed. Convert all args into an ordered tree so that # --foo fooz --too moo --foo-bar baaz --foo-arg vali # will be like the ordered tree below # foo: # <special-key>: fooz # bar: ### <-- foo-bar is not foo/bar # <special-key>: baaz # arg: ### <-- arg comes after bar # <special-key>: val # too: # <special-key>: moo logger.debug("Parsing: %s", self.args) logger.debug("DocString for Generate: %s", self._doc_string) try: self.parsed = docopt(self._doc_string, options_first=True, argv=self.args) except DocoptExit: logger.error(self._doc_string) return False logger.info("Parsed \n%s", self.parsed) self._prepare_defaults() if not self._apply_rules(): logger.error("Error while validating rules: check args %s", ' \n'.join(self.args)) return False logger.debug("New Args: %s", self.args) logger.info("After applying rules Parsed: \n%s", self.parsed) self.output_file = utils.extract_value(self.parsed, '<output-file>', optional=False) self.extra_vars = utils.extract_value(self.parsed, '--extra-vars') # filter only options; [ --foo, fooz, --bar baz ] -> [--foo, --bar] options = [x for x in self.args + self.defaults if x.startswith('--')] settings = OrderedTree(delimiter='-') for option in options: # iterate options to preserve order of args option = option.split('=')[0] value = self.parsed.get(option) if not value: continue key = option[2:] + settings.delimiter + VALUES_KEY settings[key] = value logger.debug("%s: %s", key, value) logger.debug( yaml_utils.to_yaml("Directory structure from args:", settings)) self.settings = settings return True
def test_del(): t = OrderedTree() t['foo.bar.baz'] = 'good' t['foo.bar.boz'] = 'good' t['foo.zoo.yoo'] = 'good' print_yaml("initial tree", t) assert 'foo' in t assert 'foo.bar.baz' in t assert 'foo.bar.boz' in t assert 'foo.zoo.yoo' in t del t['foo.bar.boz'] print_yaml("del foo.bar.boz", t) x = t['foo'] del x['bar'] assert 'foo.bar' not in t assert 'foo.bar.baz' not in t print_yaml("del foo.bar", t) del x['zoo'] assert 'foo' in t assert 'foo.zoo' not in t print_yaml("del foo.zoo", t) del t['foo'] assert 'foo' not in t print_yaml("del foo", t)
def test_init_dict(): d = {'foo': 'bar'} tree = OrderedTree('.', **d) print_yaml("tree init using a dict", tree) assert 'foo' in tree assert tree['foo'] == 'bar' d2 = {'foo': {'bar': 'baz'}} tree = OrderedTree('.', **d2) print_yaml("tree init using a complex dict", tree) assert 'foo.bar' in tree assert tree['foo.bar'] == 'baz' import configure x = configure.Configuration({'foo': 'bar'}) assert is_dict(x)
def _load(self): if self._loaded: return self._all_settings = OrderedTree('/') self._file_list = [] self._invalid_paths = [] self._create_file_list(self._settings, "", self._file_list) logging.info("files to load :\n %s", '\n'.join(self._file_list)) logging.info("invalid files :\n %s", '\n'.join(self._invalid_paths)) if self._invalid_paths: raise OptionError(self._invalid_paths) for f in self._file_list: self._load_file(f) self._loaded = True
def test_in(): _enable_logging() tree = OrderedTree() tree['a.b.c.d'] = 'foo' assert 'a' in tree assert 'a.b' in tree assert 'a.b.c' in tree assert 'a.b.c.d' in tree assert 'a.b.c.d.foo' not in tree assert 'x' not in tree assert 'x.y.z' not in tree print_yaml("tree", tree)
def _merge_extra_vars(self, loader): if not self.extra_vars or len(self.extra_vars) == 0: return for var in self.extra_vars: if var.startswith('@'): loader.load_file(var[1:]) # remove @ elif '=' in var: key, val = var.split('=', 1) tree = OrderedTree(delimiter='.') tree[key] = val loader.merge(tree) else: raise KeyValueError(var, "No = found between key and value")
def _update_extra_vars(extra_vars, loader): if not extra_vars or len(extra_vars) == 0: return for var in extra_vars: if var.startswith('@'): loader.load_file(var[1:]) elif '=' in var: key, val = var.split('=', 1) tree = OrderedTree(delimiter='.') tree[key] = val loader.update(tree) else: raise KeyValueError(var, "No = found between key and value")
def test_monkey_patch_merge(): """ Monkey patch configure so that merge will append lists instead of replacing them """ src_dict = { "d1": [1, 2, 3], "s": "foo", "a": [1, 2, 3], "nested_dict": { "d1": "ok", "d": "ok" } } src = Configuration.from_dict(src_dict) print_yaml("Src", src) other_dict = { "d2": [1, 3, 5], "s": "bar", "a": [3, 2, 8], "nested_dict": { "d1": "ok", "d": "ok" } } src = Configuration.from_dict(src_dict) print_yaml("Src", src) other = Configuration.from_dict(other_dict) print_yaml("Other", src) merged = deepcopy(src).merge(other) print_yaml("src", merged) print_yaml("Merged", src) assert merged['d1'] == [1, 2, 3] assert merged['d2'] == [1, 3, 5] assert merged['a'] == [1, 2, 3, 3, 2, 8] assert merged['s'] == 'bar' src = Configuration.from_string("""array: [1, 2, 3] """) print_yaml("Original config", src) other = Configuration.from_string("""array: [2, 3, 8, 9] """) merged = deepcopy(src).merge(other) print_yaml("Merged", merged) assert merged['array'] == [1, 2, 3, 2, 3, 8, 9] # ### test overwrite overwrite = Configuration.from_string(""" array: !overwrite [0, 0, 0] """) print_yaml("Overwrite", overwrite) merged = deepcopy(src).merge(overwrite) print_yaml('Merge with overwrite', merged) assert merged['array'] == [0, 0, 0] another_overwrite = Configuration.from_string( "array: !overwrite [1, 1, 1] ") print_yaml("Another Overwrite", another_overwrite) merged = merged.merge(another_overwrite) print_yaml('Merge with another overwrite', merged) assert merged['array'] == [1, 1, 1] # extend overwritten print_yaml("Extending src", src) merged = merged.merge(src) print_yaml('Merge with src', merged) assert merged['array'] == [1, 1, 1, 1, 2, 3] from ksgen.tree import OrderedTree tree = OrderedTree(delimiter='.') tree.update(merged) print_yaml("Merged", tree)
class Loader(object): def __init__(self, config_dir, settings): self._settings = settings self._config_dir = config_dir self._loaded = False self._all_settings = None self._file_list = None self._invalid_paths = None self._all_settings = OrderedTree('!') def settings(self): self.load() LookupDirective.lookup_table = self._all_settings return self._all_settings def load_file(self, f): self.load() cfg = load_configuration(f, self._config_dir) self._all_settings.merge(cfg) def merge(self, tree): self._all_settings.merge(tree) def load(self): if self._loaded: return self._file_list = [] self._invalid_paths = [] self._create_file_list(self._settings, self._file_list) logger.info( "\nList of files to load :\n - %s", '\n - '.join( [x[len(self._config_dir) + 1:] for x in self._file_list])) if self._invalid_paths: logger.info("invalid files :\n %s", '\n'.join(self._invalid_paths)) raise OptionError(self._invalid_paths) all_cfg = Configuration.from_dict({}) for f in self._file_list: cfg = load_configuration(f, self._config_dir) try: del cfg[DEFAULTS_TAG] except KeyError: pass else: logger.debug("Successfully removed default traces from %s" % f) all_cfg.merge(cfg) self._all_settings.merge(all_cfg) self._loaded = True def _create_file_list(self, settings, file_list, parent_path=""): """ Appends list of files to be process to self._file_list and list of invalid file paths to self._invalid_paths """ logger.debug('settings:\n %s \n parent: %s \n files: %s', settings, parent_path, file_list) for key, sub_tree in settings.items(): # ignore the special key value if key == VALUES_KEY: continue logger.debug("key: %s, subtree: %s", key, sub_tree) path = "%(parent_path)s%(key)s%(sep)s%(file)s" % { 'parent_path': parent_path, 'key': key, 'sep': os.sep, 'file': sub_tree[VALUES_KEY] } abs_file_path = os.path.abspath(self._config_dir + os.sep + path + '.yml') file_list.append(abs_file_path) logger.debug('path: %s', abs_file_path) if not os.path.exists(abs_file_path): self._invalid_paths.append(abs_file_path) # recurse if there are sub settings if (isinstance(sub_tree, dict) and len(sub_tree.keys()) > 1): logger.debug('recursing into: sub-tree: %s', sub_tree) self._create_file_list(sub_tree, file_list, path + os.sep)
def test_yaml_dump(): tree = OrderedTree() tree['os'] = ['Sunil', 'Thaha'] tree['os.test'] = ['Sunil', 'Thaha'] logging.debug(tree) print_yaml("tree:", tree)
def generate(config_dir, args): """ Usage: generate [options] <output-file> generate [--extra-vars=KEY_PAIR]... [options] <output-file> Options: --extra-vars=<val>... Provide extra vars {options} """ logging.debug("config_dir: %s, args: %s", config_dir, args) doc_string = generate.__doc__.format( options=docstring.Generator(config_dir).generate()) logging.debug("Parsing: %s", args) logging.debug("DocString for Generate: %s", doc_string) parsed = docopt(doc_string, options_first=True, argv=args) logging.info("Parsed: \n%s", parsed) output_file = _extract_value_for_option(parsed, '<output-file>') extra_vars = _extract_value_for_option(parsed, '--extra-vars', must_exist=False) # create the settings tree and preserve the order in which arguments # are passed. Convert all args into an ordered tree so that # --foo fooz --too moo --foo-bar baaz --foo-arg vali # will be like the ordered tree below # foo: # <special-key>: fooz # bar: ### <-- foo-bar is not foo/bar # <special-key>: baaz # arg: ### <-- arg comes after bar # <special-key>: val # too: # <special-key>: moo settings_tree = OrderedTree(delimiter='-') # filter only options; [ --foo, fooz, --bar baz ] -> [--foo, --bar] options = [x for x in args if x.startswith('--')] for option in options: # iterate options to preserve order of args option = option.split('=')[0] value = parsed.get(option) if not value: continue key = option[2:] + '-' + settings.VALUES_KEY settings_tree[key] = value logging.debug("%s: %s", key, value) logging.debug( yaml_utils.to_yaml("Directory structure from args:", settings_tree)) loader = settings.Loader(config_dir, settings_tree) _update_extra_vars(extra_vars, loader) all_settings = loader.settings_tree() logging.debug("\n" + yaml.safe_dump(all_settings, default_flow_style=False)) logging.info("Writing to file: %s", output_file) with open(output_file, 'w') as out: out.write(yaml.safe_dump(all_settings, default_flow_style=False))
class Loader(object): def __init__(self, config_dir, settings): self._settings = settings self._config_dir = config_dir self._loaded = False self._all_settings = None self._file_list = None self._invalid_paths = None def settings_tree(self): self._load() return self._all_settings def load_file(self, f): self._load() self._load_file(f) def update(self, tree): self._load() self._all_settings.update(tree) def _load(self): if self._loaded: return self._all_settings = OrderedTree('/') self._file_list = [] self._invalid_paths = [] self._create_file_list(self._settings, "", self._file_list) logging.info("files to load :\n %s", '\n'.join(self._file_list)) logging.info("invalid files :\n %s", '\n'.join(self._invalid_paths)) if self._invalid_paths: raise OptionError(self._invalid_paths) for f in self._file_list: self._load_file(f) self._loaded = True def _load_file(self, f): logging.debug('Loading file: %s', f) cfg = Configuration.from_file(f).configure() self._all_settings.update(cfg) def _create_file_list(self, settings, parent_path, file_list): """ Appends list of files to be process to self._file_list and list of invalid file paths to self._invalid_paths """ logging.debug('settings:\n %s \n parent: %s \n files: %s', settings, parent_path, file_list) for key, sub_tree in settings.items(): # ignore the special key value if key == VALUES_KEY: continue logging.debug("key: %s, subtree: %s", key, sub_tree) path = "%(parent_path)s%(key)s%(sep)s%(file)s" % { 'parent_path': parent_path, 'key': key, 'sep': os.sep, 'file': sub_tree[VALUES_KEY] } abs_file_path = os.path.abspath(self._config_dir + os.sep + path + '.yml') file_list.append(abs_file_path) logging.debug('path: %s', abs_file_path) if not os.path.exists(abs_file_path): self._invalid_paths.append(abs_file_path) # recurse if there are sub settings if (isinstance(sub_tree, dict) and len(sub_tree.keys()) > 1): logging.debug('recursing into: sub-tree: %s', sub_tree) self._create_file_list(sub_tree, path + os.sep, file_list)
class Loader(object): def __init__(self, config_dir, settings): self._settings = settings self._config_dir = config_dir self._loaded = False self._all_settings = None self._file_list = None self._invalid_paths = None self._all_settings = OrderedTree('!') def settings(self): self.load() LookupDirective.lookup_table = self._all_settings return self._all_settings def load_file(self, f): self.load() cfg = load_configuration(f, self._config_dir) self._all_settings.merge(cfg) def merge(self, tree): self._all_settings.merge(tree) def load(self): if self._loaded: return self._file_list = [] self._invalid_paths = [] self._create_file_list(self._settings, self._file_list) logger.info( "\nList of files to load :\n - %s", '\n - '.join([ x[len(self._config_dir) + 1:] for x in self._file_list ])) if self._invalid_paths: logger.info("invalid files :\n %s", '\n'.join(self._invalid_paths)) raise OptionError(self._invalid_paths) all_cfg = Configuration.from_dict({}) for f in self._file_list: cfg = load_configuration(f, self._config_dir) try: del cfg[DEFAULTS_TAG] except KeyError: pass else: logger.debug("Successfully removed default traces from %s" % f) all_cfg.merge(cfg) self._all_settings.merge(all_cfg) self._loaded = True def _create_file_list(self, settings, file_list, parent_path=""): """ Appends list of files to be process to self._file_list and list of invalid file paths to self._invalid_paths """ logger.debug('settings:\n %s \n parent: %s \n files: %s', settings, parent_path, file_list) for key, sub_tree in settings.items(): # ignore the special key value if key == VALUES_KEY: continue logger.debug("key: %s, subtree: %s", key, sub_tree) path = "%(parent_path)s%(key)s%(sep)s%(file)s" % { 'parent_path': parent_path, 'key': key, 'sep': os.sep, 'file': sub_tree[VALUES_KEY] } abs_file_path = os.path.abspath( self._config_dir + os.sep + path + '.yml') file_list.append(abs_file_path) logger.debug('path: %s', abs_file_path) if not os.path.exists(abs_file_path): self._invalid_paths.append(abs_file_path) # recurse if there are sub settings if (isinstance(sub_tree, dict) and len(sub_tree.keys()) > 1): logger.debug('recursing into: sub-tree: %s', sub_tree) self._create_file_list(sub_tree, file_list, path + os.sep)