Exemple #1
0
def test_ref():
    src_yaml = """
    foo: bar
    ref_foo: !ref:foo
    """

    src = Configuration.from_string(src_yaml)
    print_yaml("yaml for src", src)
    assert src.ref_foo == src.foo

    missing_ref_yaml = """
    foo: bar
    ref_foo: !ref:bar
    """

    raised_key_error = True
    with pytest.raises(KeyError):
        Configuration.from_string(missing_ref_yaml)
        raised_key_error = False
    assert raised_key_error
    logger.debug("Raised KeyError")

    # use previous context ...
    ref_src_foo_yaml = """
    ref_src_foo:  !ref:foo
    """
    ref_src_foo = Configuration.from_string(ref_src_foo_yaml, configure=False)
    merged = deepcopy(src).merge(ref_src_foo)
    merged.configure()
    print_yaml("yaml that refs src", merged)
Exemple #2
0
def test_overwrite_tag():
    src_yaml = """
    foo: bar
    """

    overwrite_fail_yaml = """
    foo: [1, 2, 3]
    """
    src = Configuration.from_string(src_yaml)
    print_yaml("Src", src)

    overwrite_fail = Configuration.from_string(overwrite_fail_yaml)
    print_yaml("Overwrite fail", overwrite_fail)

    error_raised = True
    with pytest.raises(ConfigurationError):
        logger.debug("Going to merge configs that should fail")
        merged = deepcopy(src).merge(overwrite_fail)
        logger.critical("You should never see this")
        error_raised = False
    assert error_raised
    logger.debug("Raised ConfigurationError")
    assert src.foo == 'bar'

    # ### use overwrite to overwrite src.foo
    overwrite_yaml = """
    foo: !overwrite [1, 2, 3]
    """
    overwrite = Configuration.from_string(overwrite_yaml)
    print_yaml("Overwrite", overwrite)

    merged = deepcopy(src).merge(overwrite)
    print_yaml("Merged", merged)
Exemple #3
0
def test_lookup():
    src_yaml = """
    foo: bar
    ref_foo: !lookup foo
    """

    src = Configuration.from_string(src_yaml, configure=False)
    from ksgen.yaml_utils import LookupDirective
    LookupDirective.lookup_table = src
    print_yaml("yaml for src", src)

    # use previous context ...
    ref_src_foo_yaml = """
    ref_src_foo:  !lookup foo
    missing_lookup: !lookup non.existent.key
    """
    ref_src_foo = Configuration.from_string(ref_src_foo_yaml, configure=False)
    merged = deepcopy(src).merge(ref_src_foo)
    LookupDirective.lookup_table = merged
    print_yaml("yaml that refs src", merged)

    import yaml
    final_config = Configuration.from_string(yaml.safe_dump(merged))
    print_yaml("yaml that refs src", final_config)
    assert final_config.ref_foo == src.foo
    assert final_config.ref_src_foo == src.foo
    assert final_config.missing_lookup == '{{ non.existent.key }}'
Exemple #4
0
def test_merge():
    d1 = {
        "d1": [1, 2, 3],
        "s": "foo",
        "a": [1, 2, 3],
        "nested_dict": {
            "d1": "ok",
            "d": "ok"
        }
    }
    c1 = Configuration.from_dict(d1)
    print_yaml("d1", d1)

    d2 = {
        "d2": "foobar",
        "s": "foobar",
        "a": [3, 4, 5],
        "nested_dict": {
            "d2": "ok",
            "d": {
                "foo": "bar"
            }
        }
    }
    c2 = Configuration.from_dict(d2)
    print_yaml("d2", d2)

    c3 = c1.merge(c2)
    print_yaml("Merged", dict(c3))
Exemple #5
0
def test_lookup():
    src_yaml = """
    foo: bar
    ref_foo: !lookup foo
    """

    src = Configuration.from_string(src_yaml, configure=False)
    from ksgen.yaml_utils import LookupDirective
    LookupDirective.lookup_table = src
    print_yaml("yaml for src", src)

    # use previous context ...
    ref_src_foo_yaml = """
    ref_src_foo:  !lookup foo
    missing_lookup: !lookup non.existent.key
    """
    ref_src_foo = Configuration.from_string(ref_src_foo_yaml, configure=False)
    merged = deepcopy(src).merge(ref_src_foo)
    LookupDirective.lookup_table = merged
    print_yaml("yaml that refs src", merged)

    import yaml
    final_config = Configuration.from_string(yaml.safe_dump(merged))
    print_yaml("yaml that refs src", final_config)
    assert final_config.ref_foo == src.foo
    assert final_config.ref_src_foo == src.foo
    assert final_config.missing_lookup == '{{ non.existent.key }}'
Exemple #6
0
def test_overwrite_tag():
    src_yaml = """
    foo: bar
    """

    overwrite_fail_yaml = """
    foo: [1, 2, 3]
    """
    src = Configuration.from_string(src_yaml)
    print_yaml("Src", src)

    overwrite_fail = Configuration.from_string(overwrite_fail_yaml)
    print_yaml("Overwrite fail", overwrite_fail)

    error_raised = True
    with pytest.raises(ConfigurationError):
        logger.debug("Going to merge configs that should fail")
        merged = deepcopy(src).merge(overwrite_fail)
        logger.critical("You should never see this")
        error_raised = False
    assert error_raised
    logger.debug("Raised ConfigurationError")
    assert src.foo == 'bar'

    # ### use overwrite to overwrite src.foo
    overwrite_yaml = """
    foo: !overwrite [1, 2, 3]
    """
    overwrite = Configuration.from_string(overwrite_yaml)
    print_yaml("Overwrite", overwrite)

    merged = deepcopy(src).merge(overwrite)
    print_yaml("Merged", merged)
Exemple #7
0
def test_ref():
    src_yaml = """
    foo: bar
    ref_foo: !ref:foo
    """

    src = Configuration.from_string(src_yaml)
    print_yaml("yaml for src", src)
    assert src.ref_foo == src.foo

    missing_ref_yaml = """
    foo: bar
    ref_foo: !ref:bar
    """

    raised_key_error = True
    with pytest.raises(KeyError):
        Configuration.from_string(missing_ref_yaml)
        raised_key_error = False
    assert raised_key_error
    logger.debug("Raised KeyError")

    # use previous context ...
    ref_src_foo_yaml = """
    ref_src_foo:  !ref:foo
    """
    ref_src_foo = Configuration.from_string(ref_src_foo_yaml, configure=False)
    merged = deepcopy(src).merge(ref_src_foo)
    merged.configure()
    print_yaml("yaml that refs src", merged)
Exemple #8
0
def test_merge_error():
    src_dict = {
        "d1": [1, 2, 3],
        "s": "foo",
        "a": [1, 2, 3],
        "nested_dict": {
            "d1": "ok",
            "d": "ok"
        }
    }

    other_dict = {
        "d2": "foobar",
        "s": "foobar",
        "a": [3, 4, 5],
        "nested_dict": {
            "d2": "ok",
            "d": {            # ### raises ConfigError
                "foo": "bar"
            }
        }
    }
    src = Configuration.from_dict(src_dict)
    print_yaml("src", src)

    other = Configuration.from_dict(other_dict)
    print_yaml("other", other)

    with pytest.raises(ConfigurationError):
        merged = deepcopy(src).merge(other)
        print_yaml("Merged", dict(merged))
    logger.info("Merge raised configuration error")
Exemple #9
0
def test_merge():
    d1 = {
        "d1": [1, 2, 3],
        "s": "foo",
        "a": [1, 2, 3],
        "nested_dict": {
            "d1": "ok",
            "d": "ok"
        }
    }
    c1 = Configuration.from_dict(d1)
    print_yaml("d1", d1)

    d2 = {
        "d2": "foobar",
        "s": "foobar",
        "a": [3, 4, 5],
        "nested_dict": {
            "d2": "ok",
            "d": {
                "foo": "bar"
            }
        }
    }
    c2 = Configuration.from_dict(d2)
    print_yaml("d2", d2)

    c3 = c1.merge(c2)
    print_yaml("Merged", dict(c3))
Exemple #10
0
def test_merge_error():
    src_dict = {
        "d1": [1, 2, 3],
        "s": "foo",
        "a": [1, 2, 3],
        "nested_dict": {
            "d1": "ok",
            "d": "ok"
        }
    }

    other_dict = {
        "d2": "foobar",
        "s": "foobar",
        "a": [3, 4, 5],
        "nested_dict": {
            "d2": "ok",
            "d": {            # ### raises ConfigError
                "foo": "bar"
            }
        }
    }
    src = Configuration.from_dict(src_dict)
    print_yaml("src", src)

    other = Configuration.from_dict(other_dict)
    print_yaml("other", other)

    with pytest.raises(ConfigurationError):
        merged = deepcopy(src).merge(other)
        print_yaml("Merged", dict(merged))
    logger.info("Merge raised configuration error")
Exemple #11
0
 def __init__(self, url: str, ssh_keyfile: str, keypass: Credential):
     self.__config = Configuration()
     self.__config.load()
     self.__raw_version = self.__config[SettingKeyNames.CBS_VERSION]
     self.__url = url
     self.__ssh_keyfile = ssh_keyfile
     self.__ssh_keypass = keypass
     (self.__version, self.__build) = CouchbaseServerInstaller._parse_version(self.__raw_version)
Exemple #12
0
def test_lookups_in_extends():
    src = Configuration.from_file(TEST_DIR + '/data/extends/extends.yml')
    print_yaml("Extends using !lookup:", src)
    src.configure()
    from ksgen.yaml_utils import LookupDirective
    LookupDirective.lookup_table = src
    print_yaml("Extends using !lookup:", src)
Exemple #13
0
def main():
    """ Run the script """
    os.system('cp ../config.xml config/config.xml')
    os.system('cp ../network.xml config/network.xml')
    config = Configuration('../config.ini')
    results = begin_all_sims(config)
    save_results(results, 'rawResults.pkl')
Exemple #14
0
    def load(self):
        if self._loaded:
            return

        self._file_list = []
        self._invalid_paths = []
        self._create_file_list(self._settings, self._file_list)

        logger.info(
            "\nList of files to load :\n  - %s", '\n  - '.join(
                [x[len(self._config_dir) + 1:] for x in self._file_list]))

        if self._invalid_paths:
            logger.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
            raise OptionError(self._invalid_paths)

        all_cfg = Configuration.from_dict({})
        for f in self._file_list:
            cfg = load_configuration(f, self._config_dir)

            try:
                del cfg[DEFAULTS_TAG]
            except KeyError:
                pass
            else:
                logger.debug("Successfully removed default traces from %s" % f)

            all_cfg.merge(cfg)
        self._all_settings.merge(all_cfg)
        self._loaded = True
Exemple #15
0
def load_configuration(environment, config_name, config_dir):
    """

    :param environment: Environment name
    :type environment: str
    :param config_name: Config file prefix
    :type config_name: str
    :param config_dir: Config directory
    :type config_dir: str
    :return: Configuration
    """
    config_filename = '{}_{}.yaml'.format(config_name, environment)
    config_filename = os.path.join(config_dir, config_filename)

    if not os.path.isfile(config_filename):
        config_filename = '{}.yaml'.format(config_name)
        config_filename = os.path.join(config_dir, config_filename)

    config = Configuration.from_file(config_filename)
    config.configure()

    def to_dict(mapping):
        return {
            k: (to_dict(v) if isinstance(v, Mapping) else v)
            for k, v in mapping.items()
        }

    return to_dict(config)
Exemple #16
0
    def load(self):
        if self._loaded:
            return

        self._file_list = []
        self._invalid_paths = []
        self._create_file_list(self._settings, self._file_list)

        logger.info(
            "\nList of files to load :\n  - %s",
            '\n  - '.join([
                x[len(self._config_dir) + 1:] for x in self._file_list
            ]))

        if self._invalid_paths:
            logger.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
            raise OptionError(self._invalid_paths)

        all_cfg = Configuration.from_dict({})
        for f in self._file_list:
            cfg = load_configuration(f, self._config_dir)

            try:
                del cfg[DEFAULTS_TAG]
            except KeyError:
                pass
            else:
                logger.debug("Successfully removed default traces from %s" % f)

            all_cfg.merge(cfg)
        self._all_settings.merge(all_cfg)
        self._loaded = True
Exemple #17
0
def test_lookups_in_extends():
    src = Configuration.from_file(TEST_DIR + '/data/extends/extends.yml')
    print_yaml("Extends using !lookup:", src)
    src.configure()
    from ksgen.yaml_utils import LookupDirective
    LookupDirective.lookup_table = src
    print_yaml("Extends using !lookup:", src)
Exemple #18
0
def checkValidRoles(h):
    result = True
    definedRoles = Configuration.from_file('config/roles.yaml').configure()
    for r in h["roles"]:
        if not r in definedRoles:
            print "==> {0} is not a valid role.".format(r)
            result = False
    return result
Exemple #19
0
    def test_load_from_file(self):
        filename = path.join(path.dirname(__file__), 'examples', 'example.default.conf')
        c = Configuration.from_file(filename)
        c.configure()

        self.assertEqual(c.a, 1)
        self.assertIsInstance(c.b, timedelta)
        self.assertEqual(c.b, timedelta(days=1))
Exemple #20
0
    def test_load_from_file(self):
        filename = path.join(path.dirname(__file__), 'examples', 'example.default.conf')
        c = Configuration.from_file(filename)
        c.configure()

        self.assertEqual(c.a, 1)
        self.assertIsInstance(c.b, timedelta)
        self.assertEqual(c.b, timedelta(days=1))
Exemple #21
0
def load_variables(file_path, namespace):
    file_path = Template(file_path).render(os.environ)
    variables = dict(
        Configuration.from_file(os.path.abspath(file_path)).configure()
    )
    staticconf.DictConfiguration(
        variables,
        namespace=namespace,
        flatten=False)
Exemple #22
0
def test_array_extend():
    src_dict = {
        "src": [11, 12, 13],
        "merge": [100, 101, 102],
        'nested_dict': {
            'src': [111, 112, 113],
            'merge': [1000, 1001, 1002]
        }
    }

    other_dict = {
        "other": [22, 22, 23],
        "merge": [200, 202, 202],
        'nested_dict': {
            'other': [222, 222, 223],
            'merge': [2000, 2002, 2002]
        }
    }

    src = Configuration.from_dict(src_dict)
    print_yaml("Src", src)

    other = Configuration.from_dict(other_dict)
    print_yaml("Other", other)

    merged = deepcopy(src).merge(other)
    print_yaml("merged", merged)

    print_yaml("Src after merge", src)
    print_yaml("Other after merge", other)

    assert verify_key_val(src, src_dict, 'merge')
    assert verify_key_val(src, src_dict, 'src')
    assert verify_key_val(src, src_dict, 'nested_dict.merge')
    assert verify_key_val(src, src_dict, 'nested_dict.src')

    with pytest.raises(KeyError):
        verify_key_val(src, src_dict, 'other')

    assert verify_key_val(merged, src_dict, 'src')
    assert verify_key_val(merged, other_dict, 'other')
    assert merged['merge'] == src_dict['merge'] + other_dict['merge']
    assert verify_key_val(merged, src_dict, 'nested_dict.src')
    assert verify_key_val(merged, other_dict, 'nested_dict.other')
Exemple #23
0
def test_array_extend():
    src_dict = {
        "src": [11, 12, 13],
        "merge": [100, 101, 102],
        'nested_dict': {
            'src': [111, 112, 113],
            'merge': [1000, 1001, 1002]
        }
    }

    other_dict = {
        "other": [22, 22, 23],
        "merge": [200, 202, 202],
        'nested_dict': {
            'other': [222, 222, 223],
            'merge': [2000, 2002, 2002]
        }
    }

    src = Configuration.from_dict(src_dict)
    print_yaml("Src", src)

    other = Configuration.from_dict(other_dict)
    print_yaml("Other", other)

    merged = deepcopy(src).merge(other)
    print_yaml("merged", merged)

    print_yaml("Src after merge", src)
    print_yaml("Other after merge", other)

    assert verify_key_val(src, src_dict, 'merge')
    assert verify_key_val(src, src_dict, 'src')
    assert verify_key_val(src, src_dict, 'nested_dict.merge')
    assert verify_key_val(src, src_dict, 'nested_dict.src')

    with pytest.raises(KeyError):
        verify_key_val(src, src_dict, 'other')

    assert verify_key_val(merged, src_dict, 'src')
    assert verify_key_val(merged, other_dict, 'other')
    assert merged['merge'] == src_dict['merge'] + other_dict['merge']
    assert verify_key_val(merged, src_dict, 'nested_dict.src')
    assert verify_key_val(merged, other_dict, 'nested_dict.other')
Exemple #24
0
def test_simple_merge():
    src_dict = {
        "merge": "src merge",
        "src": 'src only',
        'nested_dict': {
            'merge': 'nested src merge',
            'src': 'nested src only'
        }
    }

    other_dict = {
        "merge": "other merge",
        "other": 'other only',
        'nested_dict': {
            'merge': 'nested other merge',
            'other': 'nested other only'
        }
    }

    src = Configuration.from_dict(src_dict)
    print_yaml("Src", src)

    other = Configuration.from_dict(other_dict)
    print_yaml("Other", other)

    merged = deepcopy(src).merge(other)
    print_yaml("merged", merged)

    print_yaml("Src after merge", src)
    print_yaml("Other after merge", other)

    assert verify_key_val(src, src_dict, 'merge')
    assert verify_key_val(src, src_dict, 'src')
    assert verify_key_val(src, src_dict, 'nested_dict.merge')
    assert verify_key_val(src, src_dict, 'nested_dict.src')

    with pytest.raises(KeyError):
        verify_key_val(src, src_dict, 'other')

    assert verify_key_val(merged, other_dict, 'merge')
    assert verify_key_val(merged, src_dict, 'src')
    assert verify_key_val(merged, other_dict, 'other')
    return
def validate_configuration(options):
    # must have 3 sections common



if __name__ == '__main__':
    parser = build_argument_parser()
    options = parser.parse_args()
    config = Configuration.from_file(options.config)
    
Exemple #26
0
def load_yaml(file_path, namespace):
    file_path = Template(file_path).render(os.environ)
    yaml_dict = Configuration.from_file(
            os.path.abspath(file_path),
            configure=False
        )
    staticconf.DictConfiguration(
        yaml_dict,
        namespace=namespace,
        flatten=False)
Exemple #27
0
def test_simple_merge():
    src_dict = {
        "merge": "src merge",
        "src": 'src only',
        'nested_dict': {
            'merge': 'nested src merge',
            'src': 'nested src only'
        }
    }

    other_dict = {
        "merge": "other merge",
        "other": 'other only',
        'nested_dict': {
            'merge': 'nested other merge',
            'other': 'nested other only'
        }
    }

    src = Configuration.from_dict(src_dict)
    print_yaml("Src", src)

    other = Configuration.from_dict(other_dict)
    print_yaml("Other", other)

    merged = deepcopy(src).merge(other)
    print_yaml("merged", merged)

    print_yaml("Src after merge", src)
    print_yaml("Other after merge", other)

    assert verify_key_val(src, src_dict, 'merge')
    assert verify_key_val(src, src_dict, 'src')
    assert verify_key_val(src, src_dict, 'nested_dict.merge')
    assert verify_key_val(src, src_dict, 'nested_dict.src')

    with pytest.raises(KeyError):
        verify_key_val(src, src_dict, 'other')

    assert verify_key_val(merged, other_dict, 'merge')
    assert verify_key_val(merged, src_dict, 'src')
    assert verify_key_val(merged, other_dict, 'other')
    return
def configuration_loader(filename):
    from collections import Mapping
    from configure import Configuration
    config = Configuration.from_file(filename)
    config.configure()

    def to_dict(mapping):
        return {k: (to_dict(v) if isinstance(v, Mapping) else v) for k, v in mapping.items()}

    return to_dict(config)
Exemple #29
0
def test_limit_chars():
    src_yaml = """
    substr: !limit_chars [ 'abcdefghijklmnopqrstuvwxyz', 7 ]
    zero: !limit_chars [ 'abcdefghijklmnopqrstuvwxyz', 0 ]

    """
    src = Configuration.from_string(src_yaml)
    print_yaml("Src", src)

    assert src.substr == 'abcdefg'
    assert src.zero == ''
def test_limit_chars():
    src_yaml = """
    substr: !limit_chars [ 'abcdefghijklmnopqrstuvwxyz', 7 ]
    zero: !limit_chars [ 'abcdefghijklmnopqrstuvwxyz', 0 ]

    """
    src = Configuration.from_string(src_yaml)
    print_yaml("Src", src)

    assert src.substr == 'abcdefg'
    assert src.zero == ''
Exemple #31
0
def main():
    """ Main execution point """
    try:
        source_folder = sys.argv[1]
        destination_folder = sys.argv[2]
    except Exception:
        print('Please provide the path to source and destination folders.')
    else:
        folders = ['packages', 'router', 'routing_algos']
        config = Configuration('../config.ini')
        copy_folders(source_folder, destination_folder, folders)
        setup(destination_folder, folders, config)
Exemple #32
0
def validate(config_path):
    try:
        config = Configuration.configure(config_path)
    except ConfigurationError as ce:
        logger.exception(ce)
    try:
        with open('config.schema.yaml', 'r') as f:
            schema = yaml.load(f)
    except:
        
    schema = yaml.load('')
    v = DEConfigValidator
Exemple #33
0
def test_random():
    src_yaml = """
    random: !random 8

    """
    random_so_far = set()

    for x in range(2 ** 12):
        src = Configuration.from_string(src_yaml)
        if src.random in random_so_far:
            print len(random_so_far)
            assert src.random not in random_so_far
        random_so_far.add(src.random)
Exemple #34
0
def test_random():
    src_yaml = """
    random: !random 8

    """
    random_so_far = set()

    for x in range(2 ** 12):
        src = Configuration.from_string(src_yaml)
        if src.random in random_so_far:
            print len(random_so_far)
            assert src.random not in random_so_far
        random_so_far.add(src.random)
Exemple #35
0
def test_merge_lookup():
    lookup_yaml = """
    foo: bar
    merge: !lookup foo
    """

    lookup = Configuration.from_string(lookup_yaml)
    print_yaml("Lookup yaml", lookup)

    new_yaml = """
    merge: baz
    """
    new_values = Configuration.from_string(new_yaml)
    print_yaml("merge", new_values)

    error_raised = True
    with pytest.raises(ConfigurationError):
        logger.debug("Going to merge configs that should fail")
        deepcopy(lookup).merge(new_values)
        logger.critical("You should never see this")
        error_raised = False
    assert error_raised
    logger.debug("Raised ConfigurationError")
 def lineReceived(self, line):
     stdout.write("Server%d Received Data: %s\n" %( Configuration.getMyID(), line))
     delMsgs = self.algorithm.receiveMsg(line, self.node)
     #return the jsonmsg to various nodes 
     #send jsonmsg to nodes to notify them 
     #it's for conflicts
     #self.send2Node(0, "fake conflict notification")
     if delMsgs:
         for delMsg in delMsgs: 
             #put delete event into the log and execute delete
             e = self.node.createEvent(delMsg)
         for i in range(self.algorithm.n):#@TODO this line is broadcast  
             jsonmsg = self.algorithm.sendMsg2Node(i) 
             self.send2Node(i, jsonmsg)
Exemple #37
0
def test_merge_lookup():
    lookup_yaml = """
    foo: bar
    merge: !lookup foo
    """

    lookup = Configuration.from_string(lookup_yaml)
    print_yaml("Lookup yaml", lookup)

    new_yaml = """
    merge: baz
    """
    new_values = Configuration.from_string(new_yaml)
    print_yaml("merge", new_values)

    error_raised = True
    with pytest.raises(ConfigurationError):
        logger.debug("Going to merge configs that should fail")
        deepcopy(lookup).merge(new_values)
        logger.critical("You should never see this")
        error_raised = False
    assert error_raised
    logger.debug("Raised ConfigurationError")
Exemple #38
0
def test_env():
    src_yaml = """
    user: !env [USER]
    invalid: !env [DOES_NOT_EXIST, ~]
    same_user: !env [ DOES_NOT_EXIST, !env [USER, baaaz] ]
    default: !env [ DOES_NOT_EXIST, !env [FOOO, default] ]

    """
    src = Configuration.from_string(src_yaml)
    print_yaml("Src", src)

    assert src.user is not None
    assert src.invalid is None
    assert src.same_user == src.user
    assert src.default == 'default'
Exemple #39
0
def test_dict_order():
    yaml = """
        too: boo
        loo: too
        foo: bar
        moo:
            - soo
    """
    cfg = Configuration.from_string(yaml).configure()
    assert cfg.keys() == ['too', 'loo', 'foo', 'moo']
    for k in cfg.iterkeys():
        logger.debug('%s', k)
    for k, v in cfg.iteritems():
        logger.debug('%s: %s', k, v)

    print_yaml("src", cfg)
Exemple #40
0
def load_configuration(file_path, rel_dir=None):
    """ Return the Configuration for the file_path. Also logs an error if
        there is an error while parsing the file.
        If the optional rel_dir is passed the error message would print
        the file_path relative to rel_dir instead of current directory
    """

    logger.debug('Loading file: %s', file_path)
    try:
        return Configuration.from_file(file_path).configure()
    except ConfigurationError as e:
        rel_dir = rel_dir or os.curdir
        logger.error("Error loading: %s; reason: %s",
                     os.path.relpath(file_path, rel_dir), e)
        raise
    return None
Exemple #41
0
def load_configuration(file_path, rel_dir=None):
    """ Return the Configuration for the file_path. Also logs an error if
        there is an error while parsing the file.
        If the optional rel_dir is passed the error message would print
        the file_path relative to rel_dir instead of current directory
    """

    logger.debug('Loading file: %s', file_path)
    try:
        return Configuration.from_file(file_path).configure()
    except ConfigurationError as e:
        rel_dir = rel_dir or os.curdir
        logger.error("Error loading: %s; reason: %s",
                     os.path.relpath(file_path, rel_dir), e)
        raise
    return None
Exemple #42
0
def test_dict_order():
    yaml = """
        too: boo
        loo: too
        foo: bar
        moo:
            - soo
    """
    cfg = Configuration.from_string(yaml).configure()
    assert cfg.keys() == ['too', 'loo', 'foo', 'moo']
    for k in cfg.iterkeys():
        logger.debug('%s', k)
    for k, v in cfg.iteritems():
        logger.debug('%s: %s', k, v)

    print_yaml("src", cfg)
def test_env():
    src_yaml = """
    user: !env [USER]
    invalid: !env [DOES_NOT_EXIST1, ~]
    same_user: !env [ DOES_NOT_EXIST2, !env [USER, baaaz] ]
    default: !env [ DOES_NOT_EXIST3, !env [FOOO, default] ]
    home_short: !env [ HOMExxx, '/my/home/under/the/bridge/', 7 ]

    """
    src = Configuration.from_string(src_yaml)
    print_yaml("Src", src)

    assert src.user is not None
    assert src.invalid is None
    assert src.same_user == src.user
    assert src.default == 'default'
    assert src.home_short == '/my/hom'
Exemple #44
0
    def _load_defaults(self, path, value):
        param = '-'.join(path[len(self.config_dir + os.sep):].split('/')[::2])
        if not self.parsed['--' + param]:
            logging.warning(
                "\'--%s\' hasn't been provided, using \'%s\' as default" %
                (param, value))
            self.defaults.append(''.join(['--', param, '=', str(value)]))
        else:
            value = self.parsed['--' + param]

        file_path = path + os.sep + str(value) + '.yml'
        loaded_file = Configuration.from_file(file_path)

        if loaded_file.get(DEFAULTS_TAG):
            path += os.sep + str(value)
            for sub_key, sub_value in loaded_file[DEFAULTS_TAG].iteritems():
                self._load_defaults(path + os.sep + sub_key, sub_value)
Exemple #45
0
def test_env():
    src_yaml = """
    user: !env [USER]
    invalid: !env [DOES_NOT_EXIST1, ~]
    same_user: !env [ DOES_NOT_EXIST2, !env [USER, baaaz] ]
    default: !env [ DOES_NOT_EXIST3, !env [FOOO, default] ]
    home_short: !env [ HOMExxx, '/my/home/under/the/bridge/', 7 ]

    """
    src = Configuration.from_string(src_yaml)
    print_yaml("Src", src)

    assert src.user is not None
    assert src.invalid is None
    assert src.same_user == src.user
    assert src.default == 'default'
    assert src.home_short == '/my/hom'
Exemple #46
0
    def _load_defaults(self, path, value):
        param = '-'.join(path[len(self.config_dir + os.sep):].split('/')[::2])
        if not self.parsed['--' + param]:
            logging.warning(
                "\'--%s\' hasn't been provided, using \'%s\' as default" % (
                    param, value))
            self.defaults.append(''.join(['--', param, '=', str(value)]))
        else:
            value = self.parsed['--' + param]

        file_path = path + os.sep + str(value) + '.yml'
        loaded_file = Configuration.from_file(file_path)

        if loaded_file.get(DEFAULTS_TAG):
            path += os.sep + str(value)
            for sub_key, sub_value in loaded_file[DEFAULTS_TAG].iteritems():
                self._load_defaults(path + os.sep + sub_key, sub_value)
 def sendMsg2Node(self, nodek):
     #@TODO load log from database
     log = self.dc.getLogs(0,sys.maxint)
     NP = {} #partial log
     ES = {} #event lists
     matrix =  [[0 for _ in range(self.n) ] for _ in range(self.n)]
     for i in range(self.n):
         for j in range(self.n):
             matrix[i][j] = self.dc.getTime(i, j)
     for (id,name,time,content) in log:
         event = Event(id,name,time,content)
         if not self.__hasRec(matrix, event, nodek):
            ES[event.name] = (event.time, event.node, event.content)  
            logging.debug((event.time, event.node, event.content))
     NP["matrix"] = matrix
     NP["events"] = ES 
     NP["senderID"] = Configuration.getMyID() 
     NP["receiverID"] = nodek 
     logging.debug(json.dumps(NP))
     return json.dumps(NP)
Exemple #48
0
def getVars(self):
    """ Used at setup of test """
    config = Configuration.from_file('./config.yml').configure()

    self.testbrowser  = config['browser']
    self.host = config['sharehost']	
    self.port = config['shareport']		
    self.sharehost = config['sharehost']
    self.shareport = config['shareport']
    self.repohost  = config['repohost']
    self.repoport  = config['repoport']
    self.host	     = config['sharehost']	

    if config['https'] is True:
       uri = 'https'
    else:	
       uri = 'http'

    if self.shareport is None:
       self.url = uri + '://' + self.sharehost 
    else:
       self.url = uri + '://' + self.sharehost + ':' + str(self.shareport)

    if self.repoport is None:
   		 self.repourl = uri + '://' + self.repohost
    else:
       self.repourl = uri + '://' + self.repohost + ':' + str(self.repoport)

    self.username  = config['user']
    self.password  = config['passwd']
    self.loginurl  = self.url + config['loginurl']
    self.photopath = config['photopath']
    self.cmisurl   = config['cmisurl']
    self.cmisatom  = self.repourl + config['cmisatom']
    self.ftpurl    = config['sharehost']
    self.ftpport   = config['ftpport']
    self.imap_host = config['imap_host']
    self.imap_port = config['imap_port']

    return self
Exemple #49
0
    def load(self):
        if self._loaded:
            return

        self._file_list = []
        self._invalid_paths = []
        self._create_file_list(self._settings, self._file_list)

        logger.info(
            "\nList of files to load :\n  - %s", '\n  - '.join(
                [x[len(self._config_dir) + 1:] for x in self._file_list]))

        if self._invalid_paths:
            logger.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
            raise OptionError(self._invalid_paths)

        all_cfg = Configuration.from_dict({})
        for f in self._file_list:
            cfg = load_configuration(f, self._config_dir)
            all_cfg.merge(cfg)
        self._all_settings.merge(all_cfg)
        self._loaded = True
Exemple #50
0
    def load(self):
        if self._loaded:
            return

        self._file_list = []
        self._invalid_paths = []
        self._create_file_list(self._settings, self._file_list)

        logger.info(
            "\nList of files to load :\n  - %s",
            '\n  - '.join([
                x[len(self._config_dir) + 1:] for x in self._file_list
            ]))

        if self._invalid_paths:
            logger.info("invalid files :\n %s", '\n'.join(self._invalid_paths))
            raise OptionError(self._invalid_paths)

        all_cfg = Configuration.from_dict({})
        for f in self._file_list:
            cfg = load_configuration(f, self._config_dir)
            all_cfg.merge(cfg)
        self._all_settings.merge(all_cfg)
        self._loaded = True
Exemple #51
0
def load_yaml_config(filename):
    filename = os.path.abspath(filename)
    config = Configuration.from_file(filename).configure()
    return config
Exemple #52
0
# -*- coding: utf-8 -*-
import os
import warnings
from configure import Configuration


try:
    environment = os.environ['APPLICATION_ENV']
except:
    warnings.warn('catch error when get os environment APPLICATION_EVN, '
                  'use default prod instead')
    environment = 'prod'

path = os.path.dirname(__file__)
config_file = '{0}/config/config_{1}.yml'.format(path, environment)
settings = Configuration.from_file(config_file).configure()
 def __init__(self):
     self.n = Configuration.getN()
     self.C = 0
     self.ID = Configuration.getMyID()
     self.dc = DataConn()
Exemple #54
0
 def _load_file(self, f):
     logging.debug('Loading file: %s', f)
     cfg = Configuration.from_file(f).configure()
     self._all_settings.update(cfg)
Exemple #55
0
 def config(self, v, ctx=None):
     return Configuration.from_string(v.strip(), ctx=ctx)
        elif options.odma_mode.lower() == "st_1024":
            options.predefined_config = "hdl_unit_sim.odma_st_1024.defconfig"
        else:
            options.predefined_config = "hdl_unit_sim.odma.defconfig"
    else:
        options.predefined_config = "hdl_unit_sim.bridge.defconfig"

if __name__ == '__main__':
    msg.ok_msg_blue("--------> WELCOME to IBM OpenCAPI Acceleration Framework")
    question_and_answer = qa.QuestionAndAnswer(options)

    question_and_answer.ask(qa.ask_clean_str)
    if options.clean:
        env_clean(ocaccel_workflow_log)

    cfg = Configuration(options)
    cfg.log = ocaccel_workflow_log
    question_and_answer.cfg = cfg
    question_and_answer.ask(qa.ask_configure_str)
    if not options.no_configure:
        cfg.configure()

    # In unit sim mode, all configurations are handled automatically, no need to update the cfg
    if not options.unit_sim:
        cfg.update_cfg()

    if not options.no_env_check:
        env_check(options)

    question_and_answer.ask(qa.ask_make_model_str)
    if not options.no_make_model and options.simulator.lower() != "nosim":
# from configure import Include
# Function for programatically representing includes to the configure module
# def represent_Include(dumper, include):
#     return dumper.represent_scalar(u'!include:%s' % include.filename, '')
# yaml.add_representer(Include, represent_Include)

# Load the base configuration YAML
# with open(appfile, 'r') as stream:
#     app = yaml.load(stream)

# Auto load realms and sites directories
# for stem in ['realm', 'site']:
#     plural = stem + 's'
#     # basedir = os.path.join(rootdir, plural)
#     app[plural] = app.get(plural, {}) or {}
#     for dir in os.listdir(os.path.join(appdir, plural)):
#         relative = os.path.join(plural, dir, stem + '.yaml')
#         if os.path.isfile(os.path.join(appdir, relative)):
#             app[plural][dir] = Include(relative)

# config = Configuration.from_string(yaml.dump(app), pwd=os.path.abspath(appdir))

config = Configuration.from_file(appfile)

print config



print Just(1) & Nothing
Either
Maybe
Exemple #58
0
    def running_config_POST(self, arg):
        try:
            # Expectation:
            # {
            #  'credentials': {'tested-rc': '<STRING>', 'tested-passwd': '<STRING>',
            #                  'testing-rc': '<STRING>', 'testing-passwd': '<STRING>'},
            #  'kb_cfg': {<USER_OVERRIDED_CONFIGS>},
            #  'topo_cfg': {<TOPOLOGY_CONFIGS>}
            #  'tenants_cfg': {<TENANT_AND_USER_LISTS_FOR_REUSING>}
            # }
            user_config = json.loads(arg)

            # Parsing credentials from application input
            cred_config = user_config["credentials"]
            cred_tested = Credentials(openrc_contents=cred_config["tested-rc"], pwd=cred_config["tested-passwd"])
            if "testing-rc" in cred_config and cred_config["testing-rc"] != cred_config["tested-rc"]:
                cred_testing = Credentials(openrc_contents=cred_config["testing-rc"], pwd=cred_config["testing-passwd"])
            else:
                # Use the same openrc file for both cases
                cred_testing = cred_tested

            session_id = hashlib.md5(str(cred_config)).hexdigest()
            kb_config = KBConfig()
            if KBSessionManager.has(session_id):
                response.status = 403
                response.text = u"Session is already existed."
                return response.text

            # Parsing server and client configs from application input
            # Save the public key into a temporary file
            if "public_key" in user_config["kb_cfg"]:
                pubkey_filename = "/tmp/kb_public_key.pub"
                f = open(pubkey_filename, "w")
                f.write(user_config["kb_cfg"]["public_key_file"])
                f.close()
                kb_config.config_scale["public_key_file"] = pubkey_filename

            if "prompt_before_run" in user_config["kb_cfg"]:
                kb_config.config_scale["prompt_before_run"] = False

            if user_config["kb_cfg"]:
                alt_config = Configuration.from_string(user_config["kb_cfg"]).configure()
                kb_config.config_scale = kb_config.config_scale.merge(alt_config)

            # Parsing topology configs from application input
            if "topo_cfg" in user_config:
                topo_cfg = Configuration.from_string(user_config["topo_cfg"]).configure()
            else:
                topo_cfg = None

            # Parsing tenants configs from application input
            if "tenants_list" in user_config:
                tenants_list = Configuration.from_string(user_config["tenants_list"]).configure()
            else:
                tenants_list = None
        except Exception:
            response.status = 400
            response.text = u"Error while parsing configurations: \n%s" % (traceback.format_exc)
            return response.text

        logging.setup("kloudbuster", logfile="/tmp/kb_log_%s" % session_id)
        kb_config.init_with_rest_api(
            cred_tested=cred_tested, cred_testing=cred_testing, topo_cfg=topo_cfg, tenants_list=tenants_list
        )

        kb_session = KBSession()
        kb_session.kb_config = kb_config
        KBSessionManager.add(session_id, kb_session)

        return str(session_id)
Exemple #59
0
import subprocess
from subprocess import Popen, PIPE, STDOUT
import os
import boto.cloudformation
import boto.s3.connection
import gzip
import logging
import datetime
import sys
from configure import Configuration


folderlocation = os.path.dirname(os.path.realpath(__file__))

#Load Connection Configuration File
ConnectionConfig = Configuration.from_file(os.path.join(folderlocation, 'ConnectionConfig.yaml')).configure()

# Load SQL Server Queries File
queries = Configuration.from_file(os.path.join(folderlocation, 'SQLServerQueries.yaml')).configure()

#Load Redshift Configuration File
RedshiftConfig = Configuration.from_file(os.path.join(folderlocation, 'RedShiftTablesConfig.yaml')).configure()

#Set the date for logging
processdate = datetime.datetime.today().strftime('%Y%m%d')


#Set the package variables

#Packagename
packagename = sys.argv[1]
Exemple #60
0
class CouchbaseServerInstaller:
    __config: Configuration
    __url: str
    __version: str
    __build: str
    __raw_version: str
    __ssh_keyfile: str
    __ssh_keypass: Credential

    @staticmethod
    def version_to_code(version: str):
        try:
            parsed_version = Version(version)
        except InvalidVersion:
            print("Non-numeric version {} received, interpreting as codename...".format(version))
            return version

        if parsed_version >= Version("7.0"):
            return "cheshire-cat"

        print(colored("This script uses features introduced in 6.5, earlier versions not supported...", "red"))
        raise UnsupportedException("Unsupported version of Couchbase Server requested")

    @staticmethod
    def _parse_version(version: str):
        version_build = version.split("-")
        if len(version_build) == 2:
            return (CouchbaseServerInstaller._version_to_code(version_build[0]), version_build[1])

        return (version, None)

    @staticmethod
    def _generate_filename(version: str, build: str):
        if build is None:
            return "couchbase-server-enterprise-{}-centos7.x86_64.rpm".format(version)

        return "couchbase-server-enterprise-{}-{}-centos7.x86_64.rpm".format(version, build)

    def __init__(self, url: str, ssh_keyfile: str, keypass: Credential):
        self.__config = Configuration()
        self.__config.load()
        self.__raw_version = self.__config[SettingKeyNames.CBS_VERSION]
        self.__url = url
        self.__ssh_keyfile = ssh_keyfile
        self.__ssh_keypass = keypass
        (self.__version, self.__build) = CouchbaseServerInstaller._parse_version(self.__raw_version)

    def download(self):
        filename = CouchbaseServerInstaller._generate_filename(self.__version, self.__build)
        if not Path(filename).exists():
            print("Downloading Couchbase Server {}...".format(self.__raw_version))
            url = self._generate_download_url(self.__version, self.__build, filename)
            wget.download(url, filename)

    def install(self):
        filename = CouchbaseServerInstaller._generate_filename(self.__version, self.__build)
        if not Path(filename).exists():
            raise Exception("Unable to find installer, please call download first")

        print("Installing Couchbase Server to {}...".format(self.__url))
        ssh_client = SSHClient()
        ssh_client.load_system_host_keys()
        ssh_client.set_missing_host_key_policy(WarningPolicy())
        ssh_connect(ssh_client, self.__url, self.__ssh_keyfile, str(self.__ssh_keypass))
        (_, stdout, _) = ssh_client.exec_command("test -f {}".format(filename))
        if stdout.channel.recv_exit_status() == 0:
            print("Install file already present on remote host, skipping upload...")
        else:
            print("Uploading file to remote host...")
            sftp = ssh_client.open_sftp()
            sftp_upload(sftp, filename, filename)
            sftp.close()

        ssh_command(ssh_client, self.__url, "sudo yum install -y {}".format(filename))
        print("Install finished!")

    def _generate_download_url(self, version: str, build: str, filename: str):
        # All access via VPN or company network
        if build is not None:
            return "http://latestbuilds.service.couchbase.com/builds/latestbuilds/couchbase-server/{}/{}/{}".format(
                   version, build, filename)

        return "http://latestbuilds.service.couchbase.com/builds/releases/{}/{}".format(version, filename)