Ejemplo n.º 1
0
    def test_compare_sls_vs_yaml_with_jinja(self):
        tpl = '{{ data }}'
        env = jinja2.Environment()
        src = '{foo: 1, bar: 2, baz: {qux: true}}'

        sls_src = env.from_string(tpl).render(data=yamlex.deserialize(src))
        yml_src = env.from_string(tpl).render(data=yaml.deserialize(src))

        sls_data = yamlex.deserialize(sls_src)
        yml_data = yaml.deserialize(yml_src)

        # ensure that sls & yaml have the same base
        assert isinstance(sls_data, dict)
        assert isinstance(yml_data, dict)
        # The below has been commented out because something the loader test
        # is modifying the yaml renderer to render things to unicode. Without
        # running the loader test, the below passes. Even reloading the module
        # from disk does not reset its internal state (per the Python docs).
        ##
        #assert sls_data == yml_data

        # ensure that sls is ordered, while yaml not
        assert isinstance(sls_data, OrderedDict)
        assert not isinstance(yml_data, OrderedDict)

        # prove that yaml does not handle well with OrderedDict
        # while sls is jinja friendly.
        obj = OrderedDict([
            ('foo', 1),
            ('bar', 2),
            ('baz', {'qux': True})
        ])

        sls_obj = yamlex.deserialize(yamlex.serialize(obj))
        try:
            yml_obj = yaml.deserialize(yaml.serialize(obj))
        except SerializationError:
            # BLAAM! yaml was unable to serialize OrderedDict,
            # but it's not the purpose of the current test.
            yml_obj = obj.copy()

        sls_src = env.from_string(tpl).render(data=sls_obj)
        yml_src = env.from_string(tpl).render(data=yml_obj)

        final_obj = yaml.deserialize(sls_src)
        assert obj == final_obj

        # BLAAM! yml_src is not valid !
        final_obj = OrderedDict(yaml.deserialize(yml_src))
        assert obj != final_obj
Ejemplo n.º 2
0
    def test_compare_sls_vs_yaml_with_jinja(self):
        tpl = '{{ data }}'
        env = jinja2.Environment()
        src = '{foo: 1, bar: 2, baz: {qux: true}}'

        sls_src = env.from_string(tpl).render(data=yamlex.deserialize(src))
        yml_src = env.from_string(tpl).render(data=yaml.deserialize(src))

        sls_data = yamlex.deserialize(sls_src)
        yml_data = yaml.deserialize(yml_src)

        # ensure that sls & yaml have the same base
        assert isinstance(sls_data, dict)
        assert isinstance(yml_data, dict)
        assert sls_data == yml_data

        # ensure that sls is ordered, while yaml not
        assert isinstance(sls_data, OrderedDict)
        assert not isinstance(yml_data, OrderedDict)

        # prove that yaml does not handle well with OrderedDict
        # while sls is jinja friendly.
        obj = OrderedDict([
            ('foo', 1),
            ('bar', 2),
            ('baz', {'qux': True})
        ])

        sls_obj = yamlex.deserialize(yamlex.serialize(obj))
        try:
            yml_obj = yaml.deserialize(yaml.serialize(obj))
        except SerializationError:
            # BLAAM! yaml was unable to serialize OrderedDict,
            # but it's not the purpose of the current test.
            yml_obj = obj.copy()

        sls_src = env.from_string(tpl).render(data=sls_obj)
        yml_src = env.from_string(tpl).render(data=yml_obj)

        final_obj = yaml.deserialize(sls_src)
        assert obj == final_obj

        # BLAAM! yml_src is not valid !
        final_obj = OrderedDict(yaml.deserialize(yml_src))
        assert obj != final_obj
Ejemplo n.º 3
0
    def test_serialize_yaml(self):
        data = {
            "foo": "bar"
        }
        serialized = yaml.serialize(data)
        assert serialized == '{foo: bar}', serialized

        deserialized = yaml.deserialize(serialized)
        assert deserialized == data, deserialized
Ejemplo n.º 4
0
    def test_compare_sls_vs_yaml(self):
        src = '{foo: 1, bar: 2, baz: {qux: true}}'
        sls_data = yamlex.deserialize(src)
        yml_data = yaml.deserialize(src)

        # ensure that sls & yaml have the same base
        assert isinstance(sls_data, dict)
        assert isinstance(yml_data, dict)
        assert sls_data == yml_data

        # ensure that sls is ordered, while yaml not
        assert isinstance(sls_data, OrderedDict)
        assert not isinstance(yml_data, OrderedDict)
Ejemplo n.º 5
0
def _get_template(path, option_key):
    '''
    Get the contents of a template file and provide it as a module type

    :param path: path to the template.yml file
    :type  path: ``str``

    :param option_key: The unique key of this template
    :type  option_key: ``str``

    :returns: Details about the template
    :rtype: ``tuple``
    '''
    with salt.utils.fopen(path, "r") as template_f:
        template = deserialize(template_f)
        info = (option_key, template.get('description', ''), template)
    return info
Ejemplo n.º 6
0
def salt_syndic_master_factory(
    request,
    salt_factories,
    base_env_state_tree_root_dir,
    base_env_pillar_tree_root_dir,
    prod_env_state_tree_root_dir,
    prod_env_pillar_tree_root_dir,
):
    root_dir = salt_factories.get_root_dir_for_daemon("syndic_master")
    conf_dir = root_dir / "conf"
    conf_dir.mkdir(exist_ok=True)

    with salt.utils.files.fopen(
        os.path.join(RUNTIME_VARS.CONF_DIR, "syndic_master")
    ) as rfh:
        config_defaults = yaml.deserialize(rfh.read())

        tests_known_hosts_file = str(root_dir / "salt_ssh_known_hosts")
        with salt.utils.files.fopen(tests_known_hosts_file, "w") as known_hosts:
            known_hosts.write("")

    config_defaults["root_dir"] = str(root_dir)
    config_defaults["known_hosts_file"] = tests_known_hosts_file
    config_defaults["syndic_master"] = "localhost"
    config_defaults["transport"] = request.config.getoption("--transport")

    config_overrides = {}
    ext_pillar = []
    if salt.utils.platform.is_windows():
        ext_pillar.append(
            {"cmd_yaml": "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))}
        )
    else:
        ext_pillar.append(
            {"cmd_yaml": "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))}
        )

    # We need to copy the extension modules into the new master root_dir or
    # it will be prefixed by it
    extension_modules_path = str(root_dir / "extension_modules")
    if not os.path.exists(extension_modules_path):
        shutil.copytree(
            os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
            extension_modules_path,
        )

    # Copy the autosign_file to the new  master root_dir
    autosign_file_path = str(root_dir / "autosign_file")
    shutil.copyfile(
        os.path.join(RUNTIME_VARS.FILES, "autosign_file"), autosign_file_path
    )
    # all read, only owner write
    autosign_file_permissions = (
        stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR
    )
    os.chmod(autosign_file_path, autosign_file_permissions)

    config_overrides.update(
        {
            "ext_pillar": ext_pillar,
            "extension_modules": extension_modules_path,
            "file_roots": {
                "base": [
                    str(base_env_state_tree_root_dir),
                    os.path.join(RUNTIME_VARS.FILES, "file", "base"),
                ],
                # Alternate root to test __env__ choices
                "prod": [
                    str(prod_env_state_tree_root_dir),
                    os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
                ],
            },
            "pillar_roots": {
                "base": [
                    str(base_env_pillar_tree_root_dir),
                    os.path.join(RUNTIME_VARS.FILES, "pillar", "base"),
                ],
                "prod": [str(prod_env_pillar_tree_root_dir)],
            },
        }
    )

    factory = salt_factories.get_salt_master_daemon(
        "syndic_master",
        order_masters=True,
        config_defaults=config_defaults,
        config_overrides=config_overrides,
        extra_cli_arguments_after_first_start_failure=["--log-level=debug"],
    )
    return factory
Ejemplo n.º 7
0
def from_filenames_collection_modifyitems(config, items):
    from_filenames = config.getoption("--from-filenames")
    if not from_filenames:
        # Don't do anything
        return

    test_categories_paths = (
        (TESTS_DIR / "integration").relative_to(CODE_DIR),
        (TESTS_DIR / "multimaster").relative_to(CODE_DIR),
        (TESTS_DIR / "unit").relative_to(CODE_DIR),
        (PYTESTS_DIR / "e2e").relative_to(CODE_DIR),
        (PYTESTS_DIR / "functional").relative_to(CODE_DIR),
        (PYTESTS_DIR / "integration").relative_to(CODE_DIR),
        (PYTESTS_DIR / "unit").relative_to(CODE_DIR),
    )

    test_module_paths = set()
    from_filenames_listing = set()
    for path in [pathlib.Path(path.strip()) for path in from_filenames.split(",")]:
        if path.is_absolute():
            # In this case, this path is considered to be a file containing a line separated list
            # of files to consider
            with salt.utils.files.fopen(str(path)) as rfh:
                for line in rfh:
                    line_path = pathlib.Path(line.strip())
                    if not line_path.exists():
                        continue
                    from_filenames_listing.add(line_path)
            continue
        from_filenames_listing.add(path)

    filename_map = yaml.deserialize((TESTS_DIR / "filename_map.yml").read_text())
    # Let's add the match all rule
    for rule, matches in filename_map.items():
        if rule == "*":
            for match in matches:
                test_module_paths.add(_match_to_test_file(match))
            break

    # Let's now go through the list of files gathered
    for filename in from_filenames_listing:
        if str(filename).startswith("tests/"):
            # Tests in the listing don't require additional matching and will be added to the
            # list of tests to run
            test_module_paths.add(filename)
            continue
        if filename.name == "setup.py" or str(filename).startswith("salt/"):
            if path.name == "__init__.py":
                # No direct macthing
                continue
            # Now let's try a direct match between the passed file and possible test modules
            for test_categories_path in test_categories_paths:
                test_module_path = test_categories_path / "test_{}".format(path.name)
                if test_module_path.is_file():
                    test_module_paths.add(test_module_path)
                    continue

            # Do we have an entry in tests/filename_map.yml
            for rule, matches in filename_map.items():
                if rule == "*":
                    continue
                elif "|" in rule:
                    # This is regex
                    if re.match(rule, str(filename)):
                        for match in matches:
                            test_module_paths.add(_match_to_test_file(match))
                elif "*" in rule or "\\" in rule:
                    # Glob matching
                    for filerule in CODE_DIR.glob(rule):
                        if not filerule.exists():
                            continue
                        filerule = filerule.relative_to(CODE_DIR)
                        if filerule != filename:
                            continue
                        for match in matches:
                            test_module_paths.add(_match_to_test_file(match))
                else:
                    if str(filename) != rule:
                        continue
                    # Direct file paths as rules
                    filerule = pathlib.Path(rule)
                    if not filerule.exists():
                        continue
                    for match in matches:
                        test_module_paths.add(_match_to_test_file(match))
            continue
        else:
            log.debug("Don't know what to do with path %s", filename)

    selected = []
    deselected = []
    for item in items:
        itempath = pathlib.Path(str(item.fspath)).resolve().relative_to(CODE_DIR)
        if itempath in test_module_paths:
            selected.append(item)
        else:
            deselected.append(item)

    items[:] = selected
    if deselected:
        config.hook.pytest_deselected(items=deselected)
Ejemplo n.º 8
0
def salt_master_factory(
    request,
    salt_factories,
    salt_minion_id,
    salt_sub_minion_id,
    base_env_state_tree_root_dir,
    base_env_pillar_tree_root_dir,
    prod_env_state_tree_root_dir,
    prod_env_pillar_tree_root_dir,
    ext_pillar_file_tree_root_dir,
    sdb_etcd_port,
    vault_port,
    reactor_event,
    master_id,
    salt_auth_account_1_factory,
    salt_auth_account_2_factory,
    salt_netapi_account_factory,
    salt_eauth_account_factory,
    salt_auto_account_factory,
):
    root_dir = salt_factories.get_root_dir_for_daemon(master_id)
    conf_dir = root_dir / "conf"
    conf_dir.mkdir(exist_ok=True)

    with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.CONF_DIR, "master")) as rfh:
        config_defaults = yaml.deserialize(rfh.read())

        tests_known_hosts_file = str(root_dir / "salt_ssh_known_hosts")
        with salt.utils.files.fopen(tests_known_hosts_file, "w") as known_hosts:
            known_hosts.write("")

    config_defaults["root_dir"] = str(root_dir)
    config_defaults["known_hosts_file"] = tests_known_hosts_file
    config_defaults["syndic_master"] = "localhost"
    config_defaults["transport"] = request.config.getoption("--transport")
    config_defaults["reactor"] = [
        {reactor_event.tag: [str(reactor_event.sls_path)]},
    ]

    nodegroups = {
        "min": salt_minion_id,
        "sub_min": salt_sub_minion_id,
        "mins": "N@min or N@sub_min",
        "list_nodegroup": [salt_minion_id, salt_sub_minion_id],
        "multiline_nodegroup": [salt_minion_id, "or", salt_sub_minion_id],
        "one_minion_list": [salt_minion_id],
        "redundant_minions": "N@min or N@mins",
        "nodegroup_loop_a": "N@nodegroup_loop_b",
        "nodegroup_loop_b": "N@nodegroup_loop_a",
        "missing_minion": "L@{},ghostminion".format(salt_minion_id),
        "list_group": "N@multiline_nodegroup",
        "one_list_group": "N@one_minion_list",
        "list_group2": "N@list_nodegroup",
    }
    config_defaults["nodegroups"] = nodegroups
    config_defaults["sdbetcd"] = {
        "driver": "etcd",
        "etcd.host": "127.0.0.1",
        "etcd.port": sdb_etcd_port,
    }
    config_defaults["vault"] = {
        "url": "http://127.0.0.1:{}".format(vault_port),
        "auth": {"method": "token", "token": "testsecret", "uses": 0},
        "policies": ["testpolicy"],
    }

    # Config settings to test `event_return`
    config_defaults["returner_dirs"] = []
    config_defaults["returner_dirs"].append(
        os.path.join(RUNTIME_VARS.FILES, "returners")
    )
    config_defaults["event_return"] = "runtests_noop"
    config_overrides = {"pytest-master": {"log": {"level": "DEBUG"}}}
    ext_pillar = []
    if salt.utils.platform.is_windows():
        ext_pillar.append(
            {"cmd_yaml": "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))}
        )
    else:
        ext_pillar.append(
            {"cmd_yaml": "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))}
        )
    ext_pillar.append(
        {
            "file_tree": {
                "root_dir": str(ext_pillar_file_tree_root_dir),
                "follow_dir_links": False,
                "keep_newline": True,
            }
        }
    )
    config_overrides["pillar_opts"] = True
    config_overrides["external_auth"] = {
        "pam": {
            salt_auth_account_1_factory.username: ["test.*"],
            "{}%".format(salt_auth_account_2_factory.group_name): [
                "@wheel",
                "@runner",
                "test.*",
            ],
            salt_netapi_account_factory.username: ["@wheel", "@runner", "test.*"],
            salt_eauth_account_factory.username: ["@wheel", "@runner", "test.*"],
        },
        "auto": {
            salt_netapi_account_factory.username: [
                "@wheel",
                "@runner",
                "test.*",
                "grains.*",
            ],
            salt_auto_account_factory.username: ["@wheel", "@runner", "test.*"],
            "*": ["@wheel", "@runner", "test.*"],
        },
    }

    # We need to copy the extension modules into the new master root_dir or
    # it will be prefixed by it
    extension_modules_path = str(root_dir / "extension_modules")
    if not os.path.exists(extension_modules_path):
        shutil.copytree(
            os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
            extension_modules_path,
        )

    # Copy the autosign_file to the new  master root_dir
    autosign_file_path = str(root_dir / "autosign_file")
    shutil.copyfile(
        os.path.join(RUNTIME_VARS.FILES, "autosign_file"), autosign_file_path
    )
    # all read, only owner write
    autosign_file_permissions = (
        stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR
    )
    os.chmod(autosign_file_path, autosign_file_permissions)

    config_overrides.update(
        {
            "ext_pillar": ext_pillar,
            "extension_modules": extension_modules_path,
            "file_roots": {
                "base": [
                    str(base_env_state_tree_root_dir),
                    os.path.join(RUNTIME_VARS.FILES, "file", "base"),
                ],
                # Alternate root to test __env__ choices
                "prod": [
                    str(prod_env_state_tree_root_dir),
                    os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
                ],
            },
            "pillar_roots": {
                "base": [
                    str(base_env_pillar_tree_root_dir),
                    os.path.join(RUNTIME_VARS.FILES, "pillar", "base"),
                ],
                "prod": [str(prod_env_pillar_tree_root_dir)],
            },
        }
    )

    # Let's copy over the test cloud config files and directories into the running master config directory
    for entry in os.listdir(RUNTIME_VARS.CONF_DIR):
        if not entry.startswith("cloud"):
            continue
        source = os.path.join(RUNTIME_VARS.CONF_DIR, entry)
        dest = str(conf_dir / entry)
        if os.path.isdir(source):
            shutil.copytree(source, dest)
        else:
            shutil.copyfile(source, dest)

    factory = salt_factories.salt_master_daemon(
        master_id,
        defaults=config_defaults,
        overrides=config_overrides,
        extra_cli_arguments_after_first_start_failure=["--log-level=debug"],
    )
    return factory
Ejemplo n.º 9
0
def salt_master_factory(
    salt_factories,
    salt_syndic_master_factory,
    base_env_state_tree_root_dir,
    base_env_pillar_tree_root_dir,
    prod_env_state_tree_root_dir,
    prod_env_pillar_tree_root_dir,
):
    root_dir = salt_factories.get_root_dir_for_daemon("master")
    conf_dir = root_dir / "conf"
    conf_dir.mkdir(exist_ok=True)

    with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.CONF_DIR, "master")) as rfh:
        config_defaults = yaml.deserialize(rfh.read())

        tests_known_hosts_file = str(root_dir / "salt_ssh_known_hosts")
        with salt.utils.files.fopen(tests_known_hosts_file, "w") as known_hosts:
            known_hosts.write("")

    config_defaults["root_dir"] = str(root_dir)
    config_defaults["known_hosts_file"] = tests_known_hosts_file
    config_defaults["syndic_master"] = "localhost"
    config_defaults["transport"] = salt_syndic_master_factory.config["transport"]
    config_defaults["reactor"] = [
        {"salt/test/reactor": [os.path.join(RUNTIME_VARS.FILES, "reactor-test.sls")]}
    ]

    config_overrides = {}
    ext_pillar = []
    if salt.utils.platform.is_windows():
        ext_pillar.append(
            {"cmd_yaml": "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))}
        )
    else:
        ext_pillar.append(
            {"cmd_yaml": "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))}
        )
    ext_pillar.append(
        {
            "file_tree": {
                "root_dir": os.path.join(RUNTIME_VARS.PILLAR_DIR, "base", "file_tree"),
                "follow_dir_links": False,
                "keep_newline": True,
            }
        }
    )
    config_overrides["pillar_opts"] = True

    # We need to copy the extension modules into the new master root_dir or
    # it will be prefixed by it
    extension_modules_path = str(root_dir / "extension_modules")
    if not os.path.exists(extension_modules_path):
        shutil.copytree(
            os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
            extension_modules_path,
        )

    # Copy the autosign_file to the new  master root_dir
    autosign_file_path = str(root_dir / "autosign_file")
    shutil.copyfile(
        os.path.join(RUNTIME_VARS.FILES, "autosign_file"), autosign_file_path
    )
    # all read, only owner write
    autosign_file_permissions = (
        stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR
    )
    os.chmod(autosign_file_path, autosign_file_permissions)

    config_overrides.update(
        {
            "ext_pillar": ext_pillar,
            "extension_modules": extension_modules_path,
            "file_roots": {
                "base": [
                    str(base_env_state_tree_root_dir),
                    os.path.join(RUNTIME_VARS.FILES, "file", "base"),
                ],
                # Alternate root to test __env__ choices
                "prod": [
                    str(prod_env_state_tree_root_dir),
                    os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
                ],
            },
            "pillar_roots": {
                "base": [
                    str(base_env_pillar_tree_root_dir),
                    os.path.join(RUNTIME_VARS.FILES, "pillar", "base"),
                ],
                "prod": [str(prod_env_pillar_tree_root_dir)],
            },
        }
    )

    # Let's copy over the test cloud config files and directories into the running master config directory
    for entry in os.listdir(RUNTIME_VARS.CONF_DIR):
        if not entry.startswith("cloud"):
            continue
        source = os.path.join(RUNTIME_VARS.CONF_DIR, entry)
        dest = str(conf_dir / entry)
        if os.path.isdir(source):
            shutil.copytree(source, dest)
        else:
            shutil.copyfile(source, dest)

    factory = salt_syndic_master_factory.get_salt_master_daemon(
        "master",
        config_defaults=config_defaults,
        config_overrides=config_overrides,
        extra_cli_arguments_after_first_start_failure=["--log-level=debug"],
    )
    return factory
Ejemplo n.º 10
0
def session_mm_minion_default_options(session_minion_default_options):
    opts = session_minion_default_options.copy()
    with salt.utils.files.fopen(
            os.path.join(RUNTIME_VARS.CONF_DIR, 'mm_sub_minion')) as rfh:
        opts.update(yaml.deserialize(rfh.read()))
    return opts
Ejemplo n.º 11
0
def salt_master_config(request, salt_factories, salt_syndic_master_config):
    root_dir = salt_factories._get_root_dir_for_daemon("master")
    conf_dir = root_dir.join("conf").ensure(dir=True)

    with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.CONF_DIR,
                                             "master")) as rfh:
        config_defaults = yaml.deserialize(rfh.read())

        tests_known_hosts_file = root_dir.join("salt_ssh_known_hosts").strpath
        with salt.utils.files.fopen(tests_known_hosts_file,
                                    "w") as known_hosts:
            known_hosts.write("")

    config_defaults["root_dir"] = root_dir.strpath
    config_defaults["known_hosts_file"] = tests_known_hosts_file
    config_defaults["syndic_master"] = "localhost"
    config_defaults["transport"] = request.config.getoption("--transport")
    config_defaults["reactor"] = [{
        "salt/test/reactor":
        [os.path.join(RUNTIME_VARS.FILES, "reactor-test.sls")]
    }]

    config_overrides = {}
    ext_pillar = []
    if salt.utils.platform.is_windows():
        ext_pillar.append({
            "cmd_yaml":
            "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))
        })
    else:
        ext_pillar.append({
            "cmd_yaml":
            "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))
        })
    ext_pillar.append({
        "file_tree": {
            "root_dir": os.path.join(RUNTIME_VARS.PILLAR_DIR, "base",
                                     "file_tree"),
            "follow_dir_links": False,
            "keep_newline": True,
        }
    })
    config_overrides["pillar_opts"] = True

    # We need to copy the extension modules into the new master root_dir or
    # it will be prefixed by it
    extension_modules_path = root_dir.join("extension_modules").strpath
    if not os.path.exists(extension_modules_path):
        shutil.copytree(
            os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
            extension_modules_path,
        )

    # Copy the autosign_file to the new  master root_dir
    autosign_file_path = root_dir.join("autosign_file").strpath
    shutil.copyfile(os.path.join(RUNTIME_VARS.FILES, "autosign_file"),
                    autosign_file_path)
    # all read, only owner write
    autosign_file_permissions = (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
                                 | stat.S_IWUSR)
    os.chmod(autosign_file_path, autosign_file_permissions)

    config_overrides.update({
        "ext_pillar": ext_pillar,
        "extension_modules": extension_modules_path,
        "file_roots": {
            "base": [
                RUNTIME_VARS.TMP_STATE_TREE,
                os.path.join(RUNTIME_VARS.FILES, "file", "base"),
            ],
            # Alternate root to test __env__ choices
            "prod": [
                RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
                os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
            ],
        },
        "pillar_roots": {
            "base": [
                RUNTIME_VARS.TMP_PILLAR_TREE,
                os.path.join(RUNTIME_VARS.FILES, "pillar", "base"),
            ],
            "prod": [RUNTIME_VARS.TMP_PRODENV_PILLAR_TREE],
        },
    })

    # Let's copy over the test cloud config files and directories into the running master config directory
    for entry in os.listdir(RUNTIME_VARS.CONF_DIR):
        if not entry.startswith("cloud"):
            continue
        source = os.path.join(RUNTIME_VARS.CONF_DIR, entry)
        dest = conf_dir.join(entry).strpath
        if os.path.isdir(source):
            shutil.copytree(source, dest)
        else:
            shutil.copyfile(source, dest)

    return salt_factories.configure_master(
        request,
        "master",
        master_of_masters_id="syndic_master",
        config_defaults=config_defaults,
        config_overrides=config_overrides,
    )
Ejemplo n.º 12
0
def salt_syndic_master_config(request, salt_factories):
    root_dir = salt_factories._get_root_dir_for_daemon("syndic_master")

    with salt.utils.files.fopen(
            os.path.join(RUNTIME_VARS.CONF_DIR, "syndic_master")) as rfh:
        config_defaults = yaml.deserialize(rfh.read())

        tests_known_hosts_file = root_dir.join("salt_ssh_known_hosts").strpath
        with salt.utils.files.fopen(tests_known_hosts_file,
                                    "w") as known_hosts:
            known_hosts.write("")

    config_defaults["root_dir"] = root_dir.strpath
    config_defaults["known_hosts_file"] = tests_known_hosts_file
    config_defaults["syndic_master"] = "localhost"
    config_defaults["transport"] = request.config.getoption("--transport")

    config_overrides = {}
    ext_pillar = []
    if salt.utils.platform.is_windows():
        ext_pillar.append({
            "cmd_yaml":
            "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))
        })
    else:
        ext_pillar.append({
            "cmd_yaml":
            "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))
        })

    # We need to copy the extension modules into the new master root_dir or
    # it will be prefixed by it
    extension_modules_path = root_dir.join("extension_modules").strpath
    if not os.path.exists(extension_modules_path):
        shutil.copytree(
            os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
            extension_modules_path,
        )

    # Copy the autosign_file to the new  master root_dir
    autosign_file_path = root_dir.join("autosign_file").strpath
    shutil.copyfile(os.path.join(RUNTIME_VARS.FILES, "autosign_file"),
                    autosign_file_path)
    # all read, only owner write
    autosign_file_permissions = (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
                                 | stat.S_IWUSR)
    os.chmod(autosign_file_path, autosign_file_permissions)

    config_overrides.update({
        "ext_pillar": ext_pillar,
        "extension_modules": extension_modules_path,
        "file_roots": {
            "base": [
                RUNTIME_VARS.TMP_STATE_TREE,
                os.path.join(RUNTIME_VARS.FILES, "file", "base"),
            ],
            # Alternate root to test __env__ choices
            "prod": [
                RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
                os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
            ],
        },
        "pillar_roots": {
            "base": [
                RUNTIME_VARS.TMP_PILLAR_TREE,
                os.path.join(RUNTIME_VARS.FILES, "pillar", "base"),
            ],
            "prod": [RUNTIME_VARS.TMP_PRODENV_PILLAR_TREE],
        },
    })
    return salt_factories.configure_master(
        request,
        "syndic_master",
        order_masters=True,
        config_defaults=config_defaults,
        config_overrides=config_overrides,
    )
Ejemplo n.º 13
0
def from_filenames_collection_modifyitems(config, items):
    from_filenames = config.getoption("--from-filenames")
    if not from_filenames:
        # Don't do anything
        return

    log.info(
        "Calculating test modules to run based on the paths in --from-filenames"
    )
    from_filenames_paths = set()
    for path in [path.strip() for path in from_filenames.split(",")]:
        # Make sure that, no matter what kind of path we're passed, Windows or Posix path,
        # we resolve it to the platform slash separator
        properly_slashed_path = pathlib.Path(
            path.replace("\\", os.sep).replace("/", os.sep))
        if not properly_slashed_path.exists():
            log.info(
                "The path %s(%s) passed in --from-filenames does not exist",
                path,
                properly_slashed_path,
            )
            continue
        if properly_slashed_path.is_absolute():
            # In this case, this path is considered to be a file containing a line separated list
            # of files to consider
            with salt.utils.files.fopen(str(path)) as rfh:
                for line in rfh:
                    line_path = pathlib.Path(line.strip().replace(
                        "\\", os.sep).replace("/", os.sep))
                    if not line_path.exists():
                        log.info(
                            "The path %s contained in %s passed in --from-filenames does not exist",
                            line_path,
                            properly_slashed_path,
                        )
                        continue
                    from_filenames_paths.add(line_path)
            continue
        from_filenames_paths.add(properly_slashed_path)

    # Let's start collecting test modules
    test_module_paths = set()

    filename_map = yaml.deserialize(
        (TESTS_DIR / "filename_map.yml").read_text())
    # Let's add the match all rule
    for rule, matches in filename_map.items():
        if rule == "*":
            for match in matches:
                test_module_paths.add(_match_to_test_file(match))
            break

    # Let's now go through the list of files gathered
    for path in from_filenames_paths:
        if path.as_posix().startswith("tests/"):
            if path.name == "conftest.py":
                # This is not a test module, but consider any test_*.py files in child directories
                for match in path.parent.rglob("test_*.py"):
                    test_module_paths.add(match)
                continue
            # Tests in the listing don't require additional matching and will be added to the
            # list of tests to run
            test_module_paths.add(path)
            continue
        if path.name == "setup.py" or path.as_posix().startswith("salt/"):
            if path.name == "__init__.py":
                # No direct matching
                continue

            # Let's try a direct match between the passed file and possible test modules
            glob_patterns = (
                # salt/version.py ->
                #    tests/unit/test_version.py
                #    tests/pytests/unit/test_version.py
                "**/test_{}".format(path.name),
                # salt/modules/grains.py ->
                #    tests/pytests/integration/modules/grains/tests_*.py
                # salt/modules/saltutil.py ->
                #    tests/pytests/integration/modules/saltutil/test_*.py
                "**/{}/test_*.py".format(path.stem),
                # salt/modules/config.py ->
                #    tests/unit/modules/test_config.py
                #    tests/integration/modules/test_config.py
                #    tests/pytests/unit/modules/test_config.py
                #    tests/pytests/integration/modules/test_config.py
                "**/{}/test_{}".format(path.parent.name, path.name),
            )
            for pattern in glob_patterns:
                for match in TESTS_DIR.rglob(pattern):
                    relative_path = match.relative_to(CODE_DIR)
                    log.info("Glob pattern %r matched '%s'", pattern,
                             relative_path)
                    test_module_paths.add(relative_path)

            # Do we have an entry in tests/filename_map.yml
            for rule, matches in filename_map.items():
                if rule == "*":
                    continue
                elif "|" in rule:
                    # This is regex
                    if re.match(rule, path.as_posix()):
                        for match in matches:
                            test_module_paths.add(_match_to_test_file(match))
                elif "*" in rule or "\\" in rule:
                    # Glob matching
                    for filerule in CODE_DIR.glob(rule):
                        if not filerule.exists():
                            continue
                        filerule = filerule.relative_to(CODE_DIR)
                        if filerule != path:
                            continue
                        for match in matches:
                            test_module_paths.add(_match_to_test_file(match))
                else:
                    if path.as_posix() != rule:
                        continue
                    # Direct file paths as rules
                    filerule = pathlib.Path(rule)
                    if not filerule.exists():
                        continue
                    for match in matches:
                        test_module_paths.add(_match_to_test_file(match))
            continue
        else:
            log.info("Don't know what to do with path %s", path)

    log.info(
        "Collected the following paths from --from-filenames processing:\n%s",
        "\n".join(sorted(map(str, test_module_paths))),
    )
    selected = []
    deselected = []
    for item in items:
        itempath = pathlib.Path(str(
            item.fspath)).resolve().relative_to(CODE_DIR)
        if itempath in test_module_paths:
            selected.append(item)
        else:
            deselected.append(item)

    items[:] = selected
    if deselected:
        config.hook.pytest_deselected(items=deselected)