def test_compare_sls_vs_yaml_with_jinja(self): tpl = '{{ data }}' env = jinja2.Environment() src = '{foo: 1, bar: 2, baz: {qux: true}}' sls_src = env.from_string(tpl).render(data=yamlex.deserialize(src)) yml_src = env.from_string(tpl).render(data=yaml.deserialize(src)) sls_data = yamlex.deserialize(sls_src) yml_data = yaml.deserialize(yml_src) # ensure that sls & yaml have the same base assert isinstance(sls_data, dict) assert isinstance(yml_data, dict) # The below has been commented out because something the loader test # is modifying the yaml renderer to render things to unicode. Without # running the loader test, the below passes. Even reloading the module # from disk does not reset its internal state (per the Python docs). ## #assert sls_data == yml_data # ensure that sls is ordered, while yaml not assert isinstance(sls_data, OrderedDict) assert not isinstance(yml_data, OrderedDict) # prove that yaml does not handle well with OrderedDict # while sls is jinja friendly. obj = OrderedDict([ ('foo', 1), ('bar', 2), ('baz', {'qux': True}) ]) sls_obj = yamlex.deserialize(yamlex.serialize(obj)) try: yml_obj = yaml.deserialize(yaml.serialize(obj)) except SerializationError: # BLAAM! yaml was unable to serialize OrderedDict, # but it's not the purpose of the current test. yml_obj = obj.copy() sls_src = env.from_string(tpl).render(data=sls_obj) yml_src = env.from_string(tpl).render(data=yml_obj) final_obj = yaml.deserialize(sls_src) assert obj == final_obj # BLAAM! yml_src is not valid ! final_obj = OrderedDict(yaml.deserialize(yml_src)) assert obj != final_obj
def test_compare_sls_vs_yaml_with_jinja(self): tpl = '{{ data }}' env = jinja2.Environment() src = '{foo: 1, bar: 2, baz: {qux: true}}' sls_src = env.from_string(tpl).render(data=yamlex.deserialize(src)) yml_src = env.from_string(tpl).render(data=yaml.deserialize(src)) sls_data = yamlex.deserialize(sls_src) yml_data = yaml.deserialize(yml_src) # ensure that sls & yaml have the same base assert isinstance(sls_data, dict) assert isinstance(yml_data, dict) assert sls_data == yml_data # ensure that sls is ordered, while yaml not assert isinstance(sls_data, OrderedDict) assert not isinstance(yml_data, OrderedDict) # prove that yaml does not handle well with OrderedDict # while sls is jinja friendly. obj = OrderedDict([ ('foo', 1), ('bar', 2), ('baz', {'qux': True}) ]) sls_obj = yamlex.deserialize(yamlex.serialize(obj)) try: yml_obj = yaml.deserialize(yaml.serialize(obj)) except SerializationError: # BLAAM! yaml was unable to serialize OrderedDict, # but it's not the purpose of the current test. yml_obj = obj.copy() sls_src = env.from_string(tpl).render(data=sls_obj) yml_src = env.from_string(tpl).render(data=yml_obj) final_obj = yaml.deserialize(sls_src) assert obj == final_obj # BLAAM! yml_src is not valid ! final_obj = OrderedDict(yaml.deserialize(yml_src)) assert obj != final_obj
def test_serialize_yaml(self): data = { "foo": "bar" } serialized = yaml.serialize(data) assert serialized == '{foo: bar}', serialized deserialized = yaml.deserialize(serialized) assert deserialized == data, deserialized
def test_compare_sls_vs_yaml(self): src = '{foo: 1, bar: 2, baz: {qux: true}}' sls_data = yamlex.deserialize(src) yml_data = yaml.deserialize(src) # ensure that sls & yaml have the same base assert isinstance(sls_data, dict) assert isinstance(yml_data, dict) assert sls_data == yml_data # ensure that sls is ordered, while yaml not assert isinstance(sls_data, OrderedDict) assert not isinstance(yml_data, OrderedDict)
def _get_template(path, option_key): ''' Get the contents of a template file and provide it as a module type :param path: path to the template.yml file :type path: ``str`` :param option_key: The unique key of this template :type option_key: ``str`` :returns: Details about the template :rtype: ``tuple`` ''' with salt.utils.fopen(path, "r") as template_f: template = deserialize(template_f) info = (option_key, template.get('description', ''), template) return info
def salt_syndic_master_factory( request, salt_factories, base_env_state_tree_root_dir, base_env_pillar_tree_root_dir, prod_env_state_tree_root_dir, prod_env_pillar_tree_root_dir, ): root_dir = salt_factories.get_root_dir_for_daemon("syndic_master") conf_dir = root_dir / "conf" conf_dir.mkdir(exist_ok=True) with salt.utils.files.fopen( os.path.join(RUNTIME_VARS.CONF_DIR, "syndic_master") ) as rfh: config_defaults = yaml.deserialize(rfh.read()) tests_known_hosts_file = str(root_dir / "salt_ssh_known_hosts") with salt.utils.files.fopen(tests_known_hosts_file, "w") as known_hosts: known_hosts.write("") config_defaults["root_dir"] = str(root_dir) config_defaults["known_hosts_file"] = tests_known_hosts_file config_defaults["syndic_master"] = "localhost" config_defaults["transport"] = request.config.getoption("--transport") config_overrides = {} ext_pillar = [] if salt.utils.platform.is_windows(): ext_pillar.append( {"cmd_yaml": "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))} ) else: ext_pillar.append( {"cmd_yaml": "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))} ) # We need to copy the extension modules into the new master root_dir or # it will be prefixed by it extension_modules_path = str(root_dir / "extension_modules") if not os.path.exists(extension_modules_path): shutil.copytree( os.path.join(RUNTIME_VARS.FILES, "extension_modules"), extension_modules_path, ) # Copy the autosign_file to the new master root_dir autosign_file_path = str(root_dir / "autosign_file") shutil.copyfile( os.path.join(RUNTIME_VARS.FILES, "autosign_file"), autosign_file_path ) # all read, only owner write autosign_file_permissions = ( stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR ) os.chmod(autosign_file_path, autosign_file_permissions) config_overrides.update( { "ext_pillar": ext_pillar, "extension_modules": extension_modules_path, "file_roots": { "base": [ str(base_env_state_tree_root_dir), os.path.join(RUNTIME_VARS.FILES, "file", "base"), ], # Alternate root to test __env__ choices "prod": [ str(prod_env_state_tree_root_dir), os.path.join(RUNTIME_VARS.FILES, "file", "prod"), ], }, "pillar_roots": { "base": [ str(base_env_pillar_tree_root_dir), os.path.join(RUNTIME_VARS.FILES, "pillar", "base"), ], "prod": [str(prod_env_pillar_tree_root_dir)], }, } ) factory = salt_factories.get_salt_master_daemon( "syndic_master", order_masters=True, config_defaults=config_defaults, config_overrides=config_overrides, extra_cli_arguments_after_first_start_failure=["--log-level=debug"], ) return factory
def from_filenames_collection_modifyitems(config, items): from_filenames = config.getoption("--from-filenames") if not from_filenames: # Don't do anything return test_categories_paths = ( (TESTS_DIR / "integration").relative_to(CODE_DIR), (TESTS_DIR / "multimaster").relative_to(CODE_DIR), (TESTS_DIR / "unit").relative_to(CODE_DIR), (PYTESTS_DIR / "e2e").relative_to(CODE_DIR), (PYTESTS_DIR / "functional").relative_to(CODE_DIR), (PYTESTS_DIR / "integration").relative_to(CODE_DIR), (PYTESTS_DIR / "unit").relative_to(CODE_DIR), ) test_module_paths = set() from_filenames_listing = set() for path in [pathlib.Path(path.strip()) for path in from_filenames.split(",")]: if path.is_absolute(): # In this case, this path is considered to be a file containing a line separated list # of files to consider with salt.utils.files.fopen(str(path)) as rfh: for line in rfh: line_path = pathlib.Path(line.strip()) if not line_path.exists(): continue from_filenames_listing.add(line_path) continue from_filenames_listing.add(path) filename_map = yaml.deserialize((TESTS_DIR / "filename_map.yml").read_text()) # Let's add the match all rule for rule, matches in filename_map.items(): if rule == "*": for match in matches: test_module_paths.add(_match_to_test_file(match)) break # Let's now go through the list of files gathered for filename in from_filenames_listing: if str(filename).startswith("tests/"): # Tests in the listing don't require additional matching and will be added to the # list of tests to run test_module_paths.add(filename) continue if filename.name == "setup.py" or str(filename).startswith("salt/"): if path.name == "__init__.py": # No direct macthing continue # Now let's try a direct match between the passed file and possible test modules for test_categories_path in test_categories_paths: test_module_path = test_categories_path / "test_{}".format(path.name) if test_module_path.is_file(): test_module_paths.add(test_module_path) continue # Do we have an entry in tests/filename_map.yml for rule, matches in filename_map.items(): if rule == "*": continue elif "|" in rule: # This is regex if re.match(rule, str(filename)): for match in matches: test_module_paths.add(_match_to_test_file(match)) elif "*" in rule or "\\" in rule: # Glob matching for filerule in CODE_DIR.glob(rule): if not filerule.exists(): continue filerule = filerule.relative_to(CODE_DIR) if filerule != filename: continue for match in matches: test_module_paths.add(_match_to_test_file(match)) else: if str(filename) != rule: continue # Direct file paths as rules filerule = pathlib.Path(rule) if not filerule.exists(): continue for match in matches: test_module_paths.add(_match_to_test_file(match)) continue else: log.debug("Don't know what to do with path %s", filename) selected = [] deselected = [] for item in items: itempath = pathlib.Path(str(item.fspath)).resolve().relative_to(CODE_DIR) if itempath in test_module_paths: selected.append(item) else: deselected.append(item) items[:] = selected if deselected: config.hook.pytest_deselected(items=deselected)
def salt_master_factory( request, salt_factories, salt_minion_id, salt_sub_minion_id, base_env_state_tree_root_dir, base_env_pillar_tree_root_dir, prod_env_state_tree_root_dir, prod_env_pillar_tree_root_dir, ext_pillar_file_tree_root_dir, sdb_etcd_port, vault_port, reactor_event, master_id, salt_auth_account_1_factory, salt_auth_account_2_factory, salt_netapi_account_factory, salt_eauth_account_factory, salt_auto_account_factory, ): root_dir = salt_factories.get_root_dir_for_daemon(master_id) conf_dir = root_dir / "conf" conf_dir.mkdir(exist_ok=True) with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.CONF_DIR, "master")) as rfh: config_defaults = yaml.deserialize(rfh.read()) tests_known_hosts_file = str(root_dir / "salt_ssh_known_hosts") with salt.utils.files.fopen(tests_known_hosts_file, "w") as known_hosts: known_hosts.write("") config_defaults["root_dir"] = str(root_dir) config_defaults["known_hosts_file"] = tests_known_hosts_file config_defaults["syndic_master"] = "localhost" config_defaults["transport"] = request.config.getoption("--transport") config_defaults["reactor"] = [ {reactor_event.tag: [str(reactor_event.sls_path)]}, ] nodegroups = { "min": salt_minion_id, "sub_min": salt_sub_minion_id, "mins": "N@min or N@sub_min", "list_nodegroup": [salt_minion_id, salt_sub_minion_id], "multiline_nodegroup": [salt_minion_id, "or", salt_sub_minion_id], "one_minion_list": [salt_minion_id], "redundant_minions": "N@min or N@mins", "nodegroup_loop_a": "N@nodegroup_loop_b", "nodegroup_loop_b": "N@nodegroup_loop_a", "missing_minion": "L@{},ghostminion".format(salt_minion_id), "list_group": "N@multiline_nodegroup", "one_list_group": "N@one_minion_list", "list_group2": "N@list_nodegroup", } config_defaults["nodegroups"] = nodegroups config_defaults["sdbetcd"] = { "driver": "etcd", "etcd.host": "127.0.0.1", "etcd.port": sdb_etcd_port, } config_defaults["vault"] = { "url": "http://127.0.0.1:{}".format(vault_port), "auth": {"method": "token", "token": "testsecret", "uses": 0}, "policies": ["testpolicy"], } # Config settings to test `event_return` config_defaults["returner_dirs"] = [] config_defaults["returner_dirs"].append( os.path.join(RUNTIME_VARS.FILES, "returners") ) config_defaults["event_return"] = "runtests_noop" config_overrides = {"pytest-master": {"log": {"level": "DEBUG"}}} ext_pillar = [] if salt.utils.platform.is_windows(): ext_pillar.append( {"cmd_yaml": "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))} ) else: ext_pillar.append( {"cmd_yaml": "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))} ) ext_pillar.append( { "file_tree": { "root_dir": str(ext_pillar_file_tree_root_dir), "follow_dir_links": False, "keep_newline": True, } } ) config_overrides["pillar_opts"] = True config_overrides["external_auth"] = { "pam": { salt_auth_account_1_factory.username: ["test.*"], "{}%".format(salt_auth_account_2_factory.group_name): [ "@wheel", "@runner", "test.*", ], salt_netapi_account_factory.username: ["@wheel", "@runner", "test.*"], salt_eauth_account_factory.username: ["@wheel", "@runner", "test.*"], }, "auto": { salt_netapi_account_factory.username: [ "@wheel", "@runner", "test.*", "grains.*", ], salt_auto_account_factory.username: ["@wheel", "@runner", "test.*"], "*": ["@wheel", "@runner", "test.*"], }, } # We need to copy the extension modules into the new master root_dir or # it will be prefixed by it extension_modules_path = str(root_dir / "extension_modules") if not os.path.exists(extension_modules_path): shutil.copytree( os.path.join(RUNTIME_VARS.FILES, "extension_modules"), extension_modules_path, ) # Copy the autosign_file to the new master root_dir autosign_file_path = str(root_dir / "autosign_file") shutil.copyfile( os.path.join(RUNTIME_VARS.FILES, "autosign_file"), autosign_file_path ) # all read, only owner write autosign_file_permissions = ( stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR ) os.chmod(autosign_file_path, autosign_file_permissions) config_overrides.update( { "ext_pillar": ext_pillar, "extension_modules": extension_modules_path, "file_roots": { "base": [ str(base_env_state_tree_root_dir), os.path.join(RUNTIME_VARS.FILES, "file", "base"), ], # Alternate root to test __env__ choices "prod": [ str(prod_env_state_tree_root_dir), os.path.join(RUNTIME_VARS.FILES, "file", "prod"), ], }, "pillar_roots": { "base": [ str(base_env_pillar_tree_root_dir), os.path.join(RUNTIME_VARS.FILES, "pillar", "base"), ], "prod": [str(prod_env_pillar_tree_root_dir)], }, } ) # Let's copy over the test cloud config files and directories into the running master config directory for entry in os.listdir(RUNTIME_VARS.CONF_DIR): if not entry.startswith("cloud"): continue source = os.path.join(RUNTIME_VARS.CONF_DIR, entry) dest = str(conf_dir / entry) if os.path.isdir(source): shutil.copytree(source, dest) else: shutil.copyfile(source, dest) factory = salt_factories.salt_master_daemon( master_id, defaults=config_defaults, overrides=config_overrides, extra_cli_arguments_after_first_start_failure=["--log-level=debug"], ) return factory
def salt_master_factory( salt_factories, salt_syndic_master_factory, base_env_state_tree_root_dir, base_env_pillar_tree_root_dir, prod_env_state_tree_root_dir, prod_env_pillar_tree_root_dir, ): root_dir = salt_factories.get_root_dir_for_daemon("master") conf_dir = root_dir / "conf" conf_dir.mkdir(exist_ok=True) with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.CONF_DIR, "master")) as rfh: config_defaults = yaml.deserialize(rfh.read()) tests_known_hosts_file = str(root_dir / "salt_ssh_known_hosts") with salt.utils.files.fopen(tests_known_hosts_file, "w") as known_hosts: known_hosts.write("") config_defaults["root_dir"] = str(root_dir) config_defaults["known_hosts_file"] = tests_known_hosts_file config_defaults["syndic_master"] = "localhost" config_defaults["transport"] = salt_syndic_master_factory.config["transport"] config_defaults["reactor"] = [ {"salt/test/reactor": [os.path.join(RUNTIME_VARS.FILES, "reactor-test.sls")]} ] config_overrides = {} ext_pillar = [] if salt.utils.platform.is_windows(): ext_pillar.append( {"cmd_yaml": "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))} ) else: ext_pillar.append( {"cmd_yaml": "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))} ) ext_pillar.append( { "file_tree": { "root_dir": os.path.join(RUNTIME_VARS.PILLAR_DIR, "base", "file_tree"), "follow_dir_links": False, "keep_newline": True, } } ) config_overrides["pillar_opts"] = True # We need to copy the extension modules into the new master root_dir or # it will be prefixed by it extension_modules_path = str(root_dir / "extension_modules") if not os.path.exists(extension_modules_path): shutil.copytree( os.path.join(RUNTIME_VARS.FILES, "extension_modules"), extension_modules_path, ) # Copy the autosign_file to the new master root_dir autosign_file_path = str(root_dir / "autosign_file") shutil.copyfile( os.path.join(RUNTIME_VARS.FILES, "autosign_file"), autosign_file_path ) # all read, only owner write autosign_file_permissions = ( stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR ) os.chmod(autosign_file_path, autosign_file_permissions) config_overrides.update( { "ext_pillar": ext_pillar, "extension_modules": extension_modules_path, "file_roots": { "base": [ str(base_env_state_tree_root_dir), os.path.join(RUNTIME_VARS.FILES, "file", "base"), ], # Alternate root to test __env__ choices "prod": [ str(prod_env_state_tree_root_dir), os.path.join(RUNTIME_VARS.FILES, "file", "prod"), ], }, "pillar_roots": { "base": [ str(base_env_pillar_tree_root_dir), os.path.join(RUNTIME_VARS.FILES, "pillar", "base"), ], "prod": [str(prod_env_pillar_tree_root_dir)], }, } ) # Let's copy over the test cloud config files and directories into the running master config directory for entry in os.listdir(RUNTIME_VARS.CONF_DIR): if not entry.startswith("cloud"): continue source = os.path.join(RUNTIME_VARS.CONF_DIR, entry) dest = str(conf_dir / entry) if os.path.isdir(source): shutil.copytree(source, dest) else: shutil.copyfile(source, dest) factory = salt_syndic_master_factory.get_salt_master_daemon( "master", config_defaults=config_defaults, config_overrides=config_overrides, extra_cli_arguments_after_first_start_failure=["--log-level=debug"], ) return factory
def session_mm_minion_default_options(session_minion_default_options): opts = session_minion_default_options.copy() with salt.utils.files.fopen( os.path.join(RUNTIME_VARS.CONF_DIR, 'mm_sub_minion')) as rfh: opts.update(yaml.deserialize(rfh.read())) return opts
def salt_master_config(request, salt_factories, salt_syndic_master_config): root_dir = salt_factories._get_root_dir_for_daemon("master") conf_dir = root_dir.join("conf").ensure(dir=True) with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.CONF_DIR, "master")) as rfh: config_defaults = yaml.deserialize(rfh.read()) tests_known_hosts_file = root_dir.join("salt_ssh_known_hosts").strpath with salt.utils.files.fopen(tests_known_hosts_file, "w") as known_hosts: known_hosts.write("") config_defaults["root_dir"] = root_dir.strpath config_defaults["known_hosts_file"] = tests_known_hosts_file config_defaults["syndic_master"] = "localhost" config_defaults["transport"] = request.config.getoption("--transport") config_defaults["reactor"] = [{ "salt/test/reactor": [os.path.join(RUNTIME_VARS.FILES, "reactor-test.sls")] }] config_overrides = {} ext_pillar = [] if salt.utils.platform.is_windows(): ext_pillar.append({ "cmd_yaml": "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml")) }) else: ext_pillar.append({ "cmd_yaml": "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml")) }) ext_pillar.append({ "file_tree": { "root_dir": os.path.join(RUNTIME_VARS.PILLAR_DIR, "base", "file_tree"), "follow_dir_links": False, "keep_newline": True, } }) config_overrides["pillar_opts"] = True # We need to copy the extension modules into the new master root_dir or # it will be prefixed by it extension_modules_path = root_dir.join("extension_modules").strpath if not os.path.exists(extension_modules_path): shutil.copytree( os.path.join(RUNTIME_VARS.FILES, "extension_modules"), extension_modules_path, ) # Copy the autosign_file to the new master root_dir autosign_file_path = root_dir.join("autosign_file").strpath shutil.copyfile(os.path.join(RUNTIME_VARS.FILES, "autosign_file"), autosign_file_path) # all read, only owner write autosign_file_permissions = (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR) os.chmod(autosign_file_path, autosign_file_permissions) config_overrides.update({ "ext_pillar": ext_pillar, "extension_modules": extension_modules_path, "file_roots": { "base": [ RUNTIME_VARS.TMP_STATE_TREE, os.path.join(RUNTIME_VARS.FILES, "file", "base"), ], # Alternate root to test __env__ choices "prod": [ RUNTIME_VARS.TMP_PRODENV_STATE_TREE, os.path.join(RUNTIME_VARS.FILES, "file", "prod"), ], }, "pillar_roots": { "base": [ RUNTIME_VARS.TMP_PILLAR_TREE, os.path.join(RUNTIME_VARS.FILES, "pillar", "base"), ], "prod": [RUNTIME_VARS.TMP_PRODENV_PILLAR_TREE], }, }) # Let's copy over the test cloud config files and directories into the running master config directory for entry in os.listdir(RUNTIME_VARS.CONF_DIR): if not entry.startswith("cloud"): continue source = os.path.join(RUNTIME_VARS.CONF_DIR, entry) dest = conf_dir.join(entry).strpath if os.path.isdir(source): shutil.copytree(source, dest) else: shutil.copyfile(source, dest) return salt_factories.configure_master( request, "master", master_of_masters_id="syndic_master", config_defaults=config_defaults, config_overrides=config_overrides, )
def salt_syndic_master_config(request, salt_factories): root_dir = salt_factories._get_root_dir_for_daemon("syndic_master") with salt.utils.files.fopen( os.path.join(RUNTIME_VARS.CONF_DIR, "syndic_master")) as rfh: config_defaults = yaml.deserialize(rfh.read()) tests_known_hosts_file = root_dir.join("salt_ssh_known_hosts").strpath with salt.utils.files.fopen(tests_known_hosts_file, "w") as known_hosts: known_hosts.write("") config_defaults["root_dir"] = root_dir.strpath config_defaults["known_hosts_file"] = tests_known_hosts_file config_defaults["syndic_master"] = "localhost" config_defaults["transport"] = request.config.getoption("--transport") config_overrides = {} ext_pillar = [] if salt.utils.platform.is_windows(): ext_pillar.append({ "cmd_yaml": "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml")) }) else: ext_pillar.append({ "cmd_yaml": "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml")) }) # We need to copy the extension modules into the new master root_dir or # it will be prefixed by it extension_modules_path = root_dir.join("extension_modules").strpath if not os.path.exists(extension_modules_path): shutil.copytree( os.path.join(RUNTIME_VARS.FILES, "extension_modules"), extension_modules_path, ) # Copy the autosign_file to the new master root_dir autosign_file_path = root_dir.join("autosign_file").strpath shutil.copyfile(os.path.join(RUNTIME_VARS.FILES, "autosign_file"), autosign_file_path) # all read, only owner write autosign_file_permissions = (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR) os.chmod(autosign_file_path, autosign_file_permissions) config_overrides.update({ "ext_pillar": ext_pillar, "extension_modules": extension_modules_path, "file_roots": { "base": [ RUNTIME_VARS.TMP_STATE_TREE, os.path.join(RUNTIME_VARS.FILES, "file", "base"), ], # Alternate root to test __env__ choices "prod": [ RUNTIME_VARS.TMP_PRODENV_STATE_TREE, os.path.join(RUNTIME_VARS.FILES, "file", "prod"), ], }, "pillar_roots": { "base": [ RUNTIME_VARS.TMP_PILLAR_TREE, os.path.join(RUNTIME_VARS.FILES, "pillar", "base"), ], "prod": [RUNTIME_VARS.TMP_PRODENV_PILLAR_TREE], }, }) return salt_factories.configure_master( request, "syndic_master", order_masters=True, config_defaults=config_defaults, config_overrides=config_overrides, )
def from_filenames_collection_modifyitems(config, items): from_filenames = config.getoption("--from-filenames") if not from_filenames: # Don't do anything return log.info( "Calculating test modules to run based on the paths in --from-filenames" ) from_filenames_paths = set() for path in [path.strip() for path in from_filenames.split(",")]: # Make sure that, no matter what kind of path we're passed, Windows or Posix path, # we resolve it to the platform slash separator properly_slashed_path = pathlib.Path( path.replace("\\", os.sep).replace("/", os.sep)) if not properly_slashed_path.exists(): log.info( "The path %s(%s) passed in --from-filenames does not exist", path, properly_slashed_path, ) continue if properly_slashed_path.is_absolute(): # In this case, this path is considered to be a file containing a line separated list # of files to consider with salt.utils.files.fopen(str(path)) as rfh: for line in rfh: line_path = pathlib.Path(line.strip().replace( "\\", os.sep).replace("/", os.sep)) if not line_path.exists(): log.info( "The path %s contained in %s passed in --from-filenames does not exist", line_path, properly_slashed_path, ) continue from_filenames_paths.add(line_path) continue from_filenames_paths.add(properly_slashed_path) # Let's start collecting test modules test_module_paths = set() filename_map = yaml.deserialize( (TESTS_DIR / "filename_map.yml").read_text()) # Let's add the match all rule for rule, matches in filename_map.items(): if rule == "*": for match in matches: test_module_paths.add(_match_to_test_file(match)) break # Let's now go through the list of files gathered for path in from_filenames_paths: if path.as_posix().startswith("tests/"): if path.name == "conftest.py": # This is not a test module, but consider any test_*.py files in child directories for match in path.parent.rglob("test_*.py"): test_module_paths.add(match) continue # Tests in the listing don't require additional matching and will be added to the # list of tests to run test_module_paths.add(path) continue if path.name == "setup.py" or path.as_posix().startswith("salt/"): if path.name == "__init__.py": # No direct matching continue # Let's try a direct match between the passed file and possible test modules glob_patterns = ( # salt/version.py -> # tests/unit/test_version.py # tests/pytests/unit/test_version.py "**/test_{}".format(path.name), # salt/modules/grains.py -> # tests/pytests/integration/modules/grains/tests_*.py # salt/modules/saltutil.py -> # tests/pytests/integration/modules/saltutil/test_*.py "**/{}/test_*.py".format(path.stem), # salt/modules/config.py -> # tests/unit/modules/test_config.py # tests/integration/modules/test_config.py # tests/pytests/unit/modules/test_config.py # tests/pytests/integration/modules/test_config.py "**/{}/test_{}".format(path.parent.name, path.name), ) for pattern in glob_patterns: for match in TESTS_DIR.rglob(pattern): relative_path = match.relative_to(CODE_DIR) log.info("Glob pattern %r matched '%s'", pattern, relative_path) test_module_paths.add(relative_path) # Do we have an entry in tests/filename_map.yml for rule, matches in filename_map.items(): if rule == "*": continue elif "|" in rule: # This is regex if re.match(rule, path.as_posix()): for match in matches: test_module_paths.add(_match_to_test_file(match)) elif "*" in rule or "\\" in rule: # Glob matching for filerule in CODE_DIR.glob(rule): if not filerule.exists(): continue filerule = filerule.relative_to(CODE_DIR) if filerule != path: continue for match in matches: test_module_paths.add(_match_to_test_file(match)) else: if path.as_posix() != rule: continue # Direct file paths as rules filerule = pathlib.Path(rule) if not filerule.exists(): continue for match in matches: test_module_paths.add(_match_to_test_file(match)) continue else: log.info("Don't know what to do with path %s", path) log.info( "Collected the following paths from --from-filenames processing:\n%s", "\n".join(sorted(map(str, test_module_paths))), ) selected = [] deselected = [] for item in items: itempath = pathlib.Path(str( item.fspath)).resolve().relative_to(CODE_DIR) if itempath in test_module_paths: selected.append(item) else: deselected.append(item) items[:] = selected if deselected: config.hook.pytest_deselected(items=deselected)