def test_compare_sls_vs_yaml_with_jinja(self):
tpl = '{{ data }}'
env = jinja2.Environment()
src = '{foo: 1, bar: 2, baz: {qux: true}}'
sls_src = env.from_string(tpl).render(data=yamlex.deserialize(src))
yml_src = env.from_string(tpl).render(data=yaml.deserialize(src))
sls_data = yamlex.deserialize(sls_src)
yml_data = yaml.deserialize(yml_src)
# ensure that sls & yaml have the same base
assert isinstance(sls_data, dict)
assert isinstance(yml_data, dict)
# The below has been commented out because something the loader test
# is modifying the yaml renderer to render things to unicode. Without
# running the loader test, the below passes. Even reloading the module
# from disk does not reset its internal state (per the Python docs).
##
#assert sls_data == yml_data
# ensure that sls is ordered, while yaml not
assert isinstance(sls_data, OrderedDict)
assert not isinstance(yml_data, OrderedDict)
# prove that yaml does not handle well with OrderedDict
# while sls is jinja friendly.
obj = OrderedDict([
('foo', 1),
('bar', 2),
('baz', {'qux': True})
])
sls_obj = yamlex.deserialize(yamlex.serialize(obj))
try:
yml_obj = yaml.deserialize(yaml.serialize(obj))
except SerializationError:
# BLAAM! yaml was unable to serialize OrderedDict,
# but it's not the purpose of the current test.
yml_obj = obj.copy()
sls_src = env.from_string(tpl).render(data=sls_obj)
yml_src = env.from_string(tpl).render(data=yml_obj)
final_obj = yaml.deserialize(sls_src)
assert obj == final_obj
# BLAAM! yml_src is not valid !
final_obj = OrderedDict(yaml.deserialize(yml_src))
assert obj != final_obj
def test_compare_sls_vs_yaml_with_jinja(self):
tpl = '{{ data }}'
env = jinja2.Environment()
src = '{foo: 1, bar: 2, baz: {qux: true}}'
sls_src = env.from_string(tpl).render(data=yamlex.deserialize(src))
yml_src = env.from_string(tpl).render(data=yaml.deserialize(src))
sls_data = yamlex.deserialize(sls_src)
yml_data = yaml.deserialize(yml_src)
# ensure that sls & yaml have the same base
assert isinstance(sls_data, dict)
assert isinstance(yml_data, dict)
assert sls_data == yml_data
# ensure that sls is ordered, while yaml not
assert isinstance(sls_data, OrderedDict)
assert not isinstance(yml_data, OrderedDict)
# prove that yaml does not handle well with OrderedDict
# while sls is jinja friendly.
obj = OrderedDict([
('foo', 1),
('bar', 2),
('baz', {'qux': True})
])
sls_obj = yamlex.deserialize(yamlex.serialize(obj))
try:
yml_obj = yaml.deserialize(yaml.serialize(obj))
except SerializationError:
# BLAAM! yaml was unable to serialize OrderedDict,
# but it's not the purpose of the current test.
yml_obj = obj.copy()
sls_src = env.from_string(tpl).render(data=sls_obj)
yml_src = env.from_string(tpl).render(data=yml_obj)
final_obj = yaml.deserialize(sls_src)
assert obj == final_obj
# BLAAM! yml_src is not valid !
final_obj = OrderedDict(yaml.deserialize(yml_src))
assert obj != final_obj
def test_serialize_yaml(self):
data = {
"foo": "bar"
}
serialized = yaml.serialize(data)
assert serialized == '{foo: bar}', serialized
deserialized = yaml.deserialize(serialized)
assert deserialized == data, deserialized
def test_compare_sls_vs_yaml(self):
src = '{foo: 1, bar: 2, baz: {qux: true}}'
sls_data = yamlex.deserialize(src)
yml_data = yaml.deserialize(src)
# ensure that sls & yaml have the same base
assert isinstance(sls_data, dict)
assert isinstance(yml_data, dict)
assert sls_data == yml_data
# ensure that sls is ordered, while yaml not
assert isinstance(sls_data, OrderedDict)
assert not isinstance(yml_data, OrderedDict)
def _get_template(path, option_key):
'''
Get the contents of a template file and provide it as a module type
:param path: path to the template.yml file
:type path: ``str``
:param option_key: The unique key of this template
:type option_key: ``str``
:returns: Details about the template
:rtype: ``tuple``
'''
with salt.utils.fopen(path, "r") as template_f:
template = deserialize(template_f)
info = (option_key, template.get('description', ''), template)
return info
def salt_syndic_master_factory(
request,
salt_factories,
base_env_state_tree_root_dir,
base_env_pillar_tree_root_dir,
prod_env_state_tree_root_dir,
prod_env_pillar_tree_root_dir,
):
root_dir = salt_factories.get_root_dir_for_daemon("syndic_master")
conf_dir = root_dir / "conf"
conf_dir.mkdir(exist_ok=True)
with salt.utils.files.fopen(
os.path.join(RUNTIME_VARS.CONF_DIR, "syndic_master")
) as rfh:
config_defaults = yaml.deserialize(rfh.read())
tests_known_hosts_file = str(root_dir / "salt_ssh_known_hosts")
with salt.utils.files.fopen(tests_known_hosts_file, "w") as known_hosts:
known_hosts.write("")
config_defaults["root_dir"] = str(root_dir)
config_defaults["known_hosts_file"] = tests_known_hosts_file
config_defaults["syndic_master"] = "localhost"
config_defaults["transport"] = request.config.getoption("--transport")
config_overrides = {}
ext_pillar = []
if salt.utils.platform.is_windows():
ext_pillar.append(
{"cmd_yaml": "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))}
)
else:
ext_pillar.append(
{"cmd_yaml": "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))}
)
# We need to copy the extension modules into the new master root_dir or
# it will be prefixed by it
extension_modules_path = str(root_dir / "extension_modules")
if not os.path.exists(extension_modules_path):
shutil.copytree(
os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
extension_modules_path,
)
# Copy the autosign_file to the new master root_dir
autosign_file_path = str(root_dir / "autosign_file")
shutil.copyfile(
os.path.join(RUNTIME_VARS.FILES, "autosign_file"), autosign_file_path
)
# all read, only owner write
autosign_file_permissions = (
stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR
)
os.chmod(autosign_file_path, autosign_file_permissions)
config_overrides.update(
{
"ext_pillar": ext_pillar,
"extension_modules": extension_modules_path,
"file_roots": {
"base": [
str(base_env_state_tree_root_dir),
os.path.join(RUNTIME_VARS.FILES, "file", "base"),
],
# Alternate root to test __env__ choices
"prod": [
str(prod_env_state_tree_root_dir),
os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
],
},
"pillar_roots": {
"base": [
str(base_env_pillar_tree_root_dir),
os.path.join(RUNTIME_VARS.FILES, "pillar", "base"),
],
"prod": [str(prod_env_pillar_tree_root_dir)],
},
}
)
factory = salt_factories.get_salt_master_daemon(
"syndic_master",
order_masters=True,
config_defaults=config_defaults,
config_overrides=config_overrides,
extra_cli_arguments_after_first_start_failure=["--log-level=debug"],
)
return factory
def from_filenames_collection_modifyitems(config, items):
from_filenames = config.getoption("--from-filenames")
if not from_filenames:
# Don't do anything
return
test_categories_paths = (
(TESTS_DIR / "integration").relative_to(CODE_DIR),
(TESTS_DIR / "multimaster").relative_to(CODE_DIR),
(TESTS_DIR / "unit").relative_to(CODE_DIR),
(PYTESTS_DIR / "e2e").relative_to(CODE_DIR),
(PYTESTS_DIR / "functional").relative_to(CODE_DIR),
(PYTESTS_DIR / "integration").relative_to(CODE_DIR),
(PYTESTS_DIR / "unit").relative_to(CODE_DIR),
)
test_module_paths = set()
from_filenames_listing = set()
for path in [pathlib.Path(path.strip()) for path in from_filenames.split(",")]:
if path.is_absolute():
# In this case, this path is considered to be a file containing a line separated list
# of files to consider
with salt.utils.files.fopen(str(path)) as rfh:
for line in rfh:
line_path = pathlib.Path(line.strip())
if not line_path.exists():
continue
from_filenames_listing.add(line_path)
continue
from_filenames_listing.add(path)
filename_map = yaml.deserialize((TESTS_DIR / "filename_map.yml").read_text())
# Let's add the match all rule
for rule, matches in filename_map.items():
if rule == "*":
for match in matches:
test_module_paths.add(_match_to_test_file(match))
break
# Let's now go through the list of files gathered
for filename in from_filenames_listing:
if str(filename).startswith("tests/"):
# Tests in the listing don't require additional matching and will be added to the
# list of tests to run
test_module_paths.add(filename)
continue
if filename.name == "setup.py" or str(filename).startswith("salt/"):
if path.name == "__init__.py":
# No direct macthing
continue
# Now let's try a direct match between the passed file and possible test modules
for test_categories_path in test_categories_paths:
test_module_path = test_categories_path / "test_{}".format(path.name)
if test_module_path.is_file():
test_module_paths.add(test_module_path)
continue
# Do we have an entry in tests/filename_map.yml
for rule, matches in filename_map.items():
if rule == "*":
continue
elif "|" in rule:
# This is regex
if re.match(rule, str(filename)):
for match in matches:
test_module_paths.add(_match_to_test_file(match))
elif "*" in rule or "\\" in rule:
# Glob matching
for filerule in CODE_DIR.glob(rule):
if not filerule.exists():
continue
filerule = filerule.relative_to(CODE_DIR)
if filerule != filename:
continue
for match in matches:
test_module_paths.add(_match_to_test_file(match))
else:
if str(filename) != rule:
continue
# Direct file paths as rules
filerule = pathlib.Path(rule)
if not filerule.exists():
continue
for match in matches:
test_module_paths.add(_match_to_test_file(match))
continue
else:
log.debug("Don't know what to do with path %s", filename)
selected = []
deselected = []
for item in items:
itempath = pathlib.Path(str(item.fspath)).resolve().relative_to(CODE_DIR)
if itempath in test_module_paths:
selected.append(item)
else:
deselected.append(item)
items[:] = selected
if deselected:
config.hook.pytest_deselected(items=deselected)
def salt_master_factory(
request,
salt_factories,
salt_minion_id,
salt_sub_minion_id,
base_env_state_tree_root_dir,
base_env_pillar_tree_root_dir,
prod_env_state_tree_root_dir,
prod_env_pillar_tree_root_dir,
ext_pillar_file_tree_root_dir,
sdb_etcd_port,
vault_port,
reactor_event,
master_id,
salt_auth_account_1_factory,
salt_auth_account_2_factory,
salt_netapi_account_factory,
salt_eauth_account_factory,
salt_auto_account_factory,
):
root_dir = salt_factories.get_root_dir_for_daemon(master_id)
conf_dir = root_dir / "conf"
conf_dir.mkdir(exist_ok=True)
with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.CONF_DIR, "master")) as rfh:
config_defaults = yaml.deserialize(rfh.read())
tests_known_hosts_file = str(root_dir / "salt_ssh_known_hosts")
with salt.utils.files.fopen(tests_known_hosts_file, "w") as known_hosts:
known_hosts.write("")
config_defaults["root_dir"] = str(root_dir)
config_defaults["known_hosts_file"] = tests_known_hosts_file
config_defaults["syndic_master"] = "localhost"
config_defaults["transport"] = request.config.getoption("--transport")
config_defaults["reactor"] = [
{reactor_event.tag: [str(reactor_event.sls_path)]},
]
nodegroups = {
"min": salt_minion_id,
"sub_min": salt_sub_minion_id,
"mins": "N@min or N@sub_min",
"list_nodegroup": [salt_minion_id, salt_sub_minion_id],
"multiline_nodegroup": [salt_minion_id, "or", salt_sub_minion_id],
"one_minion_list": [salt_minion_id],
"redundant_minions": "N@min or N@mins",
"nodegroup_loop_a": "N@nodegroup_loop_b",
"nodegroup_loop_b": "N@nodegroup_loop_a",
"missing_minion": "L@{},ghostminion".format(salt_minion_id),
"list_group": "N@multiline_nodegroup",
"one_list_group": "N@one_minion_list",
"list_group2": "N@list_nodegroup",
}
config_defaults["nodegroups"] = nodegroups
config_defaults["sdbetcd"] = {
"driver": "etcd",
"etcd.host": "127.0.0.1",
"etcd.port": sdb_etcd_port,
}
config_defaults["vault"] = {
"url": "http://127.0.0.1:{}".format(vault_port),
"auth": {"method": "token", "token": "testsecret", "uses": 0},
"policies": ["testpolicy"],
}
# Config settings to test `event_return`
config_defaults["returner_dirs"] = []
config_defaults["returner_dirs"].append(
os.path.join(RUNTIME_VARS.FILES, "returners")
)
config_defaults["event_return"] = "runtests_noop"
config_overrides = {"pytest-master": {"log": {"level": "DEBUG"}}}
ext_pillar = []
if salt.utils.platform.is_windows():
ext_pillar.append(
{"cmd_yaml": "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))}
)
else:
ext_pillar.append(
{"cmd_yaml": "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))}
)
ext_pillar.append(
{
"file_tree": {
"root_dir": str(ext_pillar_file_tree_root_dir),
"follow_dir_links": False,
"keep_newline": True,
}
}
)
config_overrides["pillar_opts"] = True
config_overrides["external_auth"] = {
"pam": {
salt_auth_account_1_factory.username: ["test.*"],
"{}%".format(salt_auth_account_2_factory.group_name): [
"@wheel",
"@runner",
"test.*",
],
salt_netapi_account_factory.username: ["@wheel", "@runner", "test.*"],
salt_eauth_account_factory.username: ["@wheel", "@runner", "test.*"],
},
"auto": {
salt_netapi_account_factory.username: [
"@wheel",
"@runner",
"test.*",
"grains.*",
],
salt_auto_account_factory.username: ["@wheel", "@runner", "test.*"],
"*": ["@wheel", "@runner", "test.*"],
},
}
# We need to copy the extension modules into the new master root_dir or
# it will be prefixed by it
extension_modules_path = str(root_dir / "extension_modules")
if not os.path.exists(extension_modules_path):
shutil.copytree(
os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
extension_modules_path,
)
# Copy the autosign_file to the new master root_dir
autosign_file_path = str(root_dir / "autosign_file")
shutil.copyfile(
os.path.join(RUNTIME_VARS.FILES, "autosign_file"), autosign_file_path
)
# all read, only owner write
autosign_file_permissions = (
stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR
)
os.chmod(autosign_file_path, autosign_file_permissions)
config_overrides.update(
{
"ext_pillar": ext_pillar,
"extension_modules": extension_modules_path,
"file_roots": {
"base": [
str(base_env_state_tree_root_dir),
os.path.join(RUNTIME_VARS.FILES, "file", "base"),
],
# Alternate root to test __env__ choices
"prod": [
str(prod_env_state_tree_root_dir),
os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
],
},
"pillar_roots": {
"base": [
str(base_env_pillar_tree_root_dir),
os.path.join(RUNTIME_VARS.FILES, "pillar", "base"),
],
"prod": [str(prod_env_pillar_tree_root_dir)],
},
}
)
# Let's copy over the test cloud config files and directories into the running master config directory
for entry in os.listdir(RUNTIME_VARS.CONF_DIR):
if not entry.startswith("cloud"):
continue
source = os.path.join(RUNTIME_VARS.CONF_DIR, entry)
dest = str(conf_dir / entry)
if os.path.isdir(source):
shutil.copytree(source, dest)
else:
shutil.copyfile(source, dest)
factory = salt_factories.salt_master_daemon(
master_id,
defaults=config_defaults,
overrides=config_overrides,
extra_cli_arguments_after_first_start_failure=["--log-level=debug"],
)
return factory
def salt_master_factory(
salt_factories,
salt_syndic_master_factory,
base_env_state_tree_root_dir,
base_env_pillar_tree_root_dir,
prod_env_state_tree_root_dir,
prod_env_pillar_tree_root_dir,
):
root_dir = salt_factories.get_root_dir_for_daemon("master")
conf_dir = root_dir / "conf"
conf_dir.mkdir(exist_ok=True)
with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.CONF_DIR, "master")) as rfh:
config_defaults = yaml.deserialize(rfh.read())
tests_known_hosts_file = str(root_dir / "salt_ssh_known_hosts")
with salt.utils.files.fopen(tests_known_hosts_file, "w") as known_hosts:
known_hosts.write("")
config_defaults["root_dir"] = str(root_dir)
config_defaults["known_hosts_file"] = tests_known_hosts_file
config_defaults["syndic_master"] = "localhost"
config_defaults["transport"] = salt_syndic_master_factory.config["transport"]
config_defaults["reactor"] = [
{"salt/test/reactor": [os.path.join(RUNTIME_VARS.FILES, "reactor-test.sls")]}
]
config_overrides = {}
ext_pillar = []
if salt.utils.platform.is_windows():
ext_pillar.append(
{"cmd_yaml": "type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))}
)
else:
ext_pillar.append(
{"cmd_yaml": "cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))}
)
ext_pillar.append(
{
"file_tree": {
"root_dir": os.path.join(RUNTIME_VARS.PILLAR_DIR, "base", "file_tree"),
"follow_dir_links": False,
"keep_newline": True,
}
}
)
config_overrides["pillar_opts"] = True
# We need to copy the extension modules into the new master root_dir or
# it will be prefixed by it
extension_modules_path = str(root_dir / "extension_modules")
if not os.path.exists(extension_modules_path):
shutil.copytree(
os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
extension_modules_path,
)
# Copy the autosign_file to the new master root_dir
autosign_file_path = str(root_dir / "autosign_file")
shutil.copyfile(
os.path.join(RUNTIME_VARS.FILES, "autosign_file"), autosign_file_path
)
# all read, only owner write
autosign_file_permissions = (
stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IWUSR
)
os.chmod(autosign_file_path, autosign_file_permissions)
config_overrides.update(
{
"ext_pillar": ext_pillar,
"extension_modules": extension_modules_path,
"file_roots": {
"base": [
str(base_env_state_tree_root_dir),
os.path.join(RUNTIME_VARS.FILES, "file", "base"),
],
# Alternate root to test __env__ choices
"prod": [
str(prod_env_state_tree_root_dir),
os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
],
},
"pillar_roots": {
"base": [
str(base_env_pillar_tree_root_dir),
os.path.join(RUNTIME_VARS.FILES, "pillar", "base"),
],
"prod": [str(prod_env_pillar_tree_root_dir)],
},
}
)
# Let's copy over the test cloud config files and directories into the running master config directory
for entry in os.listdir(RUNTIME_VARS.CONF_DIR):
if not entry.startswith("cloud"):
continue
source = os.path.join(RUNTIME_VARS.CONF_DIR, entry)
dest = str(conf_dir / entry)
if os.path.isdir(source):
shutil.copytree(source, dest)
else:
shutil.copyfile(source, dest)
factory = salt_syndic_master_factory.get_salt_master_daemon(
"master",
config_defaults=config_defaults,
config_overrides=config_overrides,
extra_cli_arguments_after_first_start_failure=["--log-level=debug"],
)
return factory
def session_mm_minion_default_options(session_minion_default_options):
opts = session_minion_default_options.copy()
with salt.utils.files.fopen(
os.path.join(RUNTIME_VARS.CONF_DIR, 'mm_sub_minion')) as rfh:
opts.update(yaml.deserialize(rfh.read()))
return opts
def salt_master_config(request, salt_factories, salt_syndic_master_config):
root_dir = salt_factories._get_root_dir_for_daemon("master")
conf_dir = root_dir.join("conf").ensure(dir=True)
with salt.utils.files.fopen(os.path.join(RUNTIME_VARS.CONF_DIR,
"master")) as rfh:
config_defaults = yaml.deserialize(rfh.read())
tests_known_hosts_file = root_dir.join("salt_ssh_known_hosts").strpath
with salt.utils.files.fopen(tests_known_hosts_file,
"w") as known_hosts:
known_hosts.write("")
config_defaults["root_dir"] = root_dir.strpath
config_defaults["known_hosts_file"] = tests_known_hosts_file
config_defaults["syndic_master"] = "localhost"
config_defaults["transport"] = request.config.getoption("--transport")
config_defaults["reactor"] = [{
"salt/test/reactor":
[os.path.join(RUNTIME_VARS.FILES, "reactor-test.sls")]
}]
config_overrides = {}
ext_pillar = []
if salt.utils.platform.is_windows():
ext_pillar.append({
"cmd_yaml":
"type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))
})
else:
ext_pillar.append({
"cmd_yaml":
"cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))
})
ext_pillar.append({
"file_tree": {
"root_dir": os.path.join(RUNTIME_VARS.PILLAR_DIR, "base",
"file_tree"),
"follow_dir_links": False,
"keep_newline": True,
}
})
config_overrides["pillar_opts"] = True
# We need to copy the extension modules into the new master root_dir or
# it will be prefixed by it
extension_modules_path = root_dir.join("extension_modules").strpath
if not os.path.exists(extension_modules_path):
shutil.copytree(
os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
extension_modules_path,
)
# Copy the autosign_file to the new master root_dir
autosign_file_path = root_dir.join("autosign_file").strpath
shutil.copyfile(os.path.join(RUNTIME_VARS.FILES, "autosign_file"),
autosign_file_path)
# all read, only owner write
autosign_file_permissions = (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
| stat.S_IWUSR)
os.chmod(autosign_file_path, autosign_file_permissions)
config_overrides.update({
"ext_pillar": ext_pillar,
"extension_modules": extension_modules_path,
"file_roots": {
"base": [
RUNTIME_VARS.TMP_STATE_TREE,
os.path.join(RUNTIME_VARS.FILES, "file", "base"),
],
# Alternate root to test __env__ choices
"prod": [
RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
],
},
"pillar_roots": {
"base": [
RUNTIME_VARS.TMP_PILLAR_TREE,
os.path.join(RUNTIME_VARS.FILES, "pillar", "base"),
],
"prod": [RUNTIME_VARS.TMP_PRODENV_PILLAR_TREE],
},
})
# Let's copy over the test cloud config files and directories into the running master config directory
for entry in os.listdir(RUNTIME_VARS.CONF_DIR):
if not entry.startswith("cloud"):
continue
source = os.path.join(RUNTIME_VARS.CONF_DIR, entry)
dest = conf_dir.join(entry).strpath
if os.path.isdir(source):
shutil.copytree(source, dest)
else:
shutil.copyfile(source, dest)
return salt_factories.configure_master(
request,
"master",
master_of_masters_id="syndic_master",
config_defaults=config_defaults,
config_overrides=config_overrides,
)
def salt_syndic_master_config(request, salt_factories):
root_dir = salt_factories._get_root_dir_for_daemon("syndic_master")
with salt.utils.files.fopen(
os.path.join(RUNTIME_VARS.CONF_DIR, "syndic_master")) as rfh:
config_defaults = yaml.deserialize(rfh.read())
tests_known_hosts_file = root_dir.join("salt_ssh_known_hosts").strpath
with salt.utils.files.fopen(tests_known_hosts_file,
"w") as known_hosts:
known_hosts.write("")
config_defaults["root_dir"] = root_dir.strpath
config_defaults["known_hosts_file"] = tests_known_hosts_file
config_defaults["syndic_master"] = "localhost"
config_defaults["transport"] = request.config.getoption("--transport")
config_overrides = {}
ext_pillar = []
if salt.utils.platform.is_windows():
ext_pillar.append({
"cmd_yaml":
"type {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))
})
else:
ext_pillar.append({
"cmd_yaml":
"cat {}".format(os.path.join(RUNTIME_VARS.FILES, "ext.yaml"))
})
# We need to copy the extension modules into the new master root_dir or
# it will be prefixed by it
extension_modules_path = root_dir.join("extension_modules").strpath
if not os.path.exists(extension_modules_path):
shutil.copytree(
os.path.join(RUNTIME_VARS.FILES, "extension_modules"),
extension_modules_path,
)
# Copy the autosign_file to the new master root_dir
autosign_file_path = root_dir.join("autosign_file").strpath
shutil.copyfile(os.path.join(RUNTIME_VARS.FILES, "autosign_file"),
autosign_file_path)
# all read, only owner write
autosign_file_permissions = (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
| stat.S_IWUSR)
os.chmod(autosign_file_path, autosign_file_permissions)
config_overrides.update({
"ext_pillar": ext_pillar,
"extension_modules": extension_modules_path,
"file_roots": {
"base": [
RUNTIME_VARS.TMP_STATE_TREE,
os.path.join(RUNTIME_VARS.FILES, "file", "base"),
],
# Alternate root to test __env__ choices
"prod": [
RUNTIME_VARS.TMP_PRODENV_STATE_TREE,
os.path.join(RUNTIME_VARS.FILES, "file", "prod"),
],
},
"pillar_roots": {
"base": [
RUNTIME_VARS.TMP_PILLAR_TREE,
os.path.join(RUNTIME_VARS.FILES, "pillar", "base"),
],
"prod": [RUNTIME_VARS.TMP_PRODENV_PILLAR_TREE],
},
})
return salt_factories.configure_master(
request,
"syndic_master",
order_masters=True,
config_defaults=config_defaults,
config_overrides=config_overrides,
)
def from_filenames_collection_modifyitems(config, items):
from_filenames = config.getoption("--from-filenames")
if not from_filenames:
# Don't do anything
return
log.info(
"Calculating test modules to run based on the paths in --from-filenames"
)
from_filenames_paths = set()
for path in [path.strip() for path in from_filenames.split(",")]:
# Make sure that, no matter what kind of path we're passed, Windows or Posix path,
# we resolve it to the platform slash separator
properly_slashed_path = pathlib.Path(
path.replace("\\", os.sep).replace("/", os.sep))
if not properly_slashed_path.exists():
log.info(
"The path %s(%s) passed in --from-filenames does not exist",
path,
properly_slashed_path,
)
continue
if properly_slashed_path.is_absolute():
# In this case, this path is considered to be a file containing a line separated list
# of files to consider
with salt.utils.files.fopen(str(path)) as rfh:
for line in rfh:
line_path = pathlib.Path(line.strip().replace(
"\\", os.sep).replace("/", os.sep))
if not line_path.exists():
log.info(
"The path %s contained in %s passed in --from-filenames does not exist",
line_path,
properly_slashed_path,
)
continue
from_filenames_paths.add(line_path)
continue
from_filenames_paths.add(properly_slashed_path)
# Let's start collecting test modules
test_module_paths = set()
filename_map = yaml.deserialize(
(TESTS_DIR / "filename_map.yml").read_text())
# Let's add the match all rule
for rule, matches in filename_map.items():
if rule == "*":
for match in matches:
test_module_paths.add(_match_to_test_file(match))
break
# Let's now go through the list of files gathered
for path in from_filenames_paths:
if path.as_posix().startswith("tests/"):
if path.name == "conftest.py":
# This is not a test module, but consider any test_*.py files in child directories
for match in path.parent.rglob("test_*.py"):
test_module_paths.add(match)
continue
# Tests in the listing don't require additional matching and will be added to the
# list of tests to run
test_module_paths.add(path)
continue
if path.name == "setup.py" or path.as_posix().startswith("salt/"):
if path.name == "__init__.py":
# No direct matching
continue
# Let's try a direct match between the passed file and possible test modules
glob_patterns = (
# salt/version.py ->
# tests/unit/test_version.py
# tests/pytests/unit/test_version.py
"**/test_{}".format(path.name),
# salt/modules/grains.py ->
# tests/pytests/integration/modules/grains/tests_*.py
# salt/modules/saltutil.py ->
# tests/pytests/integration/modules/saltutil/test_*.py
"**/{}/test_*.py".format(path.stem),
# salt/modules/config.py ->
# tests/unit/modules/test_config.py
# tests/integration/modules/test_config.py
# tests/pytests/unit/modules/test_config.py
# tests/pytests/integration/modules/test_config.py
"**/{}/test_{}".format(path.parent.name, path.name),
)
for pattern in glob_patterns:
for match in TESTS_DIR.rglob(pattern):
relative_path = match.relative_to(CODE_DIR)
log.info("Glob pattern %r matched '%s'", pattern,
relative_path)
test_module_paths.add(relative_path)
# Do we have an entry in tests/filename_map.yml
for rule, matches in filename_map.items():
if rule == "*":
continue
elif "|" in rule:
# This is regex
if re.match(rule, path.as_posix()):
for match in matches:
test_module_paths.add(_match_to_test_file(match))
elif "*" in rule or "\\" in rule:
# Glob matching
for filerule in CODE_DIR.glob(rule):
if not filerule.exists():
continue
filerule = filerule.relative_to(CODE_DIR)
if filerule != path:
continue
for match in matches:
test_module_paths.add(_match_to_test_file(match))
else:
if path.as_posix() != rule:
continue
# Direct file paths as rules
filerule = pathlib.Path(rule)
if not filerule.exists():
continue
for match in matches:
test_module_paths.add(_match_to_test_file(match))
continue
else:
log.info("Don't know what to do with path %s", path)
log.info(
"Collected the following paths from --from-filenames processing:\n%s",
"\n".join(sorted(map(str, test_module_paths))),
)
selected = []
deselected = []
for item in items:
itempath = pathlib.Path(str(
item.fspath)).resolve().relative_to(CODE_DIR)
if itempath in test_module_paths:
selected.append(item)
else:
deselected.append(item)
items[:] = selected
if deselected:
config.hook.pytest_deselected(items=deselected)