def list_states(self, saltenv):
'''
Return a list of all available sls modules on the master for a given
environment
'''
limit_traversal = self.opts.get(u'fileserver_limit_traversal', False)
states = []
if limit_traversal:
if saltenv not in self.opts[u'file_roots']:
log.warning(
u'During an attempt to list states for saltenv \'%s\', '
u'the environment could not be found in the configured '
u'file roots', saltenv
)
return states
for path in self.opts[u'file_roots'][saltenv]:
for root, dirs, files in os.walk(path, topdown=True):
log.debug(
u'Searching for states in dirs %s and files %s',
dirs, files
)
if not [filename.endswith(u'.sls') for filename in files]:
# Use shallow copy so we don't disturb the memory used by os.walk. Otherwise this breaks!
del dirs[:]
else:
for found_file in files:
stripped_root = os.path.relpath(root, path)
if salt.utils.platform.is_windows():
stripped_root = stripped_root.replace(u'\\', u'/')
stripped_root = stripped_root.replace(u'/', u'.')
if found_file.endswith((u'.sls')):
if found_file.endswith(u'init.sls'):
if stripped_root.endswith(u'.'):
stripped_root = stripped_root.rstrip(u'.')
states.append(stripped_root)
else:
if not stripped_root.endswith(u'.'):
stripped_root += u'.'
if stripped_root.startswith(u'.'):
stripped_root = stripped_root.lstrip(u'.')
states.append(stripped_root + found_file[:-4])
else:
for path in self.file_list(saltenv):
if salt.utils.platform.is_windows():
path = path.replace(u'\\', u'/')
if path.endswith(u'.sls'):
# is an sls module!
if path.endswith(u'/init.sls'):
states.append(path.replace(u'/', u'.')[:-9])
else:
states.append(path.replace(u'/', u'.')[:-4])
return states
def create(path, saltenv=None):
"""
join `path` and `saltenv` into a 'salt://' URL.
"""
path = path.replace("\\", "/")
if salt.utils.platform.is_windows():
path = salt.utils.path.sanitize_win_path(path)
path = salt.utils.data.decode(path)
query = "saltenv={}".format(saltenv) if saltenv else ""
url = salt.utils.data.decode(urlunparse(("file", "", path, "", query, "")))
return "salt://{}".format(url[len("file:///"):])
def _translate_sep(path):
'''
Translate path separators for Windows masterless minions
'''
return path.replace('\\', '/') if os.path.sep == '\\' else path
def _translate_sep(path):
"""
Translate path separators for Windows masterless minions
"""
return path.replace("\\", "/") if os.path.sep == "\\" else path
def _file_lists(load, form):
"""
Return a dict containing the file lists for files, dirs, emtydirs and symlinks
"""
if "env" in load:
# "env" is not supported; Use "saltenv".
load.pop("env")
saltenv = load["saltenv"]
actual_saltenv = saltenv
if saltenv not in __opts__["file_roots"]:
if "__env__" in __opts__["file_roots"]:
log.debug(
"salt environment '%s' maps to __env__ file_roots directory",
saltenv)
saltenv = "__env__"
else:
return []
list_cachedir = os.path.join(__opts__["cachedir"], "file_lists", "roots")
if not os.path.isdir(list_cachedir):
try:
os.makedirs(list_cachedir)
except OSError:
log.critical("Unable to make cachedir %s", list_cachedir)
return []
list_cache = os.path.join(
list_cachedir,
"{}.p".format(salt.utils.files.safe_filename_leaf(actual_saltenv)),
)
w_lock = os.path.join(
list_cachedir,
".{}.w".format(salt.utils.files.safe_filename_leaf(actual_saltenv)),
)
cache_match, refresh_cache, save_cache = salt.fileserver.check_file_list_cache(
__opts__, form, list_cache, w_lock)
if cache_match is not None:
return cache_match
if refresh_cache:
ret = {"files": set(), "dirs": set(), "empty_dirs": set(), "links": {}}
def _add_to(tgt, fs_root, parent_dir, items):
"""
Add the files to the target set
"""
def _translate_sep(path):
"""
Translate path separators for Windows masterless minions
"""
return path.replace("\\", "/") if os.path.sep == "\\" else path
for item in items:
abs_path = os.path.join(parent_dir, item)
log.trace("roots: Processing %s", abs_path)
is_link = salt.utils.path.islink(abs_path)
log.trace("roots: %s is %sa link", abs_path,
"not " if not is_link else "")
if is_link and __opts__["fileserver_ignoresymlinks"]:
continue
rel_path = _translate_sep(os.path.relpath(abs_path, fs_root))
log.trace("roots: %s relative path is %s", abs_path, rel_path)
if salt.fileserver.is_file_ignored(__opts__, rel_path):
continue
tgt.add(rel_path)
if os.path.isdir(abs_path):
try:
if not os.listdir(abs_path):
ret["empty_dirs"].add(rel_path)
except OSError:
log.debug("Unable to list dir: %s", abs_path)
if is_link:
link_dest = salt.utils.path.readlink(abs_path)
log.trace("roots: %s symlink destination is %s", abs_path,
link_dest)
if salt.utils.platform.is_windows(
) and link_dest.startswith("\\\\"):
# Symlink points to a network path. Since you can't
# join UNC and non-UNC paths, just assume the original
# path.
log.trace(
"roots: %s is a UNC path, using %s instead",
link_dest,
abs_path,
)
link_dest = abs_path
if link_dest.startswith(".."):
joined = os.path.join(abs_path, link_dest)
else:
joined = os.path.join(os.path.dirname(abs_path),
link_dest)
rel_dest = _translate_sep(
os.path.relpath(
os.path.realpath(os.path.normpath(joined)),
os.path.realpath(fs_root),
))
log.trace("roots: %s relative path is %s", abs_path,
rel_dest)
if not rel_dest.startswith(".."):
# Only count the link if it does not point
# outside of the root dir of the fileserver
# (i.e. the "path" variable)
ret["links"][rel_path] = link_dest
else:
if not __opts__["fileserver_followsymlinks"]:
ret["links"][rel_path] = link_dest
for path in __opts__["file_roots"][saltenv]:
if saltenv == "__env__":
path = path.replace("__env__", actual_saltenv)
for root, dirs, files in salt.utils.path.os_walk(
path, followlinks=__opts__["fileserver_followsymlinks"]):
_add_to(ret["dirs"], path, root, dirs)
_add_to(ret["files"], path, root, files)
ret["files"] = sorted(ret["files"])
ret["dirs"] = sorted(ret["dirs"])
ret["empty_dirs"] = sorted(ret["empty_dirs"])
if save_cache:
try:
salt.fileserver.write_file_list_cache(__opts__, ret,
list_cache, w_lock)
except NameError:
# Catch msgpack error in salt-ssh
pass
return ret.get(form, [])
# Shouldn't get here, but if we do, this prevents a TypeError
return []
def from_filenames_collection_modifyitems(config, items):
from_filenames = config.getoption("--from-filenames")
if not from_filenames:
# Don't do anything
return
log.info(
"Calculating test modules to run based on the paths in --from-filenames"
)
from_filenames_paths = set()
for path in [path.strip() for path in from_filenames.split(",")]:
# Make sure that, no matter what kind of path we're passed, Windows or Posix path,
# we resolve it to the platform slash separator
properly_slashed_path = pathlib.Path(
path.replace("\\", os.sep).replace("/", os.sep))
if not properly_slashed_path.exists():
log.info(
"The path %s(%s) passed in --from-filenames does not exist",
path,
properly_slashed_path,
)
continue
if properly_slashed_path.is_absolute():
# In this case, this path is considered to be a file containing a line separated list
# of files to consider
with salt.utils.files.fopen(str(path)) as rfh:
for line in rfh:
line_path = pathlib.Path(line.strip().replace(
"\\", os.sep).replace("/", os.sep))
if not line_path.exists():
log.info(
"The path %s contained in %s passed in --from-filenames does not exist",
line_path,
properly_slashed_path,
)
continue
from_filenames_paths.add(line_path)
continue
from_filenames_paths.add(properly_slashed_path)
# Let's start collecting test modules
test_module_paths = set()
filename_map = yaml.deserialize(
(TESTS_DIR / "filename_map.yml").read_text())
# Let's add the match all rule
for rule, matches in filename_map.items():
if rule == "*":
for match in matches:
test_module_paths.add(_match_to_test_file(match))
break
# Let's now go through the list of files gathered
for path in from_filenames_paths:
if path.as_posix().startswith("tests/"):
if path.name == "conftest.py":
# This is not a test module, but consider any test_*.py files in child directories
for match in path.parent.rglob("test_*.py"):
test_module_paths.add(match)
continue
# Tests in the listing don't require additional matching and will be added to the
# list of tests to run
test_module_paths.add(path)
continue
if path.name == "setup.py" or path.as_posix().startswith("salt/"):
if path.name == "__init__.py":
# No direct matching
continue
# Let's try a direct match between the passed file and possible test modules
glob_patterns = (
# salt/version.py ->
# tests/unit/test_version.py
# tests/pytests/unit/test_version.py
"**/test_{}".format(path.name),
# salt/modules/grains.py ->
# tests/pytests/integration/modules/grains/tests_*.py
# salt/modules/saltutil.py ->
# tests/pytests/integration/modules/saltutil/test_*.py
"**/{}/test_*.py".format(path.stem),
# salt/modules/config.py ->
# tests/unit/modules/test_config.py
# tests/integration/modules/test_config.py
# tests/pytests/unit/modules/test_config.py
# tests/pytests/integration/modules/test_config.py
"**/{}/test_{}".format(path.parent.name, path.name),
)
for pattern in glob_patterns:
for match in TESTS_DIR.rglob(pattern):
relative_path = match.relative_to(CODE_DIR)
log.info("Glob pattern %r matched '%s'", pattern,
relative_path)
test_module_paths.add(relative_path)
# Do we have an entry in tests/filename_map.yml
for rule, matches in filename_map.items():
if rule == "*":
continue
elif "|" in rule:
# This is regex
if re.match(rule, path.as_posix()):
for match in matches:
test_module_paths.add(_match_to_test_file(match))
elif "*" in rule or "\\" in rule:
# Glob matching
for filerule in CODE_DIR.glob(rule):
if not filerule.exists():
continue
filerule = filerule.relative_to(CODE_DIR)
if filerule != path:
continue
for match in matches:
test_module_paths.add(_match_to_test_file(match))
else:
if path.as_posix() != rule:
continue
# Direct file paths as rules
filerule = pathlib.Path(rule)
if not filerule.exists():
continue
for match in matches:
test_module_paths.add(_match_to_test_file(match))
continue
else:
log.info("Don't know what to do with path %s", path)
log.info(
"Collected the following paths from --from-filenames processing:\n%s",
"\n".join(sorted(map(str, test_module_paths))),
)
selected = []
deselected = []
for item in items:
itempath = pathlib.Path(str(
item.fspath)).resolve().relative_to(CODE_DIR)
if itempath in test_module_paths:
selected.append(item)
else:
deselected.append(item)
items[:] = selected
if deselected:
config.hook.pytest_deselected(items=deselected)
