def list_states(self, saltenv): ''' Return a list of all available sls modules on the master for a given environment ''' limit_traversal = self.opts.get(u'fileserver_limit_traversal', False) states = [] if limit_traversal: if saltenv not in self.opts[u'file_roots']: log.warning( u'During an attempt to list states for saltenv \'%s\', ' u'the environment could not be found in the configured ' u'file roots', saltenv ) return states for path in self.opts[u'file_roots'][saltenv]: for root, dirs, files in os.walk(path, topdown=True): log.debug( u'Searching for states in dirs %s and files %s', dirs, files ) if not [filename.endswith(u'.sls') for filename in files]: # Use shallow copy so we don't disturb the memory used by os.walk. Otherwise this breaks! del dirs[:] else: for found_file in files: stripped_root = os.path.relpath(root, path) if salt.utils.platform.is_windows(): stripped_root = stripped_root.replace(u'\\', u'/') stripped_root = stripped_root.replace(u'/', u'.') if found_file.endswith((u'.sls')): if found_file.endswith(u'init.sls'): if stripped_root.endswith(u'.'): stripped_root = stripped_root.rstrip(u'.') states.append(stripped_root) else: if not stripped_root.endswith(u'.'): stripped_root += u'.' if stripped_root.startswith(u'.'): stripped_root = stripped_root.lstrip(u'.') states.append(stripped_root + found_file[:-4]) else: for path in self.file_list(saltenv): if salt.utils.platform.is_windows(): path = path.replace(u'\\', u'/') if path.endswith(u'.sls'): # is an sls module! if path.endswith(u'/init.sls'): states.append(path.replace(u'/', u'.')[:-9]) else: states.append(path.replace(u'/', u'.')[:-4]) return states
def create(path, saltenv=None): """ join `path` and `saltenv` into a 'salt://' URL. """ path = path.replace("\\", "/") if salt.utils.platform.is_windows(): path = salt.utils.path.sanitize_win_path(path) path = salt.utils.data.decode(path) query = "saltenv={}".format(saltenv) if saltenv else "" url = salt.utils.data.decode(urlunparse(("file", "", path, "", query, ""))) return "salt://{}".format(url[len("file:///"):])
def _translate_sep(path): ''' Translate path separators for Windows masterless minions ''' return path.replace('\\', '/') if os.path.sep == '\\' else path
def _translate_sep(path): """ Translate path separators for Windows masterless minions """ return path.replace("\\", "/") if os.path.sep == "\\" else path
def _file_lists(load, form): """ Return a dict containing the file lists for files, dirs, emtydirs and symlinks """ if "env" in load: # "env" is not supported; Use "saltenv". load.pop("env") saltenv = load["saltenv"] actual_saltenv = saltenv if saltenv not in __opts__["file_roots"]: if "__env__" in __opts__["file_roots"]: log.debug( "salt environment '%s' maps to __env__ file_roots directory", saltenv) saltenv = "__env__" else: return [] list_cachedir = os.path.join(__opts__["cachedir"], "file_lists", "roots") if not os.path.isdir(list_cachedir): try: os.makedirs(list_cachedir) except OSError: log.critical("Unable to make cachedir %s", list_cachedir) return [] list_cache = os.path.join( list_cachedir, "{}.p".format(salt.utils.files.safe_filename_leaf(actual_saltenv)), ) w_lock = os.path.join( list_cachedir, ".{}.w".format(salt.utils.files.safe_filename_leaf(actual_saltenv)), ) cache_match, refresh_cache, save_cache = salt.fileserver.check_file_list_cache( __opts__, form, list_cache, w_lock) if cache_match is not None: return cache_match if refresh_cache: ret = {"files": set(), "dirs": set(), "empty_dirs": set(), "links": {}} def _add_to(tgt, fs_root, parent_dir, items): """ Add the files to the target set """ def _translate_sep(path): """ Translate path separators for Windows masterless minions """ return path.replace("\\", "/") if os.path.sep == "\\" else path for item in items: abs_path = os.path.join(parent_dir, item) log.trace("roots: Processing %s", abs_path) is_link = salt.utils.path.islink(abs_path) log.trace("roots: %s is %sa link", abs_path, "not " if not is_link else "") if is_link and __opts__["fileserver_ignoresymlinks"]: continue rel_path = _translate_sep(os.path.relpath(abs_path, fs_root)) log.trace("roots: %s relative path is %s", abs_path, rel_path) if salt.fileserver.is_file_ignored(__opts__, rel_path): continue tgt.add(rel_path) if os.path.isdir(abs_path): try: if not os.listdir(abs_path): ret["empty_dirs"].add(rel_path) except OSError: log.debug("Unable to list dir: %s", abs_path) if is_link: link_dest = salt.utils.path.readlink(abs_path) log.trace("roots: %s symlink destination is %s", abs_path, link_dest) if salt.utils.platform.is_windows( ) and link_dest.startswith("\\\\"): # Symlink points to a network path. Since you can't # join UNC and non-UNC paths, just assume the original # path. log.trace( "roots: %s is a UNC path, using %s instead", link_dest, abs_path, ) link_dest = abs_path if link_dest.startswith(".."): joined = os.path.join(abs_path, link_dest) else: joined = os.path.join(os.path.dirname(abs_path), link_dest) rel_dest = _translate_sep( os.path.relpath( os.path.realpath(os.path.normpath(joined)), os.path.realpath(fs_root), )) log.trace("roots: %s relative path is %s", abs_path, rel_dest) if not rel_dest.startswith(".."): # Only count the link if it does not point # outside of the root dir of the fileserver # (i.e. the "path" variable) ret["links"][rel_path] = link_dest else: if not __opts__["fileserver_followsymlinks"]: ret["links"][rel_path] = link_dest for path in __opts__["file_roots"][saltenv]: if saltenv == "__env__": path = path.replace("__env__", actual_saltenv) for root, dirs, files in salt.utils.path.os_walk( path, followlinks=__opts__["fileserver_followsymlinks"]): _add_to(ret["dirs"], path, root, dirs) _add_to(ret["files"], path, root, files) ret["files"] = sorted(ret["files"]) ret["dirs"] = sorted(ret["dirs"]) ret["empty_dirs"] = sorted(ret["empty_dirs"]) if save_cache: try: salt.fileserver.write_file_list_cache(__opts__, ret, list_cache, w_lock) except NameError: # Catch msgpack error in salt-ssh pass return ret.get(form, []) # Shouldn't get here, but if we do, this prevents a TypeError return []
def from_filenames_collection_modifyitems(config, items): from_filenames = config.getoption("--from-filenames") if not from_filenames: # Don't do anything return log.info( "Calculating test modules to run based on the paths in --from-filenames" ) from_filenames_paths = set() for path in [path.strip() for path in from_filenames.split(",")]: # Make sure that, no matter what kind of path we're passed, Windows or Posix path, # we resolve it to the platform slash separator properly_slashed_path = pathlib.Path( path.replace("\\", os.sep).replace("/", os.sep)) if not properly_slashed_path.exists(): log.info( "The path %s(%s) passed in --from-filenames does not exist", path, properly_slashed_path, ) continue if properly_slashed_path.is_absolute(): # In this case, this path is considered to be a file containing a line separated list # of files to consider with salt.utils.files.fopen(str(path)) as rfh: for line in rfh: line_path = pathlib.Path(line.strip().replace( "\\", os.sep).replace("/", os.sep)) if not line_path.exists(): log.info( "The path %s contained in %s passed in --from-filenames does not exist", line_path, properly_slashed_path, ) continue from_filenames_paths.add(line_path) continue from_filenames_paths.add(properly_slashed_path) # Let's start collecting test modules test_module_paths = set() filename_map = yaml.deserialize( (TESTS_DIR / "filename_map.yml").read_text()) # Let's add the match all rule for rule, matches in filename_map.items(): if rule == "*": for match in matches: test_module_paths.add(_match_to_test_file(match)) break # Let's now go through the list of files gathered for path in from_filenames_paths: if path.as_posix().startswith("tests/"): if path.name == "conftest.py": # This is not a test module, but consider any test_*.py files in child directories for match in path.parent.rglob("test_*.py"): test_module_paths.add(match) continue # Tests in the listing don't require additional matching and will be added to the # list of tests to run test_module_paths.add(path) continue if path.name == "setup.py" or path.as_posix().startswith("salt/"): if path.name == "__init__.py": # No direct matching continue # Let's try a direct match between the passed file and possible test modules glob_patterns = ( # salt/version.py -> # tests/unit/test_version.py # tests/pytests/unit/test_version.py "**/test_{}".format(path.name), # salt/modules/grains.py -> # tests/pytests/integration/modules/grains/tests_*.py # salt/modules/saltutil.py -> # tests/pytests/integration/modules/saltutil/test_*.py "**/{}/test_*.py".format(path.stem), # salt/modules/config.py -> # tests/unit/modules/test_config.py # tests/integration/modules/test_config.py # tests/pytests/unit/modules/test_config.py # tests/pytests/integration/modules/test_config.py "**/{}/test_{}".format(path.parent.name, path.name), ) for pattern in glob_patterns: for match in TESTS_DIR.rglob(pattern): relative_path = match.relative_to(CODE_DIR) log.info("Glob pattern %r matched '%s'", pattern, relative_path) test_module_paths.add(relative_path) # Do we have an entry in tests/filename_map.yml for rule, matches in filename_map.items(): if rule == "*": continue elif "|" in rule: # This is regex if re.match(rule, path.as_posix()): for match in matches: test_module_paths.add(_match_to_test_file(match)) elif "*" in rule or "\\" in rule: # Glob matching for filerule in CODE_DIR.glob(rule): if not filerule.exists(): continue filerule = filerule.relative_to(CODE_DIR) if filerule != path: continue for match in matches: test_module_paths.add(_match_to_test_file(match)) else: if path.as_posix() != rule: continue # Direct file paths as rules filerule = pathlib.Path(rule) if not filerule.exists(): continue for match in matches: test_module_paths.add(_match_to_test_file(match)) continue else: log.info("Don't know what to do with path %s", path) log.info( "Collected the following paths from --from-filenames processing:\n%s", "\n".join(sorted(map(str, test_module_paths))), ) selected = [] deselected = [] for item in items: itempath = pathlib.Path(str( item.fspath)).resolve().relative_to(CODE_DIR) if itempath in test_module_paths: selected.append(item) else: deselected.append(item) items[:] = selected if deselected: config.hook.pytest_deselected(items=deselected)