def _get_all_files(self, path, *exclude):
"""
Walk implementation. Version in python 2.x and 3.x works differently.
"""
files = list()
dirs = list()
links = list()
if os.access(path, os.R_OK):
for obj in os.listdir(path):
obj = os.path.join(path, obj)
valid = True
for ex_obj in exclude:
if obj.startswith(str(ex_obj)):
valid = False
continue
if not valid or not os.path.exists(obj) or not os.access(obj, os.R_OK):
continue
if salt.utils.path.islink(obj):
links.append(obj)
elif os.path.isdir(obj):
dirs.append(obj)
f_obj, d_obj, l_obj = self._get_all_files(obj, *exclude)
files.extend(f_obj)
dirs.extend(d_obj)
links.extend(l_obj)
elif os.path.isfile(obj):
files.append(obj)
return sorted(files), sorted(dirs), sorted(links)
def file_list(*packages):
'''
List the files that belong to a package. Not specifying any packages will
return a list of _every_ file on the system's package database (not
generally recommended).
CLI Examples:
.. code-block:: bash
salt '*' pkg.file_list httpd
salt '*' pkg.file_list httpd postfix
salt '*' pkg.file_list
'''
output = file_dict(*packages)
files = []
for package in list(output['packages'].values()):
files.extend(package)
return {'errors': output['errors'], 'files': files}
def _locate_repo_files(repo, rewrite=False):
"""
Find what file a repo is called in.
Helper function for add_repo() and del_repo()
repo
url of the repo to locate (persistent).
rewrite
Whether to remove matching repository settings during this process.
Returns a list of absolute paths.
"""
ret_val = []
files = []
conf_dirs = ["/etc/xbps.d/", "/usr/share/xbps.d/"]
name_glob = "*.conf"
# Matches a line where first printing is "repository" and there is an equals
# sign before the repo, an optional forwardslash at the end of the repo name,
# and it's possible for there to be a comment after repository=repo
regex = re.compile(r"\s*repository\s*=\s*" + repo + r"/?\s*(#.*)?$")
for cur_dir in conf_dirs:
files.extend(glob.glob(cur_dir + name_glob))
for filename in files:
write_buff = []
with salt.utils.files.fopen(filename, "r") as cur_file:
for line in cur_file:
if regex.match(salt.utils.stringutils.to_unicode(line)):
ret_val.append(filename)
else:
write_buff.append(line)
if rewrite and filename in ret_val:
if len(write_buff) > 0:
with salt.utils.files.fopen(filename, "w") as rewrite_file:
rewrite_file.writelines(write_buff)
else: # Prune empty files
os.remove(filename)
return ret_val
def file_list(*packages, **kwargs): # pylint: disable=unused-argument
"""
List the files that belong to a package. Not specifying any packages will
return a list of _every_ file on the system's package database (not
generally recommended).
CLI Examples:
.. code-block:: bash
salt '*' pkg.file_list httpd
salt '*' pkg.file_list httpd postfix
salt '*' pkg.file_list
"""
output = file_dict(*packages)
files = []
for package in list(output["packages"].values()):
files.extend(package)
return {"errors": output["errors"], "files": files}
def _refresh_file_mapping(self):
"""
refresh the mapping of the FS on disk
"""
# map of suffix to description for imp
if (self.opts.get("cython_enable", True) is True
and ".pyx" not in self.suffix_map):
try:
global pyximport
pyximport = __import__("pyximport") # pylint: disable=import-error
pyximport.install()
# add to suffix_map so file_mapping will pick it up
self.suffix_map[".pyx"] = tuple()
if ".pyx" not in self.suffix_order:
self.suffix_order.append(".pyx")
except ImportError:
log.info("Cython is enabled in the options but not present "
"in the system path. Skipping Cython modules.")
# Allow for zipimport of modules
if (self.opts.get("enable_zip_modules", True) is True
and ".zip" not in self.suffix_map):
self.suffix_map[".zip"] = tuple()
if ".zip" not in self.suffix_order:
self.suffix_order.append(".zip")
# allow for module dirs
self.suffix_map[""] = ("", "", MODULE_KIND_PKG_DIRECTORY)
# create mapping of filename (without suffix) to (path, suffix)
# The files are added in order of priority, so order *must* be retained.
self.file_mapping = salt.utils.odict.OrderedDict()
opt_match = []
def _replace_pre_ext(obj):
"""
Hack so we can get the optimization level that we replaced (if
any) out of the re.sub call below. We use a list here because
it is a persistent data structure that we will be able to
access after re.sub is called.
"""
opt_match.append(obj)
return ""
for mod_dir in self.module_dirs:
try:
# Make sure we have a sorted listdir in order to have
# expectable override results
files = sorted(x for x in os.listdir(mod_dir)
if x != "__pycache__")
except OSError:
continue # Next mod_dir
try:
pycache_files = [
os.path.join("__pycache__", x) for x in sorted(
os.listdir(os.path.join(mod_dir, "__pycache__")))
]
except OSError:
pass
else:
files.extend(pycache_files)
for filename in files:
try:
dirname, basename = os.path.split(filename)
if basename.startswith("_"):
# skip private modules
# log messages omitted for obviousness
continue # Next filename
f_noext, ext = os.path.splitext(basename)
f_noext = PY3_PRE_EXT.sub(_replace_pre_ext, f_noext)
try:
opt_level = int(opt_match.pop().group(1).rsplit(
"-", 1)[-1])
except (AttributeError, IndexError, ValueError):
# No regex match or no optimization level matched
opt_level = 0
try:
opt_index = self.opts["optimization_order"].index(
opt_level)
except KeyError:
log.trace(
"Disallowed optimization level %d for module "
"name '%s', skipping. Add %d to the "
"'optimization_order' config option if you "
"do not want to ignore this optimization "
"level.",
opt_level,
f_noext,
opt_level,
)
continue
# make sure it is a suffix we support
if ext not in self.suffix_map:
continue # Next filename
if f_noext in self.disabled:
log.trace(
"Skipping %s, it is disabled by configuration",
filename)
continue # Next filename
fpath = os.path.join(mod_dir, filename)
# if its a directory, lets allow us to load that
if ext == "":
# is there something __init__?
subfiles = os.listdir(fpath)
for suffix in self.suffix_order:
if "" == suffix:
continue # Next suffix (__init__ must have a suffix)
init_file = "__init__{}".format(suffix)
if init_file in subfiles:
break
else:
continue # Next filename
try:
curr_ext = self.file_mapping[f_noext][1]
curr_opt_index = self.file_mapping[f_noext][2]
except KeyError:
pass
else:
if "" in (curr_ext, ext) and curr_ext != ext:
log.error(
"Module/package collision: '%s' and '%s'",
fpath,
self.file_mapping[f_noext][0],
)
if ext == ".pyc" and curr_ext == ".pyc":
# Check the optimization level
if opt_index >= curr_opt_index:
# Module name match, but a higher-priority
# optimization level was already matched, skipping.
continue
elif not curr_ext or self.suffix_order.index(
ext) >= self.suffix_order.index(curr_ext):
# Match found but a higher-priorty match already
# exists, so skip this.
continue
if not dirname and ext == ".pyc":
# On Python 3, we should only load .pyc files from the
# __pycache__ subdirectory (i.e. when dirname is not an
# empty string).
continue
# Made it this far - add it
self.file_mapping[f_noext] = (fpath, ext, opt_index)
except OSError:
continue
for smod in self.static_modules:
f_noext = smod.split(".")[-1]
self.file_mapping[f_noext] = (smod, ".o", 0)
