def test_egg_install(self):
"""
test installing an egg package
"""
# Let's create the testing virtualenv
with VirtualEnv() as venv:
ret = self.run_function(
"cmd.run",
[
"{0} setup.py install --prefix={1}".format(
venv.venv_python, venv.venv_dir)
],
cwd=RUNTIME_VARS.CODE_DIR,
)
self._clean_build()
lib_dir = os.path.join(venv.venv_dir, "lib")
for _dir in os.listdir(lib_dir):
site_pkg = os.path.join(lib_dir, _dir, "site-packages")
for _file in os.listdir(site_pkg):
if _file.startswith("salt-"):
egg = os.path.join(venv.venv_dir, _file)
assert os.path.exists(
os.path.join(site_pkg, _file, "salt",
"_version.py"))
break
# Let's ensure the version is correct
pip_ver = self.run_function("pip.list",
bin_env=venv.venv_dir).get("salt")
egg_ver = [
x for x in egg.split("/")[-1:][0].split("-")
if re.search(r"^\d.\d*", x)
][0]
assert pip_ver == egg_ver.replace("_", "-")
def test_wheel_build(self):
"""
test building a bdist_wheel package
"""
# Let's create the testing virtualenv
with VirtualEnv() as venv:
ret = self.run_function(
"cmd.run",
[
"{0} setup.py bdist_wheel --dist-dir={1}".format(
venv.venv_python, venv.venv_dir)
],
cwd=RUNTIME_VARS.CODE_DIR,
)
for _file in os.listdir(venv.venv_dir):
if _file.endswith("whl"):
whl = os.path.join(venv.venv_dir, _file)
break
ret = self.run_function("pip.install",
pkgs=whl,
bin_env=venv.venv_dir)
# Let's ensure the version is correct
pip_ver = self.run_function("pip.list",
bin_env=venv.venv_dir).get("salt")
whl_ver = [
x for x in whl.split("/")[-1:][0].split("-")
if re.search(r"^\d.\d*", x)
][0]
assert pip_ver == whl_ver.replace("_", "-")
def test_wheel_build(self):
'''
test building a bdist_wheel package
'''
# Let's create the testing virtualenv
with VirtualEnv() as venv:
ret = self.run_function('cmd.run', [
'{0} setup.py bdist_wheel --dist-dir={1}'.format(
venv.venv_python, venv.venv_dir)
],
cwd=RUNTIME_VARS.CODE_DIR)
for _file in os.listdir(venv.venv_dir):
if _file.endswith('whl'):
whl = os.path.join(venv.venv_dir, _file)
break
ret = self.run_function('pip.install',
pkgs=whl,
bin_env=venv.venv_dir)
# Let's ensure the version is correct
pip_ver = self.run_function('pip.list',
bin_env=venv.venv_dir).get('salt')
whl_ver = [
x for x in whl.split('/')[-1:][0].split('-')
if re.search(r'^\d.\d*', x)
][0]
assert pip_ver == whl_ver.replace('_', '-')
def test_egg_install(self):
'''
test installing an egg package
'''
# Let's create the testing virtualenv
with VirtualEnv() as venv:
ret = self.run_function('cmd.run', [
'{0} setup.py install --prefix={1}'.format(
venv.venv_python, venv.venv_dir)
],
cwd=RUNTIME_VARS.CODE_DIR)
self._clean_build()
lib_dir = os.path.join(venv.venv_dir, 'lib')
for _dir in os.listdir(lib_dir):
site_pkg = os.path.join(lib_dir, _dir, 'site-packages')
for _file in os.listdir(site_pkg):
if _file.startswith('salt-'):
egg = os.path.join(venv.venv_dir, _file)
assert os.path.exists(
os.path.join(site_pkg, _file, 'salt',
'_version.py'))
break
# Let's ensure the version is correct
pip_ver = self.run_function('pip.list',
bin_env=venv.venv_dir).get('salt')
egg_ver = [
x for x in egg.split('/')[-1:][0].split('-')
if re.search(r'^\d.\d*', x)
][0]
assert pip_ver == egg_ver.replace('_', '-')
def test_list_available_packages(modules, pip_version, tmp_path):
if sys.version_info < (3, 6) and pip_version == "pip>=21.0":
pytest.skip("{} is not avaiable on Py3.5".format(pip_version))
with VirtualEnv(venv_dir=tmp_path, pip_requirement=pip_version) as virtualenv:
virtualenv.install("-U", pip_version)
package_name = "pep8"
available_versions = modules.pip.list_all_versions(
package_name, bin_env=str(virtualenv.venv_bin_dir)
)
assert available_versions
def test_wheel_build(self, tempdir):
"""
test building a bdist_wheel package
"""
# Let's create the testing virtualenv
with VirtualEnv() as venv:
venv.run(venv.venv_python,
"setup.py",
"clean",
cwd=RUNTIME_VARS.CODE_DIR)
venv.run(
venv.venv_python,
"setup.py",
"bdist_wheel",
"--dist-dir",
tempdir,
cwd=RUNTIME_VARS.CODE_DIR,
)
for _file in os.listdir(tempdir):
if _file.endswith("whl"):
whl = os.path.join(tempdir, _file)
break
venv.install("--ignore-installed", whl)
# Let's ensure the version is correct
cmd = venv.run(venv.venv_python, "-m", "pip", "list", "--format",
"json")
for details in json.loads(cmd.stdout):
if details["name"] != "salt":
continue
pip_ver = details["version"]
break
else:
self.fail("Salt was not found installed")
whl_ver = [
x for x in whl.split("/")[-1:][0].split("-")
if re.search(r"^\d.\d*", x)
][0]
whl_ver_cmp = whl_ver.replace("_", "-")
assert pip_ver == whl_ver.replace("_", "-"), "{} != {}".format(
pip_ver, whl_ver_cmp)
def test_thin_dir(self):
"""
Test the thin dir to make sure salt-call can run
Run salt call via a python in a new virtual environment to ensure
salt-call has all dependencies needed.
"""
# This was previously an integration test and is now here, as a unit test.
# Should actually be a functional test
with VirtualEnv() as venv:
salt.utils.thin.gen_thin(str(venv.venv_dir))
thin_dir = venv.venv_dir / "thin"
thin_archive = thin_dir / "thin.tgz"
tar = tarfile.open(str(thin_archive))
tar.extractall(str(thin_dir))
tar.close()
ret = venv.run(
venv.venv_python, str(thin_dir / "salt-call"), "--version", check=False,
)
assert ret.exitcode == 0, ret
def test_man_pages(self):
"""
Make sure that man pages are installed
"""
# Map filenames to search strings which should be in the manpage
manpages = {
"salt-cp.1":
["salt-cp Documentation", "copies files from the master"],
"salt-cloud.1": [
"Salt Cloud Command",
"Provision virtual machines in the cloud",
],
"salt-call.1":
["salt-call Documentation", "run module functions locally"],
"salt-api.1": [
"salt-api Command",
"Start interfaces used to remotely connect",
],
"salt-unity.1":
["salt-unity Command", "unified invocation wrapper"],
"salt-syndic.1":
["salt-syndic Documentation", "Salt syndic daemon"],
"salt-ssh.1":
["salt-ssh Documentation", "executed using only SSH"],
"salt-run.1":
["salt-run Documentation", "frontend command for executing"],
"salt-proxy.1":
["salt-proxy Documentation", "proxies these commands"],
"salt-minion.1":
["salt-minion Documentation", "Salt minion daemon"],
"salt-master.1":
["salt-master Documentation", "Salt master daemon"],
"salt-key.1": [
"salt-key Documentation",
"management of Salt server public keys",
],
"salt.1": ["allows for commands to be executed"],
"salt.7": ["Salt Documentation"],
"spm.1": [
"Salt Package Manager Command",
"command for managing Salt packages",
],
}
with VirtualEnv() as venv:
rootdir = os.path.join(venv.venv_dir, "installed")
venv.run(
venv.venv_python,
"setup.py",
"install",
"--root={}".format(rootdir),
cwd=RUNTIME_VARS.CODE_DIR,
)
manpage_fns = set(manpages)
manpage_paths = {}
for root, _, files in os.walk(rootdir):
if not manpage_fns:
# All manpages found, no need to keep walking
break
# Using list because we will be modifying the set during iteration
for manpage_fn in list(manpage_fns):
if manpage_fn in files:
manpage_path = salt.utils.path.join(root, manpage_fn)
manpage_paths[manpage_fn] = manpage_path
manpage_fns.remove(manpage_fn)
assert (
not manpage_fns
), "The following manpages were not found under {}: {}".format(
rootdir, ", ".join(sorted(manpage_fns)))
failed = {}
for manpage in sorted(manpages):
with salt.utils.files.fopen(manpage_paths[manpage]) as fp_:
contents = salt.utils.stringutils.to_unicode(fp_.read())
# Check for search string in contents
for search_string in manpages[manpage]:
if search_string not in contents:
failed.setdefault(manpage, []).append(
"No match for search string '{}' found in {}".
format(search_string, manpage_paths[manpage]))
# Check for correct install dir
path = "/man{}/".format(manpage.rsplit(".", 1)[-1])
if path not in manpage_paths[manpage]:
failed.setdefault(manpage, []).append(
"{} not found in manpage path {}".format(
path, manpage_paths[manpage]))
assert not failed, "One or more manpages failed:\n{}".format(
pprint.pformat(failed))
def test_importable_installation_error(self):
extra_requirements = []
for name, version in salt.version.dependency_information():
if name in ["PyYAML"]:
extra_requirements.append("{}=={}".format(name, version))
failures = {}
pip_version_requirements = [
# Latest pip 18
"<19.0",
# Latest pip 19
"<20.0",
# Latest pip 20
"<21.0",
# Latest pip
None,
]
code = dedent("""\
import sys
import traceback
try:
import salt.states.pip_state
salt.states.pip_state.InstallationError
except ImportError as exc:
traceback.print_exc(file=sys.stdout)
sys.stdout.flush()
sys.exit(1)
except AttributeError as exc:
traceback.print_exc(file=sys.stdout)
sys.stdout.flush()
sys.exit(2)
except Exception as exc:
traceback.print_exc(exc, file=sys.stdout)
sys.stdout.flush()
sys.exit(3)
sys.exit(0)
""")
for requirement in list(pip_version_requirements):
try:
with VirtualEnv() as venv:
venv.install(*extra_requirements)
if requirement:
venv.install("pip{}".format(requirement))
try:
subprocess.check_output([venv.venv_python, "-c", code])
except subprocess.CalledProcessError as exc:
if exc.returncode == 1:
failures[
requirement] = "Failed to import pip:\n{}".format(
exc.output)
elif exc.returncode == 2:
failures[
requirement] = "Failed to import InstallationError from pip:\n{}".format(
exc.output)
else:
failures[requirement] = exc.output
except Exception as exc: # pylint: disable=broad-except
failures[requirement] = str(exc)
if failures:
errors = ""
for requirement, exception in failures.items():
errors += "pip{}: {}\n\n".format(requirement or "", exception)
self.fail(
"Failed to get InstallationError exception under at least one pip"
" version:\n{}".format(errors))
def virtualenv(setup_tests_path, pip_temp_dir):
venv_dir = setup_tests_path / ".venv"
try:
yield VirtualEnv(venv_dir=venv_dir, env={"TMPDIR": str(pip_temp_dir)})
finally:
shutil.rmtree(str(venv_dir), ignore_errors=True)
def test_wheel(self, tempdir):
"""
test building and installing a bdist_wheel package
"""
# Let's create the testing virtualenv
with VirtualEnv() as venv:
venv.run(venv.venv_python, "setup.py", "clean", cwd=RUNTIME_VARS.CODE_DIR)
venv.run(
venv.venv_python,
"setup.py",
"bdist_wheel",
"--dist-dir",
tempdir,
cwd=RUNTIME_VARS.CODE_DIR,
)
venv.run(venv.venv_python, "setup.py", "clean", cwd=RUNTIME_VARS.CODE_DIR)
salt_generated_package = list(pathlib.Path(tempdir).glob("*.whl"))
if not salt_generated_package:
self.fail("Could not find the generated wheel file")
salt_generated_package = salt_generated_package[0]
# Assert generate wheel version matches what salt reports as its version
whl_ver = [
x
for x in salt_generated_package.name.split("-")
if re.search(r"^\d.\d*", x)
][0]
whl_ver_cmp = whl_ver.replace("_", "-")
salt_ver_cmp = salt.version.__version__.replace("/", "-")
assert whl_ver_cmp == salt_ver_cmp, "{} != {}".format(
whl_ver_cmp, salt_ver_cmp
)
# Because bdist_wheel supports pep517, we don't have to pre-install Salt's
# dependencies before installing the wheel package
venv.install(str(salt_generated_package))
# Let's ensure the version is correct
cmd = venv.run(venv.venv_python, "-m", "pip", "list", "--format", "json")
for details in json.loads(cmd.stdout):
if details["name"] != "salt":
continue
installed_version = details["version"]
break
else:
self.fail("Salt was not found installed")
# Let's compare the installed version with the version salt reports
assert installed_version == salt_ver_cmp, "{} != {}".format(
installed_version, salt_ver_cmp
)
# Let's also ensure we have a salt/_version.py from the installed salt wheel
subdir = [
"lib",
"python{}.{}".format(*sys.version_info),
"site-packages",
"salt",
]
if salt.utils.platform.is_windows():
subdir.pop(1)
installed_salt_path = pathlib.Path(venv.venv_dir)
installed_salt_path = installed_salt_path.joinpath(*subdir)
assert installed_salt_path.is_dir()
salt_generated_version_file_path = installed_salt_path / "_version.py"
assert salt_generated_version_file_path.is_file()
def test_setup_install(self, tempdir):
"""
test installing directly from source
"""
# Let's create the testing virtualenv
with VirtualEnv() as venv:
venv.run(venv.venv_python, "setup.py", "clean", cwd=RUNTIME_VARS.CODE_DIR)
# Setuptools installs pre-release packages if we don't pin to an exact version
# Let's download and install requirements before, running salt's install test
venv.run(
venv.venv_python,
"-m",
"pip",
"download",
"--dest",
tempdir,
RUNTIME_VARS.CODE_DIR,
)
packages = []
for fname in os.listdir(tempdir):
packages.append(os.path.join(tempdir, fname))
venv.install(*packages)
for package in packages:
os.unlink(package)
venv.run(
venv.venv_python,
"setup.py",
"install",
"--prefix",
venv.venv_dir,
cwd=RUNTIME_VARS.CODE_DIR,
)
venv.run(venv.venv_python, "setup.py", "clean", cwd=RUNTIME_VARS.CODE_DIR)
# Let's ensure the version is correct
cmd = venv.run(venv.venv_python, "-m", "pip", "list", "--format", "json")
for details in json.loads(cmd.stdout):
if details["name"] != "salt":
continue
installed_version = details["version"]
break
else:
self.fail("Salt was not found installed")
salt_ver_cmp = salt.version.__version__.replace("/", "-")
# Let's compare the installed version with the version salt reports
assert installed_version == salt_ver_cmp, "{} != {}".format(
installed_version, salt_ver_cmp
)
# Let's also ensure we have a salt/_version.py from the installed salt
subdir = [
"lib",
"python{}.{}".format(*sys.version_info),
"site-packages",
]
if salt.utils.platform.is_windows():
subdir.pop(1)
site_packages_dir = pathlib.Path(venv.venv_dir)
site_packages_dir = site_packages_dir.joinpath(*subdir)
assert site_packages_dir.is_dir()
installed_salt_path = list(site_packages_dir.glob("salt*.egg"))
if not installed_salt_path:
self.fail("Failed to find the installed salt path")
installed_salt_path = installed_salt_path[0] / "salt"
salt_generated_version_file_path = installed_salt_path / "_version.py"
assert salt_generated_version_file_path.is_file()
def test_sdist(self, tempdir):
"""
test building and installing a sdist package
"""
# Let's create the testing virtualenv
with VirtualEnv() as venv:
venv.run(venv.venv_python, "setup.py", "clean", cwd=RUNTIME_VARS.CODE_DIR)
# Setuptools installs pre-release packages if we don't pin to an exact version
# Let's download and install requirements before, running salt's install test
venv.run(
venv.venv_python,
"-m",
"pip",
"download",
"--dest",
tempdir,
RUNTIME_VARS.CODE_DIR,
)
packages = []
for fname in os.listdir(tempdir):
packages.append(os.path.join(tempdir, fname))
venv.install(*packages)
for package in packages:
os.unlink(package)
venv.run(
venv.venv_python,
"setup.py",
"sdist",
"--dist-dir",
tempdir,
cwd=RUNTIME_VARS.CODE_DIR,
)
venv.run(venv.venv_python, "setup.py", "clean", cwd=RUNTIME_VARS.CODE_DIR)
salt_generated_package = list(pathlib.Path(tempdir).glob("*.tar.gz"))
if not salt_generated_package:
self.fail("Could not find the generated sdist file")
salt_generated_package = salt_generated_package[0]
log.info("Generated sdist file: %s", salt_generated_package.name)
# Assert generated sdist version matches what salt reports as its version
sdist_ver_cmp = salt_generated_package.name.split(".tar.gz")[0].split(
"salt-"
)[-1]
salt_ver_cmp = salt.version.__version__.replace("/", "-")
assert sdist_ver_cmp == salt_ver_cmp, "{} != {}".format(
sdist_ver_cmp, salt.version.__version__
)
venv.install(str(salt_generated_package))
# Let's also ensure we have a salt/_version.py from the installed salt wheel
subdir = [
"lib",
"python{}.{}".format(*sys.version_info),
"site-packages",
"salt",
]
if salt.utils.platform.is_windows():
subdir.pop(1)
installed_salt_path = pathlib.Path(venv.venv_dir)
installed_salt_path = installed_salt_path.joinpath(*subdir)
assert installed_salt_path.is_dir()
salt_generated_version_file_path = installed_salt_path / "_version.py"
assert salt_generated_version_file_path.is_file()
with salt_generated_version_file_path.open() as rfh:
log.debug("_version.py contents:\n >>>>>>\n%s\n <<<<<<", rfh.read())
# Let's ensure the version is correct
cmd = venv.run(venv.venv_python, "-m", "pip", "list", "--format", "json")
for details in json.loads(cmd.stdout):
if details["name"] != "salt":
continue
installed_version = details["version"]
break
else:
self.fail("Salt was not found installed")
# Let's compare the installed version with the version salt reports
assert installed_version == salt_ver_cmp, "{} != {}".format(
installed_version, salt_ver_cmp
)
def test_egg(self, tempdir):
"""
test building and installing a bdist_egg package
"""
# TODO: We should actually dissallow generating an egg file
# Let's create the testing virtualenv
with VirtualEnv() as venv:
venv.run(venv.venv_python, "setup.py", "clean", cwd=RUNTIME_VARS.CODE_DIR)
# Setuptools installs pre-release packages if we don't pin to an exact version
# Let's download and install requirements before, running salt's install test
venv.run(
venv.venv_python,
"-m",
"pip",
"download",
"--dest",
tempdir,
RUNTIME_VARS.CODE_DIR,
)
packages = []
for fname in os.listdir(tempdir):
packages.append(os.path.join(tempdir, fname))
venv.install(*packages)
for package in packages:
os.unlink(package)
venv.run(
venv.venv_python,
"setup.py",
"bdist_egg",
"--dist-dir",
tempdir,
cwd=RUNTIME_VARS.CODE_DIR,
)
venv.run(venv.venv_python, "setup.py", "clean", cwd=RUNTIME_VARS.CODE_DIR)
salt_generated_package = list(pathlib.Path(tempdir).glob("*.egg"))
if not salt_generated_package:
self.fail("Could not find the generated egg file")
salt_generated_package = salt_generated_package[0]
# Assert generate wheel version matches what salt reports as its version
egg_ver = [
x
for x in salt_generated_package.name.split("-")
if re.search(r"^\d.\d*", x)
][0]
egg_ver_cmp = egg_ver.replace("_", "-")
salt_ver_cmp = salt.version.__version__.replace("/", "-")
assert egg_ver_cmp == salt_ver_cmp, "{} != {}".format(
egg_ver_cmp, salt_ver_cmp
)
# We cannot pip install an egg file, let's go old school
venv.run(
venv.venv_python, "-m", "easy_install", str(salt_generated_package)
)
# Let's ensure the version is correct
cmd = venv.run(venv.venv_python, "-m", "pip", "list", "--format", "json")
for details in json.loads(cmd.stdout):
if details["name"] != "salt":
continue
installed_version = details["version"]
break
else:
self.fail("Salt was not found installed")
# Let's compare the installed version with the version salt reports
assert installed_version == salt_ver_cmp, "{} != {}".format(
installed_version, salt_ver_cmp
)
# Let's also ensure we have a salt/_version.py from the installed salt egg
subdir = [
"lib",
"python{}.{}".format(*sys.version_info),
"site-packages",
]
if salt.utils.platform.is_windows():
subdir.pop(1)
site_packages_dir = pathlib.Path(venv.venv_dir)
site_packages_dir = site_packages_dir.joinpath(*subdir)
assert site_packages_dir.is_dir()
installed_salt_path = list(site_packages_dir.glob("salt*.egg"))
if not installed_salt_path:
self.fail("Failed to find the installed salt path")
log.debug("Installed salt path glob matches: %s", installed_salt_path)
installed_salt_path = installed_salt_path[0] / "salt"
assert installed_salt_path.is_dir()
salt_generated_version_file_path = installed_salt_path / "_version.py"
assert (
salt_generated_version_file_path.is_file()
), "{} is not a file".format(salt_generated_version_file_path)
def test_egg_install(self, tempdir):
"""
test installing an egg package
"""
# Let's create the testing virtualenv
with VirtualEnv() as venv:
venv.run(venv.venv_python,
"setup.py",
"clean",
cwd=RUNTIME_VARS.CODE_DIR)
# Setuptools installs pre-release packages if we don't pin to an exact version
# Let's download and install requirements before, running salt's install test
venv.run(
venv.venv_python,
"-m",
"pip",
"download",
"--dest",
tempdir,
RUNTIME_VARS.CODE_DIR,
)
packages = []
for fname in os.listdir(tempdir):
packages.append(os.path.join(tempdir, fname))
venv.install(*packages)
venv.run(
venv.venv_python,
"setup.py",
"install",
"--prefix",
venv.venv_dir,
cwd=RUNTIME_VARS.CODE_DIR,
)
venv.run(venv.venv_python,
"setup.py",
"clean",
cwd=RUNTIME_VARS.CODE_DIR)
lib_dir = os.path.join(venv.venv_dir, "lib")
for _dir in os.listdir(lib_dir):
site_pkg = os.path.join(lib_dir, _dir, "site-packages")
for _file in os.listdir(site_pkg):
if _file.startswith("salt-"):
egg = os.path.join(venv.venv_dir, _file)
assert os.path.exists(
os.path.join(site_pkg, _file, "salt",
"_version.py"))
break
# Let's ensure the version is correct
cmd = venv.run(venv.venv_python, "-m", "pip", "list", "--format",
"json")
for details in json.loads(cmd.stdout):
if details["name"] != "salt":
continue
pip_ver = details["version"]
break
else:
self.fail("Salt was not found installed")
egg_ver = [
x for x in egg.split("/")[-1:][0].split("-")
if re.search(r"^\d.\d*", x)
][0]
assert pip_ver == egg_ver.replace("_", "-")
def virtualenv(tmp_path):
return VirtualEnv(venv_dir=str(tmp_path / ".venv"))