def setUp(self):
if salt.utils.platform.is_darwin and six.PY3:
self.patched_environ = patched_environ(
__cleanup__=["__PYVENV_LAUNCHER__"])
self.patched_environ.__enter__()
self.addCleanup(self.patched_environ.__exit__)
super(Base, self).setUp()
self._remove_dir()
shutil.copytree(self.root, self.tdir)
for idx in BOOT_INIT:
path = os.path.join(self.rdir, "{0}_bootstrap.py".format(idx))
for fname in BOOT_INIT[idx]:
shutil.copy2(path, os.path.join(self.tdir, fname))
def setUp(self):
super().setUp()
self.venv_test_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
# Remove the venv test directory
self.addCleanup(shutil.rmtree, self.venv_test_dir, ignore_errors=True)
self.venv_dir = os.path.join(self.venv_test_dir, "venv")
self.pip_temp = os.path.join(self.venv_test_dir, ".pip-temp")
if not os.path.isdir(self.pip_temp):
os.makedirs(self.pip_temp)
self.patched_environ = patched_environ(
PIP_SOURCE_DIR="",
PIP_BUILD_DIR="",
__cleanup__=[k for k in os.environ if k.startswith("PIP_")],
)
self.patched_environ.__enter__()
self.addCleanup(self.patched_environ.__exit__)
def setUpClass(cls):
cls.tmp_repo_dir = os.path.join(RUNTIME_VARS.TMP, 'gitfs_root')
if salt.utils.platform.is_windows():
cls.tmp_repo_dir = cls.tmp_repo_dir.replace('\\', '/')
cls.tmp_cachedir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
cls.tmp_sock_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
try:
shutil.rmtree(cls.tmp_repo_dir)
except OSError as exc:
if exc.errno == errno.EACCES:
log.error("Access error removing file %s", cls.tmp_repo_dir)
elif exc.errno != errno.ENOENT:
raise
shutil.copytree(
salt.ext.six.text_type(RUNTIME_VARS.BASE_FILES),
salt.ext.six.text_type(cls.tmp_repo_dir + '/')
)
repo = git.Repo.init(cls.tmp_repo_dir)
try:
if salt.utils.platform.is_windows():
username = salt.utils.win_functions.get_current_user()
else:
username = pwd.getpwuid(os.geteuid()).pw_name
except AttributeError:
log.error(
'Unable to get effective username, falling back to \'root\'.'
)
username = str('root')
with patched_environ(USERNAME=username):
repo.index.add([x for x in os.listdir(cls.tmp_repo_dir)
if x != '.git'])
repo.index.commit('Test')
# Add another branch with unicode characters in the name
repo.create_head(UNICODE_ENVNAME, 'HEAD')
# Add a tag
repo.create_tag(TAG_NAME, 'HEAD')
# Older GitPython versions do not have a close method.
if hasattr(repo, 'close'):
repo.close()
def setUp(self):
super(PipModuleTest, self).setUp()
self.venv_test_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
# Remove the venv test directory
self.addCleanup(shutil.rmtree, self.venv_test_dir, ignore_errors=True)
self.venv_dir = os.path.join(self.venv_test_dir, 'venv')
self.pip_temp = os.path.join(self.venv_test_dir, '.pip-temp')
# Remove the pip-temp directory
self.addCleanup(shutil.rmtree, self.pip_temp, ignore_errors=True)
if not os.path.isdir(self.pip_temp):
os.makedirs(self.pip_temp)
self.patched_environ = patched_environ(
PIP_SOURCE_DIR='',
PIP_BUILD_DIR='',
__cleanup__=[k for k in os.environ if k.startswith('PIP_')])
self.patched_environ.__enter__()
self.addCleanup(self.patched_environ.__exit__)
for item in ('venv_dir', 'venv_test_dir', 'pip_temp'):
self.addCleanup(delattr, self, item)
def test_pip_installed_errors(self):
venv_dir = os.path.join(RUNTIME_VARS.TMP, "pip-installed-errors")
self.addCleanup(shutil.rmtree, venv_dir, ignore_errors=True)
# Since we don't have the virtualenv created, pip.installed will
# throw an error.
# Example error strings:
# * "Error installing 'pep8': /tmp/pip-installed-errors: not found"
# * "Error installing 'pep8': /bin/sh: 1: /tmp/pip-installed-errors: not found"
# * "Error installing 'pep8': /bin/bash: /tmp/pip-installed-errors: No such file or directory"
with patched_environ(SHELL="/bin/sh"):
ret = self.run_function("state.sls", mods="pip-installed-errors")
self.assertSaltFalseReturn(ret)
self.assertSaltCommentRegexpMatches(ret, "Error installing 'pep8':")
# We now create the missing virtualenv
ret = self.run_function("virtualenv.create", [venv_dir])
self.assertEqual(ret["retcode"], 0)
# The state should not have any issues running now
ret = self.run_function("state.sls", mods="pip-installed-errors")
self.assertSaltTrueReturn(ret)
def test_pip_installed_errors(self):
venv_dir = os.path.join(RUNTIME_VARS.TMP, 'pip-installed-errors')
# Since we don't have the virtualenv created, pip.installed will
# throw an error.
# Example error strings:
# * "Error installing 'pep8': /tmp/pip-installed-errors: not found"
# * "Error installing 'pep8': /bin/sh: 1: /tmp/pip-installed-errors: not found"
# * "Error installing 'pep8': /bin/bash: /tmp/pip-installed-errors: No such file or directory"
with patched_environ(SHELL='/bin/sh'):
ret = self.run_function('state.sls', mods='pip-installed-errors')
self.assertSaltFalseReturn(ret)
self.assertSaltCommentRegexpMatches(ret,
'Error installing \'pep8\':')
# We now create the missing virtualenv
ret = self._create_virtualenv(venv_dir)
self.assertEqual(ret['retcode'], 0)
# The state should not have any issues running now
ret = self.run_function('state.sls', mods='pip-installed-errors')
self.assertSaltTrueReturn(ret)
class WebserverMixin(SaltClientMixin, SaltReturnAssertsMixin):
"""
Functions to stand up an nginx + uWSGI + git-http-backend webserver to
serve up git repos for tests.
"""
nginx_proc = uwsgi_proc = None
prep_states_ran = False
@classmethod
def setUpClass(cls): # pylint: disable=arguments-differ
"""
Set up all the webserver paths. Designed to be run once in a
setUpClass function.
'''
cls.root_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
cls.config_dir = os.path.join(cls.root_dir, 'config')
cls.nginx_conf = os.path.join(cls.config_dir, 'nginx.conf')
cls.uwsgi_conf = os.path.join(cls.config_dir, 'uwsgi.yml')
cls.git_dir = os.path.join(cls.root_dir, 'git')
cls.repo_dir = os.path.join(cls.git_dir, 'repos')
cls.venv_dir = os.path.join(cls.root_dir, 'venv')
cls.uwsgi_bin = os.path.join(cls.venv_dir, 'bin', 'uwsgi')
cls.nginx_port = cls.uwsgi_port = get_unused_localhost_port()
while cls.uwsgi_port == cls.nginx_port:
# Ensure we don't hit a corner case in which two sucessive calls to
# get_unused_localhost_port() return identical port numbers.
cls.uwsgi_port = get_unused_localhost_port()
cls.url = "http://127.0.0.1:{port}/repo.git".format(port=cls.nginx_port)
cls.url_extra_repo = "http://127.0.0.1:{port}/extra_repo.git".format(
port=cls.nginx_port
)
cls.ext_opts = {"url": cls.url, "url_extra_repo": cls.url_extra_repo}
# Add auth params if present (if so this will trigger the spawned
# server to turn on HTTP basic auth).
for credential_param in ("user", "password"):
if hasattr(cls, credential_param):
cls.ext_opts[credential_param] = getattr(cls, credential_param)
auth_enabled = hasattr(cls, "username") and hasattr(cls, "password")
pillar = {
"git_pillar": {
"config_dir": cls.config_dir,
"git_dir": cls.git_dir,
"venv_dir": cls.venv_dir,
"root_dir": cls.root_dir,
"nginx_port": cls.nginx_port,
"uwsgi_port": cls.uwsgi_port,
"auth_enabled": auth_enabled,
}
}
# Different libexec dir for git backend on Debian-based systems
git_core = "/usr/libexec/git-core"
if not os.path.exists(git_core):
git_core = "/usr/lib/git-core"
if not os.path.exists(git_core):
cls.tearDownClass()
raise AssertionError(
"{} not found. Either git is not installed, or the test "
"class needs to be updated.".format(git_core)
)
pillar["git_pillar"]["git-http-backend"] = os.path.join(
git_core, "git-http-backend"
)
try:
if cls.prep_states_ran is False:
ret = cls.cls_run_function(
"state.apply", mods="git_pillar.http", pillar=pillar
)
assert next(six.itervalues(ret))["result"] is True
cls.prep_states_ran = True
log.info("%s: States applied", cls.__name__)
if cls.uwsgi_proc is not None:
if not psutil.pid_exists(cls.uwsgi_proc.pid):
log.warning(
"%s: uWsgi started but appears to be dead now. Will try to restart it.",
cls.__name__,
)
cls.uwsgi_proc = None
if cls.uwsgi_proc is None:
cls.uwsgi_proc = start_daemon(
cls.uwsgi_bin, cls.config_dir, cls.uwsgi_port, UwsgiDaemon
)
log.info("%s: %s started", cls.__name__, cls.uwsgi_bin)
if cls.nginx_proc is not None:
if not psutil.pid_exists(cls.nginx_proc.pid):
log.warning(
"%s: nginx started but appears to be dead now. Will try to restart it.",
cls.__name__,
)
cls.nginx_proc = None
if cls.nginx_proc is None:
cls.nginx_proc = start_daemon(
"nginx", cls.config_dir, cls.nginx_port, NginxDaemon
)
log.info("%s: nginx started", cls.__name__)
except AssertionError:
cls.tearDownClass()
six.reraise(*sys.exc_info())
@classmethod
def tearDownClass(cls):
if cls.nginx_proc is not None:
log.info(
"[%s] Stopping %s",
cls.nginx_proc.log_prefix,
cls.nginx_proc.__class__.__name__,
)
terminate_process(cls.nginx_proc.pid, kill_children=True, slow_stop=True)
log.info(
"[%s] %s stopped",
cls.nginx_proc.log_prefix,
cls.nginx_proc.__class__.__name__,
)
cls.nginx_proc = None
if cls.uwsgi_proc is not None:
log.info(
"[%s] Stopping %s",
cls.uwsgi_proc.log_prefix,
cls.uwsgi_proc.__class__.__name__,
)
terminate_process(cls.uwsgi_proc.pid, kill_children=True, slow_stop=True)
log.info(
"[%s] %s stopped",
cls.uwsgi_proc.log_prefix,
cls.uwsgi_proc.__class__.__name__,
)
cls.uwsgi_proc = None
shutil.rmtree(cls.root_dir, ignore_errors=True)
cls.prep_states_ran = False
super(WebserverMixin, cls).tearDownClass()
class GitTestBase(ModuleCase):
"""
Base class for all gitfs/git_pillar tests. Must be subclassed and paired
with either SSHDMixin or WebserverMixin to provide the server.
"""
maxDiff = None
git_opts = '-c user.name="Foo Bar" -c [email protected]'
ext_opts = {}
def make_repo(self, root_dir, user="******"):
raise NotImplementedError()
class GitFSTestBase(GitTestBase, LoaderModuleMockMixin):
"""
Base class for all gitfs tests
"""
@requires_system_grains
def setup_loader_modules(self, grains): # pylint: disable=W0221
return {gitfs: {"__opts__": copy.copy(_OPTS), "__grains__": grains}}
def make_repo(self, root_dir, user="******"):
raise NotImplementedError()
class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin):
"""
Base class for all git_pillar tests
"""
bare_repo = bare_repo_backup = bare_extra_repo = bare_extra_repo_backup = None
admin_repo = admin_repo_backup = admin_extra_repo = admin_extra_repo_backup = None
@requires_system_grains
def setup_loader_modules(self, grains): # pylint: disable=W0221
return {git_pillar: {"__opts__": copy.copy(_OPTS), "__grains__": grains}}
def get_pillar(self, ext_pillar_conf):
"""
Run git_pillar with the specified configuration
'''
cachedir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
self.addCleanup(shutil.rmtree, cachedir, ignore_errors=True)
ext_pillar_opts = {"optimization_order": [0, 1, 2]}
ext_pillar_opts.update(
salt.utils.yaml.safe_load(
ext_pillar_conf.format(
cachedir=cachedir,
extmods=os.path.join(cachedir, "extmods"),
**self.ext_opts
)
)
)
with patch.dict(git_pillar.__opts__, ext_pillar_opts):
return git_pillar.ext_pillar(
"minion", {}, *ext_pillar_opts["ext_pillar"][0]["git"]
)
def make_repo(self, root_dir, user="******"):
log.info("Creating test Git repo....")
self.bare_repo = os.path.join(root_dir, "repo.git")
self.bare_repo_backup = "{}.backup".format(self.bare_repo)
self.admin_repo = os.path.join(root_dir, "admin")
self.admin_repo_backup = "{}.backup".format(self.admin_repo)
for dirname in (self.bare_repo, self.admin_repo):
shutil.rmtree(dirname, ignore_errors=True)
if os.path.exists(self.bare_repo_backup) and os.path.exists(
self.admin_repo_backup
):
shutil.copytree(self.bare_repo_backup, self.bare_repo)
shutil.copytree(self.admin_repo_backup, self.admin_repo)
return
# Create bare repo
self.run_function("git.init", [self.bare_repo], user=user, bare=True)
# Clone bare repo
self.run_function("git.clone", [self.admin_repo], url=self.bare_repo, user=user)
def _push(branch, message):
self.run_function("git.add", [self.admin_repo, "."], user=user)
self.run_function(
"git.commit",
[self.admin_repo, message],
user=user,
git_opts=self.git_opts,
)
self.run_function(
"git.push", [self.admin_repo], remote="origin", ref=branch, user=user,
)
with salt.utils.files.fopen(
os.path.join(self.admin_repo, "top.sls"), "w"
) as fp_:
fp_.write(
textwrap.dedent(
"""\
base:
'*':
- foo
"""
)
)
with salt.utils.files.fopen(
os.path.join(self.admin_repo, "foo.sls"), "w"
) as fp_:
fp_.write(
textwrap.dedent(
"""\
branch: master
mylist:
- master
mydict:
master: True
nested_list:
- master
nested_dict:
master: True
"""
)
)
# Add another file to be referenced using git_pillar_includes
with salt.utils.files.fopen(
os.path.join(self.admin_repo, "bar.sls"), "w"
) as fp_:
fp_.write("included_pillar: True\n")
# Add another file in subdir
os.mkdir(os.path.join(self.admin_repo, "subdir"))
with salt.utils.files.fopen(
os.path.join(self.admin_repo, "subdir", "bar.sls"), "w"
) as fp_:
fp_.write("from_subdir: True\n")
_push("master", "initial commit")
# Do the same with different values for "dev" branch
self.run_function("git.checkout", [self.admin_repo], user=user, opts="-b dev")
# The bar.sls shouldn't be in any branch but master
self.run_function("git.rm", [self.admin_repo, "bar.sls"], user=user)
with salt.utils.files.fopen(
os.path.join(self.admin_repo, "top.sls"), "w"
) as fp_:
fp_.write(
textwrap.dedent(
"""\
dev:
'*':
- foo
"""
)
)
with salt.utils.files.fopen(
os.path.join(self.admin_repo, "foo.sls"), "w"
) as fp_:
fp_.write(
textwrap.dedent(
"""\
branch: dev
mylist:
- dev
mydict:
dev: True
nested_list:
- dev
nested_dict:
dev: True
"""
)
)
_push("dev", "add dev branch")
# Create just a top file in a separate repo, to be mapped to the base
# env and referenced using git_pillar_includes
self.run_function(
"git.checkout", [self.admin_repo], user=user, opts="-b top_only"
)
# The top.sls should be the only file in this branch
self.run_function(
"git.rm",
[self.admin_repo, "foo.sls", os.path.join("subdir", "bar.sls")],
user=user,
)
with salt.utils.files.fopen(
os.path.join(self.admin_repo, "top.sls"), "w"
) as fp_:
fp_.write(
textwrap.dedent(
"""\
base:
'*':
- bar
"""
)
)
_push("top_only", "add top_only branch")
# Create just another top file in a separate repo, to be mapped to the base
# env and including mounted.bar
self.run_function(
"git.checkout", [self.admin_repo], user=user, opts="-b top_mounted"
)
# The top.sls should be the only file in this branch
with salt.utils.files.fopen(
os.path.join(self.admin_repo, "top.sls"), "w"
) as fp_:
fp_.write(
textwrap.dedent(
"""\
base:
'*':
- mounted.bar
"""
)
)
_push("top_mounted", "add top_mounted branch")
shutil.copytree(self.bare_repo, self.bare_repo_backup)
shutil.copytree(self.admin_repo, self.admin_repo_backup)
log.info("Test Git repo created.")
def make_extra_repo(self, root_dir, user="******"):
log.info("Creating extra test Git repo....")
self.bare_extra_repo = os.path.join(root_dir, "extra_repo.git")
self.bare_extra_repo_backup = "{}.backup".format(self.bare_extra_repo)
self.admin_extra_repo = os.path.join(root_dir, "admin_extra")
self.admin_extra_repo_backup = "{}.backup".format(self.admin_extra_repo)
for dirname in (self.bare_extra_repo, self.admin_extra_repo):
shutil.rmtree(dirname, ignore_errors=True)
if os.path.exists(self.bare_extra_repo_backup) and os.path.exists(
self.admin_extra_repo_backup
):
shutil.copytree(self.bare_extra_repo_backup, self.bare_extra_repo)
shutil.copytree(self.admin_extra_repo_backup, self.admin_extra_repo)
return
# Create bare extra repo
self.run_function("git.init", [self.bare_extra_repo], user=user, bare=True)
# Clone bare repo
self.run_function(
"git.clone", [self.admin_extra_repo], url=self.bare_extra_repo, user=user
)
def _push(branch, message):
self.run_function("git.add", [self.admin_extra_repo, "."], user=user)
self.run_function(
"git.commit",
[self.admin_extra_repo, message],
user=user,
git_opts=self.git_opts,
)
self.run_function(
"git.push",
[self.admin_extra_repo],
remote="origin",
ref=branch,
user=user,
)
with salt.utils.files.fopen(
os.path.join(self.admin_extra_repo, "top.sls"), "w"
) as fp_:
fp_.write(
textwrap.dedent(
"""\
"{{saltenv}}":
'*':
- motd
- nowhere.foo
"""
)
)
with salt.utils.files.fopen(
os.path.join(self.admin_extra_repo, "motd.sls"), "w"
) as fp_:
fp_.write(
textwrap.dedent(
"""\
motd: The force will be with you. Always.
"""
)
)
_push("master", "initial commit")
shutil.copytree(self.bare_extra_repo, self.bare_extra_repo_backup)
shutil.copytree(self.admin_extra_repo, self.admin_extra_repo_backup)
log.info("Extra test Git repo created.")
@classmethod
def tearDownClass(cls):
super(GitPillarTestBase, cls).tearDownClass()
for dirname in (
cls.admin_repo,
cls.admin_repo_backup,
cls.admin_extra_repo,
cls.admin_extra_repo_backup,
cls.bare_repo,
cls.bare_repo_backup,
cls.bare_extra_repo,
cls.bare_extra_repo_backup,
):
if dirname is not None:
shutil.rmtree(dirname, ignore_errors=True)
class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin):
"""
Base class for GitPython and Pygit2 SSH tests
"""
id_rsa_nopass = id_rsa_withpass = None
git_ssh = "/tmp/git_ssh"
def setUp(self):
"""
Create the SSH server and user, and create the git repo
"""
log.info("%s.setUp() started...", self.__class__.__name__)
super(GitPillarSSHTestBase, self).setUp()
root_dir = os.path.expanduser("~{0}".format(self.username))
if root_dir.startswith("~"):
raise AssertionError(
"Unable to resolve homedir for user '{0}'".format(self.username)
)
self.make_repo(root_dir, user=self.username)
self.make_extra_repo(root_dir, user=self.username)
log.info("%s.setUp() complete.", self.__class__.__name__)
def get_pillar(self, ext_pillar_conf):
"""
Wrap the parent class' get_pillar() func in logic that temporarily
changes the GIT_SSH to use our custom script, ensuring that the
passphraselsess key is used to auth without needing to modify the root
user's ssh config file.
'''
with patched_environ(GIT_SSH=self.git_ssh):
return super(GitPillarSSHTestBase, self).get_pillar(ext_pillar_conf)
def setUp(self):
super(SaltnadoTestCase, self).setUp()
self.patched_environ = patched_environ(ASYNC_TEST_TIMEOUT='30')
self.patched_environ.__enter__()
self.addCleanup(self.patched_environ.__exit__)
def setUp(self):
super().setUp()
self.patched_environ = patched_environ(ASYNC_TEST_TIMEOUT="30")
self.patched_environ.__enter__()
self.addCleanup(self.patched_environ.__exit__)
