def setUp(self): if salt.utils.platform.is_darwin and six.PY3: self.patched_environ = patched_environ( __cleanup__=["__PYVENV_LAUNCHER__"]) self.patched_environ.__enter__() self.addCleanup(self.patched_environ.__exit__) super(Base, self).setUp() self._remove_dir() shutil.copytree(self.root, self.tdir) for idx in BOOT_INIT: path = os.path.join(self.rdir, "{0}_bootstrap.py".format(idx)) for fname in BOOT_INIT[idx]: shutil.copy2(path, os.path.join(self.tdir, fname))
def setUp(self): super().setUp() self.venv_test_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) # Remove the venv test directory self.addCleanup(shutil.rmtree, self.venv_test_dir, ignore_errors=True) self.venv_dir = os.path.join(self.venv_test_dir, "venv") self.pip_temp = os.path.join(self.venv_test_dir, ".pip-temp") if not os.path.isdir(self.pip_temp): os.makedirs(self.pip_temp) self.patched_environ = patched_environ( PIP_SOURCE_DIR="", PIP_BUILD_DIR="", __cleanup__=[k for k in os.environ if k.startswith("PIP_")], ) self.patched_environ.__enter__() self.addCleanup(self.patched_environ.__exit__)
def setUpClass(cls): cls.tmp_repo_dir = os.path.join(RUNTIME_VARS.TMP, 'gitfs_root') if salt.utils.platform.is_windows(): cls.tmp_repo_dir = cls.tmp_repo_dir.replace('\\', '/') cls.tmp_cachedir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) cls.tmp_sock_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) try: shutil.rmtree(cls.tmp_repo_dir) except OSError as exc: if exc.errno == errno.EACCES: log.error("Access error removing file %s", cls.tmp_repo_dir) elif exc.errno != errno.ENOENT: raise shutil.copytree( salt.ext.six.text_type(RUNTIME_VARS.BASE_FILES), salt.ext.six.text_type(cls.tmp_repo_dir + '/') ) repo = git.Repo.init(cls.tmp_repo_dir) try: if salt.utils.platform.is_windows(): username = salt.utils.win_functions.get_current_user() else: username = pwd.getpwuid(os.geteuid()).pw_name except AttributeError: log.error( 'Unable to get effective username, falling back to \'root\'.' ) username = str('root') with patched_environ(USERNAME=username): repo.index.add([x for x in os.listdir(cls.tmp_repo_dir) if x != '.git']) repo.index.commit('Test') # Add another branch with unicode characters in the name repo.create_head(UNICODE_ENVNAME, 'HEAD') # Add a tag repo.create_tag(TAG_NAME, 'HEAD') # Older GitPython versions do not have a close method. if hasattr(repo, 'close'): repo.close()
def setUp(self): super(PipModuleTest, self).setUp() self.venv_test_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) # Remove the venv test directory self.addCleanup(shutil.rmtree, self.venv_test_dir, ignore_errors=True) self.venv_dir = os.path.join(self.venv_test_dir, 'venv') self.pip_temp = os.path.join(self.venv_test_dir, '.pip-temp') # Remove the pip-temp directory self.addCleanup(shutil.rmtree, self.pip_temp, ignore_errors=True) if not os.path.isdir(self.pip_temp): os.makedirs(self.pip_temp) self.patched_environ = patched_environ( PIP_SOURCE_DIR='', PIP_BUILD_DIR='', __cleanup__=[k for k in os.environ if k.startswith('PIP_')]) self.patched_environ.__enter__() self.addCleanup(self.patched_environ.__exit__) for item in ('venv_dir', 'venv_test_dir', 'pip_temp'): self.addCleanup(delattr, self, item)
def test_pip_installed_errors(self): venv_dir = os.path.join(RUNTIME_VARS.TMP, "pip-installed-errors") self.addCleanup(shutil.rmtree, venv_dir, ignore_errors=True) # Since we don't have the virtualenv created, pip.installed will # throw an error. # Example error strings: # * "Error installing 'pep8': /tmp/pip-installed-errors: not found" # * "Error installing 'pep8': /bin/sh: 1: /tmp/pip-installed-errors: not found" # * "Error installing 'pep8': /bin/bash: /tmp/pip-installed-errors: No such file or directory" with patched_environ(SHELL="/bin/sh"): ret = self.run_function("state.sls", mods="pip-installed-errors") self.assertSaltFalseReturn(ret) self.assertSaltCommentRegexpMatches(ret, "Error installing 'pep8':") # We now create the missing virtualenv ret = self.run_function("virtualenv.create", [venv_dir]) self.assertEqual(ret["retcode"], 0) # The state should not have any issues running now ret = self.run_function("state.sls", mods="pip-installed-errors") self.assertSaltTrueReturn(ret)
def test_pip_installed_errors(self): venv_dir = os.path.join(RUNTIME_VARS.TMP, 'pip-installed-errors') # Since we don't have the virtualenv created, pip.installed will # throw an error. # Example error strings: # * "Error installing 'pep8': /tmp/pip-installed-errors: not found" # * "Error installing 'pep8': /bin/sh: 1: /tmp/pip-installed-errors: not found" # * "Error installing 'pep8': /bin/bash: /tmp/pip-installed-errors: No such file or directory" with patched_environ(SHELL='/bin/sh'): ret = self.run_function('state.sls', mods='pip-installed-errors') self.assertSaltFalseReturn(ret) self.assertSaltCommentRegexpMatches(ret, 'Error installing \'pep8\':') # We now create the missing virtualenv ret = self._create_virtualenv(venv_dir) self.assertEqual(ret['retcode'], 0) # The state should not have any issues running now ret = self.run_function('state.sls', mods='pip-installed-errors') self.assertSaltTrueReturn(ret)
class WebserverMixin(SaltClientMixin, SaltReturnAssertsMixin): """ Functions to stand up an nginx + uWSGI + git-http-backend webserver to serve up git repos for tests. """ nginx_proc = uwsgi_proc = None prep_states_ran = False @classmethod def setUpClass(cls): # pylint: disable=arguments-differ """ Set up all the webserver paths. Designed to be run once in a setUpClass function. ''' cls.root_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) cls.config_dir = os.path.join(cls.root_dir, 'config') cls.nginx_conf = os.path.join(cls.config_dir, 'nginx.conf') cls.uwsgi_conf = os.path.join(cls.config_dir, 'uwsgi.yml') cls.git_dir = os.path.join(cls.root_dir, 'git') cls.repo_dir = os.path.join(cls.git_dir, 'repos') cls.venv_dir = os.path.join(cls.root_dir, 'venv') cls.uwsgi_bin = os.path.join(cls.venv_dir, 'bin', 'uwsgi') cls.nginx_port = cls.uwsgi_port = get_unused_localhost_port() while cls.uwsgi_port == cls.nginx_port: # Ensure we don't hit a corner case in which two sucessive calls to # get_unused_localhost_port() return identical port numbers. cls.uwsgi_port = get_unused_localhost_port() cls.url = "http://127.0.0.1:{port}/repo.git".format(port=cls.nginx_port) cls.url_extra_repo = "http://127.0.0.1:{port}/extra_repo.git".format( port=cls.nginx_port ) cls.ext_opts = {"url": cls.url, "url_extra_repo": cls.url_extra_repo} # Add auth params if present (if so this will trigger the spawned # server to turn on HTTP basic auth). for credential_param in ("user", "password"): if hasattr(cls, credential_param): cls.ext_opts[credential_param] = getattr(cls, credential_param) auth_enabled = hasattr(cls, "username") and hasattr(cls, "password") pillar = { "git_pillar": { "config_dir": cls.config_dir, "git_dir": cls.git_dir, "venv_dir": cls.venv_dir, "root_dir": cls.root_dir, "nginx_port": cls.nginx_port, "uwsgi_port": cls.uwsgi_port, "auth_enabled": auth_enabled, } } # Different libexec dir for git backend on Debian-based systems git_core = "/usr/libexec/git-core" if not os.path.exists(git_core): git_core = "/usr/lib/git-core" if not os.path.exists(git_core): cls.tearDownClass() raise AssertionError( "{} not found. Either git is not installed, or the test " "class needs to be updated.".format(git_core) ) pillar["git_pillar"]["git-http-backend"] = os.path.join( git_core, "git-http-backend" ) try: if cls.prep_states_ran is False: ret = cls.cls_run_function( "state.apply", mods="git_pillar.http", pillar=pillar ) assert next(six.itervalues(ret))["result"] is True cls.prep_states_ran = True log.info("%s: States applied", cls.__name__) if cls.uwsgi_proc is not None: if not psutil.pid_exists(cls.uwsgi_proc.pid): log.warning( "%s: uWsgi started but appears to be dead now. Will try to restart it.", cls.__name__, ) cls.uwsgi_proc = None if cls.uwsgi_proc is None: cls.uwsgi_proc = start_daemon( cls.uwsgi_bin, cls.config_dir, cls.uwsgi_port, UwsgiDaemon ) log.info("%s: %s started", cls.__name__, cls.uwsgi_bin) if cls.nginx_proc is not None: if not psutil.pid_exists(cls.nginx_proc.pid): log.warning( "%s: nginx started but appears to be dead now. Will try to restart it.", cls.__name__, ) cls.nginx_proc = None if cls.nginx_proc is None: cls.nginx_proc = start_daemon( "nginx", cls.config_dir, cls.nginx_port, NginxDaemon ) log.info("%s: nginx started", cls.__name__) except AssertionError: cls.tearDownClass() six.reraise(*sys.exc_info()) @classmethod def tearDownClass(cls): if cls.nginx_proc is not None: log.info( "[%s] Stopping %s", cls.nginx_proc.log_prefix, cls.nginx_proc.__class__.__name__, ) terminate_process(cls.nginx_proc.pid, kill_children=True, slow_stop=True) log.info( "[%s] %s stopped", cls.nginx_proc.log_prefix, cls.nginx_proc.__class__.__name__, ) cls.nginx_proc = None if cls.uwsgi_proc is not None: log.info( "[%s] Stopping %s", cls.uwsgi_proc.log_prefix, cls.uwsgi_proc.__class__.__name__, ) terminate_process(cls.uwsgi_proc.pid, kill_children=True, slow_stop=True) log.info( "[%s] %s stopped", cls.uwsgi_proc.log_prefix, cls.uwsgi_proc.__class__.__name__, ) cls.uwsgi_proc = None shutil.rmtree(cls.root_dir, ignore_errors=True) cls.prep_states_ran = False super(WebserverMixin, cls).tearDownClass() class GitTestBase(ModuleCase): """ Base class for all gitfs/git_pillar tests. Must be subclassed and paired with either SSHDMixin or WebserverMixin to provide the server. """ maxDiff = None git_opts = '-c user.name="Foo Bar" -c [email protected]' ext_opts = {} def make_repo(self, root_dir, user="******"): raise NotImplementedError() class GitFSTestBase(GitTestBase, LoaderModuleMockMixin): """ Base class for all gitfs tests """ @requires_system_grains def setup_loader_modules(self, grains): # pylint: disable=W0221 return {gitfs: {"__opts__": copy.copy(_OPTS), "__grains__": grains}} def make_repo(self, root_dir, user="******"): raise NotImplementedError() class GitPillarTestBase(GitTestBase, LoaderModuleMockMixin): """ Base class for all git_pillar tests """ bare_repo = bare_repo_backup = bare_extra_repo = bare_extra_repo_backup = None admin_repo = admin_repo_backup = admin_extra_repo = admin_extra_repo_backup = None @requires_system_grains def setup_loader_modules(self, grains): # pylint: disable=W0221 return {git_pillar: {"__opts__": copy.copy(_OPTS), "__grains__": grains}} def get_pillar(self, ext_pillar_conf): """ Run git_pillar with the specified configuration ''' cachedir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP) self.addCleanup(shutil.rmtree, cachedir, ignore_errors=True) ext_pillar_opts = {"optimization_order": [0, 1, 2]} ext_pillar_opts.update( salt.utils.yaml.safe_load( ext_pillar_conf.format( cachedir=cachedir, extmods=os.path.join(cachedir, "extmods"), **self.ext_opts ) ) ) with patch.dict(git_pillar.__opts__, ext_pillar_opts): return git_pillar.ext_pillar( "minion", {}, *ext_pillar_opts["ext_pillar"][0]["git"] ) def make_repo(self, root_dir, user="******"): log.info("Creating test Git repo....") self.bare_repo = os.path.join(root_dir, "repo.git") self.bare_repo_backup = "{}.backup".format(self.bare_repo) self.admin_repo = os.path.join(root_dir, "admin") self.admin_repo_backup = "{}.backup".format(self.admin_repo) for dirname in (self.bare_repo, self.admin_repo): shutil.rmtree(dirname, ignore_errors=True) if os.path.exists(self.bare_repo_backup) and os.path.exists( self.admin_repo_backup ): shutil.copytree(self.bare_repo_backup, self.bare_repo) shutil.copytree(self.admin_repo_backup, self.admin_repo) return # Create bare repo self.run_function("git.init", [self.bare_repo], user=user, bare=True) # Clone bare repo self.run_function("git.clone", [self.admin_repo], url=self.bare_repo, user=user) def _push(branch, message): self.run_function("git.add", [self.admin_repo, "."], user=user) self.run_function( "git.commit", [self.admin_repo, message], user=user, git_opts=self.git_opts, ) self.run_function( "git.push", [self.admin_repo], remote="origin", ref=branch, user=user, ) with salt.utils.files.fopen( os.path.join(self.admin_repo, "top.sls"), "w" ) as fp_: fp_.write( textwrap.dedent( """\ base: '*': - foo """ ) ) with salt.utils.files.fopen( os.path.join(self.admin_repo, "foo.sls"), "w" ) as fp_: fp_.write( textwrap.dedent( """\ branch: master mylist: - master mydict: master: True nested_list: - master nested_dict: master: True """ ) ) # Add another file to be referenced using git_pillar_includes with salt.utils.files.fopen( os.path.join(self.admin_repo, "bar.sls"), "w" ) as fp_: fp_.write("included_pillar: True\n") # Add another file in subdir os.mkdir(os.path.join(self.admin_repo, "subdir")) with salt.utils.files.fopen( os.path.join(self.admin_repo, "subdir", "bar.sls"), "w" ) as fp_: fp_.write("from_subdir: True\n") _push("master", "initial commit") # Do the same with different values for "dev" branch self.run_function("git.checkout", [self.admin_repo], user=user, opts="-b dev") # The bar.sls shouldn't be in any branch but master self.run_function("git.rm", [self.admin_repo, "bar.sls"], user=user) with salt.utils.files.fopen( os.path.join(self.admin_repo, "top.sls"), "w" ) as fp_: fp_.write( textwrap.dedent( """\ dev: '*': - foo """ ) ) with salt.utils.files.fopen( os.path.join(self.admin_repo, "foo.sls"), "w" ) as fp_: fp_.write( textwrap.dedent( """\ branch: dev mylist: - dev mydict: dev: True nested_list: - dev nested_dict: dev: True """ ) ) _push("dev", "add dev branch") # Create just a top file in a separate repo, to be mapped to the base # env and referenced using git_pillar_includes self.run_function( "git.checkout", [self.admin_repo], user=user, opts="-b top_only" ) # The top.sls should be the only file in this branch self.run_function( "git.rm", [self.admin_repo, "foo.sls", os.path.join("subdir", "bar.sls")], user=user, ) with salt.utils.files.fopen( os.path.join(self.admin_repo, "top.sls"), "w" ) as fp_: fp_.write( textwrap.dedent( """\ base: '*': - bar """ ) ) _push("top_only", "add top_only branch") # Create just another top file in a separate repo, to be mapped to the base # env and including mounted.bar self.run_function( "git.checkout", [self.admin_repo], user=user, opts="-b top_mounted" ) # The top.sls should be the only file in this branch with salt.utils.files.fopen( os.path.join(self.admin_repo, "top.sls"), "w" ) as fp_: fp_.write( textwrap.dedent( """\ base: '*': - mounted.bar """ ) ) _push("top_mounted", "add top_mounted branch") shutil.copytree(self.bare_repo, self.bare_repo_backup) shutil.copytree(self.admin_repo, self.admin_repo_backup) log.info("Test Git repo created.") def make_extra_repo(self, root_dir, user="******"): log.info("Creating extra test Git repo....") self.bare_extra_repo = os.path.join(root_dir, "extra_repo.git") self.bare_extra_repo_backup = "{}.backup".format(self.bare_extra_repo) self.admin_extra_repo = os.path.join(root_dir, "admin_extra") self.admin_extra_repo_backup = "{}.backup".format(self.admin_extra_repo) for dirname in (self.bare_extra_repo, self.admin_extra_repo): shutil.rmtree(dirname, ignore_errors=True) if os.path.exists(self.bare_extra_repo_backup) and os.path.exists( self.admin_extra_repo_backup ): shutil.copytree(self.bare_extra_repo_backup, self.bare_extra_repo) shutil.copytree(self.admin_extra_repo_backup, self.admin_extra_repo) return # Create bare extra repo self.run_function("git.init", [self.bare_extra_repo], user=user, bare=True) # Clone bare repo self.run_function( "git.clone", [self.admin_extra_repo], url=self.bare_extra_repo, user=user ) def _push(branch, message): self.run_function("git.add", [self.admin_extra_repo, "."], user=user) self.run_function( "git.commit", [self.admin_extra_repo, message], user=user, git_opts=self.git_opts, ) self.run_function( "git.push", [self.admin_extra_repo], remote="origin", ref=branch, user=user, ) with salt.utils.files.fopen( os.path.join(self.admin_extra_repo, "top.sls"), "w" ) as fp_: fp_.write( textwrap.dedent( """\ "{{saltenv}}": '*': - motd - nowhere.foo """ ) ) with salt.utils.files.fopen( os.path.join(self.admin_extra_repo, "motd.sls"), "w" ) as fp_: fp_.write( textwrap.dedent( """\ motd: The force will be with you. Always. """ ) ) _push("master", "initial commit") shutil.copytree(self.bare_extra_repo, self.bare_extra_repo_backup) shutil.copytree(self.admin_extra_repo, self.admin_extra_repo_backup) log.info("Extra test Git repo created.") @classmethod def tearDownClass(cls): super(GitPillarTestBase, cls).tearDownClass() for dirname in ( cls.admin_repo, cls.admin_repo_backup, cls.admin_extra_repo, cls.admin_extra_repo_backup, cls.bare_repo, cls.bare_repo_backup, cls.bare_extra_repo, cls.bare_extra_repo_backup, ): if dirname is not None: shutil.rmtree(dirname, ignore_errors=True) class GitPillarSSHTestBase(GitPillarTestBase, SSHDMixin): """ Base class for GitPython and Pygit2 SSH tests """ id_rsa_nopass = id_rsa_withpass = None git_ssh = "/tmp/git_ssh" def setUp(self): """ Create the SSH server and user, and create the git repo """ log.info("%s.setUp() started...", self.__class__.__name__) super(GitPillarSSHTestBase, self).setUp() root_dir = os.path.expanduser("~{0}".format(self.username)) if root_dir.startswith("~"): raise AssertionError( "Unable to resolve homedir for user '{0}'".format(self.username) ) self.make_repo(root_dir, user=self.username) self.make_extra_repo(root_dir, user=self.username) log.info("%s.setUp() complete.", self.__class__.__name__) def get_pillar(self, ext_pillar_conf): """ Wrap the parent class' get_pillar() func in logic that temporarily changes the GIT_SSH to use our custom script, ensuring that the passphraselsess key is used to auth without needing to modify the root user's ssh config file. ''' with patched_environ(GIT_SSH=self.git_ssh): return super(GitPillarSSHTestBase, self).get_pillar(ext_pillar_conf)
def setUp(self): super(SaltnadoTestCase, self).setUp() self.patched_environ = patched_environ(ASYNC_TEST_TIMEOUT='30') self.patched_environ.__enter__() self.addCleanup(self.patched_environ.__exit__)
def setUp(self): super().setUp() self.patched_environ = patched_environ(ASYNC_TEST_TIMEOUT="30") self.patched_environ.__enter__() self.addCleanup(self.patched_environ.__exit__)