def install_environment( repo_cmd_runner, version='default', additional_dependencies=(), ): additional_dependencies = tuple(additional_dependencies) directory = helpers.environment_dir(ENVIRONMENT_DIR, version) # Install a virtualenv with clean_path_on_failure(repo_cmd_runner.path(directory)): venv_cmd = [ sys.executable, '-m', 'virtualenv', '{{prefix}}{}'.format(directory) ] if version != 'default': venv_cmd.extend(['-p', norm_version(version)]) else: venv_cmd.extend(['-p', os.path.realpath(sys.executable)]) venv_env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1') repo_cmd_runner.run(venv_cmd, cwd='/', env=venv_env) with in_env(repo_cmd_runner, version): helpers.run_setup_cmd( repo_cmd_runner, ('pip', 'install', '.') + additional_dependencies, )
def env_prefix(self): return ". '{{prefix}}{0}activate' &&".format( virtualenv.path_locations( helpers.environment_dir(ENVIRONMENT_DIR, self.language_version) )[-1].rstrip(os.sep) + os.sep, 'activate', )
def install_environment( repo_cmd_runner, version='default', additional_dependencies=None, ): assert repo_cmd_runner.exists('package.json') directory = helpers.environment_dir(ENVIRONMENT_DIR, version) env_dir = repo_cmd_runner.path(directory) with clean_path_on_failure(env_dir): cmd = [ sys.executable, '-m', 'nodeenv', '--prebuilt', '{{prefix}}{0}'.format(directory), ] if version != 'default': cmd.extend(['-n', version]) repo_cmd_runner.run(cmd) with in_env(repo_cmd_runner, version) as node_env: node_env.run("cd '{prefix}' && npm install -g") if additional_dependencies: node_env.run( "cd '{prefix}' && npm install -g " + ' '.join( shell_escape(dep) for dep in additional_dependencies ) )
def language_is_installed(language_name, language_version): language = languages[language_name] venv = environment_dir(language.ENVIRONMENT_DIR, language_version) return ( venv is None or read_state(venv) == state(language_name, language_version) )
def _install_all(venvs, repo_url): """Tuple of (cmd_runner, language, version, deps)""" need_installed = tuple( (cmd_runner, language_name, version, deps) for cmd_runner, language_name, version, deps in venvs if not _installed(cmd_runner, language_name, version, deps) ) if need_installed: logger.info( 'Installing environment for {}.'.format(repo_url) ) logger.info('Once installed this environment will be reused.') logger.info('This may take a few minutes...') for cmd_runner, language_name, version, deps in need_installed: language = languages[language_name] venv = environment_dir(language.ENVIRONMENT_DIR, version) # There's potentially incomplete cleanup from previous runs # Clean it up! if cmd_runner.exists(venv): shutil.rmtree(cmd_runner.path(venv)) language.install_environment(cmd_runner, version, deps) # Write our state to indicate we're installed state = _state(deps) _write_installed_state(cmd_runner, venv, state)
def _installed(cmd_runner, language_name, language_version, additional_deps): language = languages[language_name] venv = environment_dir(language.ENVIRONMENT_DIR, language_version) return ( venv is None or _read_installed_state(cmd_runner, venv) == _state(additional_deps) )
def install_environment( repo_cmd_runner, version='default', additional_dependencies=(), ): # pragma: windows no cover additional_dependencies = tuple(additional_dependencies) directory = helpers.environment_dir(ENVIRONMENT_DIR, version) with clean_path_on_failure(repo_cmd_runner.path(directory)): # TODO: this currently will fail if there's no version specified and # there's no system ruby installed. Is this ok? _install_rbenv(repo_cmd_runner, version=version) with in_env(repo_cmd_runner, version): # Need to call this before installing so rbenv's directories are # set up helpers.run_setup_cmd(repo_cmd_runner, ('rbenv', 'init', '-')) if version != 'default': _install_ruby(repo_cmd_runner, version) # Need to call this after installing to set up the shims helpers.run_setup_cmd(repo_cmd_runner, ('rbenv', 'rehash')) helpers.run_setup_cmd( repo_cmd_runner, ('gem', 'build') + repo_cmd_runner.star('.gemspec'), ) helpers.run_setup_cmd( repo_cmd_runner, ( ('gem', 'install', '--no-ri', '--no-rdoc') + repo_cmd_runner.star('.gem') + additional_dependencies ), )
def _install_rbenv(prefix, version=C.DEFAULT): # pragma: windows no cover directory = helpers.environment_dir(ENVIRONMENT_DIR, version) _extract_resource('rbenv.tar.gz', prefix.path('.')) shutil.move(prefix.path('rbenv'), prefix.path(directory)) # Only install ruby-build if the version is specified if version != C.DEFAULT: plugins_dir = prefix.path(directory, 'plugins') _extract_resource('ruby-download.tar.gz', plugins_dir) _extract_resource('ruby-build.tar.gz', plugins_dir) activate_path = prefix.path(directory, 'bin', 'activate') with io.open(activate_path, 'w') as activate_file: # This is similar to how you would install rbenv to your home directory # However we do a couple things to make the executables exposed and # configure it to work in our directory. # We also modify the PS1 variable for manual debugging sake. activate_file.write( '#!/usr/bin/env bash\n' "export RBENV_ROOT='{directory}'\n" 'export PATH="$RBENV_ROOT/bin:$PATH"\n' 'eval "$(rbenv init -)"\n' 'export PS1="(rbenv)$PS1"\n' # This lets us install gems in an isolated and repeatable # directory "export GEM_HOME='{directory}/gems'\n" 'export PATH="$GEM_HOME/bin:$PATH"\n' '\n'.format(directory=prefix.path(directory)), ) # If we aren't using the system ruby, add a version here if version != C.DEFAULT: activate_file.write('export RBENV_VERSION="{}"\n'.format(version))
def install_environment( repo_cmd_runner, version='default', additional_dependencies=None, ): assert repo_cmd_runner.exists('setup.py') directory = helpers.environment_dir(ENVIRONMENT_DIR, version) # Install a virtualenv with clean_path_on_failure(repo_cmd_runner.path(directory)): venv_cmd = [ sys.executable, '-m', 'virtualenv', '{{prefix}}{0}'.format(directory) ] if version != 'default': venv_cmd.extend(['-p', norm_version(version)]) repo_cmd_runner.run(venv_cmd) with in_env(repo_cmd_runner, version) as env: env.run("cd '{prefix}' && pip install .", encoding=None) if additional_dependencies: env.run( "cd '{prefix}' && pip install " + ' '.join( shell_escape(dep) for dep in additional_dependencies ), encoding=None, )
def install_environment( repo_cmd_runner, version='default', additional_dependencies=None, ): directory = helpers.environment_dir(ENVIRONMENT_DIR, version) with clean_path_on_failure(repo_cmd_runner.path(directory)): # TODO: this currently will fail if there's no version specified and # there's no system ruby installed. Is this ok? _install_rbenv(repo_cmd_runner, version=version) with in_env(repo_cmd_runner, version) as ruby_env: if version != 'default': _install_ruby(ruby_env, version) ruby_env.run( 'cd {prefix} && gem build *.gemspec && ' 'gem install --no-ri --no-rdoc *.gem', encoding=None, ) if additional_dependencies: ruby_env.run( 'cd {prefix} && gem install --no-ri --no-rdoc ' + ' '.join( shell_escape(dep) for dep in additional_dependencies ), encoding=None, )
def install_environment( repo_cmd_runner, version='default', additional_dependencies=(), ): # pragma: windows no cover additional_dependencies = tuple(additional_dependencies) assert repo_cmd_runner.exists('package.json') directory = helpers.environment_dir(ENVIRONMENT_DIR, version) env_dir = repo_cmd_runner.path(directory) with clean_path_on_failure(env_dir): cmd = [ sys.executable, '-m', 'nodeenv', '--prebuilt', '{{prefix}}{}'.format(directory), ] if version != 'default': cmd.extend(['-n', version]) repo_cmd_runner.run(cmd) with in_env(repo_cmd_runner, version): helpers.run_setup_cmd( repo_cmd_runner, ('npm', 'install', '-g', '.') + additional_dependencies, )
def env_prefix(self): return ". '{{prefix}}{0}{1}activate' &&".format( bin_dir( helpers.environment_dir(ENVIRONMENT_DIR, self.language_version) ), os.sep, )
def in_env(repo_cmd_runner, language_version): envdir = os.path.join( repo_cmd_runner.prefix_dir, helpers.environment_dir(ENVIRONMENT_DIR, language_version), ) with envcontext(get_env_patch(envdir)): yield
def install_environment(repo_cmd_runner, version, additional_dependencies): helpers.assert_version_default('golang', version) directory = repo_cmd_runner.path( helpers.environment_dir(ENVIRONMENT_DIR, 'default'), ) with clean_path_on_failure(directory): remote = git.get_remote_url(repo_cmd_runner.path()) repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote)) # Clone into the goenv we'll create helpers.run_setup_cmd( repo_cmd_runner, ('git', 'clone', '.', repo_src_dir), ) if sys.platform == 'cygwin': # pragma: no cover _, gopath, _ = cmd_output('cygpath', '-w', directory) gopath = gopath.strip() else: gopath = directory env = dict(os.environ, GOPATH=gopath) cmd_output('go', 'get', './...', cwd=repo_src_dir, env=env) for dependency in additional_dependencies: cmd_output('go', 'get', dependency, cwd=repo_src_dir, env=env) # Same some disk space, we don't need these after installation rmtree(repo_cmd_runner.path(directory, 'src')) rmtree(repo_cmd_runner.path(directory, 'pkg'))
def language_is_installed(language_name, language_version): language = languages[language_name] directory = environment_dir( language.ENVIRONMENT_DIR, language_version, ) return ( directory is None or self.cmd_runner.exists(directory, '.installed') )
def install_environment(repo_cmd_runner, version="default", additional_dependencies=()): # pragma: windows no cover assert version == "default", "Pre-commit does not support language_version for docker " directory = repo_cmd_runner.path(helpers.environment_dir(ENVIRONMENT_DIR, "default")) # Build the swift package with clean_path_on_failure(directory): os.mkdir(directory) repo_cmd_runner.run( ("swift", "build", "-C", "{prefix}", "-c", BUILD_CONFIG, "--build-path", os.path.join(directory, BUILD_DIR)) )
def install_environment(repo_cmd_runner, version='default'): directory = helpers.environment_dir(ENVIRONMENT_DIR, version) with clean_path_on_failure(repo_cmd_runner.path(directory)): # TODO: this currently will fail if there's no version specified and # there's no system ruby installed. Is this ok? _install_rbenv(repo_cmd_runner, version=version) with in_env(repo_cmd_runner, version) as ruby_env: if version != 'default': _install_ruby(ruby_env, version) ruby_env.run( 'cd {prefix} && gem build *.gemspec' ' && gem install --no-document *.gem', )
def installed(self): lang = languages[self.language] venv = environment_dir(lang.ENVIRONMENT_DIR, self.language_version) return ( venv is None or ( ( _read_state(self.prefix, venv) == _state(self.additional_dependencies) ) and lang.healthy(self.prefix, self.language_version) ) )
def install_environment(repo_cmd_runner, version='default'): assert repo_cmd_runner.exists('setup.py') directory = helpers.environment_dir(ENVIRONMENT_DIR, version) # Install a virtualenv with clean_path_on_failure(repo_cmd_runner.path(directory)): venv_cmd = [ sys.executable, '-m', 'virtualenv', '{{prefix}}{0}'.format(directory) ] if version != 'default': venv_cmd.extend(['-p', norm_version(version)]) repo_cmd_runner.run(venv_cmd) with in_env(repo_cmd_runner, version) as env: env.run("cd '{prefix}' && pip install .")
def install_environment(prefix, version, additional_dependencies): additional_dependencies = tuple(additional_dependencies) directory = helpers.environment_dir(_dir, version) env_dir = prefix.path(directory) with clean_path_on_failure(env_dir): if version != C.DEFAULT: python = norm_version(version) else: python = os.path.realpath(sys.executable) _make_venv(env_dir, python) with in_env(prefix, version): helpers.run_setup_cmd( prefix, ('pip', 'install', '.') + additional_dependencies, )
def test_additional_golang_dependencies_installed( tempdir_factory, store, ): path = make_repo(tempdir_factory, 'golang_hooks_repo') config = make_config_from_repo(path) # A small go package deps = ['github.com/golang/example/hello'] config['hooks'][0]['additional_dependencies'] = deps repo = Repository.create(config, store) repo.require_installed() binaries = os.listdir(repo._cmd_runner.path( helpers.environment_dir(golang.ENVIRONMENT_DIR, 'default'), 'bin', )) # normalize for windows binaries = [os.path.splitext(binary)[0] for binary in binaries] assert 'hello' in binaries
def install_environment( prefix, version, additional_dependencies, ): # pragma: windows no cover helpers.assert_version_default('docker', version) helpers.assert_no_additional_deps('docker', additional_dependencies) assert_docker_available() directory = prefix.path( helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), ) # Docker doesn't really have relevant disk environment, but pre-commit # still needs to cleanup it's state files on failure with clean_path_on_failure(directory): build_docker_image(prefix, pull=True) os.mkdir(directory)
def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], ) -> None: # pragma: win32 no cover helpers.assert_version_default('docker', version) helpers.assert_no_additional_deps('docker', additional_dependencies) assert_docker_available() directory = prefix.path( helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), ) # Docker doesn't really have relevant disk environment, but pre-commit # still needs to cleanup its state files on failure with clean_path_on_failure(directory): build_docker_image(prefix, pull=True) os.mkdir(directory)
def healthy(prefix: Prefix, language_version: str) -> bool: directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version) envdir = prefix.path(directory) pyvenv_cfg = os.path.join(envdir, "pyvenv.cfg") # created with "old" virtualenv if not os.path.exists(pyvenv_cfg): return False exe_name = "python.exe" if sys.platform == "win32" else "python" py_exe = prefix.path(bin_dir(envdir), exe_name) cfg = _read_pyvenv_cfg(pyvenv_cfg) return ("version_info" in cfg and _version_info(py_exe) == cfg["version_info"] and ("base-executable" not in cfg or _version_info(cfg["base-executable"]) == cfg["version_info"]))
def test_additional_golang_dependencies_installed( tempdir_factory, store, ): path = make_repo(tempdir_factory, 'golang_hooks_repo') config = make_config_from_repo(path) # A small go package deps = ['github.com/golang/example/hello'] config['hooks'][0]['additional_dependencies'] = deps hook = _get_hook(config, store, 'golang-hook') binaries = os.listdir( hook.prefix.path( helpers.environment_dir(golang.ENVIRONMENT_DIR, C.DEFAULT), 'bin', ), ) # normalize for windows binaries = [os.path.splitext(binary)[0] for binary in binaries] assert 'hello' in binaries
def test_additional_golang_dependencies_installed( tempdir_factory, store, ): path = make_repo(tempdir_factory, 'golang_hooks_repo') config = make_config_from_repo(path) # A small go package deps = ['github.com/golang/example/hello'] config['hooks'][0]['additional_dependencies'] = deps repo = Repository.create(config, store) repo.require_installed() (prefix, _, _, _), = repo._venvs() binaries = os.listdir(prefix.path( helpers.environment_dir(golang.ENVIRONMENT_DIR, 'default'), 'bin', )) # normalize for windows binaries = [os.path.splitext(binary)[0] for binary in binaries] assert 'hello' in binaries
def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], ) -> None: directory = helpers.environment_dir(_dir, version) install = ('python', '-mpip', 'install', '.', *additional_dependencies) env_dir = prefix.path(directory) with clean_path_on_failure(env_dir): if version != C.DEFAULT: python = norm_version(version) else: python = os.path.realpath(sys.executable) _make_venv(env_dir, python) with in_env(prefix, version): helpers.run_setup_cmd(prefix, install)
def install(self): logger.info('Installing environment for {}.'.format(self.src)) logger.info('Once installed this environment will be reused.') logger.info('This may take a few minutes...') lang = languages[self.language] venv = environment_dir(lang.ENVIRONMENT_DIR, self.language_version) # There's potentially incomplete cleanup from previous runs # Clean it up! if self.prefix.exists(venv): rmtree(self.prefix.path(venv)) lang.install_environment( self.prefix, self.language_version, self.additional_dependencies, ) # Write our state to indicate we're installed _write_state(self.prefix, venv, _state(self.additional_dependencies))
def install_environment( prefix, version, additional_dependencies, ): # pragma: windows no cover helpers.assert_version_default('swift', version) helpers.assert_no_additional_deps('swift', additional_dependencies) directory = prefix.path( helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), ) # Build the swift package with clean_path_on_failure(directory): os.mkdir(directory) cmd_output( 'swift', 'build', '-C', prefix.prefix_dir, '-c', BUILD_CONFIG, '--build-path', os.path.join(directory, BUILD_DIR), )
def install(self): logger.info('Installing environment for {}.'.format(self.src)) logger.info('Once installed this environment will be reused.') logger.info('This may take a few minutes...') lang = languages[self.language] venv = environment_dir(lang.ENVIRONMENT_DIR, self.language_version) # There's potentially incomplete cleanup from previous runs # Clean it up! if self.prefix.exists(venv): rmtree(self.prefix.path(venv)) lang.install_environment( self.prefix, self.language_version, self.additional_dependencies, ) # Write our state to indicate we're installed _write_state(self.prefix, venv, _state(self.additional_dependencies))
def test_additional_rust_lib_dependencies_installed( tempdir_factory, store, ): path = make_repo(tempdir_factory, 'rust_hooks_repo') config = make_config_from_repo(path) # A small rust package with no dependencies. deps = ['shellharden:3.1.0'] config['hooks'][0]['additional_dependencies'] = deps hook = _get_hook(config, store, 'rust-hook') binaries = os.listdir( hook.prefix.path( helpers.environment_dir(rust.ENVIRONMENT_DIR, C.DEFAULT), 'bin', ), ) # normalize for windows binaries = [os.path.splitext(binary)[0] for binary in binaries] assert 'rust-hello-world' in binaries assert 'shellharden' not in binaries
def test_additional_rust_lib_dependencies_installed( tempdir_factory, store, ): path = make_repo(tempdir_factory, 'rust_hooks_repo') config = make_config_from_repo(path) # A small rust package with no dependencies. deps = ['shellharden:3.1.0'] config['hooks'][0]['additional_dependencies'] = deps repo = Repository.create(config, store) repo.require_installed() (prefix, _, _, _), = repo._venvs() binaries = os.listdir(prefix.path( helpers.environment_dir(rust.ENVIRONMENT_DIR, 'default'), 'bin', )) # normalize for windows binaries = [os.path.splitext(binary)[0] for binary in binaries] assert 'rust-hello-world' in binaries assert 'shellharden' not in binaries
def install_environment( repo_cmd_runner, version, additional_dependencies, ): # pragma: windows no cover helpers.assert_version_default('swift', version) helpers.assert_no_additional_deps('swift', additional_dependencies) directory = repo_cmd_runner.path( helpers.environment_dir(ENVIRONMENT_DIR, 'default'), ) # Build the swift package with clean_path_on_failure(directory): os.mkdir(directory) repo_cmd_runner.run(( 'swift', 'build', '-C', '{prefix}', '-c', BUILD_CONFIG, '--build-path', os.path.join(directory, BUILD_DIR), ))
def _install_rbenv( repo_cmd_runner, version='default', ): # pragma: windows no cover directory = helpers.environment_dir(ENVIRONMENT_DIR, version) with tarfile.open(resource_filename('rbenv.tar.gz')) as tf: tf.extractall(repo_cmd_runner.path('.')) shutil.move( repo_cmd_runner.path('rbenv'), repo_cmd_runner.path(directory), ) # Only install ruby-build if the version is specified if version != 'default': # ruby-download with tarfile.open(resource_filename('ruby-download.tar.gz')) as tf: tf.extractall(repo_cmd_runner.path(directory, 'plugins')) # ruby-build with tarfile.open(resource_filename('ruby-build.tar.gz')) as tf: tf.extractall(repo_cmd_runner.path(directory, 'plugins')) activate_path = repo_cmd_runner.path(directory, 'bin', 'activate') with io.open(activate_path, 'w') as activate_file: # This is similar to how you would install rbenv to your home directory # However we do a couple things to make the executables exposed and # configure it to work in our directory. # We also modify the PS1 variable for manual debugging sake. activate_file.write( '#!/usr/bin/env bash\n' "export RBENV_ROOT='{directory}'\n" 'export PATH="$RBENV_ROOT/bin:$PATH"\n' 'eval "$(rbenv init -)"\n' 'export PS1="(rbenv)$PS1"\n' # This lets us install gems in an isolated and repeatable # directory "export GEM_HOME='{directory}/gems'\n" 'export PATH="$GEM_HOME/bin:$PATH"\n' '\n'.format(directory=repo_cmd_runner.path(directory))) # If we aren't using the system ruby, add a version here if version != 'default': activate_file.write('export RBENV_VERSION="{}"\n'.format(version))
def _hook_install(hook: Hook) -> None: logger.info(f"Installing environment for {hook.src}.") logger.info("Once installed this environment will be reused.") logger.info("This may take a few minutes...") lang = languages[hook.language] assert lang.ENVIRONMENT_DIR is not None venv = environment_dir(lang.ENVIRONMENT_DIR, hook.language_version) # There's potentially incomplete cleanup from previous runs # Clean it up! if hook.prefix.exists(venv): rmtree(hook.prefix.path(venv)) lang.install_environment( hook.prefix, hook.language_version, hook.additional_dependencies, ) # Write our state to indicate we're installed _write_state(hook.prefix, venv, _state(hook.additional_dependencies))
def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], ) -> None: helpers.assert_version_default('conda', version) directory = helpers.environment_dir(ENVIRONMENT_DIR, version) env_dir = prefix.path(directory) with clean_path_on_failure(env_dir): cmd_output_b( 'conda', 'env', 'create', '-p', env_dir, '--file', 'environment.yml', cwd=prefix.prefix_dir, ) if additional_dependencies: cmd_output_b( 'conda', 'install', '-p', env_dir, *additional_dependencies, cwd=prefix.prefix_dir, )
def install_environment( repo_cmd_runner, version, additional_dependencies, ): # pragma: windows no cover assert repo_cmd_runner.exists('Dockerfile'), ( 'No Dockerfile was found in the hook repository' ) helpers.assert_version_default('docker', version) helpers.assert_no_additional_deps('docker', additional_dependencies) assert_docker_available() directory = repo_cmd_runner.path( helpers.environment_dir(ENVIRONMENT_DIR, 'default'), ) # Docker doesn't really have relevant disk environment, but pre-commit # still needs to cleanup it's state files on failure with clean_path_on_failure(directory): build_docker_image(repo_cmd_runner, pull=True) os.mkdir(directory)
def install_environment(prefix, version, additional_dependencies): additional_dependencies = tuple(additional_dependencies) directory = helpers.environment_dir(ENVIRONMENT_DIR, version) # Install a virtualenv env_dir = prefix.path(directory) with clean_path_on_failure(env_dir): venv_cmd = [sys.executable, '-m', 'virtualenv', env_dir] if version != 'default': venv_cmd.extend(['-p', norm_version(version)]) else: venv_cmd.extend(['-p', os.path.realpath(sys.executable)]) venv_env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1') cmd_output(*venv_cmd, cwd='/', env=venv_env) with in_env(prefix, version): helpers.run_setup_cmd( prefix, ('pip', 'install', '.') + additional_dependencies, )
def install_environment( repo_cmd_runner, version, additional_dependencies, ): # pragma: windows no cover assert repo_cmd_runner.exists('Dockerfile'), ( 'No Dockerfile was found in the hook repository') helpers.assert_version_default('docker', version) helpers.assert_no_additional_deps('docker', additional_dependencies) assert_docker_available() directory = repo_cmd_runner.path( helpers.environment_dir(ENVIRONMENT_DIR, 'default'), ) # Docker doesn't really have relevant disk environment, but pre-commit # still needs to cleanup it's state files on failure with clean_path_on_failure(directory): build_docker_image(repo_cmd_runner, pull=True) os.mkdir(directory)
def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], ) -> None: helpers.assert_version_default('rust', version) directory = prefix.path( helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), ) # There are two cases where we might want to specify more dependencies: # as dependencies for the library being built, and as binary packages # to be `cargo install`'d. # # Unlike e.g. Python, if we just `cargo install` a library, it won't be # used for compilation. And if we add a crate providing a binary to the # `Cargo.toml`, the binary won't be built. # # Because of this, we allow specifying "cli" dependencies by prefixing # with 'cli:'. cli_deps = { dep for dep in additional_dependencies if dep.startswith('cli:') } lib_deps = set(additional_dependencies) - cli_deps if len(lib_deps) > 0: _add_dependencies(prefix.path('Cargo.toml'), lib_deps) with clean_path_on_failure(directory): packages_to_install: Set[Tuple[str, ...]] = {('--path', '.')} for cli_dep in cli_deps: cli_dep = cli_dep[len('cli:'):] package, _, version = cli_dep.partition(':') if version != '': packages_to_install.add((package, '--version', version)) else: packages_to_install.add((package,)) for args in packages_to_install: cmd_output_b( 'cargo', 'install', '--bins', '--root', directory, *args, cwd=prefix.prefix_dir, )
def healthy(prefix: Prefix, language_version: str) -> bool: directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version) envdir = prefix.path(directory) pyvenv_cfg = os.path.join(envdir, 'pyvenv.cfg') # created with "old" virtualenv if not os.path.exists(pyvenv_cfg): return False exe_name = 'python.exe' if sys.platform == 'win32' else 'python' py_exe = prefix.path(bin_dir(envdir), exe_name) cfg = _read_pyvenv_cfg(pyvenv_cfg) return ( 'version_info' in cfg and # always use uncached lookup here in case we replaced an unhealthy env _version_info.__wrapped__(py_exe) == cfg['version_info'] and ('base-executable' not in cfg or _version_info(cfg['base-executable']) == cfg['version_info']))
def test_invalidated_virtualenv(tempdir_factory, store): # A cached virtualenv may become invalidated if the system python upgrades # This should not cause every hook in that virtualenv to fail. path = make_repo(tempdir_factory, 'python_hooks_repo') config = make_config_from_repo(path) hook = _get_hook(config, store, 'foo') # Simulate breaking of the virtualenv libdir = hook.prefix.path( helpers.environment_dir(python.ENVIRONMENT_DIR, hook.language_version), 'lib', hook.language_version, ) paths = [ os.path.join(libdir, p) for p in ('site.py', 'site.pyc', '__pycache__') ] cmd_output_b('rm', '-rf', *paths) # pre-commit should rebuild the virtualenv and it should be runnable retv, stdout, stderr = _get_hook(config, store, 'foo').run(()) assert retv == 0
def install(self): """Install the hook repository.""" def language_is_installed(language_name, language_version): language = languages[language_name] directory = environment_dir( language.ENVIRONMENT_DIR, language_version, ) return ( directory is None or self.cmd_runner.exists(directory, '.installed') ) if not all( language_is_installed(language_name, language_version) for language_name, language_version in self.languages ): logger.info( 'Installing environment for {0}.'.format(self.repo_url) ) logger.info('Once installed this environment will be reused.') logger.info('This may take a few minutes...') for language_name, language_version in self.languages: language = languages[language_name] if language_is_installed(language_name, language_version): continue directory = environment_dir( language.ENVIRONMENT_DIR, language_version, ) # There's potentially incomplete cleanup from previous runs # Clean it up! if self.cmd_runner.exists(directory): shutil.rmtree(self.cmd_runner.path(directory)) language.install_environment( self.cmd_runner, language_version, self.additional_dependencies[language_name][language_version], ) # Touch the .installed file (atomic) to indicate we've installed open(self.cmd_runner.path(directory, '.installed'), 'w').close()
def _install_rbenv(repo_cmd_runner, version='default'): directory = helpers.environment_dir(ENVIRONMENT_DIR, version) with tarfile_open(resource_filename('rbenv.tar.gz')) as tf: tf.extractall(repo_cmd_runner.path('.')) shutil.move( repo_cmd_runner.path('rbenv'), repo_cmd_runner.path(directory), ) # Only install ruby-build if the version is specified if version != 'default': # ruby-download with tarfile_open(resource_filename('ruby-download.tar.gz')) as tf: tf.extractall(repo_cmd_runner.path(directory, 'plugins')) # ruby-build with tarfile_open(resource_filename('ruby-build.tar.gz')) as tf: tf.extractall(repo_cmd_runner.path(directory, 'plugins')) activate_path = repo_cmd_runner.path(directory, 'bin', 'activate') with io.open(activate_path, 'w') as activate_file: # This is similar to how you would install rbenv to your home directory # However we do a couple things to make the executables exposed and # configure it to work in our directory. # We also modify the PS1 variable for manual debugging sake. activate_file.write( '#!/usr/bin/env bash\n' "export RBENV_ROOT='{0}'\n" 'export PATH="$RBENV_ROOT/bin:$PATH"\n' 'eval "$(rbenv init -)"\n' 'export PS1="(rbenv)$PS1"\n' # This lets us install gems in an isolated and repeatable # directory "export GEM_HOME='{0}/gems'\n" 'export PATH="$GEM_HOME/bin:$PATH"\n' '\n'.format(repo_cmd_runner.path(directory))) # If we aren't using the system ruby, add a version here if version != 'default': activate_file.write('export RBENV_VERSION="{0}"\n'.format(version))
def install_environment( repo_cmd_runner, version='default', additional_dependencies=None, ): directory = helpers.environment_dir(ENVIRONMENT_DIR, version) with clean_path_on_failure(repo_cmd_runner.path(directory)): # TODO: this currently will fail if there's no version specified and # there's no system ruby installed. Is this ok? _install_rbenv(repo_cmd_runner, version=version) with in_env(repo_cmd_runner, version) as ruby_env: if version != 'default': _install_ruby(ruby_env, version) ruby_env.run( 'cd {prefix} && gem build *.gemspec && ' 'gem install --no-ri --no-rdoc *.gem', ) if additional_dependencies: ruby_env.run('cd {prefix} && gem install --no-ri --no-rdoc ' + ' '.join( shell_escape(dep) for dep in additional_dependencies))
def install_environment(repo_cmd_runner, version, additional_dependencies): helpers.assert_version_default('golang', version) directory = repo_cmd_runner.path( helpers.environment_dir(ENVIRONMENT_DIR, 'default'), ) with clean_path_on_failure(directory): remote = git.get_remote_url(repo_cmd_runner.path()) repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote)) # Clone into the goenv we'll create helpers.run_setup_cmd( repo_cmd_runner, ('git', 'clone', '.', repo_src_dir), ) env = dict(os.environ, GOPATH=directory) cmd_output('go', 'get', './...', cwd=repo_src_dir, env=env) for dependency in additional_dependencies: cmd_output('go', 'get', dependency, cwd=repo_src_dir, env=env) # Same some disk space, we don't need these after installation rmtree(repo_cmd_runner.path(directory, 'src')) rmtree(repo_cmd_runner.path(directory, 'pkg'))
def install_environment(prefix, version, additional_dependencies): helpers.assert_version_default('rust', version) directory = prefix.path( helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), ) # There are two cases where we might want to specify more dependencies: # as dependencies for the library being built, and as binary packages # to be `cargo install`'d. # # Unlike e.g. Python, if we just `cargo install` a library, it won't be # used for compilation. And if we add a crate providing a binary to the # `Cargo.toml`, the binary won't be built. # # Because of this, we allow specifying "cli" dependencies by prefixing # with 'cli:'. cli_deps = { dep for dep in additional_dependencies if dep.startswith('cli:') } lib_deps = set(additional_dependencies) - cli_deps if len(lib_deps) > 0: _add_dependencies(prefix.path('Cargo.toml'), lib_deps) with clean_path_on_failure(directory): packages_to_install = {()} for cli_dep in cli_deps: cli_dep = cli_dep[len('cli:'):] package, _, version = cli_dep.partition(':') if version != '': packages_to_install.add((package, '--version', version)) else: packages_to_install.add((package,)) for package in packages_to_install: cmd_output( 'cargo', 'install', '--bins', '--root', directory, *package, cwd=prefix.prefix_dir )
def install_environment( repo_cmd_runner, version='default', additional_dependencies=(), ): additional_dependencies = tuple(additional_dependencies) directory = helpers.environment_dir(ENVIRONMENT_DIR, version) # Install a virtualenv with clean_path_on_failure(repo_cmd_runner.path(directory)): venv_cmd = [ sys.executable, '-m', 'virtualenv', '{{prefix}}{0}'.format(directory) ] if version != 'default': venv_cmd.extend(['-p', norm_version(version)]) repo_cmd_runner.run(venv_cmd) with in_env(repo_cmd_runner, version): helpers.run_setup_cmd( repo_cmd_runner, ('pip', 'install', '.') + additional_dependencies, )
def install_environment( repo_cmd_runner, version='default', additional_dependencies=(), ): additional_dependencies = tuple(additional_dependencies) directory = helpers.environment_dir(ENVIRONMENT_DIR, version) # Install a virtualenv with clean_path_on_failure(repo_cmd_runner.path(directory)): venv_cmd = [ sys.executable, '-m', 'virtualenv', '{{prefix}}{0}'.format(directory) ] if version != 'default': venv_cmd.extend(['-p', norm_version(version)]) repo_cmd_runner.run(venv_cmd) with in_env(repo_cmd_runner, version): helpers.run_setup_cmd( repo_cmd_runner, ('pip', 'install', '.') + additional_dependencies, )
def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], ) -> None: helpers.assert_version_default('golang', version) directory = prefix.path( helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), ) with clean_path_on_failure(directory): remote = git.get_remote_url(prefix.prefix_dir) repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote)) # Clone into the goenv we'll create cmd = ('git', 'clone', '--recursive', '.', repo_src_dir) helpers.run_setup_cmd(prefix, cmd) if sys.platform == 'cygwin': # pragma: no cover _, gopath, _ = cmd_output('cygpath', '-w', directory) gopath = gopath.strip() else: gopath = directory env = dict(os.environ, GOPATH=gopath) env.pop('GOBIN', None) cmd_output_b('go', 'install', './...', cwd=repo_src_dir, env=env) for dependency in additional_dependencies: cmd_output_b( 'go', 'install', dependency, cwd=repo_src_dir, env=env, ) # Same some disk space, we don't need these after installation rmtree(prefix.path(directory, 'src')) pkgdir = prefix.path(directory, 'pkg') if os.path.exists(pkgdir): # pragma: no cover (go<1.10) rmtree(pkgdir)
def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], ) -> None: # pragma: win32 no cover helpers.assert_version_default("swift", version) helpers.assert_no_additional_deps("swift", additional_dependencies) directory = prefix.path( helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), ) # Build the swift package with clean_path_on_failure(directory): os.mkdir(directory) cmd_output_b( "swift", "build", "-C", prefix.prefix_dir, "-c", BUILD_CONFIG, "--build-path", os.path.join(directory, BUILD_DIR), )
def install_environment( prefix: Prefix, version: str, additional_dependencies: Sequence[str], ) -> None: helpers.assert_version_default('conda', version) directory = helpers.environment_dir(ENVIRONMENT_DIR, version) env_dir = prefix.path(directory) env_yaml_path = prefix.path('environment.yml') with clean_path_on_failure(env_dir): with open(env_yaml_path) as env_file: env_yaml = yaml_load(env_file) env_yaml['dependencies'] += additional_dependencies tmp_env_file = None try: with NamedTemporaryFile( suffix='.yml', mode='w', delete=False, ) as tmp_env_file: yaml_dump(env_yaml, stream=tmp_env_file) cmd_output_b( 'conda', 'env', 'create', '-p', env_dir, '--file', tmp_env_file.name, cwd=prefix.prefix_dir, ) finally: if tmp_env_file and os.path.exists(tmp_env_file.name): os.remove(tmp_env_file.name)
def install_environment(prefix, version, additional_dependencies): helpers.assert_version_default('golang', version) directory = prefix.path( helpers.environment_dir(ENVIRONMENT_DIR, 'default'), ) with clean_path_on_failure(directory): remote = git.get_remote_url(prefix.prefix_dir) repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote)) # Clone into the goenv we'll create helpers.run_setup_cmd(prefix, ('git', 'clone', '.', repo_src_dir)) if sys.platform == 'cygwin': # pragma: no cover _, gopath, _ = cmd_output('cygpath', '-w', directory) gopath = gopath.strip() else: gopath = directory env = dict(os.environ, GOPATH=gopath) cmd_output('go', 'get', './...', cwd=repo_src_dir, env=env) for dependency in additional_dependencies: cmd_output('go', 'get', dependency, cwd=repo_src_dir, env=env) # Same some disk space, we don't need these after installation rmtree(prefix.path(directory, 'src')) rmtree(prefix.path(directory, 'pkg'))
def test_control_c_control_c_on_install(tempdir_factory, store): """Regression test for #186.""" path = make_repo(tempdir_factory, 'python_hooks_repo') config = make_config_from_repo(path) hooks = [_get_hook_no_install(config, store, 'foo')] class MyKeyboardInterrupt(KeyboardInterrupt): pass # To simulate a killed install, we'll make PythonEnv.run raise ^C # and then to simulate a second ^C during cleanup, we'll make shutil.rmtree # raise as well. with pytest.raises(MyKeyboardInterrupt): with mock.patch.object( helpers, 'run_setup_cmd', side_effect=MyKeyboardInterrupt, ): with mock.patch.object( shutil, 'rmtree', side_effect=MyKeyboardInterrupt, ): install_hook_envs(hooks, store) # Should have made an environment, however this environment is broken! hook, = hooks assert hook.prefix.exists( helpers.environment_dir(python.ENVIRONMENT_DIR, hook.language_version), ) # However, it should be perfectly runnable (reinstall after botched # install) install_hook_envs(hooks, store) ret, out = _hook_run(hook, (), color=False) assert ret == 0
def _install_all(venvs, repo_url, store): """Tuple of (prefix, language, version, deps)""" def _need_installed(): return tuple( (prefix, language_name, version, deps) for prefix, language_name, version, deps in venvs if not _installed(prefix, language_name, version, deps) ) if not _need_installed(): return with store.exclusive_lock(): # Another process may have already completed this work need_installed = _need_installed() if not need_installed: # pragma: no cover (race) return logger.info( 'Installing environment for {}.'.format(repo_url), ) logger.info('Once installed this environment will be reused.') logger.info('This may take a few minutes...') for prefix, language_name, version, deps in need_installed: language = languages[language_name] venv = environment_dir(language.ENVIRONMENT_DIR, version) # There's potentially incomplete cleanup from previous runs # Clean it up! if prefix.exists(venv): shutil.rmtree(prefix.path(venv)) language.install_environment(prefix, version, deps) # Write our state to indicate we're installed state = _state(deps) _write_state(prefix, venv, state)
def _get_env_dir(prefix: Prefix, version: str) -> str: return prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
def _envdir(prefix: Prefix, version: str) -> str: directory = helpers.environment_dir(ENVIRONMENT_DIR, version) return prefix.path(directory)
def in_env(repo_cmd_runner): envdir = repo_cmd_runner.path( helpers.environment_dir(ENVIRONMENT_DIR, 'default'), ) with envcontext(get_env_patch(envdir)): yield