def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=(),
):
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
# Install a virtualenv
with clean_path_on_failure(repo_cmd_runner.path(directory)):
venv_cmd = [
sys.executable, '-m', 'virtualenv',
'{{prefix}}{}'.format(directory)
]
if version != 'default':
venv_cmd.extend(['-p', norm_version(version)])
else:
venv_cmd.extend(['-p', os.path.realpath(sys.executable)])
venv_env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1')
repo_cmd_runner.run(venv_cmd, cwd='/', env=venv_env)
with in_env(repo_cmd_runner, version):
helpers.run_setup_cmd(
repo_cmd_runner,
('pip', 'install', '.') + additional_dependencies,
)
def env_prefix(self):
return ". '{{prefix}}{0}activate' &&".format(
virtualenv.path_locations(
helpers.environment_dir(ENVIRONMENT_DIR, self.language_version)
)[-1].rstrip(os.sep) + os.sep,
'activate',
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=None,
):
assert repo_cmd_runner.exists('package.json')
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
env_dir = repo_cmd_runner.path(directory)
with clean_path_on_failure(env_dir):
cmd = [
sys.executable, '-m', 'nodeenv', '--prebuilt',
'{{prefix}}{0}'.format(directory),
]
if version != 'default':
cmd.extend(['-n', version])
repo_cmd_runner.run(cmd)
with in_env(repo_cmd_runner, version) as node_env:
node_env.run("cd '{prefix}' && npm install -g")
if additional_dependencies:
node_env.run(
"cd '{prefix}' && npm install -g " +
' '.join(
shell_escape(dep) for dep in additional_dependencies
)
)
def language_is_installed(language_name, language_version):
language = languages[language_name]
venv = environment_dir(language.ENVIRONMENT_DIR, language_version)
return (
venv is None or
read_state(venv) == state(language_name, language_version)
)
def _install_all(venvs, repo_url):
"""Tuple of (cmd_runner, language, version, deps)"""
need_installed = tuple(
(cmd_runner, language_name, version, deps)
for cmd_runner, language_name, version, deps in venvs
if not _installed(cmd_runner, language_name, version, deps)
)
if need_installed:
logger.info(
'Installing environment for {}.'.format(repo_url)
)
logger.info('Once installed this environment will be reused.')
logger.info('This may take a few minutes...')
for cmd_runner, language_name, version, deps in need_installed:
language = languages[language_name]
venv = environment_dir(language.ENVIRONMENT_DIR, version)
# There's potentially incomplete cleanup from previous runs
# Clean it up!
if cmd_runner.exists(venv):
shutil.rmtree(cmd_runner.path(venv))
language.install_environment(cmd_runner, version, deps)
# Write our state to indicate we're installed
state = _state(deps)
_write_installed_state(cmd_runner, venv, state)
def _installed(cmd_runner, language_name, language_version, additional_deps):
language = languages[language_name]
venv = environment_dir(language.ENVIRONMENT_DIR, language_version)
return (
venv is None or
_read_installed_state(cmd_runner, venv) == _state(additional_deps)
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=(),
): # pragma: windows no cover
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with clean_path_on_failure(repo_cmd_runner.path(directory)):
# TODO: this currently will fail if there's no version specified and
# there's no system ruby installed. Is this ok?
_install_rbenv(repo_cmd_runner, version=version)
with in_env(repo_cmd_runner, version):
# Need to call this before installing so rbenv's directories are
# set up
helpers.run_setup_cmd(repo_cmd_runner, ('rbenv', 'init', '-'))
if version != 'default':
_install_ruby(repo_cmd_runner, version)
# Need to call this after installing to set up the shims
helpers.run_setup_cmd(repo_cmd_runner, ('rbenv', 'rehash'))
helpers.run_setup_cmd(
repo_cmd_runner,
('gem', 'build') + repo_cmd_runner.star('.gemspec'),
)
helpers.run_setup_cmd(
repo_cmd_runner,
(
('gem', 'install', '--no-ri', '--no-rdoc') +
repo_cmd_runner.star('.gem') + additional_dependencies
),
)
def _install_rbenv(prefix, version=C.DEFAULT): # pragma: windows no cover
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
_extract_resource('rbenv.tar.gz', prefix.path('.'))
shutil.move(prefix.path('rbenv'), prefix.path(directory))
# Only install ruby-build if the version is specified
if version != C.DEFAULT:
plugins_dir = prefix.path(directory, 'plugins')
_extract_resource('ruby-download.tar.gz', plugins_dir)
_extract_resource('ruby-build.tar.gz', plugins_dir)
activate_path = prefix.path(directory, 'bin', 'activate')
with io.open(activate_path, 'w') as activate_file:
# This is similar to how you would install rbenv to your home directory
# However we do a couple things to make the executables exposed and
# configure it to work in our directory.
# We also modify the PS1 variable for manual debugging sake.
activate_file.write(
'#!/usr/bin/env bash\n'
"export RBENV_ROOT='{directory}'\n"
'export PATH="$RBENV_ROOT/bin:$PATH"\n'
'eval "$(rbenv init -)"\n'
'export PS1="(rbenv)$PS1"\n'
# This lets us install gems in an isolated and repeatable
# directory
"export GEM_HOME='{directory}/gems'\n"
'export PATH="$GEM_HOME/bin:$PATH"\n'
'\n'.format(directory=prefix.path(directory)),
)
# If we aren't using the system ruby, add a version here
if version != C.DEFAULT:
activate_file.write('export RBENV_VERSION="{}"\n'.format(version))
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=None,
):
assert repo_cmd_runner.exists('setup.py')
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
# Install a virtualenv
with clean_path_on_failure(repo_cmd_runner.path(directory)):
venv_cmd = [
sys.executable, '-m', 'virtualenv',
'{{prefix}}{0}'.format(directory)
]
if version != 'default':
venv_cmd.extend(['-p', norm_version(version)])
repo_cmd_runner.run(venv_cmd)
with in_env(repo_cmd_runner, version) as env:
env.run("cd '{prefix}' && pip install .", encoding=None)
if additional_dependencies:
env.run(
"cd '{prefix}' && pip install " +
' '.join(
shell_escape(dep) for dep in additional_dependencies
),
encoding=None,
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=None,
):
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with clean_path_on_failure(repo_cmd_runner.path(directory)):
# TODO: this currently will fail if there's no version specified and
# there's no system ruby installed. Is this ok?
_install_rbenv(repo_cmd_runner, version=version)
with in_env(repo_cmd_runner, version) as ruby_env:
if version != 'default':
_install_ruby(ruby_env, version)
ruby_env.run(
'cd {prefix} && gem build *.gemspec && '
'gem install --no-ri --no-rdoc *.gem',
encoding=None,
)
if additional_dependencies:
ruby_env.run(
'cd {prefix} && gem install --no-ri --no-rdoc ' +
' '.join(
shell_escape(dep) for dep in additional_dependencies
),
encoding=None,
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=(),
): # pragma: windows no cover
additional_dependencies = tuple(additional_dependencies)
assert repo_cmd_runner.exists('package.json')
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
env_dir = repo_cmd_runner.path(directory)
with clean_path_on_failure(env_dir):
cmd = [
sys.executable, '-m', 'nodeenv', '--prebuilt',
'{{prefix}}{}'.format(directory),
]
if version != 'default':
cmd.extend(['-n', version])
repo_cmd_runner.run(cmd)
with in_env(repo_cmd_runner, version):
helpers.run_setup_cmd(
repo_cmd_runner,
('npm', 'install', '-g', '.') + additional_dependencies,
)
def env_prefix(self):
return ". '{{prefix}}{0}{1}activate' &&".format(
bin_dir(
helpers.environment_dir(ENVIRONMENT_DIR, self.language_version)
),
os.sep,
)
def in_env(repo_cmd_runner, language_version):
envdir = os.path.join(
repo_cmd_runner.prefix_dir,
helpers.environment_dir(ENVIRONMENT_DIR, language_version),
)
with envcontext(get_env_patch(envdir)):
yield
def install_environment(repo_cmd_runner, version, additional_dependencies):
helpers.assert_version_default('golang', version)
directory = repo_cmd_runner.path(
helpers.environment_dir(ENVIRONMENT_DIR, 'default'),
)
with clean_path_on_failure(directory):
remote = git.get_remote_url(repo_cmd_runner.path())
repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote))
# Clone into the goenv we'll create
helpers.run_setup_cmd(
repo_cmd_runner, ('git', 'clone', '.', repo_src_dir),
)
if sys.platform == 'cygwin': # pragma: no cover
_, gopath, _ = cmd_output('cygpath', '-w', directory)
gopath = gopath.strip()
else:
gopath = directory
env = dict(os.environ, GOPATH=gopath)
cmd_output('go', 'get', './...', cwd=repo_src_dir, env=env)
for dependency in additional_dependencies:
cmd_output('go', 'get', dependency, cwd=repo_src_dir, env=env)
# Same some disk space, we don't need these after installation
rmtree(repo_cmd_runner.path(directory, 'src'))
rmtree(repo_cmd_runner.path(directory, 'pkg'))
def language_is_installed(language_name, language_version):
language = languages[language_name]
directory = environment_dir(
language.ENVIRONMENT_DIR, language_version,
)
return (
directory is None or
self.cmd_runner.exists(directory, '.installed')
)
def install_environment(repo_cmd_runner, version="default", additional_dependencies=()): # pragma: windows no cover
assert version == "default", "Pre-commit does not support language_version for docker "
directory = repo_cmd_runner.path(helpers.environment_dir(ENVIRONMENT_DIR, "default"))
# Build the swift package
with clean_path_on_failure(directory):
os.mkdir(directory)
repo_cmd_runner.run(
("swift", "build", "-C", "{prefix}", "-c", BUILD_CONFIG, "--build-path", os.path.join(directory, BUILD_DIR))
)
def install_environment(repo_cmd_runner, version='default'):
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with clean_path_on_failure(repo_cmd_runner.path(directory)):
# TODO: this currently will fail if there's no version specified and
# there's no system ruby installed. Is this ok?
_install_rbenv(repo_cmd_runner, version=version)
with in_env(repo_cmd_runner, version) as ruby_env:
if version != 'default':
_install_ruby(ruby_env, version)
ruby_env.run(
'cd {prefix} && gem build *.gemspec'
' && gem install --no-document *.gem', )
def installed(self):
lang = languages[self.language]
venv = environment_dir(lang.ENVIRONMENT_DIR, self.language_version)
return (
venv is None or (
(
_read_state(self.prefix, venv) ==
_state(self.additional_dependencies)
) and
lang.healthy(self.prefix, self.language_version)
)
)
def install_environment(repo_cmd_runner, version='default'):
assert repo_cmd_runner.exists('setup.py')
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
# Install a virtualenv
with clean_path_on_failure(repo_cmd_runner.path(directory)):
venv_cmd = [
sys.executable, '-m', 'virtualenv',
'{{prefix}}{0}'.format(directory)
]
if version != 'default':
venv_cmd.extend(['-p', norm_version(version)])
repo_cmd_runner.run(venv_cmd)
with in_env(repo_cmd_runner, version) as env:
env.run("cd '{prefix}' && pip install .")
def install_environment(prefix, version, additional_dependencies):
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(_dir, version)
env_dir = prefix.path(directory)
with clean_path_on_failure(env_dir):
if version != C.DEFAULT:
python = norm_version(version)
else:
python = os.path.realpath(sys.executable)
_make_venv(env_dir, python)
with in_env(prefix, version):
helpers.run_setup_cmd(
prefix, ('pip', 'install', '.') + additional_dependencies,
)
def test_additional_golang_dependencies_installed(
tempdir_factory, store,
):
path = make_repo(tempdir_factory, 'golang_hooks_repo')
config = make_config_from_repo(path)
# A small go package
deps = ['github.com/golang/example/hello']
config['hooks'][0]['additional_dependencies'] = deps
repo = Repository.create(config, store)
repo.require_installed()
binaries = os.listdir(repo._cmd_runner.path(
helpers.environment_dir(golang.ENVIRONMENT_DIR, 'default'), 'bin',
))
# normalize for windows
binaries = [os.path.splitext(binary)[0] for binary in binaries]
assert 'hello' in binaries
def install_environment(
prefix, version, additional_dependencies,
): # pragma: windows no cover
helpers.assert_version_default('docker', version)
helpers.assert_no_additional_deps('docker', additional_dependencies)
assert_docker_available()
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
# Docker doesn't really have relevant disk environment, but pre-commit
# still needs to cleanup it's state files on failure
with clean_path_on_failure(directory):
build_docker_image(prefix, pull=True)
os.mkdir(directory)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None: # pragma: win32 no cover
helpers.assert_version_default('docker', version)
helpers.assert_no_additional_deps('docker', additional_dependencies)
assert_docker_available()
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), )
# Docker doesn't really have relevant disk environment, but pre-commit
# still needs to cleanup its state files on failure
with clean_path_on_failure(directory):
build_docker_image(prefix, pull=True)
os.mkdir(directory)
def healthy(prefix: Prefix, language_version: str) -> bool:
directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
envdir = prefix.path(directory)
pyvenv_cfg = os.path.join(envdir, "pyvenv.cfg")
# created with "old" virtualenv
if not os.path.exists(pyvenv_cfg):
return False
exe_name = "python.exe" if sys.platform == "win32" else "python"
py_exe = prefix.path(bin_dir(envdir), exe_name)
cfg = _read_pyvenv_cfg(pyvenv_cfg)
return ("version_info" in cfg
and _version_info(py_exe) == cfg["version_info"] and
("base-executable" not in cfg
or _version_info(cfg["base-executable"]) == cfg["version_info"]))
def test_additional_golang_dependencies_installed(
tempdir_factory, store,
):
path = make_repo(tempdir_factory, 'golang_hooks_repo')
config = make_config_from_repo(path)
# A small go package
deps = ['github.com/golang/example/hello']
config['hooks'][0]['additional_dependencies'] = deps
hook = _get_hook(config, store, 'golang-hook')
binaries = os.listdir(
hook.prefix.path(
helpers.environment_dir(golang.ENVIRONMENT_DIR, C.DEFAULT), 'bin',
),
)
# normalize for windows
binaries = [os.path.splitext(binary)[0] for binary in binaries]
assert 'hello' in binaries
def test_additional_golang_dependencies_installed(
tempdir_factory, store,
):
path = make_repo(tempdir_factory, 'golang_hooks_repo')
config = make_config_from_repo(path)
# A small go package
deps = ['github.com/golang/example/hello']
config['hooks'][0]['additional_dependencies'] = deps
repo = Repository.create(config, store)
repo.require_installed()
(prefix, _, _, _), = repo._venvs()
binaries = os.listdir(prefix.path(
helpers.environment_dir(golang.ENVIRONMENT_DIR, 'default'), 'bin',
))
# normalize for windows
binaries = [os.path.splitext(binary)[0] for binary in binaries]
assert 'hello' in binaries
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
directory = helpers.environment_dir(_dir, version)
install = ('python', '-mpip', 'install', '.', *additional_dependencies)
env_dir = prefix.path(directory)
with clean_path_on_failure(env_dir):
if version != C.DEFAULT:
python = norm_version(version)
else:
python = os.path.realpath(sys.executable)
_make_venv(env_dir, python)
with in_env(prefix, version):
helpers.run_setup_cmd(prefix, install)
def install(self):
logger.info('Installing environment for {}.'.format(self.src))
logger.info('Once installed this environment will be reused.')
logger.info('This may take a few minutes...')
lang = languages[self.language]
venv = environment_dir(lang.ENVIRONMENT_DIR, self.language_version)
# There's potentially incomplete cleanup from previous runs
# Clean it up!
if self.prefix.exists(venv):
rmtree(self.prefix.path(venv))
lang.install_environment(
self.prefix, self.language_version, self.additional_dependencies,
)
# Write our state to indicate we're installed
_write_state(self.prefix, venv, _state(self.additional_dependencies))
def install_environment(
prefix, version, additional_dependencies,
): # pragma: windows no cover
helpers.assert_version_default('swift', version)
helpers.assert_no_additional_deps('swift', additional_dependencies)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
# Build the swift package
with clean_path_on_failure(directory):
os.mkdir(directory)
cmd_output(
'swift', 'build',
'-C', prefix.prefix_dir,
'-c', BUILD_CONFIG,
'--build-path', os.path.join(directory, BUILD_DIR),
)
def install(self):
logger.info('Installing environment for {}.'.format(self.src))
logger.info('Once installed this environment will be reused.')
logger.info('This may take a few minutes...')
lang = languages[self.language]
venv = environment_dir(lang.ENVIRONMENT_DIR, self.language_version)
# There's potentially incomplete cleanup from previous runs
# Clean it up!
if self.prefix.exists(venv):
rmtree(self.prefix.path(venv))
lang.install_environment(
self.prefix, self.language_version, self.additional_dependencies,
)
# Write our state to indicate we're installed
_write_state(self.prefix, venv, _state(self.additional_dependencies))
def test_additional_rust_lib_dependencies_installed(
tempdir_factory, store,
):
path = make_repo(tempdir_factory, 'rust_hooks_repo')
config = make_config_from_repo(path)
# A small rust package with no dependencies.
deps = ['shellharden:3.1.0']
config['hooks'][0]['additional_dependencies'] = deps
hook = _get_hook(config, store, 'rust-hook')
binaries = os.listdir(
hook.prefix.path(
helpers.environment_dir(rust.ENVIRONMENT_DIR, C.DEFAULT), 'bin',
),
)
# normalize for windows
binaries = [os.path.splitext(binary)[0] for binary in binaries]
assert 'rust-hello-world' in binaries
assert 'shellharden' not in binaries
def test_additional_rust_lib_dependencies_installed(
tempdir_factory, store,
):
path = make_repo(tempdir_factory, 'rust_hooks_repo')
config = make_config_from_repo(path)
# A small rust package with no dependencies.
deps = ['shellharden:3.1.0']
config['hooks'][0]['additional_dependencies'] = deps
repo = Repository.create(config, store)
repo.require_installed()
(prefix, _, _, _), = repo._venvs()
binaries = os.listdir(prefix.path(
helpers.environment_dir(rust.ENVIRONMENT_DIR, 'default'), 'bin',
))
# normalize for windows
binaries = [os.path.splitext(binary)[0] for binary in binaries]
assert 'rust-hello-world' in binaries
assert 'shellharden' not in binaries
def install_environment(
repo_cmd_runner, version, additional_dependencies,
): # pragma: windows no cover
helpers.assert_version_default('swift', version)
helpers.assert_no_additional_deps('swift', additional_dependencies)
directory = repo_cmd_runner.path(
helpers.environment_dir(ENVIRONMENT_DIR, 'default'),
)
# Build the swift package
with clean_path_on_failure(directory):
os.mkdir(directory)
repo_cmd_runner.run((
'swift', 'build',
'-C', '{prefix}',
'-c', BUILD_CONFIG,
'--build-path', os.path.join(directory, BUILD_DIR),
))
def _install_rbenv(
repo_cmd_runner,
version='default',
): # pragma: windows no cover
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with tarfile.open(resource_filename('rbenv.tar.gz')) as tf:
tf.extractall(repo_cmd_runner.path('.'))
shutil.move(
repo_cmd_runner.path('rbenv'),
repo_cmd_runner.path(directory),
)
# Only install ruby-build if the version is specified
if version != 'default':
# ruby-download
with tarfile.open(resource_filename('ruby-download.tar.gz')) as tf:
tf.extractall(repo_cmd_runner.path(directory, 'plugins'))
# ruby-build
with tarfile.open(resource_filename('ruby-build.tar.gz')) as tf:
tf.extractall(repo_cmd_runner.path(directory, 'plugins'))
activate_path = repo_cmd_runner.path(directory, 'bin', 'activate')
with io.open(activate_path, 'w') as activate_file:
# This is similar to how you would install rbenv to your home directory
# However we do a couple things to make the executables exposed and
# configure it to work in our directory.
# We also modify the PS1 variable for manual debugging sake.
activate_file.write(
'#!/usr/bin/env bash\n'
"export RBENV_ROOT='{directory}'\n"
'export PATH="$RBENV_ROOT/bin:$PATH"\n'
'eval "$(rbenv init -)"\n'
'export PS1="(rbenv)$PS1"\n'
# This lets us install gems in an isolated and repeatable
# directory
"export GEM_HOME='{directory}/gems'\n"
'export PATH="$GEM_HOME/bin:$PATH"\n'
'\n'.format(directory=repo_cmd_runner.path(directory)))
# If we aren't using the system ruby, add a version here
if version != 'default':
activate_file.write('export RBENV_VERSION="{}"\n'.format(version))
def _hook_install(hook: Hook) -> None:
logger.info(f"Installing environment for {hook.src}.")
logger.info("Once installed this environment will be reused.")
logger.info("This may take a few minutes...")
lang = languages[hook.language]
assert lang.ENVIRONMENT_DIR is not None
venv = environment_dir(lang.ENVIRONMENT_DIR, hook.language_version)
# There's potentially incomplete cleanup from previous runs
# Clean it up!
if hook.prefix.exists(venv):
rmtree(hook.prefix.path(venv))
lang.install_environment(
hook.prefix, hook.language_version, hook.additional_dependencies,
)
# Write our state to indicate we're installed
_write_state(hook.prefix, venv, _state(hook.additional_dependencies))
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('conda', version)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
env_dir = prefix.path(directory)
with clean_path_on_failure(env_dir):
cmd_output_b(
'conda', 'env', 'create', '-p', env_dir, '--file',
'environment.yml', cwd=prefix.prefix_dir,
)
if additional_dependencies:
cmd_output_b(
'conda', 'install', '-p', env_dir, *additional_dependencies,
cwd=prefix.prefix_dir,
)
def install_environment(
repo_cmd_runner, version, additional_dependencies,
): # pragma: windows no cover
assert repo_cmd_runner.exists('Dockerfile'), (
'No Dockerfile was found in the hook repository'
)
helpers.assert_version_default('docker', version)
helpers.assert_no_additional_deps('docker', additional_dependencies)
assert_docker_available()
directory = repo_cmd_runner.path(
helpers.environment_dir(ENVIRONMENT_DIR, 'default'),
)
# Docker doesn't really have relevant disk environment, but pre-commit
# still needs to cleanup it's state files on failure
with clean_path_on_failure(directory):
build_docker_image(repo_cmd_runner, pull=True)
os.mkdir(directory)
def install_environment(prefix, version, additional_dependencies):
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
# Install a virtualenv
env_dir = prefix.path(directory)
with clean_path_on_failure(env_dir):
venv_cmd = [sys.executable, '-m', 'virtualenv', env_dir]
if version != 'default':
venv_cmd.extend(['-p', norm_version(version)])
else:
venv_cmd.extend(['-p', os.path.realpath(sys.executable)])
venv_env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1')
cmd_output(*venv_cmd, cwd='/', env=venv_env)
with in_env(prefix, version):
helpers.run_setup_cmd(
prefix,
('pip', 'install', '.') + additional_dependencies,
)
def install_environment(
repo_cmd_runner,
version,
additional_dependencies,
): # pragma: windows no cover
assert repo_cmd_runner.exists('Dockerfile'), (
'No Dockerfile was found in the hook repository')
helpers.assert_version_default('docker', version)
helpers.assert_no_additional_deps('docker', additional_dependencies)
assert_docker_available()
directory = repo_cmd_runner.path(
helpers.environment_dir(ENVIRONMENT_DIR, 'default'), )
# Docker doesn't really have relevant disk environment, but pre-commit
# still needs to cleanup it's state files on failure
with clean_path_on_failure(directory):
build_docker_image(repo_cmd_runner, pull=True)
os.mkdir(directory)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('rust', version)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
# There are two cases where we might want to specify more dependencies:
# as dependencies for the library being built, and as binary packages
# to be `cargo install`'d.
#
# Unlike e.g. Python, if we just `cargo install` a library, it won't be
# used for compilation. And if we add a crate providing a binary to the
# `Cargo.toml`, the binary won't be built.
#
# Because of this, we allow specifying "cli" dependencies by prefixing
# with 'cli:'.
cli_deps = {
dep for dep in additional_dependencies if dep.startswith('cli:')
}
lib_deps = set(additional_dependencies) - cli_deps
if len(lib_deps) > 0:
_add_dependencies(prefix.path('Cargo.toml'), lib_deps)
with clean_path_on_failure(directory):
packages_to_install: Set[Tuple[str, ...]] = {('--path', '.')}
for cli_dep in cli_deps:
cli_dep = cli_dep[len('cli:'):]
package, _, version = cli_dep.partition(':')
if version != '':
packages_to_install.add((package, '--version', version))
else:
packages_to_install.add((package,))
for args in packages_to_install:
cmd_output_b(
'cargo', 'install', '--bins', '--root', directory, *args,
cwd=prefix.prefix_dir,
)
def healthy(prefix: Prefix, language_version: str) -> bool:
directory = helpers.environment_dir(ENVIRONMENT_DIR, language_version)
envdir = prefix.path(directory)
pyvenv_cfg = os.path.join(envdir, 'pyvenv.cfg')
# created with "old" virtualenv
if not os.path.exists(pyvenv_cfg):
return False
exe_name = 'python.exe' if sys.platform == 'win32' else 'python'
py_exe = prefix.path(bin_dir(envdir), exe_name)
cfg = _read_pyvenv_cfg(pyvenv_cfg)
return (
'version_info' in cfg and
# always use uncached lookup here in case we replaced an unhealthy env
_version_info.__wrapped__(py_exe) == cfg['version_info']
and ('base-executable' not in cfg
or _version_info(cfg['base-executable']) == cfg['version_info']))
def test_invalidated_virtualenv(tempdir_factory, store):
# A cached virtualenv may become invalidated if the system python upgrades
# This should not cause every hook in that virtualenv to fail.
path = make_repo(tempdir_factory, 'python_hooks_repo')
config = make_config_from_repo(path)
hook = _get_hook(config, store, 'foo')
# Simulate breaking of the virtualenv
libdir = hook.prefix.path(
helpers.environment_dir(python.ENVIRONMENT_DIR, hook.language_version),
'lib', hook.language_version,
)
paths = [
os.path.join(libdir, p) for p in ('site.py', 'site.pyc', '__pycache__')
]
cmd_output_b('rm', '-rf', *paths)
# pre-commit should rebuild the virtualenv and it should be runnable
retv, stdout, stderr = _get_hook(config, store, 'foo').run(())
assert retv == 0
def install(self):
"""Install the hook repository."""
def language_is_installed(language_name, language_version):
language = languages[language_name]
directory = environment_dir(
language.ENVIRONMENT_DIR, language_version,
)
return (
directory is None or
self.cmd_runner.exists(directory, '.installed')
)
if not all(
language_is_installed(language_name, language_version)
for language_name, language_version in self.languages
):
logger.info(
'Installing environment for {0}.'.format(self.repo_url)
)
logger.info('Once installed this environment will be reused.')
logger.info('This may take a few minutes...')
for language_name, language_version in self.languages:
language = languages[language_name]
if language_is_installed(language_name, language_version):
continue
directory = environment_dir(
language.ENVIRONMENT_DIR, language_version,
)
# There's potentially incomplete cleanup from previous runs
# Clean it up!
if self.cmd_runner.exists(directory):
shutil.rmtree(self.cmd_runner.path(directory))
language.install_environment(
self.cmd_runner, language_version,
self.additional_dependencies[language_name][language_version],
)
# Touch the .installed file (atomic) to indicate we've installed
open(self.cmd_runner.path(directory, '.installed'), 'w').close()
def _install_rbenv(repo_cmd_runner, version='default'):
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with tarfile_open(resource_filename('rbenv.tar.gz')) as tf:
tf.extractall(repo_cmd_runner.path('.'))
shutil.move(
repo_cmd_runner.path('rbenv'),
repo_cmd_runner.path(directory),
)
# Only install ruby-build if the version is specified
if version != 'default':
# ruby-download
with tarfile_open(resource_filename('ruby-download.tar.gz')) as tf:
tf.extractall(repo_cmd_runner.path(directory, 'plugins'))
# ruby-build
with tarfile_open(resource_filename('ruby-build.tar.gz')) as tf:
tf.extractall(repo_cmd_runner.path(directory, 'plugins'))
activate_path = repo_cmd_runner.path(directory, 'bin', 'activate')
with io.open(activate_path, 'w') as activate_file:
# This is similar to how you would install rbenv to your home directory
# However we do a couple things to make the executables exposed and
# configure it to work in our directory.
# We also modify the PS1 variable for manual debugging sake.
activate_file.write(
'#!/usr/bin/env bash\n'
"export RBENV_ROOT='{0}'\n"
'export PATH="$RBENV_ROOT/bin:$PATH"\n'
'eval "$(rbenv init -)"\n'
'export PS1="(rbenv)$PS1"\n'
# This lets us install gems in an isolated and repeatable
# directory
"export GEM_HOME='{0}/gems'\n"
'export PATH="$GEM_HOME/bin:$PATH"\n'
'\n'.format(repo_cmd_runner.path(directory)))
# If we aren't using the system ruby, add a version here
if version != 'default':
activate_file.write('export RBENV_VERSION="{0}"\n'.format(version))
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=None,
):
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with clean_path_on_failure(repo_cmd_runner.path(directory)):
# TODO: this currently will fail if there's no version specified and
# there's no system ruby installed. Is this ok?
_install_rbenv(repo_cmd_runner, version=version)
with in_env(repo_cmd_runner, version) as ruby_env:
if version != 'default':
_install_ruby(ruby_env, version)
ruby_env.run(
'cd {prefix} && gem build *.gemspec && '
'gem install --no-ri --no-rdoc *.gem', )
if additional_dependencies:
ruby_env.run('cd {prefix} && gem install --no-ri --no-rdoc ' +
' '.join(
shell_escape(dep)
for dep in additional_dependencies))
def install_environment(repo_cmd_runner, version, additional_dependencies):
helpers.assert_version_default('golang', version)
directory = repo_cmd_runner.path(
helpers.environment_dir(ENVIRONMENT_DIR, 'default'),
)
with clean_path_on_failure(directory):
remote = git.get_remote_url(repo_cmd_runner.path())
repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote))
# Clone into the goenv we'll create
helpers.run_setup_cmd(
repo_cmd_runner, ('git', 'clone', '.', repo_src_dir),
)
env = dict(os.environ, GOPATH=directory)
cmd_output('go', 'get', './...', cwd=repo_src_dir, env=env)
for dependency in additional_dependencies:
cmd_output('go', 'get', dependency, cwd=repo_src_dir, env=env)
# Same some disk space, we don't need these after installation
rmtree(repo_cmd_runner.path(directory, 'src'))
rmtree(repo_cmd_runner.path(directory, 'pkg'))
def install_environment(prefix, version, additional_dependencies):
helpers.assert_version_default('rust', version)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
# There are two cases where we might want to specify more dependencies:
# as dependencies for the library being built, and as binary packages
# to be `cargo install`'d.
#
# Unlike e.g. Python, if we just `cargo install` a library, it won't be
# used for compilation. And if we add a crate providing a binary to the
# `Cargo.toml`, the binary won't be built.
#
# Because of this, we allow specifying "cli" dependencies by prefixing
# with 'cli:'.
cli_deps = {
dep for dep in additional_dependencies if dep.startswith('cli:')
}
lib_deps = set(additional_dependencies) - cli_deps
if len(lib_deps) > 0:
_add_dependencies(prefix.path('Cargo.toml'), lib_deps)
with clean_path_on_failure(directory):
packages_to_install = {()}
for cli_dep in cli_deps:
cli_dep = cli_dep[len('cli:'):]
package, _, version = cli_dep.partition(':')
if version != '':
packages_to_install.add((package, '--version', version))
else:
packages_to_install.add((package,))
for package in packages_to_install:
cmd_output(
'cargo', 'install', '--bins', '--root', directory, *package,
cwd=prefix.prefix_dir
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=(),
):
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
# Install a virtualenv
with clean_path_on_failure(repo_cmd_runner.path(directory)):
venv_cmd = [
sys.executable, '-m', 'virtualenv',
'{{prefix}}{0}'.format(directory)
]
if version != 'default':
venv_cmd.extend(['-p', norm_version(version)])
repo_cmd_runner.run(venv_cmd)
with in_env(repo_cmd_runner, version):
helpers.run_setup_cmd(
repo_cmd_runner,
('pip', 'install', '.') + additional_dependencies,
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=(),
):
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
# Install a virtualenv
with clean_path_on_failure(repo_cmd_runner.path(directory)):
venv_cmd = [
sys.executable, '-m', 'virtualenv',
'{{prefix}}{0}'.format(directory)
]
if version != 'default':
venv_cmd.extend(['-p', norm_version(version)])
repo_cmd_runner.run(venv_cmd)
with in_env(repo_cmd_runner, version):
helpers.run_setup_cmd(
repo_cmd_runner,
('pip', 'install', '.') + additional_dependencies,
)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('golang', version)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), )
with clean_path_on_failure(directory):
remote = git.get_remote_url(prefix.prefix_dir)
repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote))
# Clone into the goenv we'll create
cmd = ('git', 'clone', '--recursive', '.', repo_src_dir)
helpers.run_setup_cmd(prefix, cmd)
if sys.platform == 'cygwin': # pragma: no cover
_, gopath, _ = cmd_output('cygpath', '-w', directory)
gopath = gopath.strip()
else:
gopath = directory
env = dict(os.environ, GOPATH=gopath)
env.pop('GOBIN', None)
cmd_output_b('go', 'install', './...', cwd=repo_src_dir, env=env)
for dependency in additional_dependencies:
cmd_output_b(
'go',
'install',
dependency,
cwd=repo_src_dir,
env=env,
)
# Same some disk space, we don't need these after installation
rmtree(prefix.path(directory, 'src'))
pkgdir = prefix.path(directory, 'pkg')
if os.path.exists(pkgdir): # pragma: no cover (go<1.10)
rmtree(pkgdir)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None: # pragma: win32 no cover
helpers.assert_version_default("swift", version)
helpers.assert_no_additional_deps("swift", additional_dependencies)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), )
# Build the swift package
with clean_path_on_failure(directory):
os.mkdir(directory)
cmd_output_b(
"swift",
"build",
"-C",
prefix.prefix_dir,
"-c",
BUILD_CONFIG,
"--build-path",
os.path.join(directory, BUILD_DIR),
)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('conda', version)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
env_dir = prefix.path(directory)
env_yaml_path = prefix.path('environment.yml')
with clean_path_on_failure(env_dir):
with open(env_yaml_path) as env_file:
env_yaml = yaml_load(env_file)
env_yaml['dependencies'] += additional_dependencies
tmp_env_file = None
try:
with NamedTemporaryFile(
suffix='.yml',
mode='w',
delete=False,
) as tmp_env_file:
yaml_dump(env_yaml, stream=tmp_env_file)
cmd_output_b(
'conda',
'env',
'create',
'-p',
env_dir,
'--file',
tmp_env_file.name,
cwd=prefix.prefix_dir,
)
finally:
if tmp_env_file and os.path.exists(tmp_env_file.name):
os.remove(tmp_env_file.name)
def install_environment(prefix, version, additional_dependencies):
helpers.assert_version_default('golang', version)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, 'default'), )
with clean_path_on_failure(directory):
remote = git.get_remote_url(prefix.prefix_dir)
repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote))
# Clone into the goenv we'll create
helpers.run_setup_cmd(prefix, ('git', 'clone', '.', repo_src_dir))
if sys.platform == 'cygwin': # pragma: no cover
_, gopath, _ = cmd_output('cygpath', '-w', directory)
gopath = gopath.strip()
else:
gopath = directory
env = dict(os.environ, GOPATH=gopath)
cmd_output('go', 'get', './...', cwd=repo_src_dir, env=env)
for dependency in additional_dependencies:
cmd_output('go', 'get', dependency, cwd=repo_src_dir, env=env)
# Same some disk space, we don't need these after installation
rmtree(prefix.path(directory, 'src'))
rmtree(prefix.path(directory, 'pkg'))
def test_control_c_control_c_on_install(tempdir_factory, store):
"""Regression test for #186."""
path = make_repo(tempdir_factory, 'python_hooks_repo')
config = make_config_from_repo(path)
hooks = [_get_hook_no_install(config, store, 'foo')]
class MyKeyboardInterrupt(KeyboardInterrupt):
pass
# To simulate a killed install, we'll make PythonEnv.run raise ^C
# and then to simulate a second ^C during cleanup, we'll make shutil.rmtree
# raise as well.
with pytest.raises(MyKeyboardInterrupt):
with mock.patch.object(
helpers,
'run_setup_cmd',
side_effect=MyKeyboardInterrupt,
):
with mock.patch.object(
shutil,
'rmtree',
side_effect=MyKeyboardInterrupt,
):
install_hook_envs(hooks, store)
# Should have made an environment, however this environment is broken!
hook, = hooks
assert hook.prefix.exists(
helpers.environment_dir(python.ENVIRONMENT_DIR,
hook.language_version), )
# However, it should be perfectly runnable (reinstall after botched
# install)
install_hook_envs(hooks, store)
ret, out = _hook_run(hook, (), color=False)
assert ret == 0
def _install_all(venvs, repo_url, store):
"""Tuple of (prefix, language, version, deps)"""
def _need_installed():
return tuple(
(prefix, language_name, version, deps)
for prefix, language_name, version, deps in venvs
if not _installed(prefix, language_name, version, deps)
)
if not _need_installed():
return
with store.exclusive_lock():
# Another process may have already completed this work
need_installed = _need_installed()
if not need_installed: # pragma: no cover (race)
return
logger.info(
'Installing environment for {}.'.format(repo_url),
)
logger.info('Once installed this environment will be reused.')
logger.info('This may take a few minutes...')
for prefix, language_name, version, deps in need_installed:
language = languages[language_name]
venv = environment_dir(language.ENVIRONMENT_DIR, version)
# There's potentially incomplete cleanup from previous runs
# Clean it up!
if prefix.exists(venv):
shutil.rmtree(prefix.path(venv))
language.install_environment(prefix, version, deps)
# Write our state to indicate we're installed
state = _state(deps)
_write_state(prefix, venv, state)
def _get_env_dir(prefix: Prefix, version: str) -> str:
return prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
def _envdir(prefix: Prefix, version: str) -> str:
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
return prefix.path(directory)
def in_env(repo_cmd_runner):
envdir = repo_cmd_runner.path(
helpers.environment_dir(ENVIRONMENT_DIR, 'default'),
)
with envcontext(get_env_patch(envdir)):
yield
