def install_environment(repo_cmd_runner, version, additional_dependencies):
helpers.assert_version_default('golang', version)
directory = repo_cmd_runner.path(
helpers.environment_dir(ENVIRONMENT_DIR, 'default'),
)
with clean_path_on_failure(directory):
remote = git.get_remote_url(repo_cmd_runner.path())
repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote))
# Clone into the goenv we'll create
helpers.run_setup_cmd(
repo_cmd_runner, ('git', 'clone', '.', repo_src_dir),
)
if sys.platform == 'cygwin': # pragma: no cover
_, gopath, _ = cmd_output('cygpath', '-w', directory)
gopath = gopath.strip()
else:
gopath = directory
env = dict(os.environ, GOPATH=gopath)
cmd_output('go', 'get', './...', cwd=repo_src_dir, env=env)
for dependency in additional_dependencies:
cmd_output('go', 'get', dependency, cwd=repo_src_dir, env=env)
# Same some disk space, we don't need these after installation
rmtree(repo_cmd_runner.path(directory, 'src'))
rmtree(repo_cmd_runner.path(directory, 'pkg'))
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=None,
):
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with clean_path_on_failure(repo_cmd_runner.path(directory)):
# TODO: this currently will fail if there's no version specified and
# there's no system ruby installed. Is this ok?
_install_rbenv(repo_cmd_runner, version=version)
with in_env(repo_cmd_runner, version) as ruby_env:
if version != 'default':
_install_ruby(ruby_env, version)
ruby_env.run(
'cd {prefix} && gem build *.gemspec && '
'gem install --no-ri --no-rdoc *.gem',
encoding=None,
)
if additional_dependencies:
ruby_env.run(
'cd {prefix} && gem install --no-ri --no-rdoc ' +
' '.join(
shell_escape(dep) for dep in additional_dependencies
),
encoding=None,
)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with clean_path_on_failure(prefix.path(directory)):
if version != 'system': # pragma: win32 no cover
_install_rbenv(prefix, version)
with in_env(prefix, version):
# Need to call this before installing so rbenv's directories
# are set up
helpers.run_setup_cmd(prefix, ('rbenv', 'init', '-'))
# XXX: this will *always* fail if `version == C.DEFAULT`
_install_ruby(prefix, version)
# Need to call this after installing to set up the shims
helpers.run_setup_cmd(prefix, ('rbenv', 'rehash'))
with in_env(prefix, version):
helpers.run_setup_cmd(
prefix,
('gem', 'build', *prefix.star('.gemspec')),
)
helpers.run_setup_cmd(
prefix,
(
'gem',
'install',
'--no-document',
'--no-format-executable',
*prefix.star('.gem'),
*additional_dependencies,
),
)
def install_environment(
prefix,
version,
additional_dependencies,
): # pragma: windows no cover
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with clean_path_on_failure(prefix.path(directory)):
# TODO: this currently will fail if there's no version specified and
# there's no system ruby installed. Is this ok?
_install_rbenv(prefix, version=version)
with in_env(prefix, version):
# Need to call this before installing so rbenv's directories are
# set up
helpers.run_setup_cmd(prefix, ('rbenv', 'init', '-'))
if version != 'default':
_install_ruby(prefix, version)
# Need to call this after installing to set up the shims
helpers.run_setup_cmd(prefix, ('rbenv', 'rehash'))
helpers.run_setup_cmd(
prefix,
('gem', 'build') + prefix.star('.gemspec'),
)
helpers.run_setup_cmd(
prefix,
('gem', 'install', '--no-ri', '--no-rdoc') +
prefix.star('.gem') + additional_dependencies,
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=(),
):
additional_dependencies = tuple(additional_dependencies)
assert repo_cmd_runner.exists('package.json')
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
env_dir = repo_cmd_runner.path(directory)
with clean_path_on_failure(env_dir):
cmd = [
sys.executable,
'-m',
'nodeenv',
'--prebuilt',
'{{prefix}}{}'.format(directory),
]
if version != 'default':
cmd.extend(['-n', version])
repo_cmd_runner.run(cmd)
with in_env(repo_cmd_runner, version):
helpers.run_setup_cmd(
repo_cmd_runner,
('npm', 'install', '-g', '.') + additional_dependencies,
)
def clone(self, url, sha):
"""Clone the given url and checkout the specific sha."""
self.require_created()
# Check if we already exist
with sqlite3.connect(self.db_path) as db:
result = db.execute(
'SELECT path FROM repos WHERE repo = ? AND ref = ?',
[url, sha],
).fetchone()
if result:
return result[0]
logger.info('Initializing environment for {0}.'.format(url))
dir = tempfile.mkdtemp(prefix='repo', dir=self.directory)
with clean_path_on_failure(dir):
cmd_output(
'git', 'clone', '--no-checkout', url, dir, env=no_git_env(),
)
with cwd(dir):
cmd_output('git', 'reset', sha, '--hard', env=no_git_env())
# Update our db with the created repo
with sqlite3.connect(self.db_path) as db:
db.execute(
'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)',
[url, sha, dir],
)
return dir
def install_environment(prefix, version, additional_dependencies):
helpers.assert_version_default('golang', version)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), )
with clean_path_on_failure(directory):
remote = git.get_remote_url(prefix.prefix_dir)
repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote))
# Clone into the goenv we'll create
helpers.run_setup_cmd(prefix, ('git', 'clone', '.', repo_src_dir))
if sys.platform == 'cygwin': # pragma: no cover
_, gopath, _ = cmd_output('cygpath', '-w', directory)
gopath = gopath.strip()
else:
gopath = directory
env = dict(os.environ, GOPATH=gopath)
cmd_output('go', 'get', './...', cwd=repo_src_dir, env=env)
for dependency in additional_dependencies:
cmd_output('go', 'get', dependency, cwd=repo_src_dir, env=env)
# Same some disk space, we don't need these after installation
rmtree(prefix.path(directory, 'src'))
pkgdir = prefix.path(directory, 'pkg')
if os.path.exists(pkgdir): # pragma: no cover (go<1.10)
rmtree(pkgdir)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=None,
):
assert repo_cmd_runner.exists('package.json')
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
env_dir = repo_cmd_runner.path(directory)
with clean_path_on_failure(env_dir):
cmd = [
sys.executable,
'-m',
'nodeenv',
'--prebuilt',
'{{prefix}}{0}'.format(directory),
]
if version != 'default':
cmd.extend(['-n', version])
repo_cmd_runner.run(cmd)
with in_env(repo_cmd_runner, version) as node_env:
node_env.run("cd '{prefix}' && npm install -g")
if additional_dependencies:
node_env.run("cd '{prefix}' && npm install -g " + ' '.join(
shell_escape(dep) for dep in additional_dependencies))
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None: # pragma: win32 no cover
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with clean_path_on_failure(prefix.path(directory)):
# TODO: this currently will fail if there's no version specified and
# there's no system ruby installed. Is this ok?
_install_rbenv(prefix, version=version)
with in_env(prefix, version):
# Need to call this before installing so rbenv's directories are
# set up
helpers.run_setup_cmd(prefix, ("rbenv", "init", "-"))
if version != C.DEFAULT:
_install_ruby(prefix, version)
# Need to call this after installing to set up the shims
helpers.run_setup_cmd(prefix, ("rbenv", "rehash"))
helpers.run_setup_cmd(
prefix,
("gem", "build", *prefix.star(".gemspec")),
)
helpers.run_setup_cmd(
prefix,
(
"gem",
"install",
"--no-document",
*prefix.star(".gem"),
*additional_dependencies,
),
)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with clean_path_on_failure(prefix.path(directory)):
if version != "system": # pragma: win32 no cover
_install_rbenv(prefix, version)
with in_env(prefix, version):
# Need to call this before installing so rbenv's directories
# are set up
helpers.run_setup_cmd(prefix, ("rbenv", "init", "-"))
# XXX: this will *always* fail if `version == C.DEFAULT`
_install_ruby(prefix, version)
# Need to call this after installing to set up the shims
helpers.run_setup_cmd(prefix, ("rbenv", "rehash"))
with in_env(prefix, version):
helpers.run_setup_cmd(
prefix,
("gem", "build", *prefix.star(".gemspec")),
)
helpers.run_setup_cmd(
prefix,
(
"gem",
"install",
"--no-document",
*prefix.star(".gem"),
*additional_dependencies,
),
)
def _new_repo(self, repo, ref, make_strategy):
self.require_created()
# Check if we already exist
with sqlite3.connect(self.db_path) as db:
result = db.execute(
'SELECT path FROM repos WHERE repo = ? AND ref = ?',
[repo, ref],
).fetchone()
if result:
return result[0]
logger.info('Initializing environment for {}.'.format(repo))
directory = tempfile.mkdtemp(prefix='repo', dir=self.directory)
with clean_path_on_failure(directory):
make_strategy(directory)
# Update our db with the created repo
with sqlite3.connect(self.db_path) as db:
db.execute(
'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)',
[repo, ref, directory],
)
return directory
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None: # pragma: win32 no cover
helpers.assert_version_default('coursier', version)
helpers.assert_no_additional_deps('coursier', additional_dependencies)
envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
channel = prefix.path('.pre-commit-channel')
with clean_path_on_failure(envdir):
for app_descriptor in os.listdir(channel):
_, app_file = os.path.split(app_descriptor)
app, _ = os.path.splitext(app_file)
helpers.run_setup_cmd(
prefix,
(
'cs',
'install',
'--default-channels=false',
f'--channel={channel}',
app,
f'--dir={envdir}',
),
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=None,
):
assert repo_cmd_runner.exists('setup.py')
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
# Install a virtualenv
with clean_path_on_failure(repo_cmd_runner.path(directory)):
venv_cmd = [
sys.executable, '-m', 'virtualenv',
'{{prefix}}{0}'.format(directory)
]
if version != 'default':
venv_cmd.extend(['-p', norm_version(version)])
repo_cmd_runner.run(venv_cmd)
with in_env(repo_cmd_runner, version) as env:
env.run("cd '{prefix}' && pip install .", encoding=None)
if additional_dependencies:
env.run(
"cd '{prefix}' && pip install " +
' '.join(
shell_escape(dep) for dep in additional_dependencies
),
encoding=None,
)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default("golang", version)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT), )
with clean_path_on_failure(directory):
remote = git.get_remote_url(prefix.prefix_dir)
repo_src_dir = os.path.join(directory, "src", guess_go_dir(remote))
# Clone into the goenv we'll create
helpers.run_setup_cmd(prefix, ("git", "clone", ".", repo_src_dir))
if sys.platform == "cygwin": # pragma: no cover
_, gopath, _ = cmd_output("cygpath", "-w", directory)
gopath = gopath.strip()
else:
gopath = directory
env = dict(os.environ, GOPATH=gopath)
env.pop("GOBIN", None)
cmd_output_b("go", "get", "./...", cwd=repo_src_dir, env=env)
for dependency in additional_dependencies:
cmd_output_b("go", "get", dependency, cwd=repo_src_dir, env=env)
# Same some disk space, we don't need these after installation
rmtree(prefix.path(directory, "src"))
pkgdir = prefix.path(directory, "pkg")
if os.path.exists(pkgdir): # pragma: no cover (go<1.10)
rmtree(pkgdir)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=(),
): # pragma: windows no cover
additional_dependencies = tuple(additional_dependencies)
assert repo_cmd_runner.exists('package.json')
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
env_dir = repo_cmd_runner.path(directory)
with clean_path_on_failure(env_dir):
cmd = [
sys.executable, '-m', 'nodeenv', '--prebuilt',
'{{prefix}}{}'.format(directory),
]
if version != 'default':
cmd.extend(['-n', version])
repo_cmd_runner.run(cmd)
with in_env(repo_cmd_runner, version):
helpers.run_setup_cmd(
repo_cmd_runner,
('npm', 'install', '-g', '.') + additional_dependencies,
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=None,
):
assert repo_cmd_runner.exists('package.json')
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
env_dir = repo_cmd_runner.path(directory)
with clean_path_on_failure(env_dir):
cmd = [
sys.executable, '-m', 'nodeenv', '--prebuilt',
'{{prefix}}{0}'.format(directory),
]
if version != 'default':
cmd.extend(['-n', version])
repo_cmd_runner.run(cmd)
with in_env(repo_cmd_runner, version) as node_env:
node_env.run("cd '{prefix}' && npm install -g")
if additional_dependencies:
node_env.run(
"cd '{prefix}' && npm install -g " +
' '.join(
shell_escape(dep) for dep in additional_dependencies
)
)
def install_environment(prefix, version, additional_dependencies):
additional_dependencies = tuple(additional_dependencies)
assert prefix.exists('package.json')
envdir = _envdir(prefix, version)
# https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath
if sys.platform == 'win32': # pragma: no cover
envdir = '\\\\?\\' + os.path.normpath(envdir)
with clean_path_on_failure(envdir):
cmd = [
sys.executable,
'-mnodeenv',
'--prebuilt',
'--clean-src',
envdir,
]
if version != 'default':
cmd.extend(['-n', version])
cmd_output(*cmd)
with in_env(prefix, version):
helpers.run_setup_cmd(
prefix,
('npm', 'install', '-g', '.') + additional_dependencies,
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=(),
):
helpers.assert_version_default('golang', version)
directory = repo_cmd_runner.path(
helpers.environment_dir(ENVIRONMENT_DIR, 'default'),
)
with clean_path_on_failure(directory):
remote = git.get_remote_url(repo_cmd_runner.path())
repo_src_dir = os.path.join(directory, 'src', guess_go_dir(remote))
# Clone into the goenv we'll create
helpers.run_setup_cmd(
repo_cmd_runner, ('git', 'clone', '.', repo_src_dir),
)
env = dict(os.environ, GOPATH=directory)
cmd_output('go', 'get', './...', cwd=repo_src_dir, env=env)
for dependency in additional_dependencies:
cmd_output('go', 'get', dependency, cwd=repo_src_dir, env=env)
# Same some disk space, we don't need these after installation
rmtree(repo_cmd_runner.path(directory, 'src'))
rmtree(repo_cmd_runner.path(directory, 'pkg'))
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
additional_dependencies = tuple(additional_dependencies)
assert prefix.exists('package.json')
envdir = _envdir(prefix, version)
# https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath
if sys.platform == 'win32': # pragma: no cover
envdir = fr'\\?\{os.path.normpath(envdir)}'
with clean_path_on_failure(envdir):
cmd = [
sys.executable,
'-mnodeenv',
'--prebuilt',
'--clean-src',
envdir,
]
if version != C.DEFAULT:
cmd.extend(['-n', version])
cmd_output_b(*cmd)
with in_env(prefix, version):
# https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449
# install as if we installed from git
helpers.run_setup_cmd(prefix, ('npm', 'install'))
helpers.run_setup_cmd(
prefix,
('npm', 'install', '-g', '.', *additional_dependencies),
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=(),
):
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
# Install a virtualenv
with clean_path_on_failure(repo_cmd_runner.path(directory)):
venv_cmd = [
sys.executable, '-m', 'virtualenv',
'{{prefix}}{}'.format(directory)
]
if version != 'default':
venv_cmd.extend(['-p', norm_version(version)])
else:
venv_cmd.extend(['-p', os.path.realpath(sys.executable)])
venv_env = dict(os.environ, VIRTUALENV_NO_DOWNLOAD='1')
repo_cmd_runner.run(venv_cmd, cwd='/', env=venv_env)
with in_env(repo_cmd_runner, version):
helpers.run_setup_cmd(
repo_cmd_runner,
('pip', 'install', '.') + additional_dependencies,
)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None: # pragma: win32 no cover
helpers.assert_version_default('coursier', version)
helpers.assert_no_additional_deps('coursier', additional_dependencies)
# Support both possible executable names (either "cs" or "coursier")
executable = find_executable('cs') or find_executable('coursier')
if executable is None:
raise AssertionError(
'pre-commit requires system-installed "cs" or "coursier" '
'executables in the application search path', )
envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
channel = prefix.path('.pre-commit-channel')
with clean_path_on_failure(envdir):
for app_descriptor in os.listdir(channel):
_, app_file = os.path.split(app_descriptor)
app, _ = os.path.splitext(app_file)
helpers.run_setup_cmd(
prefix,
(
executable,
'install',
'--default-channels=false',
f'--channel={channel}',
app,
f'--dir={envdir}',
),
)
def clone(self, url, sha):
"""Clone the given url and checkout the specific sha."""
self.require_created()
# Check if we already exist
with sqlite3.connect(self.db_path) as db:
result = db.execute(
'SELECT path FROM repos WHERE repo = ? AND ref = ?',
[url, sha],
).fetchone()
if result:
return result[0]
logger.info('Initializing environment for {}.'.format(url))
dir = tempfile.mkdtemp(prefix='repo', dir=self.directory)
with clean_path_on_failure(dir):
cmd_output(
'git',
'clone',
'--no-checkout',
url,
dir,
env=no_git_env(),
)
with cwd(dir):
cmd_output('git', 'reset', sha, '--hard', env=no_git_env())
# Update our db with the created repo
with sqlite3.connect(self.db_path) as db:
db.execute(
'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)',
[url, sha, dir],
)
return dir
def _new_repo(self, repo, ref, make_strategy):
self.require_created()
def _get_result():
# Check if we already exist
with sqlite3.connect(self.db_path) as db:
result = db.execute(
'SELECT path FROM repos WHERE repo = ? AND ref = ?',
[repo, ref],
).fetchone()
if result:
return result[0]
result = _get_result()
if result:
return result
with self.exclusive_lock():
# Another process may have already completed this work
result = _get_result()
if result: # pragma: no cover (race)
return result
logger.info('Initializing environment for {}.'.format(repo))
directory = tempfile.mkdtemp(prefix='repo', dir=self.directory)
with clean_path_on_failure(directory):
make_strategy(directory)
# Update our db with the created repo
with sqlite3.connect(self.db_path) as db:
db.execute(
'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)',
[repo, ref, directory],
)
return directory
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=(),
): # pragma: windows no cover
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with clean_path_on_failure(repo_cmd_runner.path(directory)):
# TODO: this currently will fail if there's no version specified and
# there's no system ruby installed. Is this ok?
_install_rbenv(repo_cmd_runner, version=version)
with in_env(repo_cmd_runner, version):
# Need to call this before installing so rbenv's directories are
# set up
helpers.run_setup_cmd(repo_cmd_runner, ('rbenv', 'init', '-'))
if version != 'default':
_install_ruby(repo_cmd_runner, version)
# Need to call this after installing to set up the shims
helpers.run_setup_cmd(repo_cmd_runner, ('rbenv', 'rehash'))
helpers.run_setup_cmd(
repo_cmd_runner,
('gem', 'build') + repo_cmd_runner.star('.gemspec'),
)
helpers.run_setup_cmd(
repo_cmd_runner,
(
('gem', 'install', '--no-ri', '--no-rdoc') +
repo_cmd_runner.star('.gem') + additional_dependencies
),
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=None,
):
assert repo_cmd_runner.exists('setup.py')
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
# Install a virtualenv
with clean_path_on_failure(repo_cmd_runner.path(directory)):
venv_cmd = [
sys.executable, '-m', 'virtualenv',
'{{prefix}}{0}'.format(directory)
]
if version != 'default':
venv_cmd.extend(['-p', norm_version(version)])
repo_cmd_runner.run(venv_cmd)
with in_env(repo_cmd_runner, version) as env:
env.run("cd '{prefix}' && pip install .")
if additional_dependencies:
env.run(
"cd '{prefix}' && pip install " +
' '.join(
shell_escape(dep) for dep in additional_dependencies
)
)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('conda', version)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
conda_exe = _conda_exe()
env_dir = prefix.path(directory)
with clean_path_on_failure(env_dir):
cmd_output_b(
conda_exe,
'env',
'create',
'-p',
env_dir,
'--file',
'environment.yml',
cwd=prefix.prefix_dir,
)
if additional_dependencies:
cmd_output_b(
conda_exe,
'install',
'-p',
env_dir,
*additional_dependencies,
cwd=prefix.prefix_dir,
)
def _new_repo(self, repo, ref, make_strategy):
self.require_created()
# Check if we already exist
with sqlite3.connect(self.db_path) as db:
result = db.execute(
'SELECT path FROM repos WHERE repo = ? AND ref = ?',
[repo, ref],
).fetchone()
if result:
return result[0]
logger.info('Initializing environment for {}.'.format(repo))
directory = tempfile.mkdtemp(prefix='repo', dir=self.directory)
with clean_path_on_failure(directory):
make_strategy(directory)
# Update our db with the created repo
with sqlite3.connect(self.db_path) as db:
db.execute(
'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)',
[repo, ref, directory],
)
return directory
def _new_repo(self, repo, ref, deps, make_strategy):
repo = self.db_repo_name(repo, deps)
def _get_result():
# Check if we already exist
with self.connect() as db:
result = db.execute(
'SELECT path FROM repos WHERE repo = ? AND ref = ?',
(repo, ref),
).fetchone()
if result:
return result[0]
result = _get_result()
if result:
return result
with self.exclusive_lock():
# Another process may have already completed this work
result = _get_result()
if result: # pragma: no cover (race)
return result
logger.info('Initializing environment for {}.'.format(repo))
directory = tempfile.mkdtemp(prefix='repo', dir=self.directory)
with clean_path_on_failure(directory):
make_strategy(directory)
# Update our db with the created repo
with self.connect() as db:
db.execute(
'INSERT INTO repos (repo, ref, path) VALUES (?, ?, ?)',
[repo, ref, directory],
)
return directory
def install_environment(
prefix, version, additional_dependencies,
): # pragma: windows no cover
additional_dependencies = tuple(additional_dependencies)
assert prefix.exists('package.json')
envdir = _envdir(prefix, version)
# https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath
if sys.platform == 'win32': # pragma: no cover
envdir = '\\\\?\\' + os.path.normpath(envdir)
with clean_path_on_failure(envdir):
cmd = [
sys.executable, '-mnodeenv', '--prebuilt', '--clean-src', envdir,
]
if version != C.DEFAULT:
cmd.extend(['-n', version])
cmd_output(*cmd)
with in_env(prefix, version):
# https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449
# install as if we installed from git
helpers.run_setup_cmd(prefix, ('npm', 'install'))
helpers.run_setup_cmd(
prefix,
('npm', 'install', '-g', '.') + additional_dependencies,
)
def install_environment(
prefix: Prefix, version: str, additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default("conda", version)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
env_dir = prefix.path(directory)
with clean_path_on_failure(env_dir):
cmd_output_b(
"conda",
"env",
"create",
"-p",
env_dir,
"--file",
"environment.yml",
cwd=prefix.prefix_dir,
)
if additional_dependencies:
cmd_output_b(
"conda",
"install",
"-p",
env_dir,
*additional_dependencies,
cwd=prefix.prefix_dir,
)
def install_environment(repo_cmd_runner, version="default"):
with clean_path_on_failure(repo_cmd_runner.path("rbenv")):
# TODO: this currently will fail if there's no version specified and
# there's no system ruby installed. Is this ok?
_install_rbenv(repo_cmd_runner, version=version)
with in_env(repo_cmd_runner) as ruby_env:
if version != "default":
_install_ruby(ruby_env, version)
ruby_env.run("cd {prefix} && gem build *.gemspec && gem install *.gem")
def test_clean_path_on_failure_cleans_for_normal_exception(in_tmpdir):
class MyException(Exception):
pass
with pytest.raises(MyException):
with clean_path_on_failure('foo'):
os.mkdir('foo')
raise MyException
assert not os.path.exists('foo')
def install_environment(repo_cmd_runner, version="default", additional_dependencies=()): # pragma: windows no cover
assert version == "default", "Pre-commit does not support language_version for docker "
directory = repo_cmd_runner.path(helpers.environment_dir(ENVIRONMENT_DIR, "default"))
# Build the swift package
with clean_path_on_failure(directory):
os.mkdir(directory)
repo_cmd_runner.run(
("swift", "build", "-C", "{prefix}", "-c", BUILD_CONFIG, "--build-path", os.path.join(directory, BUILD_DIR))
)
def test_clean_path_on_failure_cleans_for_system_exit(in_tmpdir):
class MySystemExit(SystemExit):
pass
with pytest.raises(MySystemExit):
with clean_path_on_failure('foo'):
os.mkdir('foo')
raise MySystemExit
assert not os.path.exists('foo')
def test_clean_path_on_failure_cleans_for_system_exit(in_tmpdir):
class MySystemExit(SystemExit):
pass
with pytest.raises(MySystemExit):
with clean_path_on_failure('foo'):
os.mkdir('foo')
raise MySystemExit
assert not os.path.exists('foo')
def install_environment(repo_cmd_runner, version='default'):
with clean_path_on_failure(repo_cmd_runner.path('rbenv')):
# TODO: this currently will fail if there's no version specified and
# there's no system ruby installed. Is this ok?
_install_rbenv(repo_cmd_runner, version=version)
with in_env(repo_cmd_runner) as ruby_env:
if version != 'default':
_install_ruby(ruby_env, version)
ruby_env.run(
'cd {prefix} && gem build *.gemspec && gem install *.gem', )
def test_clean_path_on_failure_cleans_for_normal_exception(in_tmpdir):
class MyException(Exception):
pass
with pytest.raises(MyException):
with clean_path_on_failure('foo'):
os.mkdir('foo')
raise MyException
assert not os.path.exists('foo')
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
env_dir = _get_env_dir(prefix, version)
with clean_path_on_failure(env_dir):
os.makedirs(env_dir, exist_ok=True)
shutil.copy(prefix.path('renv.lock'), env_dir)
shutil.copytree(prefix.path('renv'), os.path.join(env_dir, 'renv'))
cmd_output_b(
_rscript_exec(),
'--vanilla',
'-e',
f"""\
prefix_dir <- {prefix.prefix_dir!r}
options(
repos = c(CRAN = "https://cran.rstudio.com"),
renv.consent = TRUE
)
source("renv/activate.R")
renv::restore()
activate_statement <- paste0(
'suppressWarnings({{',
'old <- setwd("', getwd(), '"); ',
'source("renv/activate.R"); ',
'setwd(old); ',
'renv::load("', getwd(), '");}})'
)
writeLines(activate_statement, 'activate.R')
is_package <- tryCatch(
{{
path_desc <- file.path(prefix_dir, 'DESCRIPTION')
suppressWarnings(desc <- read.dcf(path_desc))
"Package" %in% colnames(desc)
}},
error = function(...) FALSE
)
if (is_package) {{
renv::install(prefix_dir)
}}
""",
cwd=env_dir,
)
if additional_dependencies:
with in_env(prefix, version):
cmd_output_b(
_rscript_exec(),
*RSCRIPT_OPTS,
'-e',
'renv::install(commandArgs(trailingOnly = TRUE))',
*additional_dependencies,
cwd=env_dir,
)
def install_environment(repo_cmd_runner, version='default'):
assert repo_cmd_runner.exists('setup.py')
# Install a virtualenv
with clean_path_on_failure(repo_cmd_runner.path(ENVIRONMENT_DIR)):
venv_cmd = ['virtualenv', '{{prefix}}{0}'.format(ENVIRONMENT_DIR)]
if version != 'default':
venv_cmd.extend(['-p', version])
repo_cmd_runner.run(venv_cmd)
with in_env(repo_cmd_runner) as env:
env.run('cd {prefix} && pip install .')
def install_environment(repo_cmd_runner, version='default'):
assert repo_cmd_runner.exists('setup.py')
# Install a virtualenv
with clean_path_on_failure(repo_cmd_runner.path(ENVIRONMENT_DIR)):
venv_cmd = ['virtualenv', '{{prefix}}{0}'.format(ENVIRONMENT_DIR)]
if version != 'default':
venv_cmd.extend(['-p', version])
repo_cmd_runner.run(venv_cmd)
with in_env(repo_cmd_runner) as env:
env.run('cd {prefix} && pip install .')
def install_environment(repo_cmd_runner, version='default'):
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
with clean_path_on_failure(repo_cmd_runner.path(directory)):
# TODO: this currently will fail if there's no version specified and
# there's no system ruby installed. Is this ok?
_install_rbenv(repo_cmd_runner, version=version)
with in_env(repo_cmd_runner, version) as ruby_env:
if version != 'default':
_install_ruby(ruby_env, version)
ruby_env.run(
'cd {prefix} && gem build *.gemspec'
' && gem install --no-document *.gem', )
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('perl', version)
with clean_path_on_failure(_envdir(prefix, version)):
with in_env(prefix, version):
helpers.run_setup_cmd(
prefix,
('cpan', '-T', '.', *additional_dependencies),
)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
env_dir = _get_env_dir(prefix, version)
with clean_path_on_failure(env_dir):
os.makedirs(env_dir, exist_ok=True)
path_desc_source = prefix.path('DESCRIPTION')
if os.path.exists(path_desc_source):
shutil.copy(path_desc_source, env_dir)
shutil.copy(prefix.path('renv.lock'), env_dir)
cmd_output_b(
'Rscript',
'--vanilla',
'-e',
"""\
missing_pkgs <- setdiff(
"renv", unname(installed.packages()[, "Package"])
)
options(
repos = c(CRAN = "https://cran.rstudio.com"),
renv.consent = TRUE
)
install.packages(missing_pkgs)
renv::activate()
renv::restore()
activate_statement <- paste0(
'renv::activate("', file.path(getwd()), '"); '
)
writeLines(activate_statement, 'activate.R')
is_package <- tryCatch(
suppressWarnings(
unname(read.dcf('DESCRIPTION')[,'Type'] == "Package")
),
error = function(...) FALSE
)
if (is_package) {
renv::install(normalizePath('.'))
}
""",
cwd=env_dir,
)
if additional_dependencies:
cmd_output_b(
'Rscript',
'-e',
'renv::install(commandArgs(trailingOnly = TRUE))',
*additional_dependencies,
cwd=env_dir,
)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
python = norm_version(version)
venv_cmd = (sys.executable, '-mvirtualenv', envdir, '-p', python)
install_cmd = ('python', '-mpip', 'install', '.', *additional_dependencies)
with clean_path_on_failure(envdir):
cmd_output_b(*venv_cmd, cwd='/')
with in_env(prefix, version):
helpers.run_setup_cmd(prefix, install_cmd)
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
helpers.assert_version_default('dotnet', version)
helpers.assert_no_additional_deps('dotnet', additional_dependencies)
envdir = prefix.path(helpers.environment_dir(ENVIRONMENT_DIR, version))
with clean_path_on_failure(envdir):
build_dir = 'pre-commit-build'
# Build & pack nupkg file
helpers.run_setup_cmd(
prefix,
(
'dotnet',
'pack',
'--configuration',
'Release',
'--output',
build_dir,
),
)
# Determine tool from the packaged file <tool_name>.<version>.nupkg
build_outputs = os.listdir(os.path.join(prefix.prefix_dir, build_dir))
if len(build_outputs) != 1:
raise NotImplementedError(
f"Can't handle multiple build outputs. Got {build_outputs}", )
tool_name = build_outputs[0].split('.')[0]
# Install to bin dir
helpers.run_setup_cmd(
prefix,
(
'dotnet',
'tool',
'install',
'--tool-path',
os.path.join(envdir, BIN_DIR),
'--add-source',
build_dir,
tool_name,
),
)
# Cleanup build output
for d in ('bin', 'obj', build_dir):
rmtree(prefix.path(d))
def install_environment(
prefix: Prefix,
version: str,
additional_dependencies: Sequence[str],
) -> None:
additional_dependencies = tuple(additional_dependencies)
assert prefix.exists('package.json')
envdir = _envdir(prefix, version)
# https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx?f=255&MSPPError=-2147217396#maxpath
if sys.platform == 'win32': # pragma: no cover
envdir = fr'\\?\{os.path.normpath(envdir)}'
with clean_path_on_failure(envdir):
cmd = [
sys.executable,
'-mnodeenv',
'--prebuilt',
'--clean-src',
envdir,
]
if version != C.DEFAULT:
cmd.extend(['-n', version])
cmd_output_b(*cmd)
with in_env(prefix, version):
# https://npm.community/t/npm-install-g-git-vs-git-clone-cd-npm-install-g/5449
# install as if we installed from git
local_install_cmd = (
'npm',
'install',
'--dev',
'--prod',
'--ignore-prepublish',
'--no-progress',
'--no-save',
)
helpers.run_setup_cmd(prefix, local_install_cmd)
_, pkg, _ = cmd_output('npm', 'pack', cwd=prefix.prefix_dir)
pkg = prefix.path(pkg.strip())
install = ('npm', 'install', '-g', pkg, *additional_dependencies)
helpers.run_setup_cmd(prefix, install)
# clean these up after installation
if prefix.exists('node_modules'): # pragma: win32 no cover
rmtree(prefix.path('node_modules'))
os.remove(pkg)
def install_environment(prefix, version, additional_dependencies):
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(_dir, version)
env_dir = prefix.path(directory)
with clean_path_on_failure(env_dir):
if version != C.DEFAULT:
python = norm_version(version)
else:
python = os.path.realpath(sys.executable)
_make_venv(env_dir, python)
with in_env(prefix, version):
helpers.run_setup_cmd(
prefix, ('pip', 'install', '.') + additional_dependencies,
)
def install_environment(prefix, version, additional_dependencies):
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(_dir, version)
env_dir = prefix.path(directory)
with clean_path_on_failure(env_dir):
if version != C.DEFAULT:
python = norm_version(version)
else:
python = os.path.realpath(sys.executable)
_make_venv(env_dir, python)
with in_env(prefix, version):
helpers.run_setup_cmd(
prefix, ('pip', 'install', '.') + additional_dependencies,
)
def install_environment(repo_cmd_runner, version='default'):
assert repo_cmd_runner.exists('setup.py')
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
# Install a virtualenv
with clean_path_on_failure(repo_cmd_runner.path(directory)):
venv_cmd = [
sys.executable, '-m', 'virtualenv',
'{{prefix}}{0}'.format(directory)
]
if version != 'default':
venv_cmd.extend(['-p', norm_version(version)])
repo_cmd_runner.run(venv_cmd)
with in_env(repo_cmd_runner, version) as env:
env.run("cd '{prefix}' && pip install .")
def install_environment(
prefix, version, additional_dependencies,
): # pragma: windows no cover
helpers.assert_version_default('docker', version)
helpers.assert_no_additional_deps('docker', additional_dependencies)
assert_docker_available()
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
# Docker doesn't really have relevant disk environment, but pre-commit
# still needs to cleanup it's state files on failure
with clean_path_on_failure(directory):
build_docker_image(prefix, pull=True)
os.mkdir(directory)
def install_environment(repo_cmd_runner, version='default'):
assert repo_cmd_runner.exists('package.json')
env_dir = repo_cmd_runner.path(ENVIRONMENT_DIR)
with clean_path_on_failure(env_dir):
cmd = [
sys.executable, '-m', 'nodeenv', '--prebuilt',
'{{prefix}}{0}'.format(ENVIRONMENT_DIR),
]
if version != 'default':
cmd.extend(['-n', version])
repo_cmd_runner.run(cmd)
with in_env(repo_cmd_runner) as node_env:
node_env.run('cd {prefix} && npm install -g')
def install_environment(
prefix, version, additional_dependencies,
): # pragma: windows no cover
helpers.assert_version_default('swift', version)
helpers.assert_no_additional_deps('swift', additional_dependencies)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
# Build the swift package
with clean_path_on_failure(directory):
os.mkdir(directory)
cmd_output(
'swift', 'build',
'-C', prefix.prefix_dir,
'-c', BUILD_CONFIG,
'--build-path', os.path.join(directory, BUILD_DIR),
)
def install_environment(
repo_cmd_runner, version, additional_dependencies,
): # pragma: windows no cover
helpers.assert_version_default('swift', version)
helpers.assert_no_additional_deps('swift', additional_dependencies)
directory = repo_cmd_runner.path(
helpers.environment_dir(ENVIRONMENT_DIR, 'default'),
)
# Build the swift package
with clean_path_on_failure(directory):
os.mkdir(directory)
repo_cmd_runner.run((
'swift', 'build',
'-C', '{prefix}',
'-c', BUILD_CONFIG,
'--build-path', os.path.join(directory, BUILD_DIR),
))
def install_environment(
repo_cmd_runner, version, additional_dependencies,
): # pragma: windows no cover
assert repo_cmd_runner.exists('Dockerfile'), (
'No Dockerfile was found in the hook repository'
)
helpers.assert_version_default('docker', version)
helpers.assert_no_additional_deps('docker', additional_dependencies)
assert_docker_available()
directory = repo_cmd_runner.path(
helpers.environment_dir(ENVIRONMENT_DIR, 'default'),
)
# Docker doesn't really have relevant disk environment, but pre-commit
# still needs to cleanup it's state files on failure
with clean_path_on_failure(directory):
build_docker_image(repo_cmd_runner, pull=True)
os.mkdir(directory)
def install_environment(prefix, version, additional_dependencies):
helpers.assert_version_default('rust', version)
directory = prefix.path(
helpers.environment_dir(ENVIRONMENT_DIR, C.DEFAULT),
)
# There are two cases where we might want to specify more dependencies:
# as dependencies for the library being built, and as binary packages
# to be `cargo install`'d.
#
# Unlike e.g. Python, if we just `cargo install` a library, it won't be
# used for compilation. And if we add a crate providing a binary to the
# `Cargo.toml`, the binary won't be built.
#
# Because of this, we allow specifying "cli" dependencies by prefixing
# with 'cli:'.
cli_deps = {
dep for dep in additional_dependencies if dep.startswith('cli:')
}
lib_deps = set(additional_dependencies) - cli_deps
if len(lib_deps) > 0:
_add_dependencies(prefix.path('Cargo.toml'), lib_deps)
with clean_path_on_failure(directory):
packages_to_install = {()}
for cli_dep in cli_deps:
cli_dep = cli_dep[len('cli:'):]
package, _, version = cli_dep.partition(':')
if version != '':
packages_to_install.add((package, '--version', version))
else:
packages_to_install.add((package,))
for package in packages_to_install:
cmd_output(
'cargo', 'install', '--bins', '--root', directory, *package,
cwd=prefix.prefix_dir
)
def install_environment(
repo_cmd_runner,
version='default',
additional_dependencies=(),
):
additional_dependencies = tuple(additional_dependencies)
directory = helpers.environment_dir(ENVIRONMENT_DIR, version)
# Install a virtualenv
with clean_path_on_failure(repo_cmd_runner.path(directory)):
venv_cmd = [
sys.executable, '-m', 'virtualenv',
'{{prefix}}{0}'.format(directory)
]
if version != 'default':
venv_cmd.extend(['-p', norm_version(version)])
repo_cmd_runner.run(venv_cmd)
with in_env(repo_cmd_runner, version):
helpers.run_setup_cmd(
repo_cmd_runner,
('pip', 'install', '.') + additional_dependencies,
)
def clone(self, url, sha):
"""Clone the given url and checkout the specific sha."""
self.require_created()
# Check if we already exist
sha_path = os.path.join(self.directory, sha + '_' + hex_md5(url))
if os.path.exists(sha_path):
return os.readlink(sha_path)
logger.info('Installing environment for {0}.'.format(url))
logger.info('Once installed this environment will be reused.')
logger.info('This may take a few minutes...')
dir = tempfile.mkdtemp(prefix='repo', dir=self.directory)
with clean_path_on_failure(dir):
local['git']('clone', '--no-checkout', url, dir)
with local.cwd(dir):
local['git']('checkout', sha)
# Make a symlink from sha->repo
os.symlink(dir, sha_path)
return dir
def test_clean_on_failure_noop(in_tmpdir):
with clean_path_on_failure('foo'):
pass
def test_clean_path_on_failure_does_nothing_when_not_raising(in_tmpdir):
with clean_path_on_failure('foo'):
os.mkdir('foo')
assert os.path.exists('foo')
