def _make_env(self) -> Path:
venv_path = self._path / "venv"
_echo(
"Installing {} ({}): {}".format(
colored("green", "PDM", bold=True),
colored("yellow", self.version),
colored("cyan", "Creating virtual environment"),
)
)
try:
import venv
venv.create(venv_path, clear=False, with_pip=True)
except (ModuleNotFoundError, subprocess.CalledProcessError):
try:
import virtualenv
except ModuleNotFoundError:
python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
url = (
"https://bootstrap.pypa.io/virtualenv/"
f"{python_version}/virtualenv.pyz"
)
with TemporaryDirectory(prefix="pdm-installer-") as tempdir:
virtualenv_zip = Path(tempdir) / "virtualenv.pyz"
urllib.request.urlretrieve(url, virtualenv_zip)
_call_subprocess(
[sys.executable, str(virtualenv_zip), str(venv_path)]
)
else:
virtualenv.cli_run([str(venv_path)])
return venv_path
def _make_env(self) -> Path:
venv_path = self._path / "venv"
_echo("Installing {} ({}): {}".format(
colored("green", "PDM", bold=True),
colored("yellow", self.version),
colored("cyan", "Creating virtual environment"),
))
try:
import venv
except ModuleNotFoundError:
try:
import virtualenv
except ModuleNotFoundError:
tmpdir = TemporaryDirectory()
atexit.register(tmpdir.cleanup)
_call_subprocess([
sys.executable,
os.path.dirname(pip_location),
"install",
"virtualenv",
"-t",
tmpdir.name,
])
sys.path.insert(0, tmpdir.name)
import virtualenv
virtualenv.cli_run([str(venv_path)])
else:
venv.create(venv_path, clear=False, with_pip=True)
return venv_path
def cmd_env():
"""
Create Virtual Environment for your project and automatically install requirements.
[Options]:
Choose path or use CWD to install virtual environment
Choose a name for virtual environment
Code readability: 'join' / 'expanduser' reused from DJANGUALR settings
"""
data = Env()
print("To install in current directory [Enter]: " + os.getcwd())
choose_path = data.choose_path
name_env = data.name_env
venv_dir = os.path.join(os.path.expanduser(choose_path), name_env)
cli_run([venv_dir])
if choose_path or name_env:
print("▸ Virtualenv path: ", str(choose_path + ": " + name_env))
else:
if choose_path == "":
print("▸ Virtualenv path: ", str(os.getcwd() + ": " + name_env))
print("▸ Activate {} with 'djangular -env activate'".format(name_env))
activate_file = os.path.join(venv_dir, "bin", "activate_this.py")
with open(activate_file) as f:
code = compile(f.read(), activate_file, "exec")
exec(code, dict(__file__=activate_file))
def _create(self):
"""
create a virtual environment from the current settings
:return:
"""
cli_run(['-p', self.python, self.path])
def create_if_not_exists(self):
python = os.path.join(self.path, 'bin', 'python')
if not os.path.exists(python):
makedirs(self.path)
log.info(style_note('Creating Python virtualenv', self.path))
if hasattr(virtualenv, 'cli_run'):
# New API (in which there isn't really any API)
virtualenv.cli_run(
['--no-pip', '--no-wheel', '--no-setuptools', self.path])
else:
# Old API
virtualenv.create_environment(self.path,
no_setuptools=True,
no_pip=True)
if not os.path.exists(python + '-config'):
version = get_default_python().version
names = (
'python{}.{}-config'.format(*version),
'python{}-config'.format(*version),
'python-config',
)
prefix = getattr(sys, 'real_prefix', sys.prefix)
for name in names:
old_path = os.path.join(prefix, 'bin', name)
if os.path.exists(old_path):
for name in names:
new_path = os.path.join(self.path, 'bin', name)
self.rewrite_shebang_or_link(old_path, new_path)
break
else:
log.warning('Could not find python-config')
def __setup(venv_dir, requirements=None, verbose=False):
"""
This creates (if relevant) and activates a virtual environment. It also
allows to define requirements to be installed in this environment.
:param venv_dir: virtual environment's directory
:param requirements: list of required package OR path of the requirements
file to be used
:param verbose: displayed Pip output while installing packages
"""
__deactivate()
venv_dir = os.path.abspath(venv_dir)
if not os.path.exists(venv_dir):
# fix to issue: https://github.com/pypa/virtualenv/issues/1585
try:
virtualenv.create_environment(venv_dir)
except AttributeError:
virtualenv.cli_run([venv_dir])
__activate(venv_dir)
if isinstance(requirements, string_types):
with open(requirements) as f:
requirements = [l.strip() for l in f]
if isinstance(requirements, (tuple, list, set)):
args = ["-v"] if verbose else []
kwargs = {'prefix': venv_dir}
for req in requirements:
pkg = __install(req, *args, **kwargs)
for tl in pkg.top_level:
if hasattr(virtualenv, tl):
raise TopLevelAlreadyExists("{} ({})".format(tl, pkg.name))
m = import_module(tl)
setattr(virtualenv, tl, m)
def check_install_package(self, package, simple):
pip_args = ["--no-cache-dir"]
if simple:
pip_args.append("-i")
pip_args.append("%s/simple" % self.server_url)
else:
pip_args.append("--no-index")
pip_args.append("-f")
pip_args.append(self.temp_data_dir)
# create the virtualenv
self.venv_dir = tempfile.mkdtemp(suffix="pypisync_tests_venv")
virtualenv.cli_run([self.venv_dir])
# install the package
version = ""
try:
version = "==%s" % packaging.version.Version(package[1])
except packaging.version.InvalidVersion:
try:
version = "%s" % packaging.specifiers.SpecifierSet(package[1])
except packaging.specifiers.InvalidSpecifier:
pass
package_name = "%s%s" % (package[0], version)
self.assertEqual(
subprocess.check_call(
["pip", "install", package_name, "--prefix", self.venv_dir] +
pip_args), 0)
# cleanup the venv
shutil.rmtree(self.venv_dir, ignore_errors=True)
def make_env(self, version: str) -> Path:
self._overwrite("Installing {} ({}): {}".format(
colorize("info", "Poetry"),
colorize("b", version),
colorize("comment", "Creating environment"),
))
env_path = self._data_dir.joinpath("venv")
with temporary_directory() as tmp_dir:
subprocess.call(
[
sys.executable, "-m", "pip", "install", "virtualenv", "-t",
tmp_dir
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
sys.path.insert(0, tmp_dir)
import virtualenv
virtualenv.cli_run([str(env_path), "--clear"])
return env_path
def init_venv(config: PyConfig):
"""
Initialize a git project
Args:
config: configuration for project
"""
# run installation
cli_run([str(config.project.joinpath(".venv")), '-p', config.pyversion])
def venv(temporary_directory): # type: (Path) -> Path
venv_dir = temporary_directory / ".venv"
virtualenv.cli_run([
"--no-download",
"--no-periodic-update",
"--python",
sys.executable,
venv_dir.as_posix(),
])
return venv_dir
def create_with_virtualenv(path: Path, pretend=False):
import virtualenv
args = [str(path)]
if pretend:
virtualenv.session_via_cli(args)
else:
logger.warning("\nInstalling virtual environment, it might take a while...\n")
virtualenv.cli_run(args)
logger.report("virtualenv", path)
def _run_task(
ctx: Context,
connector_string: str,
task_name: str,
multi_envs: bool = True,
module_path: Optional[str] = None,
task_commands: Dict = TASK_COMMANDS,
**kwargs: Any,
) -> int:
"""
Run task in its own environment.
"""
cur_dir = os.getcwd()
if multi_envs:
if module_path:
os.chdir(module_path)
source_path = connector_string
else:
os.chdir(os.path.join(CONNECTORS_DIR,
f"source-{connector_string}"))
source_path = f"source_{connector_string.replace('-', '_')}"
else:
source_path = connector_string
venv_name = tempfile.mkdtemp(dir=os.curdir)
virtualenv.cli_run([venv_name])
activator = os.path.join(os.path.abspath(venv_name), "bin", "activate")
commands = []
commands.extend([
cmd.format(source_path=source_path, venv=venv_name, **kwargs)
for cmd in task_commands[task_name]
])
exit_code: int = 0
try:
with ctx.prefix(f"source {activator}"):
for command in commands:
result = ctx.run(command, echo=True, warn=True)
if result.return_code:
exit_code = 1
break
finally:
shutil.rmtree(venv_name, ignore_errors=True)
if module_path:
os.chdir(cur_dir)
return exit_code
def _test_with_test_file(test_file_name, expected_packages=set(), unexpected_packages=set(), **kwargs):
setup_path = pathlib.Path(__file__).parent / "resources" / test_file_name
with tempfile.TemporaryDirectory("setuper_tests") as test_virtualenv:
virtualenv_path = pathlib.Path(test_virtualenv)
virtualenv.cli_run([str(virtualenv_path)])
initial_packages = _get_installed_packages(virtualenv_path)
setuper.run(setup_path, pip=[str(virtualenv_path / "bin" / "python"), "-m", "pip"], **kwargs)
final_packages = _get_installed_packages(virtualenv_path)
packages = final_packages.difference(initial_packages)
if expected_packages:
assert expected_packages.issubset(packages)
if unexpected_packages:
assert not unexpected_packages.issubset(packages)
def install_packages(
kernel_name: str,
resolution_engine: str,
kernels_path: Path = Path.home().joinpath(".local/share/thoth/kernels"),
is_cli: bool = False,
is_magic_command: bool = False,
) -> None:
"""Install dependencies in the virtualenv."""
_LOGGER.info(f"kernel_name selected: {kernel_name}")
env_path = kernels_path.joinpath(kernel_name)
if not is_magic_command:
env_path.mkdir(parents=True, exist_ok=True)
package_manager: str = "micropipenv"
_LOGGER.info(
f"Installing requirements using {package_manager} in virtualenv at {env_path}."
)
# 1. Creating new environment
if is_cli or resolution_engine != "pipenv":
cli_run([str(env_path)])
# 2. Install micropipenv if not installed already
check_install = subprocess.run(
f"python3 -c \"import sys, pkgutil; sys.exit(0 if pkgutil.find_loader('{package_manager}') else 1)\"",
shell=True,
cwd=kernels_path,
capture_output=True,
)
if check_install.returncode != 0:
_LOGGER.debug("micropipenv is not installed in the host!: %r",
check_install.stderr)
_ = subprocess.run(
"pip install micropipenv",
shell=True,
cwd=kernels_path,
)
else:
_LOGGER.debug("micropipenv is already present on the host!")
# 3. Install packages using micropipenv
_ = subprocess.run(
f". {kernel_name}/bin/activate "
f"&& cd {kernel_name} && micropipenv install --dev",
shell=True,
cwd=kernels_path,
)
def _run_virtualenv(python: pathlib.Path, env_dir: pathlib.Path, prompt: str):
import virtualenv
args = [
"--python",
os.fspath(python),
"--prompt",
prompt,
os.fspath(env_dir),
]
try:
virtualenv.cli_run(args)
except Exception as e:
raise EnvironmentCreationError(e)
def create_virtualenv(workdir, git_path, orchestrator=None, templates=[]):
import virtualenv
if hasattr(virtualenv, 'create_environment'):
virtualenv.create_environment(workdir)
else:
virtualenv.cli_run([workdir])
print("[+] Update pip version ...")
subprocess.check_call(
[os.path.join(workdir, 'bin', 'pip'), 'install', '--upgrade', 'pip'],
cwd=workdir,
stderr=subprocess.STDOUT)
print("[+] Install dependencies")
subprocess.check_call([
os.path.join(workdir, 'bin', 'pip'), 'install', '--no-use-pep517',
'-r', 'requirements.txt'
],
cwd=os.path.join(git_path, 'pupy'),
stderr=subprocess.STDOUT)
shell_commands = [
'exec {1}/bin/python -OB {0}/pupy/pupysh.py --workdir {1} "$@"'.format(
shstr(git_path), shstr(workdir))
]
update_commands = [
'cd {}'.format(git_path),
'prev_ref=`git rev-parse HEAD`',
'git pull --recurse-submodules=yes --autostash --rebase',
'if (git diff --name-only $prev_ref HEAD | grep client/ >/dev/null)'
'then',
]
if orchestrator and templates:
for target in templates:
update_commands.extend([
'echo "[+] Rebuilding templates for {}"'.format(target),
'{} start -a build-pupy-{}-{}'.format(
orchestrator, target, get_place_digest(git_path))
])
else:
update_commands.extend(
['echo "[-] You must update templates manually"'])
update_commands.extend(['fi'])
return shell_commands, update_commands
def _setUp(self):
path = self.useFixture(fixtures.TempDir()).path
virtualenv.cli_run([path])
python = os.path.join(path, 'bin', 'python')
command = [python] + self.pip_cmd + ['-U']
if self.modules and len(self.modules) > 0:
command.extend(self.modules)
self.useFixture(base.CapturedSubprocess(
'mkvenv-' + self._reason, command))
self.addCleanup(delattr, self, 'path')
self.addCleanup(delattr, self, 'python')
self.path = path
self.python = python
return path, python
def setup_venv():
print('Creating environment using virtualenv [' + str(DIR_VENV) + ']...')
virtualenv.cli_run([str(DIR_VENV)])
print('Done!')
print('Installing all required python packages via pip from: ' +
str(FILE_REQUIREMENTS))
pip_install = ("source {}/bin/activate && pip install -r " +
str(FILE_REQUIREMENTS)).format(DIR_VENV)
pip_packages = ("source {}/bin/activate && pip freeze").format(DIR_VENV)
pip_install_log = subprocess.getoutput(pip_install)
pip_packages_installed = subprocess.getoutput(pip_packages)
print('Install...' + str(pip_install_log))
print('Installed packages:' + '\n' + pip_packages_installed)
print('Done!')
def commands():
import os
import virtualenv
from pythonfinder import Finder
finder = Finder()
python_exec = finder.find_python_version(this.version.major,
minor=this.version.minor)
if python_exec is None:
stop("No python installed in system.")
venv_dir = python_exec.path.parent / "rez.venv"
args = [
"--python",
str(python_exec.path),
"--no-seed", # no pip, no wheel, no setuptools
str(venv_dir)
]
# compute venv
session = virtualenv.session_via_cli(args)
if not os.path.isdir(str(session.creator.bin_dir)):
print("\tCreating virtual env..")
session = virtualenv.cli_run(args)
env.PATH.prepend(str(session.creator.bin_dir))
def test_custom_interpreter(tmp_path, monkeypatch, capfd, args_joined):
result = virtualenv.cli_run([str(tmp_path), "--activators", ""])
cmd = [sys.executable]
cmd += ["--python={}".format(result.creator.exe)] if args_joined else [
"--python", str(result.creator.exe)
]
monkeypatch.setattr(sys, "argv", cmd)
p.main()
out, _ = capfd.readouterr()
found = {i.split("==")[0] for i in out.splitlines()}
implementation = platform.python_implementation()
if implementation == "CPython":
expected = {"pip", "setuptools", "wheel"}
elif implementation == "PyPy":
expected = {
"cffi", "greenlet", "pip", "readline", "setuptools", "wheel"
}
else:
raise ValueError(implementation)
assert found == expected, out
monkeypatch.setattr(sys, "argv", cmd + ["--graph-output", "something"])
with pytest.raises(SystemExit) as context:
p.main()
out, err = capfd.readouterr()
assert context.value.code == 1
assert not out
assert err == "graphviz functionality is not supported when querying" " non-host python\n"
def test_pick_periodic_update(tmp_path, session_app_data, mocker,
for_py_version):
embed, current = get_embed_wheel("setuptools", "3.4"), get_embed_wheel(
"setuptools", for_py_version)
mocker.patch("virtualenv.seed.wheels.bundle.load_embed_wheel",
return_value=embed)
completed = datetime.now() - timedelta(days=29)
u_log = UpdateLog(
started=datetime.now() - timedelta(days=30),
completed=completed,
versions=[
NewVersion(filename=current.path,
found_date=completed,
release_date=completed)
],
periodic=True,
)
read_dict = mocker.patch(
"virtualenv.app_data.via_disk_folder.JSONStoreDisk.read",
return_value=u_log.to_dict())
result = cli_run([
str(tmp_path), "--activators", "", "--no-periodic-update",
"--no-wheel", "--no-pip"
])
assert read_dict.call_count == 1
installed = list(i.name for i in result.creator.purelib.iterdir()
if i.suffix == ".dist-info")
assert "setuptools-{}.dist-info".format(current.version) in installed
def __setup(venv_dir,
requirements=None,
force_reinstall=False,
no_cache=True,
verbose=False):
"""
This creates (if relevant) and activates a virtual environment. It also allows to define requirements to be
installed in this environment.
:param venv_dir: virtual environment's directory
:param requirements: list of required package OR path of the requirements file to be used
:param force_reinstall: force the reinstallation of required packages
:param no_cache: disable the cache
:param verbose: displayed Pip output while installing packages
"""
__deactivate()
venv_dir = os.path.abspath(venv_dir)
if not os.path.exists(venv_dir):
# fix to issue: https://github.com/pypa/virtualenv/issues/1585
try:
virtualenv.create_environment(venv_dir)
except AttributeError:
virtualenv.cli_run([venv_dir])
# other issue: https://github.com/pypa/pipenv/issues/4518
# for Python3, ImportError when using virtualenv installed with pip ;
# works while installed with apt (python3-virtualenv)
__activate(venv_dir)
if isinstance(requirements, string_types):
with open(requirements) as f:
requirements = [l.strip() for l in f]
if isinstance(requirements, (tuple, list, set)):
args = []
if force_reinstall:
args.append("--force-reinstall")
if no_cache:
args.append("--no-cache-dir")
if verbose:
args.append("-v")
kwargs = {'prefix': venv_dir}
for req in requirements:
pkg = __install(req, *args, **kwargs)
for tl in pkg.top_level:
if hasattr(virtualenv, tl):
raise TopLevelAlreadyExists("{} ({})".format(tl, pkg.name))
m = import_module(tl)
setattr(virtualenv, tl, m)
def prepare_virtualenv(packages=()):
"""
Prepares a virtual environment.
:rtype : VirtualEnvDescription
"""
vroot = get_vroot()
env_key = get_env_key(packages)
vdir = os.path.join(vroot, env_key)
vbin = os.path.join(vdir, ('bin', 'Scripts')[_windows])
vpython = os.path.join(vbin, 'python' + get_exe_suffix())
vpip = os.path.join(vbin, 'pip' + get_exe_suffix())
vpip_install = [vpython, "-m", "pip", "install", "--force-reinstall"]
if (2, 5) <= sys.version_info < (2, 6):
vpip_install.append("--insecure")
venv_description = VirtualEnvDescription(home_dir=vdir, bin_dir=vbin, python=vpython, pip=vpip, packages=packages)
print("Will install now")
print(str(venv_description))
env = get_clean_system_environment()
env['PIP_DOWNLOAD_CACHE'] = os.path.abspath(os.path.join(vroot, "pip-download-cache"))
# Cache environment
done_flag_file = os.path.join(vdir, "done")
if not os.path.exists(done_flag_file):
if os.path.exists(vdir):
shutil.rmtree(vdir)
virtualenv.cli_run([vdir])
# Update for newly created environment
if sys.version_info >= (2, 7):
_call([vpython, "-m", "pip", "install", "--upgrade", "pip", "setuptools"], env=env, cwd=get_teamcity_messages_root())
for package_spec in packages:
_call(vpip_install + [package_spec], env=env)
open(done_flag_file, 'a').close()
# Update for env. that already exists: does not take long, but may save old envs.
if sys.version_info >= (2, 7):
_call([vpython, "-m", "pip", "install", "--upgrade", "pip", "setuptools"], env=env, cwd=get_teamcity_messages_root())
_call([vpython, "setup.py", "install"], env=env, cwd=get_teamcity_messages_root())
return venv_description
def env_path(asdf_version, tmp_path_factory):
"""
Path to the virtualenv where the (old) asdf library is installed.
"""
path = tmp_path_factory.mktemp("asdf-{}-env".format(asdf_version),
numbered=False)
virtualenv.cli_run([str(path)])
assert env_run(
path,
"pip",
"install",
"asdf=={}".format(asdf_version),
capture_output=True), (
"Failed to install asdf version {}".format(asdf_version))
return path
def create_virtualenv(ctx):
tmpdir = mkdtemp(prefix="komodoenv.")
from virtualenv import cli_run
ld_library_path = os.environ.get("LD_LIBRARY_PATH")
os.environ["LD_LIBRARY_PATH"] = str(ctx.srcpath / "root" / "lib")
cli_run([
"--python",
str(ctx.src_python_path),
"--app-data",
tmpdir,
"--always-copy",
str(ctx.dstpath / "root"),
], )
if ld_library_path is None:
del os.environ["LD_LIBRARY_PATH"]
else:
os.environ["LD_LIBRARY_PATH"] = ld_library_path
def create(
self, packages: Sequence[str] | None = None, silent: bool = False
) -> None:
"""
Create the virtual environment in the project.
If packages are specified here, these will be installed
once the environment is created.
Args:
packages (Optional[List[str]], optional): Packages to install immediately
after environment creation. Defaults to None.
silent (bool, optional): Whether to discard or display output.
Defaults to False.
"""
virtualenv.cli_run(args=[str(self.project_path.joinpath(".venv")), "--quiet"])
# Install any specified packages
if packages: # pragma: no cover
self.install(packages=packages, silent=silent)
def use_virtualenv(
location: Path,
contents: Optional[List[str]] = None,
require_empty: bool = False,
):
did_exist = location.is_dir()
assert not require_empty or not did_exist, (
f"Test requires no directory already exists at {location}, "
"maybe try delete it and run again")
# create new virtualenv
virtualenv.cli_run([str(location)])
if contents:
install_into_virtualenv(location, contents)
yield
# Only cleanup if we actually created it to avoid this fixture being a bit dangerous
if not did_exist:
shutil.rmtree(location)
def create_and_install_kernel() -> Tuple[str, str]:
# Create environment
kernel_name = str(uuid.uuid4())
env_name = f"{ENVIRONMENTS_PATH}/{kernel_name}"
# venv.create(env_name, system_site_packages=True, with_pip=True)
virtualenv.cli_run([env_name, "--system-site-packages"])
# Create kernel spec
kernel_spec = {
"argv": [
f"{env_name}/bin/python",
"-m",
"ipykernel_launcher",
"-f",
"{connection_file}",
],
"display_name":
"Python 3",
"language":
"python",
}
kernel_spec_folder = os.path.join(KERNELS_SPECS_PATH, kernel_name)
kernel_spec_file = os.path.join(kernel_spec_folder, "kernel.json")
# Create kernel spec folder
if not os.path.exists(os.path.dirname(kernel_spec_file)):
try:
os.makedirs(os.path.dirname(kernel_spec_file))
except OSError as exc: # Guard against race condition
if exc.errno != errno.EEXIST:
raise
with open(kernel_spec_file, mode="w", encoding="utf-8") as f:
json.dump(kernel_spec, f)
# Install kernel
kernel_spec_manager = KernelSpecManager()
kernel_spec_manager.install_kernel_spec(source_dir=kernel_spec_folder,
kernel_name=kernel_name)
return kernel_name, env_name
def check_virtualenv(self, envdir, context):
pull_sh_path = Path(envdir) / 'bin' / 'pull.sh'
ok = False
if os.path.exists(envdir):
ok = True
# msg = "Update virtualenv in {}"
# return self.batch or click.confirm(msg.format(envdir), default=True)
else:
msg = "Create virtualenv in {}"
if self.batch or self.yes_or_no(msg.format(envdir), default=True):
# create an empty directory and fix permissions
os.makedirs(envdir)
self.check_permissions(envdir)
virtualenv.cli_run([envdir, '--python', 'python3'])
ok = True
if ok:
context.update(envdir=envdir)
if not os.path.exists(pull_sh_path):
self.jinja_write(pull_sh_path, **context)
self.make_file_executable(pull_sh_path)
return ok
def _create_isolated_env_virtualenv(path): # type: (str) -> Tuple[str, str]
"""
On Python 2 we use the virtualenv package to provision a virtual environment.
:param path: The path where to create the isolated build environment
:return: The Python executable and script folder
"""
cmd = [str(path), '--no-setuptools', '--no-wheel', '--activators', '']
result = virtualenv.cli_run(cmd, setup_logging=False)
executable = str(result.creator.exe)
script_dir = str(result.creator.script_dir)
return executable, script_dir
