def _create_isolated_env_venv(path: str) -> Tuple[str, str]: """ On Python 3 we use the venv package from the standard library. :param path: The path where to create the isolated build environment :return: The Python executable and script folder """ import venv import packaging.version symlinks = _fs_supports_symlink() try: venv.EnvBuilder(with_pip=True, symlinks=symlinks).create(path) except subprocess.CalledProcessError as exc: raise build.FailedProcessError( exc, 'Failed to create venv. Maybe try installing virtualenv.' ) from None executable, script_dir, purelib = _find_executable_and_scripts(path) # Get the version of pip in the environment pip_distribution = next( iter(metadata.distributions(name='pip', path=[purelib ]))) # type: ignore[no-untyped-call] current_pip_version = packaging.version.Version(pip_distribution.version) if platform.system() == 'Darwin' and int( platform.mac_ver()[0].split('.')[0]) >= 11: # macOS 11+ name scheme change requires 20.3. Intel macOS 11.0 can be told to report 10.16 for backwards # compatibility; but that also fixes earlier versions of pip so this is only needed for 11+. is_apple_silicon_python = platform.machine() != 'x86_64' minimum_pip_version = '21.0.1' if is_apple_silicon_python else '20.3.0' else: # PEP-517 and manylinux1 was first implemented in 19.1 minimum_pip_version = '19.1.0' if current_pip_version < packaging.version.Version(minimum_pip_version): _subprocess([ executable, '-m', 'pip', 'install', f'pip>={minimum_pip_version}' ]) # Avoid the setuptools from ensurepip to break the isolation _subprocess([executable, '-m', 'pip', 'uninstall', 'setuptools', '-y']) return executable, script_dir
def test_overwrite_existing(self): """ Test creating environment in an existing directory. """ self.create_contents(self.ENV_SUBDIRS, 'foo') venv.create(self.env_dir) for subdirs in self.ENV_SUBDIRS: fn = os.path.join(self.env_dir, *(subdirs + ('foo', ))) self.assertTrue(os.path.exists(fn)) with open(fn, 'rb') as f: self.assertEqual(f.read(), b'Still here?') builder = venv.EnvBuilder(clear=True) builder.create(self.env_dir) for subdirs in self.ENV_SUBDIRS: fn = os.path.join(self.env_dir, *(subdirs + ('foo', ))) self.assertFalse(os.path.exists(fn))
def test_unicode_in_batch_file(self): """ Test isolation from system site-packages """ rmtree(self.env_dir) env_dir = os.path.join(os.path.realpath(self.env_dir), 'ϼўТλФЙ') builder = venv.EnvBuilder(clear=True) builder.create(env_dir) activate = os.path.join(env_dir, self.bindir, 'activate.bat') envpy = os.path.join(env_dir, self.bindir, self.exe) cmd = [activate, '&', self.exe, '-c', 'print(0)'] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='oem', shell=True) out, err = p.communicate() print(err) self.assertEqual(out.strip(), '0')
def create_venv(): require_venv() folder = get_erdpy_path() ensure_folder(folder) logger.info(f"Creating virtual environment in: {folder}.") import venv builder = venv.EnvBuilder(with_pip=True) builder.clear_directory(folder) builder.create(folder) # Create symlink to "bin/activate" link_path = os.path.join(elrondsdk_path, "erdpy-activate") if os.path.exists(link_path): os.remove(link_path) os.symlink(os.path.join(folder, "bin", "activate"), link_path) logger.info(f"Virtual environment has been created in: {folder}.")
def pip_without_index( logger: Logger, current_platform: TargetPlatform, project_dir: str, wheel_distribution_archive_path: str, requirement_parser: RequirementParser, ) -> VirtualenvPip: pip = VirtualenvPip( logger=logger, target_platform=current_platform, target_directory=os.path.join(project_dir, "pip-without-index-venv"), env_builder=venv.EnvBuilder(with_pip=True), no_index=True, wheel_distribution_path=wheel_distribution_archive_path, requirement_parser=requirement_parser, ) pip.prepare_virtualenv() return pip
def test_create_environment_from_venv_no_pip(tmpdir): std_venv_dir = str(tmpdir / "stdvenv") ve_venv_dir = str(tmpdir / "vevenv") home_dir, lib_dir, inc_dir, bin_dir = virtualenv.path_locations( ve_venv_dir) builder = std_venv.EnvBuilder() ctx = builder.ensure_directories(std_venv_dir) builder.create_configuration(ctx) builder.setup_python(ctx) builder.setup_scripts(ctx) subprocess.check_call( [ctx.env_exe, virtualenv.__file__, "--no-pip", ve_venv_dir]) ve_exe = os.path.join(bin_dir, "python") out = subprocess.check_output( [ve_exe, "-c", "import sys; print(sys.real_prefix)"], universal_newlines=True) # Test against real_prefix if present - we might be running the test from a virtualenv (e.g. tox). assert out.strip() == getattr(sys, "real_prefix", sys.prefix)
def test_symlinking(self): """ Test symlinking works as expected """ for usl in (False, True): builder = venv.EnvBuilder(clear=True, symlinks=usl) builder.create(self.env_dir) fn = self.get_env_file(self.bindir, self.exe) # Don't test when False, because e.g. 'python' is always # symlinked to 'python3.3' in the env, even when symlinking in # general isn't wanted. if usl: if self.cannot_link_exe: # Symlinking is skipped when our executable is already a # special app symlink self.assertFalse(os.path.islink(fn)) else: self.assertTrue(os.path.islink(fn))
def install_virtualenv(parser_args): """ Installs virtual environment """ python_version = '.'.join(str(v) for v in sys.version_info[:2]) sys.stdout.write( f'Installing Python {python_version} virtualenv into {VE_ROOT} \n') if sys.version_info < (3, 7): raise NotImplementedError('Scheduler requires Python 3.7+') # Install virtual environment for Python 3.7+; removing the old one if it exists import venv builder = venv.EnvBuilder(system_site_packages=False, clear=True, symlinks=False, upgrade=False, with_pip=True) builder.create(VE_ROOT) ret_code = subprocess.call( [VE_SCRIPT, PROJECT_ROOT, VE_ROOT, python_version]) sys.exit(ret_code)
def main(): env_builder = venv.EnvBuilder(upgrade=True, with_pip=True) venv_dir = os.path.join(TEST_DIR, "venv") env_builder.create(venv_dir) subprocess.check_call( [os.path.join(venv_dir, "bin", "pip"), "install", "wheel"]) subprocess.check_call([ os.path.join(venv_dir, "bin", "pip"), "install", "requests", "requests-toolbelt", "pytest", "pytest-sugar" ]) with subprocess.Popen([ "kubectl", "--context=dev-gcp", "--namespace=aura", "port-forward", "svc/karapace-test", "8080:80" ]) as pf: try: time.sleep(5) return subprocess.call([os.path.join(venv_dir, "bin", "pytest")]) finally: pf.terminate()
def create_venv(venv_path): """ Creates a virtual environment in the given path. Args: venv_path (String): The path of the virtual environment to create. Returns: :class:`SimpleNamespace`: The context of the virtual environment generated. It contains information about the virtual environment such as python executable, binaries folder, etc. """ # Verify the virtual environment exists before creating it. venv_context = get_venv(venv_path) if venv_context is None: venv_builder = venv.EnvBuilder(with_pip=True) venv_builder.create(venv_path) return venv_builder.ensure_directories(venv_path) else: return venv_context
def test_upgrade(self): """ Test upgrading an existing environment directory. """ # See Issue #21643: the loop needs to run twice to ensure # that everything works on the upgrade (the first run just creates # the venv). for upgrade in (False, True): builder = venv.EnvBuilder(upgrade=upgrade) self.run_with_capture(builder.create, self.env_dir) self.isdir(self.bindir) self.isdir(self.include) self.isdir(*self.lib) fn = self.get_env_file(self.bindir, self.exe) if not os.path.exists(fn): # diagnostics for Windows buildbot failures bd = self.get_env_file(self.bindir) print('Contents of %r:' % bd) print(' %r' % os.listdir(bd)) self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
def test_upgrade_dependencies(self): builder = venv.EnvBuilder() bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' python_exe = os.path.split(sys.executable)[1] with tempfile.TemporaryDirectory() as fake_env_dir: expect_exe = os.path.normcase( os.path.join(fake_env_dir, bin_path, python_exe)) if sys.platform == 'win32': expect_exe = os.path.normcase(os.path.realpath(expect_exe)) def pip_cmd_checker(cmd): cmd[0] = os.path.normcase(cmd[0]) self.assertEqual(cmd, [ expect_exe, '-m', 'pip', 'install', '--upgrade', 'pip', 'setuptools' ]) fake_context = builder.ensure_directories(fake_env_dir) with patch('venv.subprocess.check_call', pip_cmd_checker): builder.upgrade_dependencies(fake_context)
def pip( logger: Logger, current_platform: TargetPlatform, target_directory: Path, wheel_distribution_archive_path: str, requirement_parser: RequirementParser, install_target: Path, ) -> VirtualenvPip: pip = VirtualenvPip( logger=logger, target_platform=current_platform, target_directory=str(install_target), env_builder=venv.EnvBuilder(with_pip=True), no_index=True, wheel_distribution_path=wheel_distribution_archive_path, find_links=[str(target_directory)], requirement_parser=requirement_parser, ) pip.prepare_virtualenv() return pip
def fetch(package: str, destination: str, index_url: str) -> None: venv_dir = os.path.join(destination, venv_subdir) venv_builder = venv.EnvBuilder(with_pip=True) venv_builder.create(venv_dir) write_state({ 'version': 1, 'index_url': index_url, 'client_packages': [{'name': package}], 'entry_points': [], 'index_hashes': {}, }, venv_dir) try: refresh_packages(venv_dir) except UpdateError as e: echo('unable to install:', *e.args) sys.exit(12) refresh_entry_points(destination, venv_dir)
def main(env_dir): """Add a posix 'activate' script to a virtual environment for Cygwin""" env_dir = os.path.abspath(env_dir) builder = venv.EnvBuilder() context = builder.ensure_directories(env_dir) context.env_dir = convert_path(context.env_dir) activate_path = os.path.join(context.bin_path, 'activate') if os.path.exists(activate_path): print('Script already exists: "%s"' % activate_path) return venv_path = os.path.abspath(os.path.dirname(venv.__file__)) template = os.path.join(venv_path, 'scripts', 'posix', 'activate') with open(activate_path, 'w', newline='') as activate_file: with open(template, 'r') as activate_tpl: activate_file.write( builder.replace_variables(activate_tpl.read(), context)) print('Created: "%s"' % activate_path)
def _create_env_venv(self): # type: () -> None if sys.version_info[0] == 2: raise RuntimeError('venv not available on Python 2') import venv venv.EnvBuilder(with_pip=True).create(self.path) env_scripts = self._get_env_path('scripts') if not env_scripts: raise RuntimeError("Couldn't get environment scripts path") exe = 'pypy3' if platform.python_implementation( ) == 'PyPy' else 'python' if os.name == 'nt': exe = '{}.exe'.format(exe) self._executable = os.path.join(self.path, env_scripts, exe) if not os.path.exists(self._executable): raise RuntimeError( 'Virtual environment creation failed, executable {} missing'. format(self._executable))
def test_upgrade_dependencies(self): builder = venv.EnvBuilder() bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' pip_exe = 'pip.exe' if sys.platform == 'win32' else 'pip' with tempfile.TemporaryDirectory() as fake_env_dir: def pip_cmd_checker(cmd): self.assertEqual( cmd, [ os.path.join(fake_env_dir, bin_path, pip_exe), 'install', '-U', 'pip', 'setuptools' ] ) fake_context = builder.ensure_directories(fake_env_dir) with patch('venv.subprocess.check_call', pip_cmd_checker): builder.upgrade_dependencies(fake_context)
def command_init(self) -> bool: """Download any dependencies for this package and perform other set-up functions.""" if self.clean: print("Removing old dependencies...") self.shell(f"rm -rf ./node_modules &>/dev/null") self.shell(f"rm -rf ./usr &>/dev/null") print("Installing Git Hooks...") self.shell(f"cd .git/hooks; ls ../../src/hooks | while read HOOK; do ln -sf ../../src/hooks/$HOOK $HOOK; done") print("Creating Python virtual environment...") builder = venv.EnvBuilder(symlinks=False, with_pip=True) builder.create(self.venv_dir) print("Installing Python dependencies...") self.shell(f"{self.pip} install --upgrade pip") self.shell(f"{self.pip} install -r {self.project_root}/requirements.txt") self.shell(f"{self.pip} freeze | grep -v '^-e' > {self.project_root}/.pip.lock") self.shell(f"rm -rf {self.venv_dir}/lib/python*/site-packages/mccabe*") return True
def _create_isolated_env_venv(path): # type: (str) -> Tuple[str, str] """ On Python 3 we use the venv package from the standard library. :param path: The path where to create the isolated build environment :return: The Python executable and script folder """ import venv venv.EnvBuilder(with_pip=True).create(path) executable, script_dir, purelib = _find_executable_and_scripts(path) # Get the version of pip in the environment pip_distribution = next( iter(metadata.distributions(name='pip', path=[purelib]))) pip_version = packaging.version.Version(pip_distribution.version) needs_pip_upgrade = False if pip_version < packaging.version.Version('19.1'): # PEP-517 and manylinux1 was first implemented in 19.1 needs_pip_upgrade = True elif platform.system() == 'Darwin': # macOS 11+ name scheme change requires 20.3. Intel macOS 11.0 can be told to report 10.16 for backwards # compatibility; but that also fixes earlier versions of pip so this is only needed for 11+. if int(platform.mac_ver()[0].split('.')[0]) >= 11: needs_pip_upgrade = pip_version < packaging.version.Version( '20.3') or (pip_version < packaging.version.Version( '21.0.1') # Apple Silicon macOS requires 21.0.1+ and sys.version_info >= (3, 6) and platform.machine() != 'x86_64') if needs_pip_upgrade: subprocess.check_call( [executable, '-m', 'pip', 'install', '-U', 'pip']) # Avoid the setuptools from ensurepip to break the isolation subprocess.check_call( [executable, '-m', 'pip', 'uninstall', 'setuptools', '-y']) return executable, script_dir
def get_venv(venv_path): """ Gets the context of the virtual environment located in the given path. Args: venv_path (String): The path of the virtual environment to get its context. Returns: :class:`SimpleNamespace`: The context of the virtual environment of the demo. It contains information about the virtual environment such as python executable, binaries folder, etc. `None` if there is not a valid environment in that path. """ venv_builder = venv.EnvBuilder(with_pip=True) # Get the virtual environment context. venv_context = venv_builder.ensure_directories(venv_path) if not os.path.exists(venv_context.env_exe): return None return venv_context
def use_venv( location: Path, contents: Optional[List[str]] = None, require_empty: bool = False, ): did_exist = location.is_dir() assert not require_empty or not did_exist, ( f"Test requires no directory already exists at {location}, " "maybe try delete it and run again") # create new venv venv.EnvBuilder( symlinks=True, with_pip=True, ).create(str(location)) if contents: install_into_virtualenv(location, contents) yield # Only cleanup if we actually created it to avoid this fixture being a bit dangerous if not did_exist: shutil.rmtree(location)
async def upload_src(env_name: str, file: UploadFile = File(...)): # read the zip archive content = await file.read() # extract in temp folder p = Path("envs") / env_name p.mkdir(parents=True) with zipfile.ZipFile(io.BytesIO(content), 'r') as zip: zip.extractall(path=p / "src") # create a venv with pip venv.EnvBuilder(with_pip=True).create(p / "venv") # install requirements req = p / "src" / "requirements.txt" if req.is_file(): # replace Script/pip.exe by bin/pip on linux process = subprocess.Popen( [p / "venv" / "Scripts" / "pip.exe", "install", "-r", req]) process.wait() return f"env {env_name} created"
def main(argv=None): """Recreate the versions.json file.""" if argv is None: argv = sys.argv if '--help' in argv: print(__doc__) return 0 dvm_version = 'docs-versions-menu' if not argv[-1].endswith('versions.py') and not argv[-1].startswith('--'): dvm_version = argv.pop() if dvm_version.endswith('=master'): dvm_version = DVM_REPO + '@master#egg=docs_versions_menu' venvdir = Path(__file__).parent / '.venv' builder = venv.EnvBuilder(with_pip=True) builder.create(venvdir) env = DOCS_VERSIONS_ENV_VARS.copy() env.update(os.environ) # overrides DOCS_VERSIONS_ENV_VARS try: subprocess.run( [Path('.venv') / 'bin' / 'pip', 'install', dvm_version], cwd=venvdir.parent, check=True, ) subprocess.run( [Path('.venv') / 'bin' / 'docs-versions-menu', '--debug'], cwd=venvdir.parent, check=True, env=env, ) return 0 except subprocess.CalledProcessError: return 1 finally: if '--keep-venv' not in argv: if venvdir.is_dir(): shutil.rmtree(venvdir)
def __init__(self, script_path, script_args): self._script_dir = os.path.dirname(script_path) self._script_path = script_path self._script_args = script_args self._requirements_path = os.path.join(self._script_dir, 'requirements.txt') self._venv_dir = os.path.join(self._script_dir, '.venv') self._venv_cache_path = os.path.join(self._venv_dir, '%s.cache' % MODULE_NAME) # Load the paths using virtualenv's logic. if venv: builder = venv.EnvBuilder(with_pip=True) ctx = builder.ensure_directories(self._venv_dir) self._venv_python = os.path.join(ctx.bin_path, ctx.python_exe) self._venv_pip = os.path.join(ctx.bin_path, 'pip') else: home_dir, lib_dir, inc_dir, bin_dir = virtualenv.path_locations(self._venv_dir) self._venv_python = os.path.join(bin_dir, os.path.basename(sys.executable)) self._venv_pip = os.path.join(bin_dir, 'pip') self._in_virtual_env = os.path.normcase(self._venv_python) == os.path.normcase(sys.executable) # Make sure this is the *right* virtual environment! assert self._in_virtual_env == bool(os.environ.get(self._check_environ_name)), \ 'Error detecting virtual environment!'
def make(cls, target: Path) -> "VirtualEnvironment": try: import venv builder = venv.EnvBuilder(clear=True, with_pip=True, symlinks=False) builder.ensure_directories(target) builder.create(target) except ImportError: # fallback to using virtualenv package if venv is not available, eg: ubuntu python_version = f"{sys.version_info.major}.{sys.version_info.minor}" virtualenv_bootstrap_url = ( f"https://bootstrap.pypa.io/virtualenv/{python_version}/virtualenv.pyz" ) with tempfile.TemporaryDirectory( prefix="poetry-installer") as temp_dir: virtualenv_pyz = Path(temp_dir) / "virtualenv.pyz" request = Request(virtualenv_bootstrap_url, headers={"User-Agent": "Python Poetry"}) virtualenv_pyz.write_bytes(urlopen(request).read()) cls.run(sys.executable, virtualenv_pyz, "--clear", "--always-copy", target) # We add a special file so that Poetry can detect # its own virtual environment target.joinpath("poetry_env").touch() env = cls(target) # we do this here to ensure that outdated system default pip does not trigger # older bugs env.pip("install", "--disable-pip-version-check", "--upgrade", "pip") return env
print(MODULE_FOLDER) # Project's folder creation try: os.mkdir(args.project_name) except FileExistsError: sys.exit("A folder with the project's name already exists") # Enter the project's folder os.chdir(args.project_name) # Git init os.system("git init") # Virtual environnement creation new_env = venv.EnvBuilder(with_pip=True) new_env.create(".env") # Download common modules os.system(".env/bin/pip install pylint black") # Create vscode workspace os.mkdir(".vscode") shutil.copyfile( MODULE_FOLDER + "/include/settings.json", EXECUTION_FOLDER + "/" + args.project_name + "/.vscode/settings.json", ) shutil.copyfile( MODULE_FOLDER + "/include/initproject.code-workspace",
def install(inst): """ Install the engine in one of the three possible modes """ if inst is server: # create the openquake user if necessary try: pwd.getpwnam('openquake') except KeyError: subprocess.check_call(['useradd', 'openquake']) print('Created user openquake') # create the database if not os.path.exists(inst.OQDATA): os.makedirs(inst.OQDATA) if inst is server: subprocess.check_call(['chown', 'openquake', inst.OQDATA]) # create the openquake venv if necessary if not os.path.exists(inst.VENV): # create venv venv.EnvBuilder(with_pip=True).create(inst.VENV) print('Created %s' % inst.VENV) # upgrade pip subprocess.check_call(['%s/bin/pip' % inst.VENV, 'install', 'pip', 'wheel', '--upgrade']) # install the requirements req = 'https://raw.githubusercontent.com/gem/oq-engine/master/' \ 'requirements-py%d%d-%s.txt' % (PYVER + PLATFORM[sys.platform]) subprocess.check_call(['%s/bin/pip' % inst.VENV, 'install', '-r', req]) # install the engine if inst is devel: subprocess.check_call(['%s/bin/pip' % inst.VENV, 'install', '-e', '.']) else: subprocess.check_call(['%s/bin/pip' % inst.VENV, 'install', 'openquake.engine', '--upgrade']) # create openquake.cfg if inst is server: if os.path.exists(inst.CFG): print('There is an old file %s; it will not be overwritten, ' 'but consider updating it with\n%s' % (inst.CFG, inst.CONFIG)) else: with open(inst.CFG, 'w') as cfg: cfg.write(inst.CONFIG) print('Created %s' % inst.CFG) # create symlink to oq oqreal = '%s/bin/oq' % inst.VENV if inst is server and not os.path.exists(inst.OQ): os.symlink(oqreal, inst.OQ) if inst is user: print(f'Please add an alias oq={oqreal} in your .bashrc or similar') elif inst is devel: print(f'Please activate the venv with {inst.VENV}/bin/activate') # create systemd services if inst is server and os.path.exists('/lib/systemd/system'): for service in ['dbserver', 'webui']: service_name = 'openquake-%s.service' % service service_path = '/lib/systemd/system/' + service_name if not os.path.exists(service_path): with open(service_path, 'w') as f: srv = SERVICE.format(service=service, OQDATA=inst.OQDATA) f.write(srv) subprocess.check_call(['systemctl', 'enable', service_name]) subprocess.check_call(['systemctl', 'start', service_name]) # download and unzip the demos try: with urlopen(DEMOS) as f: data = f.read() except OSError: msg = 'However, we could not download the demos from %s' % DEMOS else: th, tmp = tempfile.mkstemp(suffix='.zip') with os.fdopen(th, 'wb') as t: t.write(data) zipfile.ZipFile(tmp).extractall(inst.VENV) os.remove(tmp) path = os.path.join(inst.VENV, 'demos', 'hazard', 'AreaSourceClassicalPSHA', 'job.ini') msg = ('You can run a test calculation with the command\n' f'{oqreal} engine --run {path}') print('The engine was installed successfully.\n' + msg)
parser = argparse.ArgumentParser('Install the Alt+Shft notify automatically') parser.add_argument('install-dir', help='Directory to install to.', type=pathlib.Path, default=cwd) args = parser.parse_args() cwd = pathlib.Path(os.path.realpath(getattr(args, 'install-dir'))) sd = pathlib.Path(os.path.realpath(__file__)).parent cwd.mkdir(parents=True, exist_ok=True) print(f'Source Directory: {sd}') print(f'Target Directory: {cwd}') if cwd != sd: distutils.file_util.copy_file(sd.joinpath('main.py'), cwd.joinpath('main.py'), update=1, verbose=1) distutils.file_util.copy_file(sd.joinpath('requirements.txt'), cwd.joinpath('requirements.txt'), update=1, verbose=1) env_builder = venv.EnvBuilder(clear=True, symlinks=False, system_site_packages=False, with_pip=True) env_builder.create(cwd.joinpath('venv')) result = subprocess.run([cwd.joinpath('venv', 'bin', 'python'), '-m' 'pip', 'install', '-r', 'requirements.txt'], stdout=sys.stdout, stderr=sys.stderr) if result.returncode != 0: if result.returncode < 0: sys.exit(1) sys.exit(result.returncode) with open(sd.joinpath('alt-shift-notify.service.in'), mode='r') as file: service_script_template = file.read() service_script = service_script_template.replace('@INSTALL_DIR@', str(cwd)) with open(cwd.joinpath('alt-shift-notify.service'), mode='w') as file: file.write(service_script)
def install(): print("Copying files...") for folder in ["bin", "lib", "share", "build", "dependencies"]: sh("mkdir", "-p", folder) print("Building Python virtual environment...") venv_builder = venv.EnvBuilder(clear=True, with_pip=True) venv_builder.create("./venv") subprocess.run([ "bash", "-c", f""" source ./venv/bin/activate python3 -m pip install --upgrade -r ../dependencies/python/precompile_time.txt python3 -m pip install --upgrade -r ../dependencies/python/compile_time.txt python3 -m pip install --upgrade -r ../dependencies/python/run_time.txt """ ]) print("Installing dependencies...") with chdir("build"): for folder in ["repos", "versions"]: sh("mkdir", "-p", folder) skip_tools = re.compile( os.getenv("SKIP_TOOLS") or "Unmatchable") for tool in tools.values(): if not tool.in_install: continue if skip_tools.match(tool.name) is not None: continue installed_version = "" version_path = f"versions/{tool.name}" try: installed_version = open(version_path).read() except: pass if installed_version == tool.version_string and os.getenv( "FORCE_REINSTALL") != "1": print( f"{tool.version_string} already installed, skipping..." ) continue print(f"Installing {tool.name}...") with chdir("repos"): if not exists(tool.name): sh("git", "clone", tool.repo, tool.name) with chdir(tool.name): sh("git", "fetch") sh("git", "submodule", "update", "--init") sh("git", "checkout", tool.commit) subprocess.run([ "bash", "-c", f"""\ set -e source {install_dir}/venv/bin/activate {tool.build_script} """ ], env=run_env, check=True) with open(version_path, "w") as f: f.write(tool.version_string) path_elements.reverse() with open("env.tcl", "w") as f: f.write( textwrap.dedent(f"""\ set OL_INSTALL_DIR [file dirname [file normalize [info script]]] set ::env(OPENLANE_LOCAL_INSTALL) 1 set ::env(OL_INSTALL_DIR) "$OL_INSTALL_DIR" set ::env(PATH) "{":".join(path_elements)}:$::env(PATH)" set ::env(VIRTUAL_ENV) "$OL_INSTALL_DIR/venv" """)) with open("installed_version", "w") as f: f.write(ol_version)
def make_cross_python(self, context): """ Assemble the cross-python virtual environment """ logger.info("Creating cross-python environment") if self.cross_prefix: context.cross_env_dir = self.cross_prefix else: context.cross_env_dir = os.path.join(context.env_dir, 'cross') clear_cross = self.clear in ('default', 'cross-only', 'both') env = venv.EnvBuilder(system_site_packages=False, clear=self.clear_cross, symlinks=True, upgrade=False, with_pip=False) env.create(context.cross_env_dir) context.cross_bin_path = os.path.join(context.cross_env_dir, 'bin') context.cross_env_exe = os.path.join(context.cross_bin_path, context.python_exe) context.cross_cfg_path = os.path.join(context.cross_env_dir, 'pyvenv.cfg') context.cross_activate = os.path.join(context.cross_bin_path, 'activate') # Remove binaries. We'll run from elsewhere for exe in os.listdir(context.cross_bin_path): if not exe.startswith('activate'): utils.remove_path(os.path.join(context.cross_bin_path, exe)) # Alter pyvenv.cfg with utils.overwrite_file(context.cross_cfg_path) as out: with open(context.cross_cfg_path) as inp: for line in inp: if line.split()[0:2] == ['home', '=']: line = 'home = %s\n' % self.host_project_base out.write(line) # make a script that sets the environment variables and calls Python. # Don't do this in bin/activate, because it's a pain to set/unset # properly (and for csh, fish as well). # Note that env_exe hasn't actually been created yet. sysconfig_name = os.path.basename(self.host_sysconfigdata_file) sysconfig_name, _ = os.path.splitext(sysconfig_name) # If this venv is generated from a cross-python still in its # build directory, rather than installed, then our modifications # prevent build-python from finding its pure-Python libs, which # will cause a crash on startup. Add them back to PYTHONPATH. # Also: 'stdlib' might not be accurate if build-python is in a build # directory. stdlib = os.path.abspath(os.path.dirname(os.__file__)) context.sentinel = random.randint(0, 0xffffffff) extra_envs = list(self.extra_env_vars) # Add sysroot to various environment variables. This doesn't help # compiling, but some packages try to do manual checks for existence # of headers and libraries. This will help them find things. if self.host_sysroot: if os.path.isdir(os.path.join(self.host_sysroot, 'usr')): libs = os.path.join(self.host_sysroot, 'usr', 'lib*') inc = os.path.join(self.host_sysroot, 'usr', 'include') elif os.path.isdir(os.path.join(self.host_sysroot, 'lib')): libs = os.path.join(self.host_sysroot, 'lib*') inc = os.path.join(self.host_sysroot, 'include') else: libs = '' inc = '' libs = glob.glob(libs) if not libs: logger.warning("No libs in sysroot. Does it exist?") else: libs = os.pathsep.join(libs) extra_envs.append(('LIBRARY_PATH', ':=', libs)) if not os.path.isdir(inc): logger.warning("No include/ in sysroot. Does it exist?") else: extra_envs.append(('CPATH', ':=', inc)) utils.install_script('pywrapper.py.tmpl', context.cross_env_exe, locals()) for exe in ('python', 'python3'): exe = os.path.join(context.cross_bin_path, exe) if not os.path.exists(exe): utils.symlink(context.python_exe, exe) macosx_deployment_target = self.macosx_deployment_target # Install patches to environment utils.install_script('site.py.tmpl', os.path.join(context.lib_path, 'site.py'), locals()) self.copy_and_patch_sysconfigdata(context) # cross-python is ready. We will use build-pip to install cross-pip # because 'python -m ensurepip' is likely to get confused and think # that there's nothing to do. if self.with_cross_pip: logger.info("Installing cross-pip") # Make sure we install the same version of pip and setuptools to logger.debug("Installing: %s", context.build_pip_reqs) subprocess.check_output([ context.cross_env_exe, '-m', 'pip', '--disable-pip-version-check', 'install', '--ignore-installed', '--prefix=' + context.cross_env_dir ] + context.build_pip_reqs)