def _create_isolated_env_venv(path: str) -> Tuple[str, str]:
"""
On Python 3 we use the venv package from the standard library.
:param path: The path where to create the isolated build environment
:return: The Python executable and script folder
"""
import venv
import packaging.version
symlinks = _fs_supports_symlink()
try:
venv.EnvBuilder(with_pip=True, symlinks=symlinks).create(path)
except subprocess.CalledProcessError as exc:
raise build.FailedProcessError(
exc, 'Failed to create venv. Maybe try installing virtualenv.'
) from None
executable, script_dir, purelib = _find_executable_and_scripts(path)
# Get the version of pip in the environment
pip_distribution = next(
iter(metadata.distributions(name='pip',
path=[purelib
]))) # type: ignore[no-untyped-call]
current_pip_version = packaging.version.Version(pip_distribution.version)
if platform.system() == 'Darwin' and int(
platform.mac_ver()[0].split('.')[0]) >= 11:
# macOS 11+ name scheme change requires 20.3. Intel macOS 11.0 can be told to report 10.16 for backwards
# compatibility; but that also fixes earlier versions of pip so this is only needed for 11+.
is_apple_silicon_python = platform.machine() != 'x86_64'
minimum_pip_version = '21.0.1' if is_apple_silicon_python else '20.3.0'
else:
# PEP-517 and manylinux1 was first implemented in 19.1
minimum_pip_version = '19.1.0'
if current_pip_version < packaging.version.Version(minimum_pip_version):
_subprocess([
executable, '-m', 'pip', 'install', f'pip>={minimum_pip_version}'
])
# Avoid the setuptools from ensurepip to break the isolation
_subprocess([executable, '-m', 'pip', 'uninstall', 'setuptools', '-y'])
return executable, script_dir
def test_overwrite_existing(self):
"""
Test creating environment in an existing directory.
"""
self.create_contents(self.ENV_SUBDIRS, 'foo')
venv.create(self.env_dir)
for subdirs in self.ENV_SUBDIRS:
fn = os.path.join(self.env_dir, *(subdirs + ('foo', )))
self.assertTrue(os.path.exists(fn))
with open(fn, 'rb') as f:
self.assertEqual(f.read(), b'Still here?')
builder = venv.EnvBuilder(clear=True)
builder.create(self.env_dir)
for subdirs in self.ENV_SUBDIRS:
fn = os.path.join(self.env_dir, *(subdirs + ('foo', )))
self.assertFalse(os.path.exists(fn))
def test_unicode_in_batch_file(self):
"""
Test isolation from system site-packages
"""
rmtree(self.env_dir)
env_dir = os.path.join(os.path.realpath(self.env_dir), 'ϼўТλФЙ')
builder = venv.EnvBuilder(clear=True)
builder.create(env_dir)
activate = os.path.join(env_dir, self.bindir, 'activate.bat')
envpy = os.path.join(env_dir, self.bindir, self.exe)
cmd = [activate, '&', self.exe, '-c', 'print(0)']
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, encoding='oem',
shell=True)
out, err = p.communicate()
print(err)
self.assertEqual(out.strip(), '0')
def create_venv():
require_venv()
folder = get_erdpy_path()
ensure_folder(folder)
logger.info(f"Creating virtual environment in: {folder}.")
import venv
builder = venv.EnvBuilder(with_pip=True)
builder.clear_directory(folder)
builder.create(folder)
# Create symlink to "bin/activate"
link_path = os.path.join(elrondsdk_path, "erdpy-activate")
if os.path.exists(link_path):
os.remove(link_path)
os.symlink(os.path.join(folder, "bin", "activate"), link_path)
logger.info(f"Virtual environment has been created in: {folder}.")
def pip_without_index(
logger: Logger,
current_platform: TargetPlatform,
project_dir: str,
wheel_distribution_archive_path: str,
requirement_parser: RequirementParser,
) -> VirtualenvPip:
pip = VirtualenvPip(
logger=logger,
target_platform=current_platform,
target_directory=os.path.join(project_dir, "pip-without-index-venv"),
env_builder=venv.EnvBuilder(with_pip=True),
no_index=True,
wheel_distribution_path=wheel_distribution_archive_path,
requirement_parser=requirement_parser,
)
pip.prepare_virtualenv()
return pip
def test_create_environment_from_venv_no_pip(tmpdir):
std_venv_dir = str(tmpdir / "stdvenv")
ve_venv_dir = str(tmpdir / "vevenv")
home_dir, lib_dir, inc_dir, bin_dir = virtualenv.path_locations(
ve_venv_dir)
builder = std_venv.EnvBuilder()
ctx = builder.ensure_directories(std_venv_dir)
builder.create_configuration(ctx)
builder.setup_python(ctx)
builder.setup_scripts(ctx)
subprocess.check_call(
[ctx.env_exe, virtualenv.__file__, "--no-pip", ve_venv_dir])
ve_exe = os.path.join(bin_dir, "python")
out = subprocess.check_output(
[ve_exe, "-c", "import sys; print(sys.real_prefix)"],
universal_newlines=True)
# Test against real_prefix if present - we might be running the test from a virtualenv (e.g. tox).
assert out.strip() == getattr(sys, "real_prefix", sys.prefix)
def test_symlinking(self):
"""
Test symlinking works as expected
"""
for usl in (False, True):
builder = venv.EnvBuilder(clear=True, symlinks=usl)
builder.create(self.env_dir)
fn = self.get_env_file(self.bindir, self.exe)
# Don't test when False, because e.g. 'python' is always
# symlinked to 'python3.3' in the env, even when symlinking in
# general isn't wanted.
if usl:
if self.cannot_link_exe:
# Symlinking is skipped when our executable is already a
# special app symlink
self.assertFalse(os.path.islink(fn))
else:
self.assertTrue(os.path.islink(fn))
def install_virtualenv(parser_args):
""" Installs virtual environment """
python_version = '.'.join(str(v) for v in sys.version_info[:2])
sys.stdout.write(
f'Installing Python {python_version} virtualenv into {VE_ROOT} \n')
if sys.version_info < (3, 7):
raise NotImplementedError('Scheduler requires Python 3.7+')
# Install virtual environment for Python 3.7+; removing the old one if it exists
import venv
builder = venv.EnvBuilder(system_site_packages=False,
clear=True,
symlinks=False,
upgrade=False,
with_pip=True)
builder.create(VE_ROOT)
ret_code = subprocess.call(
[VE_SCRIPT, PROJECT_ROOT, VE_ROOT, python_version])
sys.exit(ret_code)
def main():
env_builder = venv.EnvBuilder(upgrade=True, with_pip=True)
venv_dir = os.path.join(TEST_DIR, "venv")
env_builder.create(venv_dir)
subprocess.check_call(
[os.path.join(venv_dir, "bin", "pip"), "install", "wheel"])
subprocess.check_call([
os.path.join(venv_dir, "bin", "pip"), "install", "requests",
"requests-toolbelt", "pytest", "pytest-sugar"
])
with subprocess.Popen([
"kubectl", "--context=dev-gcp", "--namespace=aura", "port-forward",
"svc/karapace-test", "8080:80"
]) as pf:
try:
time.sleep(5)
return subprocess.call([os.path.join(venv_dir, "bin", "pytest")])
finally:
pf.terminate()
def create_venv(venv_path):
"""
Creates a virtual environment in the given path.
Args:
venv_path (String): The path of the virtual environment to create.
Returns:
:class:`SimpleNamespace`: The context of the virtual environment
generated. It contains information about the virtual environment
such as python executable, binaries folder, etc.
"""
# Verify the virtual environment exists before creating it.
venv_context = get_venv(venv_path)
if venv_context is None:
venv_builder = venv.EnvBuilder(with_pip=True)
venv_builder.create(venv_path)
return venv_builder.ensure_directories(venv_path)
else:
return venv_context
def test_upgrade(self):
"""
Test upgrading an existing environment directory.
"""
# See Issue #21643: the loop needs to run twice to ensure
# that everything works on the upgrade (the first run just creates
# the venv).
for upgrade in (False, True):
builder = venv.EnvBuilder(upgrade=upgrade)
self.run_with_capture(builder.create, self.env_dir)
self.isdir(self.bindir)
self.isdir(self.include)
self.isdir(*self.lib)
fn = self.get_env_file(self.bindir, self.exe)
if not os.path.exists(fn):
# diagnostics for Windows buildbot failures
bd = self.get_env_file(self.bindir)
print('Contents of %r:' % bd)
print(' %r' % os.listdir(bd))
self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn)
def test_upgrade_dependencies(self):
builder = venv.EnvBuilder()
bin_path = 'Scripts' if sys.platform == 'win32' else 'bin'
python_exe = os.path.split(sys.executable)[1]
with tempfile.TemporaryDirectory() as fake_env_dir:
expect_exe = os.path.normcase(
os.path.join(fake_env_dir, bin_path, python_exe))
if sys.platform == 'win32':
expect_exe = os.path.normcase(os.path.realpath(expect_exe))
def pip_cmd_checker(cmd):
cmd[0] = os.path.normcase(cmd[0])
self.assertEqual(cmd, [
expect_exe, '-m', 'pip', 'install', '--upgrade', 'pip',
'setuptools'
])
fake_context = builder.ensure_directories(fake_env_dir)
with patch('venv.subprocess.check_call', pip_cmd_checker):
builder.upgrade_dependencies(fake_context)
def pip(
logger: Logger,
current_platform: TargetPlatform,
target_directory: Path,
wheel_distribution_archive_path: str,
requirement_parser: RequirementParser,
install_target: Path,
) -> VirtualenvPip:
pip = VirtualenvPip(
logger=logger,
target_platform=current_platform,
target_directory=str(install_target),
env_builder=venv.EnvBuilder(with_pip=True),
no_index=True,
wheel_distribution_path=wheel_distribution_archive_path,
find_links=[str(target_directory)],
requirement_parser=requirement_parser,
)
pip.prepare_virtualenv()
return pip
def fetch(package: str, destination: str, index_url: str) -> None:
venv_dir = os.path.join(destination, venv_subdir)
venv_builder = venv.EnvBuilder(with_pip=True)
venv_builder.create(venv_dir)
write_state({
'version': 1,
'index_url': index_url,
'client_packages': [{'name': package}],
'entry_points': [],
'index_hashes': {},
}, venv_dir)
try:
refresh_packages(venv_dir)
except UpdateError as e:
echo('unable to install:', *e.args)
sys.exit(12)
refresh_entry_points(destination, venv_dir)
def main(env_dir):
"""Add a posix 'activate' script to a virtual environment for Cygwin"""
env_dir = os.path.abspath(env_dir)
builder = venv.EnvBuilder()
context = builder.ensure_directories(env_dir)
context.env_dir = convert_path(context.env_dir)
activate_path = os.path.join(context.bin_path, 'activate')
if os.path.exists(activate_path):
print('Script already exists: "%s"' % activate_path)
return
venv_path = os.path.abspath(os.path.dirname(venv.__file__))
template = os.path.join(venv_path, 'scripts', 'posix', 'activate')
with open(activate_path, 'w', newline='') as activate_file:
with open(template, 'r') as activate_tpl:
activate_file.write(
builder.replace_variables(activate_tpl.read(), context))
print('Created: "%s"' % activate_path)
def _create_env_venv(self): # type: () -> None
if sys.version_info[0] == 2:
raise RuntimeError('venv not available on Python 2')
import venv
venv.EnvBuilder(with_pip=True).create(self.path)
env_scripts = self._get_env_path('scripts')
if not env_scripts:
raise RuntimeError("Couldn't get environment scripts path")
exe = 'pypy3' if platform.python_implementation(
) == 'PyPy' else 'python'
if os.name == 'nt':
exe = '{}.exe'.format(exe)
self._executable = os.path.join(self.path, env_scripts, exe)
if not os.path.exists(self._executable):
raise RuntimeError(
'Virtual environment creation failed, executable {} missing'.
format(self._executable))
def test_upgrade_dependencies(self):
builder = venv.EnvBuilder()
bin_path = 'Scripts' if sys.platform == 'win32' else 'bin'
pip_exe = 'pip.exe' if sys.platform == 'win32' else 'pip'
with tempfile.TemporaryDirectory() as fake_env_dir:
def pip_cmd_checker(cmd):
self.assertEqual(
cmd,
[
os.path.join(fake_env_dir, bin_path, pip_exe),
'install',
'-U',
'pip',
'setuptools'
]
)
fake_context = builder.ensure_directories(fake_env_dir)
with patch('venv.subprocess.check_call', pip_cmd_checker):
builder.upgrade_dependencies(fake_context)
def command_init(self) -> bool:
"""Download any dependencies for this package and perform other set-up functions."""
if self.clean:
print("Removing old dependencies...")
self.shell(f"rm -rf ./node_modules &>/dev/null")
self.shell(f"rm -rf ./usr &>/dev/null")
print("Installing Git Hooks...")
self.shell(f"cd .git/hooks; ls ../../src/hooks | while read HOOK; do ln -sf ../../src/hooks/$HOOK $HOOK; done")
print("Creating Python virtual environment...")
builder = venv.EnvBuilder(symlinks=False, with_pip=True)
builder.create(self.venv_dir)
print("Installing Python dependencies...")
self.shell(f"{self.pip} install --upgrade pip")
self.shell(f"{self.pip} install -r {self.project_root}/requirements.txt")
self.shell(f"{self.pip} freeze | grep -v '^-e' > {self.project_root}/.pip.lock")
self.shell(f"rm -rf {self.venv_dir}/lib/python*/site-packages/mccabe*")
return True
def _create_isolated_env_venv(path): # type: (str) -> Tuple[str, str]
"""
On Python 3 we use the venv package from the standard library.
:param path: The path where to create the isolated build environment
:return: The Python executable and script folder
"""
import venv
venv.EnvBuilder(with_pip=True).create(path)
executable, script_dir, purelib = _find_executable_and_scripts(path)
# Get the version of pip in the environment
pip_distribution = next(
iter(metadata.distributions(name='pip', path=[purelib])))
pip_version = packaging.version.Version(pip_distribution.version)
needs_pip_upgrade = False
if pip_version < packaging.version.Version('19.1'):
# PEP-517 and manylinux1 was first implemented in 19.1
needs_pip_upgrade = True
elif platform.system() == 'Darwin':
# macOS 11+ name scheme change requires 20.3. Intel macOS 11.0 can be told to report 10.16 for backwards
# compatibility; but that also fixes earlier versions of pip so this is only needed for 11+.
if int(platform.mac_ver()[0].split('.')[0]) >= 11:
needs_pip_upgrade = pip_version < packaging.version.Version(
'20.3') or (pip_version < packaging.version.Version(
'21.0.1') # Apple Silicon macOS requires 21.0.1+
and sys.version_info >=
(3, 6) and platform.machine() != 'x86_64')
if needs_pip_upgrade:
subprocess.check_call(
[executable, '-m', 'pip', 'install', '-U', 'pip'])
# Avoid the setuptools from ensurepip to break the isolation
subprocess.check_call(
[executable, '-m', 'pip', 'uninstall', 'setuptools', '-y'])
return executable, script_dir
def get_venv(venv_path):
"""
Gets the context of the virtual environment located in the given path.
Args:
venv_path (String): The path of the virtual environment to get its
context.
Returns:
:class:`SimpleNamespace`: The context of the virtual environment
of the demo. It contains information about the virtual environment
such as python executable, binaries folder, etc. `None` if there
is not a valid environment in that path.
"""
venv_builder = venv.EnvBuilder(with_pip=True)
# Get the virtual environment context.
venv_context = venv_builder.ensure_directories(venv_path)
if not os.path.exists(venv_context.env_exe):
return None
return venv_context
def use_venv(
location: Path,
contents: Optional[List[str]] = None,
require_empty: bool = False,
):
did_exist = location.is_dir()
assert not require_empty or not did_exist, (
f"Test requires no directory already exists at {location}, "
"maybe try delete it and run again")
# create new venv
venv.EnvBuilder(
symlinks=True,
with_pip=True,
).create(str(location))
if contents:
install_into_virtualenv(location, contents)
yield
# Only cleanup if we actually created it to avoid this fixture being a bit dangerous
if not did_exist:
shutil.rmtree(location)
async def upload_src(env_name: str, file: UploadFile = File(...)):
# read the zip archive
content = await file.read()
# extract in temp folder
p = Path("envs") / env_name
p.mkdir(parents=True)
with zipfile.ZipFile(io.BytesIO(content), 'r') as zip:
zip.extractall(path=p / "src")
# create a venv with pip
venv.EnvBuilder(with_pip=True).create(p / "venv")
# install requirements
req = p / "src" / "requirements.txt"
if req.is_file():
# replace Script/pip.exe by bin/pip on linux
process = subprocess.Popen(
[p / "venv" / "Scripts" / "pip.exe", "install", "-r", req])
process.wait()
return f"env {env_name} created"
def main(argv=None):
"""Recreate the versions.json file."""
if argv is None:
argv = sys.argv
if '--help' in argv:
print(__doc__)
return 0
dvm_version = 'docs-versions-menu'
if not argv[-1].endswith('versions.py') and not argv[-1].startswith('--'):
dvm_version = argv.pop()
if dvm_version.endswith('=master'):
dvm_version = DVM_REPO + '@master#egg=docs_versions_menu'
venvdir = Path(__file__).parent / '.venv'
builder = venv.EnvBuilder(with_pip=True)
builder.create(venvdir)
env = DOCS_VERSIONS_ENV_VARS.copy()
env.update(os.environ) # overrides DOCS_VERSIONS_ENV_VARS
try:
subprocess.run(
[Path('.venv') / 'bin' / 'pip', 'install', dvm_version],
cwd=venvdir.parent,
check=True,
)
subprocess.run(
[Path('.venv') / 'bin' / 'docs-versions-menu', '--debug'],
cwd=venvdir.parent,
check=True,
env=env,
)
return 0
except subprocess.CalledProcessError:
return 1
finally:
if '--keep-venv' not in argv:
if venvdir.is_dir():
shutil.rmtree(venvdir)
def __init__(self, script_path, script_args):
self._script_dir = os.path.dirname(script_path)
self._script_path = script_path
self._script_args = script_args
self._requirements_path = os.path.join(self._script_dir, 'requirements.txt')
self._venv_dir = os.path.join(self._script_dir, '.venv')
self._venv_cache_path = os.path.join(self._venv_dir, '%s.cache' % MODULE_NAME)
# Load the paths using virtualenv's logic.
if venv:
builder = venv.EnvBuilder(with_pip=True)
ctx = builder.ensure_directories(self._venv_dir)
self._venv_python = os.path.join(ctx.bin_path, ctx.python_exe)
self._venv_pip = os.path.join(ctx.bin_path, 'pip')
else:
home_dir, lib_dir, inc_dir, bin_dir = virtualenv.path_locations(self._venv_dir)
self._venv_python = os.path.join(bin_dir, os.path.basename(sys.executable))
self._venv_pip = os.path.join(bin_dir, 'pip')
self._in_virtual_env = os.path.normcase(self._venv_python) == os.path.normcase(sys.executable)
# Make sure this is the *right* virtual environment!
assert self._in_virtual_env == bool(os.environ.get(self._check_environ_name)), \
'Error detecting virtual environment!'
def make(cls, target: Path) -> "VirtualEnvironment":
try:
import venv
builder = venv.EnvBuilder(clear=True,
with_pip=True,
symlinks=False)
builder.ensure_directories(target)
builder.create(target)
except ImportError:
# fallback to using virtualenv package if venv is not available, eg: ubuntu
python_version = f"{sys.version_info.major}.{sys.version_info.minor}"
virtualenv_bootstrap_url = (
f"https://bootstrap.pypa.io/virtualenv/{python_version}/virtualenv.pyz"
)
with tempfile.TemporaryDirectory(
prefix="poetry-installer") as temp_dir:
virtualenv_pyz = Path(temp_dir) / "virtualenv.pyz"
request = Request(virtualenv_bootstrap_url,
headers={"User-Agent": "Python Poetry"})
virtualenv_pyz.write_bytes(urlopen(request).read())
cls.run(sys.executable, virtualenv_pyz, "--clear",
"--always-copy", target)
# We add a special file so that Poetry can detect
# its own virtual environment
target.joinpath("poetry_env").touch()
env = cls(target)
# we do this here to ensure that outdated system default pip does not trigger
# older bugs
env.pip("install", "--disable-pip-version-check", "--upgrade", "pip")
return env
print(MODULE_FOLDER)
# Project's folder creation
try:
os.mkdir(args.project_name)
except FileExistsError:
sys.exit("A folder with the project's name already exists")
# Enter the project's folder
os.chdir(args.project_name)
# Git init
os.system("git init")
# Virtual environnement creation
new_env = venv.EnvBuilder(with_pip=True)
new_env.create(".env")
# Download common modules
os.system(".env/bin/pip install pylint black")
# Create vscode workspace
os.mkdir(".vscode")
shutil.copyfile(
MODULE_FOLDER + "/include/settings.json",
EXECUTION_FOLDER + "/" + args.project_name + "/.vscode/settings.json",
)
shutil.copyfile(
MODULE_FOLDER + "/include/initproject.code-workspace",
def install(inst):
"""
Install the engine in one of the three possible modes
"""
if inst is server:
# create the openquake user if necessary
try:
pwd.getpwnam('openquake')
except KeyError:
subprocess.check_call(['useradd', 'openquake'])
print('Created user openquake')
# create the database
if not os.path.exists(inst.OQDATA):
os.makedirs(inst.OQDATA)
if inst is server:
subprocess.check_call(['chown', 'openquake', inst.OQDATA])
# create the openquake venv if necessary
if not os.path.exists(inst.VENV):
# create venv
venv.EnvBuilder(with_pip=True).create(inst.VENV)
print('Created %s' % inst.VENV)
# upgrade pip
subprocess.check_call(['%s/bin/pip' % inst.VENV, 'install', 'pip', 'wheel',
'--upgrade'])
# install the requirements
req = 'https://raw.githubusercontent.com/gem/oq-engine/master/' \
'requirements-py%d%d-%s.txt' % (PYVER + PLATFORM[sys.platform])
subprocess.check_call(['%s/bin/pip' % inst.VENV, 'install', '-r', req])
# install the engine
if inst is devel:
subprocess.check_call(['%s/bin/pip' % inst.VENV, 'install',
'-e', '.'])
else:
subprocess.check_call(['%s/bin/pip' % inst.VENV, 'install',
'openquake.engine', '--upgrade'])
# create openquake.cfg
if inst is server:
if os.path.exists(inst.CFG):
print('There is an old file %s; it will not be overwritten, '
'but consider updating it with\n%s' %
(inst.CFG, inst.CONFIG))
else:
with open(inst.CFG, 'w') as cfg:
cfg.write(inst.CONFIG)
print('Created %s' % inst.CFG)
# create symlink to oq
oqreal = '%s/bin/oq' % inst.VENV
if inst is server and not os.path.exists(inst.OQ):
os.symlink(oqreal, inst.OQ)
if inst is user:
print(f'Please add an alias oq={oqreal} in your .bashrc or similar')
elif inst is devel:
print(f'Please activate the venv with {inst.VENV}/bin/activate')
# create systemd services
if inst is server and os.path.exists('/lib/systemd/system'):
for service in ['dbserver', 'webui']:
service_name = 'openquake-%s.service' % service
service_path = '/lib/systemd/system/' + service_name
if not os.path.exists(service_path):
with open(service_path, 'w') as f:
srv = SERVICE.format(service=service, OQDATA=inst.OQDATA)
f.write(srv)
subprocess.check_call(['systemctl', 'enable', service_name])
subprocess.check_call(['systemctl', 'start', service_name])
# download and unzip the demos
try:
with urlopen(DEMOS) as f:
data = f.read()
except OSError:
msg = 'However, we could not download the demos from %s' % DEMOS
else:
th, tmp = tempfile.mkstemp(suffix='.zip')
with os.fdopen(th, 'wb') as t:
t.write(data)
zipfile.ZipFile(tmp).extractall(inst.VENV)
os.remove(tmp)
path = os.path.join(inst.VENV, 'demos', 'hazard',
'AreaSourceClassicalPSHA', 'job.ini')
msg = ('You can run a test calculation with the command\n'
f'{oqreal} engine --run {path}')
print('The engine was installed successfully.\n' + msg)
parser = argparse.ArgumentParser('Install the Alt+Shft notify automatically')
parser.add_argument('install-dir', help='Directory to install to.', type=pathlib.Path, default=cwd)
args = parser.parse_args()
cwd = pathlib.Path(os.path.realpath(getattr(args, 'install-dir')))
sd = pathlib.Path(os.path.realpath(__file__)).parent
cwd.mkdir(parents=True, exist_ok=True)
print(f'Source Directory: {sd}')
print(f'Target Directory: {cwd}')
if cwd != sd:
distutils.file_util.copy_file(sd.joinpath('main.py'), cwd.joinpath('main.py'), update=1, verbose=1)
distutils.file_util.copy_file(sd.joinpath('requirements.txt'), cwd.joinpath('requirements.txt'), update=1, verbose=1)
env_builder = venv.EnvBuilder(clear=True, symlinks=False, system_site_packages=False, with_pip=True)
env_builder.create(cwd.joinpath('venv'))
result = subprocess.run([cwd.joinpath('venv', 'bin', 'python'), '-m' 'pip', 'install', '-r', 'requirements.txt'], stdout=sys.stdout, stderr=sys.stderr)
if result.returncode != 0:
if result.returncode < 0:
sys.exit(1)
sys.exit(result.returncode)
with open(sd.joinpath('alt-shift-notify.service.in'), mode='r') as file:
service_script_template = file.read()
service_script = service_script_template.replace('@INSTALL_DIR@', str(cwd))
with open(cwd.joinpath('alt-shift-notify.service'), mode='w') as file:
file.write(service_script)
def install():
print("Copying files...")
for folder in ["bin", "lib", "share", "build", "dependencies"]:
sh("mkdir", "-p", folder)
print("Building Python virtual environment...")
venv_builder = venv.EnvBuilder(clear=True, with_pip=True)
venv_builder.create("./venv")
subprocess.run([
"bash", "-c", f"""
source ./venv/bin/activate
python3 -m pip install --upgrade -r ../dependencies/python/precompile_time.txt
python3 -m pip install --upgrade -r ../dependencies/python/compile_time.txt
python3 -m pip install --upgrade -r ../dependencies/python/run_time.txt
"""
])
print("Installing dependencies...")
with chdir("build"):
for folder in ["repos", "versions"]:
sh("mkdir", "-p", folder)
skip_tools = re.compile(
os.getenv("SKIP_TOOLS") or "Unmatchable")
for tool in tools.values():
if not tool.in_install:
continue
if skip_tools.match(tool.name) is not None:
continue
installed_version = ""
version_path = f"versions/{tool.name}"
try:
installed_version = open(version_path).read()
except:
pass
if installed_version == tool.version_string and os.getenv(
"FORCE_REINSTALL") != "1":
print(
f"{tool.version_string} already installed, skipping..."
)
continue
print(f"Installing {tool.name}...")
with chdir("repos"):
if not exists(tool.name):
sh("git", "clone", tool.repo, tool.name)
with chdir(tool.name):
sh("git", "fetch")
sh("git", "submodule", "update", "--init")
sh("git", "checkout", tool.commit)
subprocess.run([
"bash", "-c", f"""\
set -e
source {install_dir}/venv/bin/activate
{tool.build_script}
"""
],
env=run_env,
check=True)
with open(version_path, "w") as f:
f.write(tool.version_string)
path_elements.reverse()
with open("env.tcl", "w") as f:
f.write(
textwrap.dedent(f"""\
set OL_INSTALL_DIR [file dirname [file normalize [info script]]]
set ::env(OPENLANE_LOCAL_INSTALL) 1
set ::env(OL_INSTALL_DIR) "$OL_INSTALL_DIR"
set ::env(PATH) "{":".join(path_elements)}:$::env(PATH)"
set ::env(VIRTUAL_ENV) "$OL_INSTALL_DIR/venv"
"""))
with open("installed_version", "w") as f:
f.write(ol_version)
def make_cross_python(self, context):
"""
Assemble the cross-python virtual environment
"""
logger.info("Creating cross-python environment")
if self.cross_prefix:
context.cross_env_dir = self.cross_prefix
else:
context.cross_env_dir = os.path.join(context.env_dir, 'cross')
clear_cross = self.clear in ('default', 'cross-only', 'both')
env = venv.EnvBuilder(system_site_packages=False,
clear=self.clear_cross,
symlinks=True,
upgrade=False,
with_pip=False)
env.create(context.cross_env_dir)
context.cross_bin_path = os.path.join(context.cross_env_dir, 'bin')
context.cross_env_exe = os.path.join(context.cross_bin_path,
context.python_exe)
context.cross_cfg_path = os.path.join(context.cross_env_dir,
'pyvenv.cfg')
context.cross_activate = os.path.join(context.cross_bin_path,
'activate')
# Remove binaries. We'll run from elsewhere
for exe in os.listdir(context.cross_bin_path):
if not exe.startswith('activate'):
utils.remove_path(os.path.join(context.cross_bin_path, exe))
# Alter pyvenv.cfg
with utils.overwrite_file(context.cross_cfg_path) as out:
with open(context.cross_cfg_path) as inp:
for line in inp:
if line.split()[0:2] == ['home', '=']:
line = 'home = %s\n' % self.host_project_base
out.write(line)
# make a script that sets the environment variables and calls Python.
# Don't do this in bin/activate, because it's a pain to set/unset
# properly (and for csh, fish as well).
# Note that env_exe hasn't actually been created yet.
sysconfig_name = os.path.basename(self.host_sysconfigdata_file)
sysconfig_name, _ = os.path.splitext(sysconfig_name)
# If this venv is generated from a cross-python still in its
# build directory, rather than installed, then our modifications
# prevent build-python from finding its pure-Python libs, which
# will cause a crash on startup. Add them back to PYTHONPATH.
# Also: 'stdlib' might not be accurate if build-python is in a build
# directory.
stdlib = os.path.abspath(os.path.dirname(os.__file__))
context.sentinel = random.randint(0, 0xffffffff)
extra_envs = list(self.extra_env_vars)
# Add sysroot to various environment variables. This doesn't help
# compiling, but some packages try to do manual checks for existence
# of headers and libraries. This will help them find things.
if self.host_sysroot:
if os.path.isdir(os.path.join(self.host_sysroot, 'usr')):
libs = os.path.join(self.host_sysroot, 'usr', 'lib*')
inc = os.path.join(self.host_sysroot, 'usr', 'include')
elif os.path.isdir(os.path.join(self.host_sysroot, 'lib')):
libs = os.path.join(self.host_sysroot, 'lib*')
inc = os.path.join(self.host_sysroot, 'include')
else:
libs = ''
inc = ''
libs = glob.glob(libs)
if not libs:
logger.warning("No libs in sysroot. Does it exist?")
else:
libs = os.pathsep.join(libs)
extra_envs.append(('LIBRARY_PATH', ':=', libs))
if not os.path.isdir(inc):
logger.warning("No include/ in sysroot. Does it exist?")
else:
extra_envs.append(('CPATH', ':=', inc))
utils.install_script('pywrapper.py.tmpl', context.cross_env_exe,
locals())
for exe in ('python', 'python3'):
exe = os.path.join(context.cross_bin_path, exe)
if not os.path.exists(exe):
utils.symlink(context.python_exe, exe)
macosx_deployment_target = self.macosx_deployment_target
# Install patches to environment
utils.install_script('site.py.tmpl',
os.path.join(context.lib_path, 'site.py'),
locals())
self.copy_and_patch_sysconfigdata(context)
# cross-python is ready. We will use build-pip to install cross-pip
# because 'python -m ensurepip' is likely to get confused and think
# that there's nothing to do.
if self.with_cross_pip:
logger.info("Installing cross-pip")
# Make sure we install the same version of pip and setuptools to
logger.debug("Installing: %s", context.build_pip_reqs)
subprocess.check_output([
context.cross_env_exe, '-m', 'pip',
'--disable-pip-version-check', 'install', '--ignore-installed',
'--prefix=' + context.cross_env_dir
] + context.build_pip_reqs)
