def _generate_new_files(self):
# create the pyc files, as the build image will be R/O
cmd = [
str(self._creator.exe), "-m", "compileall",
str(self._image_dir)
]
process = Popen(cmd, stdout=PIPE, stderr=PIPE)
process.communicate()
# the root pyc is shared, so we'll not symlink that - but still add the pyc files to the RECORD for close
root_py_cache = self._image_dir / "__pycache__"
new_files = set()
if root_py_cache.exists():
new_files.update(root_py_cache.iterdir())
new_files.add(root_py_cache)
safe_delete(root_py_cache)
core_new_files = super()._generate_new_files()
# remove files that are within the image folder deeper than one level (as these will be not linked directly)
for file in core_new_files:
try:
rel = file.relative_to(self._image_dir)
if len(rel.parts) > 1:
continue
except ValueError:
pass
new_files.add(file)
return new_files
def _generate_new_files(self):
new_files = set()
installer = self._dist_info / "INSTALLER"
installer.write_text("pip\n")
new_files.add(installer)
# inject a no-op root element, as workaround for bug in https://github.com/pypa/pip/issues/7226
marker = self._image_dir / "{}.virtualenv".format(self._dist_info.stem)
marker.write_text("")
new_files.add(marker)
folder = mkdtemp()
try:
to_folder = Path(folder)
rel = os.path.relpath(ensure_text(str(self._creator.script_dir)),
ensure_text(str(self._creator.purelib)))
version_info = self._creator.interpreter.version_info
for name, module in self._console_scripts.items():
new_files.update(
Path(
os.path.normpath(
ensure_text(str(self._image_dir / rel / i.name))))
for i in self._create_console_entry_point(
name, module, to_folder, version_info))
finally:
safe_delete(folder)
return new_files
def _get_seed_wheels(self, creator, base_cache):
with base_cache.lock_for_key("wheels"):
wheels_to = base_cache.path / "wheels"
if wheels_to.exists():
safe_delete(wheels_to)
wheels_to.mkdir(parents=True, exist_ok=True)
name_to_whl, lock, fail = {}, Lock(), {}
def _get(package, version):
wheel_loader = partial(
get_wheels,
creator.interpreter.version_release_str,
wheels_to,
self.extra_search_dir,
{package: version},
self.app_data,
)
failure, result = None, None
# fallback to download in case the exact version is not available
for download in [True] if self.download else [False, True]:
failure = None
try:
result = wheel_loader(download)
if result:
break
except Exception as exception:
failure = exception
if failure:
if isinstance(failure, WheelDownloadFail):
msg = "failed to download {}".format(package)
if version is not None:
msg += " version {}".format(version)
msg += ", pip download exit code {}".format(
failure.exit_code)
output = failure.out + failure.err
if output:
msg += "\n"
msg += output
else:
msg = repr(failure)
logging.error(msg)
with lock:
fail[package] = version
else:
with lock:
name_to_whl.update(result)
package_versions = self.package_version()
threads = list(
Thread(target=_get, args=(pkg, v))
for pkg, v in package_versions.items())
for thread in threads:
thread.start()
for thread in threads:
thread.join()
if fail:
raise RuntimeError(
"seed failed due to failing to download wheels {}".format(
", ".join(fail.keys())))
yield name_to_whl
def _get_seed_wheels(self, creator, base_cache):
with base_cache.lock_for_key("wheels"):
wheels_to = base_cache.path / "wheels"
if wheels_to.exists():
safe_delete(wheels_to)
wheels_to.mkdir(parents=True, exist_ok=True)
name_to_whl, lock = {}, Lock()
def _get(package, version):
result = get_wheels(
creator.interpreter.version_release_str,
wheels_to,
self.extra_search_dir,
self.download,
{package: version},
self.app_data,
)
with lock:
name_to_whl.update(result)
threads = list(Thread(target=_get, args=(pkg, v)) for pkg, v in self.package_version().items())
for thread in threads:
thread.start()
for thread in threads:
thread.join()
yield name_to_whl
def test_populated_read_only_cache_and_symlinked_app_data(
tmp_path, current_fastest, temp_app_data, monkeypatch):
dest = tmp_path / "venv"
cmd = [
"--seeder",
"app-data",
"--creator",
current_fastest,
"--symlink-app-data",
"-vv",
str(dest),
]
assert cli_run(cmd)
subprocess.check_call(
(str(dest.joinpath("bin/python")), "-c", "import pip"))
cached_py_info._CACHE.clear() # necessary to re-trigger py info discovery
safe_delete(dest)
# should succeed with special flag when read-only
with read_only_dir(temp_app_data):
assert cli_run(["--read-only-app-data"] + cmd)
subprocess.check_call(
(str(dest.joinpath("bin/python")), "-c", "import pip"))
def run(self):
if self.dest.exists() and self.clear:
logging.debug("delete %s", self.dest)
safe_delete(self.dest)
self.create()
self.set_pyenv_cfg()
self.setup_ignore_vcs()
def _uninstall_dist(dist):
dist_base = dist.parent
logging.debug("uninstall existing distribution %s from %s", dist.stem,
dist_base)
top_txt = dist / "top_level.txt" # add top level packages at folder level
paths = {
dist.parent / i.strip()
for i in top_txt.read_text().splitlines()
} if top_txt.exists() else set()
paths.add(dist) # add the dist-info folder itself
base_dirs, record = paths.copy(
), dist / "RECORD" # collect entries in record that we did not register yet
for name in (i.split(",")[0] for i in record.read_text().splitlines()
) if record.exists() else ():
path = dist_base / name
if not any(p in base_dirs for p in path.parents
): # only add if not already added as a base dir
paths.add(path)
for path in sorted(paths): # actually remove stuff in a stable order
if path.exists():
if path.is_dir() and not path.is_symlink():
safe_delete(path)
else:
path.unlink()
def install(self, version_info):
self._extracted = True
# sync image
for filename in self._image_dir.iterdir():
into = self._creator.purelib / filename.name
if into.exists():
if into.is_dir() and not into.is_symlink():
safe_delete(into)
else:
into.unlink()
self._sync(filename, into)
# generate console executables
consoles = set()
script_dir = self._creator.script_dir
for name, module in self._console_scripts.items():
consoles.update(self._create_console_entry_point(name, module, script_dir, version_info))
logging.debug("generated console scripts %s", " ".join(i.name for i in consoles))
def test_populated_read_only_cache_and_copied_app_data(tmp_path,
current_fastest,
temp_app_data,
monkeypatch):
dest = tmp_path / "venv"
cmd = [
"--seeder",
"app-data",
"--creator",
current_fastest,
"-vv",
"-p",
"python",
str(dest),
]
assert cli_run(cmd)
cached_py_info._CACHE.clear() # necessary to re-trigger py info discovery
safe_delete(dest)
# should succeed with special flag when read-only
with read_only_dir(temp_app_data):
assert cli_run(["--read-only-app-data"] + cmd)
def reset(self):
logging.debug("reset app data folder %s", self.lock.path)
safe_delete(self.lock.path)
def clear(self):
if self._image_dir.exists():
safe_delete(self._image_dir)
def close(self):
logging.debug("remove temporary app data folder %s", self.lock.path)
safe_delete(self.lock.path)
def clean(self):
logging.debug("clean app data folder %s", self.folder.path)
safe_delete(self.folder.path)
def clear(self):
if self._image_dir.exists():
safe_delete(self._image_dir)
super(SymlinkPipInstall, self).clear()
