def _generate_new_files(self): # create the pyc files, as the build image will be R/O cmd = [ str(self._creator.exe), "-m", "compileall", str(self._image_dir) ] process = Popen(cmd, stdout=PIPE, stderr=PIPE) process.communicate() # the root pyc is shared, so we'll not symlink that - but still add the pyc files to the RECORD for close root_py_cache = self._image_dir / "__pycache__" new_files = set() if root_py_cache.exists(): new_files.update(root_py_cache.iterdir()) new_files.add(root_py_cache) safe_delete(root_py_cache) core_new_files = super()._generate_new_files() # remove files that are within the image folder deeper than one level (as these will be not linked directly) for file in core_new_files: try: rel = file.relative_to(self._image_dir) if len(rel.parts) > 1: continue except ValueError: pass new_files.add(file) return new_files
def _generate_new_files(self): new_files = set() installer = self._dist_info / "INSTALLER" installer.write_text("pip\n") new_files.add(installer) # inject a no-op root element, as workaround for bug in https://github.com/pypa/pip/issues/7226 marker = self._image_dir / "{}.virtualenv".format(self._dist_info.stem) marker.write_text("") new_files.add(marker) folder = mkdtemp() try: to_folder = Path(folder) rel = os.path.relpath(ensure_text(str(self._creator.script_dir)), ensure_text(str(self._creator.purelib))) version_info = self._creator.interpreter.version_info for name, module in self._console_scripts.items(): new_files.update( Path( os.path.normpath( ensure_text(str(self._image_dir / rel / i.name)))) for i in self._create_console_entry_point( name, module, to_folder, version_info)) finally: safe_delete(folder) return new_files
def _get_seed_wheels(self, creator, base_cache): with base_cache.lock_for_key("wheels"): wheels_to = base_cache.path / "wheels" if wheels_to.exists(): safe_delete(wheels_to) wheels_to.mkdir(parents=True, exist_ok=True) name_to_whl, lock, fail = {}, Lock(), {} def _get(package, version): wheel_loader = partial( get_wheels, creator.interpreter.version_release_str, wheels_to, self.extra_search_dir, {package: version}, self.app_data, ) failure, result = None, None # fallback to download in case the exact version is not available for download in [True] if self.download else [False, True]: failure = None try: result = wheel_loader(download) if result: break except Exception as exception: failure = exception if failure: if isinstance(failure, WheelDownloadFail): msg = "failed to download {}".format(package) if version is not None: msg += " version {}".format(version) msg += ", pip download exit code {}".format( failure.exit_code) output = failure.out + failure.err if output: msg += "\n" msg += output else: msg = repr(failure) logging.error(msg) with lock: fail[package] = version else: with lock: name_to_whl.update(result) package_versions = self.package_version() threads = list( Thread(target=_get, args=(pkg, v)) for pkg, v in package_versions.items()) for thread in threads: thread.start() for thread in threads: thread.join() if fail: raise RuntimeError( "seed failed due to failing to download wheels {}".format( ", ".join(fail.keys()))) yield name_to_whl
def _get_seed_wheels(self, creator, base_cache): with base_cache.lock_for_key("wheels"): wheels_to = base_cache.path / "wheels" if wheels_to.exists(): safe_delete(wheels_to) wheels_to.mkdir(parents=True, exist_ok=True) name_to_whl, lock = {}, Lock() def _get(package, version): result = get_wheels( creator.interpreter.version_release_str, wheels_to, self.extra_search_dir, self.download, {package: version}, self.app_data, ) with lock: name_to_whl.update(result) threads = list(Thread(target=_get, args=(pkg, v)) for pkg, v in self.package_version().items()) for thread in threads: thread.start() for thread in threads: thread.join() yield name_to_whl
def test_populated_read_only_cache_and_symlinked_app_data( tmp_path, current_fastest, temp_app_data, monkeypatch): dest = tmp_path / "venv" cmd = [ "--seeder", "app-data", "--creator", current_fastest, "--symlink-app-data", "-vv", str(dest), ] assert cli_run(cmd) subprocess.check_call( (str(dest.joinpath("bin/python")), "-c", "import pip")) cached_py_info._CACHE.clear() # necessary to re-trigger py info discovery safe_delete(dest) # should succeed with special flag when read-only with read_only_dir(temp_app_data): assert cli_run(["--read-only-app-data"] + cmd) subprocess.check_call( (str(dest.joinpath("bin/python")), "-c", "import pip"))
def run(self): if self.dest.exists() and self.clear: logging.debug("delete %s", self.dest) safe_delete(self.dest) self.create() self.set_pyenv_cfg() self.setup_ignore_vcs()
def _uninstall_dist(dist): dist_base = dist.parent logging.debug("uninstall existing distribution %s from %s", dist.stem, dist_base) top_txt = dist / "top_level.txt" # add top level packages at folder level paths = { dist.parent / i.strip() for i in top_txt.read_text().splitlines() } if top_txt.exists() else set() paths.add(dist) # add the dist-info folder itself base_dirs, record = paths.copy( ), dist / "RECORD" # collect entries in record that we did not register yet for name in (i.split(",")[0] for i in record.read_text().splitlines() ) if record.exists() else (): path = dist_base / name if not any(p in base_dirs for p in path.parents ): # only add if not already added as a base dir paths.add(path) for path in sorted(paths): # actually remove stuff in a stable order if path.exists(): if path.is_dir() and not path.is_symlink(): safe_delete(path) else: path.unlink()
def install(self, version_info): self._extracted = True # sync image for filename in self._image_dir.iterdir(): into = self._creator.purelib / filename.name if into.exists(): if into.is_dir() and not into.is_symlink(): safe_delete(into) else: into.unlink() self._sync(filename, into) # generate console executables consoles = set() script_dir = self._creator.script_dir for name, module in self._console_scripts.items(): consoles.update(self._create_console_entry_point(name, module, script_dir, version_info)) logging.debug("generated console scripts %s", " ".join(i.name for i in consoles))
def test_populated_read_only_cache_and_copied_app_data(tmp_path, current_fastest, temp_app_data, monkeypatch): dest = tmp_path / "venv" cmd = [ "--seeder", "app-data", "--creator", current_fastest, "-vv", "-p", "python", str(dest), ] assert cli_run(cmd) cached_py_info._CACHE.clear() # necessary to re-trigger py info discovery safe_delete(dest) # should succeed with special flag when read-only with read_only_dir(temp_app_data): assert cli_run(["--read-only-app-data"] + cmd)
def reset(self): logging.debug("reset app data folder %s", self.lock.path) safe_delete(self.lock.path)
def clear(self): if self._image_dir.exists(): safe_delete(self._image_dir)
def close(self): logging.debug("remove temporary app data folder %s", self.lock.path) safe_delete(self.lock.path)
def clean(self): logging.debug("clean app data folder %s", self.folder.path) safe_delete(self.folder.path)
def clear(self): if self._image_dir.exists(): safe_delete(self._image_dir) super(SymlinkPipInstall, self).clear()