def validate_dest(cls, raw_value): """No path separator in the path, valid chars and must be write-able""" def non_write_able(dest, value): common = Path(*os.path.commonprefix([value.parts, dest.parts])) raise ArgumentTypeError( "the destination {} is not write-able at {}".format( dest.relative_to(common), common ), ) # the file system must be able to encode # note in newer CPython this is always utf-8 https://www.python.org/dev/peps/pep-0529/ encoding = sys.getfilesystemencoding() refused = OrderedDict() kwargs = {"errors": "ignore"} if encoding != "mbcs" else {} for char in ensure_text(raw_value): try: trip = char.encode(encoding, **kwargs).decode(encoding) if trip == char: continue raise ValueError(trip) except ValueError: refused[char] = None if refused: raise ArgumentTypeError( "the file system codec ({}) cannot handle characters {!r} within {!r}".format( encoding, "".join(refused.keys()), raw_value, ), ) if os.pathsep in raw_value: raise ArgumentTypeError( "destination {!r} must not contain the path separator ({}) as this would break " "the activation scripts".format(raw_value, os.pathsep), ) value = Path(raw_value) if value.exists() and value.is_file(): raise ArgumentTypeError( "the destination {} already exists and is a file".format(value) ) if (3, 3) <= sys.version_info <= (3, 6): # pre 3.6 resolve is always strict, aka must exists, sidestep by using os.path operation dest = Path(os.path.realpath(raw_value)) else: dest = Path( os.path.abspath(str(value)) ).resolve() # on Windows absolute does not imply resolve so use both value = dest while dest: if dest.exists(): if os.access(ensure_text(str(dest)), os.W_OK): break else: non_write_able(dest, value) base, _ = dest.parent, dest.name if base == dest: non_write_able(dest, value) # pragma: no cover dest = base return str(value)
def sources(cls, interpreter): for src in super(CPython2Windows, cls).sources(interpreter): yield src py27_dll = Path(interpreter.system_executable).parent / "python27.dll" if py27_dll.exists(): # this might be global in the Windows folder in which case it's alright to be missing yield PathRefToDest(py27_dll, dest=cls.to_bin) libs = Path(interpreter.system_prefix) / "libs" if libs.exists(): yield PathRefToDest(libs, dest=lambda self, s: self.dest / s.name)
def sources(cls, interpreter): for src in super(PyPy3Posix, cls).sources(interpreter): yield src host_lib = Path(interpreter.system_prefix) / "lib" if host_lib.exists() and host_lib.is_dir(): for path in host_lib.iterdir(): yield PathRefToDest(path, dest=cls.to_lib)
def test_discover_ok(tmp_path, monkeypatch, suffix, impl, version, arch, into, caplog, session_app_data): caplog.set_level(logging.DEBUG) folder = tmp_path / into folder.mkdir(parents=True, exist_ok=True) name = "{}{}".format(impl, version) if arch: name += "-{}".format(arch) name += suffix dest = folder / name os.symlink(CURRENT.executable, str(dest)) pyvenv = Path(CURRENT.executable).parents[1] / "pyvenv.cfg" if pyvenv.exists(): (folder / pyvenv.name).write_text(pyvenv.read_text()) inside_folder = str(tmp_path) base = CURRENT.discover_exe(session_app_data, inside_folder) found = base.executable dest_str = str(dest) if not fs_is_case_sensitive(): found = found.lower() dest_str = dest_str.lower() assert found == dest_str assert len(caplog.messages) >= 1, caplog.text assert "get interpreter info via cmd: " in caplog.text dest.rename(dest.parent / (dest.name + "-1")) CURRENT._cache_exe_discovery.clear() with pytest.raises(RuntimeError): CURRENT.discover_exe(session_app_data, inside_folder)
class IniConfig(object): VIRTUALENV_CONFIG_FILE_ENV_VAR = six.ensure_str("VIRTUALENV_CONFIG_FILE") STATE = {None: "failed to parse", True: "active", False: "missing"} section = "virtualenv" def __init__(self): config_file = os.environ.get(self.VIRTUALENV_CONFIG_FILE_ENV_VAR, None) self.is_env_var = config_file is not None self.config_file = Path( config_file) if config_file is not None else DEFAULT_CONFIG_FILE self._cache = {} self.has_config_file = self.config_file.exists() if self.has_config_file: self.config_file = self.config_file.resolve() self.config_parser = ConfigParser.ConfigParser() try: self._load() self.has_virtualenv_section = self.config_parser.has_section( self.section) except Exception as exception: logging.error("failed to read config file %s because %r", config_file, exception) self.has_config_file = None def _load(self): with self.config_file.open("rt") as file_handler: reader = getattr(self.config_parser, "read_file" if PY3 else "readfp") reader(file_handler) def get(self, key, as_type): cache_key = key, as_type if cache_key in self._cache: return self._cache[cache_key] # noinspection PyBroadException try: source = "file" raw_value = self.config_parser.get(self.section, key.lower()) value = convert(raw_value, as_type, source) result = value, source except Exception: result = None self._cache[cache_key] = result return result def __bool__(self): return bool(self.has_config_file) and bool(self.has_virtualenv_section) @property def epilog(self): msg = "{}config file {} {} (change{} via env var {})" return msg.format( os.linesep, self.config_file, self.STATE[self.has_config_file], "d" if self.is_env_var else "", self.VIRTUALENV_CONFIG_FILE_ENV_VAR, )
def sources(cls, interpreter): for src in super(Python2, cls).sources(interpreter): yield src # install files needed to run site.py for req in cls.modules(): yield PathRefToDest(Path(interpreter.system_stdlib) / "{}.py".format(req), dest=cls.to_stdlib) comp = Path(interpreter.system_stdlib) / "{}.pyc".format(req) if comp.exists(): yield PathRefToDest(comp, dest=cls.to_stdlib)
def sources(cls, interpreter): for src in super(CPython2PosixBase, cls).sources(interpreter): yield src # check if the makefile exists and if so make it available under the virtual environment make_file = Path(interpreter.sysconfig["makefile_filename"]) if make_file.exists() and str(make_file).startswith(interpreter.prefix): under_prefix = make_file.relative_to(Path(interpreter.prefix)) yield PathRefToDest(make_file, dest=lambda self, s: self.dest / under_prefix)
def ensure_file_on_disk(path, app_data): if IS_ZIPAPP: if app_data is None: with TemporaryFile() as temp_file: dest = Path(temp_file.name) extract(path, dest) yield Path(dest) else: base = app_data / "zipapp" / "extract" / __version__ with base.lock_for_key(path.name): dest = base.path / path.name if not dest.exists(): extract(path, dest) yield dest else: yield path
def sources(cls, interpreter): for src in super(PyPy3Posix, cls).sources(interpreter): yield src # Also copy/symlink anything under prefix/lib, which, for "portable" # PyPy builds, includes the tk,tcl runtime and a number of shared # objects. In distro-specific builds or on conda this should be empty # (on PyPy3.8+ it will, like on CPython, hold the stdlib). host_lib = Path(interpreter.system_prefix) / "lib" stdlib = Path(interpreter.system_stdlib) if host_lib.exists() and host_lib.is_dir(): for path in host_lib.iterdir(): if stdlib == path: # For PyPy3.8+ the stdlib lives in lib/pypy3.8 # We need to avoid creating a symlink to it since that # will defeat the purpose of a virtualenv continue yield PathRefToDest(path, dest=cls.to_lib)
def test_py_info_cached_symlink(mocker, tmp_path, session_app_data): spy = mocker.spy(cached_py_info, "_run_subprocess") first_result = PythonInfo.from_exe(sys.executable, session_app_data) assert first_result is not None count = spy.call_count # at least two, one for the venv, one more for the host exp_count = 1 if first_result.executable == sys.executable else 2 assert count >= exp_count # at least two, one for the venv, one more for the host new_exe = tmp_path / "a" new_exe.symlink_to(sys.executable) pyvenv = Path(sys.executable).parents[1] / "pyvenv.cfg" if pyvenv.exists(): (tmp_path / pyvenv.name).write_text(pyvenv.read_text()) new_exe_str = str(new_exe) second_result = PythonInfo.from_exe(new_exe_str, session_app_data) assert second_result.executable == new_exe_str assert spy.call_count == count + 1 # no longer needed the host invocation, but the new symlink is must
def sources(cls, interpreter): for src in super(PyPy2Posix, cls).sources(interpreter): yield src host_lib = Path(interpreter.system_prefix) / "lib" if host_lib.exists(): yield PathRefToDest(host_lib, dest=lambda self, _: self.lib)
def shim(cls, interpreter): shim = Path(interpreter.system_stdlib ) / "venv" / "scripts" / "nt" / "python.exe" if shim.exists(): return shim return None
current = PythonInfo.current_system(session_app_data) core = "somethingVeryCryptic{}".format(".".join( str(i) for i in current.version_info[0:3])) name = "somethingVeryCryptic" if case == "lower": name = name.lower() elif case == "upper": name = name.upper() exe_name = "{}{}{}".format(name, current.version_info.major, ".exe" if sys.platform == "win32" else "") target = tmp_path / current.distutils_install["scripts"] target.mkdir(parents=True) executable = target / exe_name os.symlink(sys.executable, ensure_text(str(executable))) pyvenv_cfg = Path(sys.executable).parents[1] / "pyvenv.cfg" if pyvenv_cfg.exists(): (target / pyvenv_cfg.name).write_bytes(pyvenv_cfg.read_bytes()) new_path = os.pathsep.join( [str(target)] + os.environ.get(str("PATH"), str("")).split(os.pathsep)) monkeypatch.setenv(str("PATH"), new_path) interpreter = get_interpreter(core, []) assert interpreter is not None def test_discovery_via_path_not_found(tmp_path, monkeypatch): monkeypatch.setenv(str("PATH"), str(tmp_path)) interpreter = get_interpreter(uuid4().hex, []) assert interpreter is None
def __init__(self, folder): self._lock = None path = Path(folder) self.path = path.resolve() if path.exists() else path
class Creator(object): """A class that given a python Interpreter creates a virtual environment""" def __init__(self, options, interpreter): """Construct a new virtual environment creator. :param options: the CLI option as parsed from :meth:`add_parser_arguments` :param interpreter: the interpreter to create virtual environment from """ self.interpreter = interpreter self._debug = None self.dest = Path(options.dest) self.clear = options.clear self.no_vcs_ignore = options.no_vcs_ignore self.pyenv_cfg = PyEnvCfg.from_folder(self.dest) self.app_data = options.app_data self.env = options.env def __repr__(self): return ensure_str(self.__unicode__()) def __unicode__(self): return "{}({})".format( self.__class__.__name__, ", ".join("{}={}".format(k, v) for k, v in self._args())) def _args(self): return [ ("dest", ensure_text(str(self.dest))), ("clear", self.clear), ("no_vcs_ignore", self.no_vcs_ignore), ] @classmethod def can_create(cls, interpreter): """Determine if we can create a virtual environment. :param interpreter: the interpreter in question :return: ``None`` if we can't create, any other object otherwise that will be forwarded to \ :meth:`add_parser_arguments` """ return True @classmethod def add_parser_arguments(cls, parser, interpreter, meta, app_data): """Add CLI arguments for the creator. :param parser: the CLI parser :param app_data: the application data folder :param interpreter: the interpreter we're asked to create virtual environment for :param meta: value as returned by :meth:`can_create` """ parser.add_argument( "dest", help="directory to create virtualenv at", type=cls.validate_dest, ) parser.add_argument( "--clear", dest="clear", action="store_true", help= "remove the destination directory if exist before starting (will overwrite files otherwise)", default=False, ) parser.add_argument( "--no-vcs-ignore", dest="no_vcs_ignore", action="store_true", help= "don't create VCS ignore directive in the destination directory", default=False, ) @abstractmethod def create(self): """Perform the virtual environment creation.""" raise NotImplementedError @classmethod def validate_dest(cls, raw_value): """No path separator in the path, valid chars and must be write-able""" def non_write_able(dest, value): common = Path(*os.path.commonprefix([value.parts, dest.parts])) raise ArgumentTypeError( "the destination {} is not write-able at {}".format( dest.relative_to(common), common), ) # the file system must be able to encode # note in newer CPython this is always utf-8 https://www.python.org/dev/peps/pep-0529/ encoding = sys.getfilesystemencoding() refused = OrderedDict() kwargs = {"errors": "ignore"} if encoding != "mbcs" else {} for char in ensure_text(raw_value): try: trip = char.encode(encoding, **kwargs).decode(encoding) if trip == char: continue raise ValueError(trip) except ValueError: refused[char] = None if refused: raise ArgumentTypeError( "the file system codec ({}) cannot handle characters {!r} within {!r}" .format( encoding, "".join(refused.keys()), raw_value, ), ) if os.pathsep in raw_value: raise ArgumentTypeError( "destination {!r} must not contain the path separator ({}) as this would break " "the activation scripts".format(raw_value, os.pathsep), ) value = Path(raw_value) if value.exists() and value.is_file(): raise ArgumentTypeError( "the destination {} already exists and is a file".format( value)) if (3, 3) <= sys.version_info <= (3, 6): # pre 3.6 resolve is always strict, aka must exists, sidestep by using os.path operation dest = Path(os.path.realpath(raw_value)) else: dest = Path(os.path.abspath(str(value))).resolve( ) # on Windows absolute does not imply resolve so use both value = dest while dest: if dest.exists(): if os.access(ensure_text(str(dest)), os.W_OK): break else: non_write_able(dest, value) base, _ = dest.parent, dest.name if base == dest: non_write_able(dest, value) # pragma: no cover dest = base return str(value) def run(self): if self.dest.exists() and self.clear: logging.debug("delete %s", self.dest) safe_delete(self.dest) self.create() self.set_pyenv_cfg() if not self.no_vcs_ignore: self.setup_ignore_vcs() def set_pyenv_cfg(self): self.pyenv_cfg.content = OrderedDict() self.pyenv_cfg["home"] = self.interpreter.system_exec_prefix self.pyenv_cfg["implementation"] = self.interpreter.implementation self.pyenv_cfg["version_info"] = ".".join( str(i) for i in self.interpreter.version_info) self.pyenv_cfg["virtualenv"] = __version__ def setup_ignore_vcs(self): """Generate ignore instructions for version control systems.""" # mark this folder to be ignored by VCS, handle https://www.python.org/dev/peps/pep-0610/#registered-vcs git_ignore = self.dest / ".gitignore" if not git_ignore.exists(): git_ignore.write_text( dedent( """ # created by virtualenv automatically * """, ).lstrip(), ) # Mercurial - does not support the .hgignore file inside a subdirectory directly, but only if included via the # subinclude directive from root, at which point on might as well ignore the directory itself, see # https://www.selenic.com/mercurial/hgignore.5.html for more details # Bazaar - does not support ignore files in sub-directories, only at root level via .bzrignore # Subversion - does not support ignore files, requires direct manipulation with the svn tool @property def debug(self): """ :return: debug information about the virtual environment (only valid after :meth:`create` has run) """ if self._debug is None and self.exe is not None: self._debug = get_env_debug_info(self.exe, self.debug_script(), self.app_data, self.env) return self._debug # noinspection PyMethodMayBeStatic def debug_script(self): return DEBUG_SCRIPT
class Creator(object): """A class that given a python Interpreter creates a virtual environment""" def __init__(self, options, interpreter): """Construct a new virtual environment creator. :param options: the CLI option as parsed from :meth:`add_parser_arguments` :param interpreter: the interpreter to create virtual environment from """ self.interpreter = interpreter self._debug = None self.dest = Path(options.dest) self.clear = options.clear self.pyenv_cfg = PyEnvCfg.from_folder(self.dest) def __repr__(self): return six.ensure_str(self.__unicode__()) def __unicode__(self): return "{}({})".format(self.__class__.__name__, ", ".join("{}={}".format(k, v) for k, v in self._args())) def _args(self): return [ ("dest", six.ensure_text(str(self.dest))), ("clear", self.clear), ] @classmethod def can_create(cls, interpreter): """Determine if we can create a virtual environment. :param interpreter: the interpreter in question :return: ``None`` if we can't create, any other object otherwise that will be forwarded to \ :meth:`add_parser_arguments` """ return True @classmethod def add_parser_arguments(cls, parser, interpreter, meta): """Add CLI arguments for the creator. :param parser: the CLI parser :param interpreter: the interpreter we're asked to create virtual environment for :param meta: value as returned by :meth:`can_create` """ parser.add_argument( "dest", help="directory to create virtualenv at", type=cls.validate_dest, default="venv", nargs="?", ) parser.add_argument( "--clear", dest="clear", action="store_true", help="remove the destination directory if exist before starting (will overwrite files otherwise)", default=False, ) @abstractmethod def create(self): """Perform the virtual environment creation.""" raise NotImplementedError @classmethod def validate_dest(cls, raw_value): """No path separator in the path, valid chars and must be write-able""" def non_write_able(dest, value): common = Path(*os.path.commonprefix([value.parts, dest.parts])) raise ArgumentTypeError( "the destination {} is not write-able at {}".format(dest.relative_to(common), common) ) # the file system must be able to encode # note in newer CPython this is always utf-8 https://www.python.org/dev/peps/pep-0529/ encoding = sys.getfilesystemencoding() refused = OrderedDict() kwargs = {"errors": "ignore"} if encoding != "mbcs" else {} for char in six.ensure_text(raw_value): try: trip = char.encode(encoding, **kwargs).decode(encoding) if trip == char: continue raise ValueError(trip) except ValueError: refused[char] = None if refused: raise ArgumentTypeError( "the file system codec ({}) cannot handle characters {!r} within {!r}".format( encoding, "".join(refused.keys()), raw_value ) ) for char in (i for i in (os.pathsep, os.altsep) if i is not None): if char in raw_value: raise ArgumentTypeError( "destination {!r} must not contain the path separator ({}) as this would break " "the activation scripts".format(raw_value, char) ) value = Path(raw_value) if value.exists() and value.is_file(): raise ArgumentTypeError("the destination {} already exists and is a file".format(value)) if (3, 3) <= sys.version_info <= (3, 6): # pre 3.6 resolve is always strict, aka must exists, sidestep by using os.path operation dest = Path(os.path.realpath(raw_value)) else: dest = value.resolve() value = dest while dest: if dest.exists(): if os.access(six.ensure_text(str(dest)), os.W_OK): break else: non_write_able(dest, value) base, _ = dest.parent, dest.name if base == dest: non_write_able(dest, value) # pragma: no cover dest = base return str(value) def run(self): if self.dest.exists() and self.clear: logging.debug("delete %s", self.dest) def onerror(func, path, exc_info): if not os.access(path, os.W_OK): os.chmod(path, S_IWUSR) func(path) else: raise shutil.rmtree(str(self.dest), ignore_errors=True, onerror=onerror) self.create() self.set_pyenv_cfg() def set_pyenv_cfg(self): self.pyenv_cfg.content = OrderedDict() self.pyenv_cfg["home"] = self.interpreter.system_exec_prefix self.pyenv_cfg["implementation"] = self.interpreter.implementation self.pyenv_cfg["version_info"] = ".".join(str(i) for i in self.interpreter.version_info) self.pyenv_cfg["virtualenv"] = __version__ @property def debug(self): """ :return: debug information about the virtual environment (only valid after :meth:`create` has run) """ if self._debug is None and self.exe is not None: self._debug = get_env_debug_info(self.exe, self.debug_script()) return self._debug # noinspection PyMethodMayBeStatic def debug_script(self): return DEBUG_SCRIPT
class PipInstall(object): def __init__(self, wheel, creator, image_folder): self._wheel = wheel self._creator = creator self._image_dir = image_folder self._extracted = False self.__dist_info = None self._console_entry_points = None @abstractmethod def _sync(self, src, dst): raise NotImplementedError def install(self, version_info): self._extracted = True self._uninstall_previous_version() # sync image for filename in self._image_dir.iterdir(): into = self._creator.purelib / filename.name self._sync(filename, into) # generate console executables consoles = set() script_dir = self._creator.script_dir for name, module in self._console_scripts.items(): consoles.update( self._create_console_entry_point(name, module, script_dir, version_info)) logging.debug("generated console scripts %s", " ".join(i.name for i in consoles)) def build_image(self): # 1. first extract the wheel logging.debug("build install image for %s to %s", self._wheel.name, self._image_dir) with zipfile.ZipFile(str(self._wheel)) as zip_ref: self._shorten_path_if_needed(zip_ref) zip_ref.extractall(str(self._image_dir)) self._extracted = True # 2. now add additional files not present in the distribution new_files = self._generate_new_files() # 3. finally fix the records file self._fix_records(new_files) def _shorten_path_if_needed(self, zip_ref): if os.name == "nt": to_folder = str(self._image_dir) # https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation zip_max_len = max(len(i) for i in zip_ref.namelist()) path_len = zip_max_len + len(to_folder) if path_len > 260: self._image_dir.mkdir( exist_ok=True) # to get a short path must exist from virtualenv.util.path import get_short_path_name to_folder = get_short_path_name(to_folder) self._image_dir = Path(to_folder) def _records_text(self, files): record_data = "\n".join("{},,".format( os.path.relpath(ensure_text(str(rec)), ensure_text(str(self._image_dir)))) for rec in files) return record_data def _generate_new_files(self): new_files = set() installer = self._dist_info / "INSTALLER" installer.write_text("pip\n") new_files.add(installer) # inject a no-op root element, as workaround for bug in https://github.com/pypa/pip/issues/7226 marker = self._image_dir / "{}.virtualenv".format(self._dist_info.stem) marker.write_text("") new_files.add(marker) folder = mkdtemp() try: to_folder = Path(folder) rel = os.path.relpath(ensure_text(str(self._creator.script_dir)), ensure_text(str(self._creator.purelib))) version_info = self._creator.interpreter.version_info for name, module in self._console_scripts.items(): new_files.update( Path( os.path.normpath( ensure_text(str(self._image_dir / rel / i.name)))) for i in self._create_console_entry_point( name, module, to_folder, version_info)) finally: safe_delete(folder) return new_files @property def _dist_info(self): if self._extracted is False: return None # pragma: no cover if self.__dist_info is None: files = [] for filename in self._image_dir.iterdir(): files.append(filename.name) if filename.suffix == ".dist-info": self.__dist_info = filename break else: msg = "no .dist-info at {}, has {}".format( self._image_dir, ", ".join(files)) # pragma: no cover raise RuntimeError(msg) # pragma: no cover return self.__dist_info @abstractmethod def _fix_records(self, extra_record_data): raise NotImplementedError @property def _console_scripts(self): if self._extracted is False: return None # pragma: no cover if self._console_entry_points is None: self._console_entry_points = {} entry_points = self._dist_info / "entry_points.txt" if entry_points.exists(): parser = ConfigParser.ConfigParser() with entry_points.open() as file_handler: reader = getattr(parser, "read_file" if PY3 else "readfp") reader(file_handler) if "console_scripts" in parser.sections(): for name, value in parser.items("console_scripts"): match = re.match(r"(.*?)-?\d\.?\d*", name) if match: name = match.groups(1)[0] self._console_entry_points[name] = value return self._console_entry_points def _create_console_entry_point(self, name, value, to_folder, version_info): result = [] maker = ScriptMakerCustom(to_folder, version_info, self._creator.exe, name) specification = "{} = {}".format(name, value) new_files = maker.make(specification) result.extend(Path(i) for i in new_files) return result def _uninstall_previous_version(self): dist_name = self._dist_info.stem.split("-")[0] in_folders = chain.from_iterable([ i.iterdir() for i in {self._creator.purelib, self._creator.platlib} ]) paths = (p for p in in_folders if p.stem.split("-")[0] == dist_name and p.suffix == ".dist-info" and p.is_dir()) existing_dist = next(paths, None) if existing_dist is not None: self._uninstall_dist(existing_dist) @staticmethod def _uninstall_dist(dist): dist_base = dist.parent logging.debug("uninstall existing distribution %s from %s", dist.stem, dist_base) top_txt = dist / "top_level.txt" # add top level packages at folder level paths = { dist.parent / i.strip() for i in top_txt.read_text().splitlines() } if top_txt.exists() else set() paths.add(dist) # add the dist-info folder itself base_dirs, record = paths.copy( ), dist / "RECORD" # collect entries in record that we did not register yet for name in (i.split(",")[0] for i in record.read_text().splitlines() ) if record.exists() else (): path = dist_base / name if not any(p in base_dirs for p in path.parents ): # only add if not already added as a base dir paths.add(path) for path in sorted(paths): # actually remove stuff in a stable order if path.exists(): if path.is_dir() and not path.is_symlink(): safe_delete(path) else: path.unlink() def clear(self): if self._image_dir.exists(): safe_delete(self._image_dir) def has_image(self): return self._image_dir.exists() and next( self._image_dir.iterdir()) is not None