def _get_file_stash(self, path): """Stashes a file. If no root has been provided, one will be created for the directory in the user's temp directory.""" path = os.path.normcase(path) head, old_head = os.path.dirname(path), None save_dir = None while head != old_head: try: save_dir = self._save_dirs[head] break except KeyError: pass head, old_head = os.path.dirname(head), head else: # Did not find any suitable root head = os.path.dirname(path) save_dir = TempDirectory(kind='uninstall') save_dir.create() self._save_dirs[head] = save_dir relpath = os.path.relpath(path, head) if relpath and relpath != os.path.curdir: return os.path.join(save_dir.path, relpath) return save_dir.path
class EphemWheelCache(SimpleWheelCache): """A SimpleWheelCache that creates it's own temporary cache directory """ def __init__(self, format_control): self._temp_dir = TempDirectory(kind="ephem-wheel-cache") self._temp_dir.create() super(EphemWheelCache, self).__init__(self._temp_dir.path, format_control) def cleanup(self): self._temp_dir.cleanup()
class EphemWheelCache(SimpleWheelCache): """A SimpleWheelCache that creates it's own temporary cache directory """ def __init__(self, format_control): self._temp_dir = TempDirectory(kind="ephem-wheel-cache") self._temp_dir.create() super(EphemWheelCache, self).__init__( self._temp_dir.path, format_control ) def cleanup(self): self._temp_dir.cleanup()
class InstallRequirement(object): """ Represents something that may be installed later on, may have information about where to fetch the relavant requirement and also contains logic for installing the said requirement. """ def __init__( self, req, # type: Optional[Requirement] comes_from, # type: Optional[Union[str, InstallRequirement]] source_dir=None, # type: Optional[str] editable=False, # type: bool link=None, # type: Optional[Link] update=True, # type: bool markers=None, # type: Optional[Marker] use_pep517=None, # type: Optional[bool] isolated=False, # type: bool options=None, # type: Optional[Dict[str, Any]] wheel_cache=None, # type: Optional[WheelCache] constraint=False, # type: bool extras=() # type: Iterable[str] ): # type: (...) -> None assert req is None or isinstance(req, Requirement), req self.req = req self.comes_from = comes_from self.constraint = constraint if source_dir is not None: self.source_dir = os.path.normpath(os.path.abspath(source_dir)) else: self.source_dir = None self.editable = editable self._wheel_cache = wheel_cache if link is None and req and req.url: # PEP 508 URL requirement link = Link(req.url) self.link = self.original_link = link if extras: self.extras = extras elif req: self.extras = { pkg_resources.safe_extra(extra) for extra in req.extras } else: self.extras = set() if markers is None and req: markers = req.marker self.markers = markers self._egg_info_path = None # type: Optional[str] # This holds the pkg_resources.Distribution object if this requirement # is already available: self.satisfied_by = None # This hold the pkg_resources.Distribution object if this requirement # conflicts with another installed distribution: self.conflicts_with = None # Temporary build location self._temp_build_dir = TempDirectory(kind="req-build") # Used to store the global directory where the _temp_build_dir should # have been created. Cf _correct_build_location method. self._ideal_build_dir = None # type: Optional[str] # True if the editable should be updated: self.update = update # Set to True after successful installation self.install_succeeded = None # type: Optional[bool] # UninstallPathSet of uninstalled distribution (for possible rollback) self.uninstalled_pathset = None self.options = options if options else {} # Set to True after successful preparation of this requirement self.prepared = False self.is_direct = False self.isolated = isolated self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment # For PEP 517, the directory where we request the project metadata # gets stored. We need this to pass to build_wheel, so the backend # can ensure that the wheel matches the metadata (see the PEP for # details). self.metadata_directory = None # type: Optional[str] # The static build requirements (from pyproject.toml) self.pyproject_requires = None # type: Optional[List[str]] # Build requirements that we will check are available self.requirements_to_check = [] # type: List[str] # The PEP 517 backend we should use to build the project self.pep517_backend = None # type: Optional[Pep517HookCaller] # Are we using PEP 517 for this requirement? # After pyproject.toml has been loaded, the only valid values are True # and False. Before loading, None is valid (meaning "use the default"). # Setting an explicit value before loading pyproject.toml is supported, # but after loading this flag should be treated as read only. self.use_pep517 = use_pep517 def __str__(self): if self.req: s = str(self.req) if self.link: s += ' from %s' % redact_password_from_url(self.link.url) elif self.link: s = redact_password_from_url(self.link.url) else: s = '<InstallRequirement>' if self.satisfied_by is not None: s += ' in %s' % display_path(self.satisfied_by.location) if self.comes_from: if isinstance(self.comes_from, six.string_types): comes_from = self.comes_from else: comes_from = self.comes_from.from_path() if comes_from: s += ' (from %s)' % comes_from return s def __repr__(self): return '<%s object: %s editable=%r>' % ( self.__class__.__name__, str(self), self.editable) def populate_link(self, finder, upgrade, require_hashes): # type: (PackageFinder, bool, bool) -> None """Ensure that if a link can be found for this, that it is found. Note that self.link may still be None - if Upgrade is False and the requirement is already installed. If require_hashes is True, don't use the wheel cache, because cached wheels, always built locally, have different hashes than the files downloaded from the index server and thus throw false hash mismatches. Furthermore, cached wheels at present have undeterministic contents due to file modification times. """ if self.link is None: self.link = finder.find_requirement(self, upgrade) if self._wheel_cache is not None and not require_hashes: old_link = self.link self.link = self._wheel_cache.get(self.link, self.name) if old_link != self.link: logger.debug('Using cached wheel link: %s', self.link) # Things that are valid for all kinds of requirements? @property def name(self): # type: () -> Optional[str] if self.req is None: return None return native_str(pkg_resources.safe_name(self.req.name)) @property def specifier(self): # type: () -> SpecifierSet return self.req.specifier @property def is_pinned(self): # type: () -> bool """Return whether I am pinned to an exact version. For example, some-package==1.2 is pinned; some-package>1.2 is not. """ specifiers = self.specifier return (len(specifiers) == 1 and next(iter(specifiers)).operator in {'==', '==='}) @property def installed_version(self): return get_installed_version(self.name) def match_markers(self, extras_requested=None): # type: (Optional[Iterable[str]]) -> bool if not extras_requested: # Provide an extra to safely evaluate the markers # without matching any extra extras_requested = ('',) if self.markers is not None: return any( self.markers.evaluate({'extra': extra}) for extra in extras_requested) else: return True @property def has_hash_options(self): # type: () -> bool """Return whether any known-good hashes are specified as options. These activate --require-hashes mode; hashes specified as part of a URL do not. """ return bool(self.options.get('hashes', {})) def hashes(self, trust_internet=True): # type: (bool) -> Hashes """Return a hash-comparer that considers my option- and URL-based hashes to be known-good. Hashes in URLs--ones embedded in the requirements file, not ones downloaded from an index server--are almost peers with ones from flags. They satisfy --require-hashes (whether it was implicitly or explicitly activated) but do not activate it. md5 and sha224 are not allowed in flags, which should nudge people toward good algos. We always OR all hashes together, even ones from URLs. :param trust_internet: Whether to trust URL-based (#md5=...) hashes downloaded from the internet, as by populate_link() """ good_hashes = self.options.get('hashes', {}).copy() link = self.link if trust_internet else self.original_link if link and link.hash: good_hashes.setdefault(link.hash_name, []).append(link.hash) return Hashes(good_hashes) def from_path(self): # type: () -> Optional[str] """Format a nice indicator to show where this "comes from" """ if self.req is None: return None s = str(self.req) if self.comes_from: if isinstance(self.comes_from, six.string_types): comes_from = self.comes_from else: comes_from = self.comes_from.from_path() if comes_from: s += '->' + comes_from return s def build_location(self, build_dir): # type: (str) -> Optional[str] assert build_dir is not None if self._temp_build_dir.path is not None: return self._temp_build_dir.path if self.req is None: # for requirement via a path to a directory: the name of the # package is not available yet so we create a temp directory # Once run_egg_info will have run, we'll be able # to fix it via _correct_build_location # Some systems have /tmp as a symlink which confuses custom # builds (such as numpy). Thus, we ensure that the real path # is returned. self._temp_build_dir.create() self._ideal_build_dir = build_dir return self._temp_build_dir.path if self.editable: name = self.name.lower() else: name = self.name # FIXME: Is there a better place to create the build_dir? (hg and bzr # need this) if not os.path.exists(build_dir): logger.debug('Creating directory %s', build_dir) _make_build_dir(build_dir) return os.path.join(build_dir, name) def _correct_build_location(self): # type: () -> None """Move self._temp_build_dir to self._ideal_build_dir/self.req.name For some requirements (e.g. a path to a directory), the name of the package is not available until we run egg_info, so the build_location will return a temporary directory and store the _ideal_build_dir. This is only called by self.run_egg_info to fix the temporary build directory. """ if self.source_dir is not None: return assert self.req is not None assert self._temp_build_dir.path assert (self._ideal_build_dir is not None and self._ideal_build_dir.path) # type: ignore old_location = self._temp_build_dir.path self._temp_build_dir.path = None new_location = self.build_location(self._ideal_build_dir) if os.path.exists(new_location): raise InstallationError( 'A package already exists in %s; please remove it to continue' % display_path(new_location)) logger.debug( 'Moving package %s from %s to new location %s', self, display_path(old_location), display_path(new_location), ) shutil.move(old_location, new_location) self._temp_build_dir.path = new_location self._ideal_build_dir = None self.source_dir = os.path.normpath(os.path.abspath(new_location)) self._egg_info_path = None # Correct the metadata directory, if it exists if self.metadata_directory: old_meta = self.metadata_directory rel = os.path.relpath(old_meta, start=old_location) new_meta = os.path.join(new_location, rel) new_meta = os.path.normpath(os.path.abspath(new_meta)) self.metadata_directory = new_meta def remove_temporary_source(self): # type: () -> None """Remove the source files from this requirement, if they are marked for deletion""" if self.source_dir and os.path.exists( os.path.join(self.source_dir, PIP_DELETE_MARKER_FILENAME)): logger.debug('Removing source in %s', self.source_dir) rmtree(self.source_dir) self.source_dir = None self._temp_build_dir.cleanup() self.build_env.cleanup() def check_if_exists(self, use_user_site): # type: (bool) -> bool """Find an installed distribution that satisfies or conflicts with this requirement, and set self.satisfied_by or self.conflicts_with appropriately. """ if self.req is None: return False try: # get_distribution() will resolve the entire list of requirements # anyway, and we've already determined that we need the requirement # in question, so strip the marker so that we don't try to # evaluate it. no_marker = Requirement(str(self.req)) no_marker.marker = None self.satisfied_by = pkg_resources.get_distribution(str(no_marker)) if self.editable and self.satisfied_by: self.conflicts_with = self.satisfied_by # when installing editables, nothing pre-existing should ever # satisfy self.satisfied_by = None return True except pkg_resources.DistributionNotFound: return False except pkg_resources.VersionConflict: existing_dist = pkg_resources.get_distribution( self.req.name ) if use_user_site: if dist_in_usersite(existing_dist): self.conflicts_with = existing_dist elif (running_under_virtualenv() and dist_in_site_packages(existing_dist)): raise InstallationError( "Will not install to the user site because it will " "lack sys.path precedence to %s in %s" % (existing_dist.project_name, existing_dist.location) ) else: self.conflicts_with = existing_dist return True # Things valid for wheels @property def is_wheel(self): # type: () -> bool if not self.link: return False return self.link.is_wheel def move_wheel_files( self, wheeldir, # type: str root=None, # type: Optional[str] home=None, # type: Optional[str] prefix=None, # type: Optional[str] warn_script_location=True, # type: bool use_user_site=False, # type: bool pycompile=True # type: bool ): # type: (...) -> None move_wheel_files( self.name, self.req, wheeldir, user=use_user_site, home=home, root=root, prefix=prefix, pycompile=pycompile, isolated=self.isolated, warn_script_location=warn_script_location, ) # Things valid for sdists @property def setup_py_dir(self): # type: () -> str return os.path.join( self.source_dir, self.link and self.link.subdirectory_fragment or '') @property def setup_py(self): # type: () -> str assert self.source_dir, "No source dir for %s" % self setup_py = os.path.join(self.setup_py_dir, 'setup.py') # Python2 __file__ should not be unicode if six.PY2 and isinstance(setup_py, six.text_type): setup_py = setup_py.encode(sys.getfilesystemencoding()) return setup_py @property def pyproject_toml(self): # type: () -> str assert self.source_dir, "No source dir for %s" % self return make_pyproject_path(self.setup_py_dir) def load_pyproject_toml(self): # type: () -> None """Load the pyproject.toml file. After calling this routine, all of the attributes related to PEP 517 processing for this requirement have been set. In particular, the use_pep517 attribute can be used to determine whether we should follow the PEP 517 or legacy (setup.py) code path. """ pep517_data = load_pyproject_toml( self.use_pep517, self.pyproject_toml, self.setup_py, str(self) ) if pep517_data is None: self.use_pep517 = False else: self.use_pep517 = True requires, backend, check = pep517_data self.requirements_to_check = check self.pyproject_requires = requires self.pep517_backend = Pep517HookCaller(self.setup_py_dir, backend) # Use a custom function to call subprocesses self.spin_message = "" def runner(cmd, cwd=None, extra_environ=None): with open_spinner(self.spin_message) as spinner: call_subprocess( cmd, cwd=cwd, extra_environ=extra_environ, show_stdout=False, spinner=spinner ) self.spin_message = "" self.pep517_backend._subprocess_runner = runner def prepare_metadata(self): # type: () -> None """Ensure that project metadata is available. Under PEP 517, call the backend hook to prepare the metadata. Under legacy processing, call setup.py egg-info. """ assert self.source_dir with indent_log(): if self.use_pep517: self.prepare_pep517_metadata() else: self.run_egg_info() if not self.req: if isinstance(parse_version(self.metadata["Version"]), Version): op = "==" else: op = "===" self.req = Requirement( "".join([ self.metadata["Name"], op, self.metadata["Version"], ]) ) self._correct_build_location() else: metadata_name = canonicalize_name(self.metadata["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( 'Generating metadata for package %s ' 'produced metadata for project name %s. Fix your ' '#egg=%s fragments.', self.name, metadata_name, self.name ) self.req = Requirement(metadata_name) def prepare_pep517_metadata(self): # type: () -> None assert self.pep517_backend is not None metadata_dir = os.path.join( self.setup_py_dir, 'pip-wheel-metadata' ) ensure_dir(metadata_dir) with self.build_env: # Note that Pep517HookCaller implements a fallback for # prepare_metadata_for_build_wheel, so we don't have to # consider the possibility that this hook doesn't exist. backend = self.pep517_backend self.spin_message = "Preparing wheel metadata" distinfo_dir = backend.prepare_metadata_for_build_wheel( metadata_dir ) self.metadata_directory = os.path.join(metadata_dir, distinfo_dir) def run_egg_info(self): # type: () -> None if self.name: logger.debug( 'Running setup.py (path:%s) egg_info for package %s', self.setup_py, self.name, ) else: logger.debug( 'Running setup.py (path:%s) egg_info for package from %s', self.setup_py, self.link, ) script = SETUPTOOLS_SHIM % self.setup_py sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) base_cmd = [sys_executable, '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] # We can't put the .egg-info files at the root, because then the # source code will be mistaken for an installed egg, causing # problems if self.editable: egg_base_option = [] # type: List[str] else: egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info') ensure_dir(egg_info_dir) egg_base_option = ['--egg-base', 'pip-egg-info'] with self.build_env: call_subprocess( egg_info_cmd + egg_base_option, cwd=self.setup_py_dir, show_stdout=False, command_desc='python setup.py egg_info') @property def egg_info_path(self): # type: () -> str if self._egg_info_path is None: if self.editable: base = self.source_dir else: base = os.path.join(self.setup_py_dir, 'pip-egg-info') filenames = os.listdir(base) if self.editable: filenames = [] for root, dirs, files in os.walk(base): for dir in vcs.dirnames: if dir in dirs: dirs.remove(dir) # Iterate over a copy of ``dirs``, since mutating # a list while iterating over it can cause trouble. # (See https://github.com/pypa/pip/pull/462.) for dir in list(dirs): # Don't search in anything that looks like a virtualenv # environment if ( os.path.lexists( os.path.join(root, dir, 'bin', 'python') ) or os.path.exists( os.path.join( root, dir, 'Scripts', 'Python.exe' ) )): dirs.remove(dir) # Also don't search through tests elif dir == 'test' or dir == 'tests': dirs.remove(dir) filenames.extend([os.path.join(root, dir) for dir in dirs]) filenames = [f for f in filenames if f.endswith('.egg-info')] if not filenames: raise InstallationError( "Files/directories not found in %s" % base ) # if we have more than one match, we pick the toplevel one. This # can easily be the case if there is a dist folder which contains # an extracted tarball for testing purposes. if len(filenames) > 1: filenames.sort( key=lambda x: x.count(os.path.sep) + (os.path.altsep and x.count(os.path.altsep) or 0) ) self._egg_info_path = os.path.join(base, filenames[0]) return self._egg_info_path @property def metadata(self): if not hasattr(self, '_metadata'): self._metadata = get_metadata(self.get_dist()) return self._metadata def get_dist(self): # type: () -> Distribution """Return a pkg_resources.Distribution for this requirement""" if self.metadata_directory: base_dir, distinfo = os.path.split(self.metadata_directory) metadata = pkg_resources.PathMetadata( base_dir, self.metadata_directory ) dist_name = os.path.splitext(distinfo)[0] typ = pkg_resources.DistInfoDistribution else: egg_info = self.egg_info_path.rstrip(os.path.sep) base_dir = os.path.dirname(egg_info) metadata = pkg_resources.PathMetadata(base_dir, egg_info) dist_name = os.path.splitext(os.path.basename(egg_info))[0] # https://github.com/python/mypy/issues/1174 typ = pkg_resources.Distribution # type: ignore return typ( base_dir, project_name=dist_name, metadata=metadata, ) def assert_source_matches_version(self): # type: () -> None assert self.source_dir version = self.metadata['version'] if self.req.specifier and version not in self.req.specifier: logger.warning( 'Requested %s, but installing version %s', self, version, ) else: logger.debug( 'Source in %s has version %s, which satisfies requirement %s', display_path(self.source_dir), version, self, ) # For both source distributions and editables def ensure_has_source_dir(self, parent_dir): # type: (str) -> str """Ensure that a source_dir is set. This will create a temporary build dir if the name of the requirement isn't known yet. :param parent_dir: The ideal pip parent_dir for the source_dir. Generally src_dir for editables and build_dir for sdists. :return: self.source_dir """ if self.source_dir is None: self.source_dir = self.build_location(parent_dir) return self.source_dir # For editable installations def install_editable( self, install_options, # type: List[str] global_options=(), # type: Sequence[str] prefix=None # type: Optional[str] ): # type: (...) -> None logger.info('Running setup.py develop for %s', self.name) if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] if prefix: prefix_param = ['--prefix={}'.format(prefix)] install_options = list(install_options) + prefix_param with indent_log(): # FIXME: should we do --install-headers here too? with self.build_env: sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) call_subprocess( [ sys_executable, '-c', SETUPTOOLS_SHIM % self.setup_py ] + list(global_options) + ['develop', '--no-deps'] + list(install_options), cwd=self.setup_py_dir, show_stdout=False, ) self.install_succeeded = True def update_editable(self, obtain=True): # type: (bool) -> None if not self.link: logger.debug( "Cannot update repository at %s; repository location is " "unknown", self.source_dir, ) return assert self.editable assert self.source_dir if self.link.scheme == 'file': # Static paths don't get updated return assert '+' in self.link.url, "bad url: %r" % self.link.url if not self.update: return vc_type, url = self.link.url.split('+', 1) backend = vcs.get_backend(vc_type) if backend: vcs_backend = backend(self.link.url) if obtain: vcs_backend.obtain(self.source_dir) else: vcs_backend.export(self.source_dir) else: assert 0, ( 'Unexpected version control type (in %s): %s' % (self.link, vc_type)) # Top-level Actions def uninstall(self, auto_confirm=False, verbose=False, use_user_site=False): # type: (bool, bool, bool) -> Optional[UninstallPathSet] """ Uninstall the distribution currently satisfying this requirement. Prompts before removing or modifying files unless ``auto_confirm`` is True. Refuses to delete or modify files outside of ``sys.prefix`` - thus uninstallation within a virtual environment can only modify that virtual environment, even if the virtualenv is linked to global site-packages. """ if not self.check_if_exists(use_user_site): logger.warning("Skipping %s as it is not installed.", self.name) return None dist = self.satisfied_by or self.conflicts_with uninstalled_pathset = UninstallPathSet.from_dist(dist) uninstalled_pathset.remove(auto_confirm, verbose) return uninstalled_pathset def _clean_zip_name(self, name, prefix): # only used by archive. assert name.startswith(prefix + os.path.sep), ( "name %r doesn't start with prefix %r" % (name, prefix) ) name = name[len(prefix) + 1:] name = name.replace(os.path.sep, '/') return name def _get_archive_name(self, path, parentdir, rootdir): # type: (str, str, str) -> str path = os.path.join(parentdir, path) name = self._clean_zip_name(path, rootdir) return self.name + '/' + name # TODO: Investigate if this should be kept in InstallRequirement # Seems to be used only when VCS + downloads def archive(self, build_dir): # type: (str) -> None assert self.source_dir create_archive = True archive_name = '%s-%s.zip' % (self.name, self.metadata["version"]) archive_path = os.path.join(build_dir, archive_name) if os.path.exists(archive_path): response = ask_path_exists( 'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)bort ' % display_path(archive_path), ('i', 'w', 'b', 'a')) if response == 'i': create_archive = False elif response == 'w': logger.warning('Deleting %s', display_path(archive_path)) os.remove(archive_path) elif response == 'b': dest_file = backup_dir(archive_path) logger.warning( 'Backing up %s to %s', display_path(archive_path), display_path(dest_file), ) shutil.move(archive_path, dest_file) elif response == 'a': sys.exit(-1) if create_archive: zip = zipfile.ZipFile( archive_path, 'w', zipfile.ZIP_DEFLATED, allowZip64=True ) dir = os.path.normcase(os.path.abspath(self.setup_py_dir)) for dirpath, dirnames, filenames in os.walk(dir): if 'pip-egg-info' in dirnames: dirnames.remove('pip-egg-info') for dirname in dirnames: dir_arcname = self._get_archive_name(dirname, parentdir=dirpath, rootdir=dir) zipdir = zipfile.ZipInfo(dir_arcname + '/') zipdir.external_attr = 0x1ED << 16 # 0o755 zip.writestr(zipdir, '') for filename in filenames: if filename == PIP_DELETE_MARKER_FILENAME: continue file_arcname = self._get_archive_name(filename, parentdir=dirpath, rootdir=dir) filename = os.path.join(dirpath, filename) zip.write(filename, file_arcname) zip.close() logger.info('Saved %s', display_path(archive_path)) def install( self, install_options, # type: List[str] global_options=None, # type: Optional[Sequence[str]] root=None, # type: Optional[str] home=None, # type: Optional[str] prefix=None, # type: Optional[str] warn_script_location=True, # type: bool use_user_site=False, # type: bool pycompile=True # type: bool ): # type: (...) -> None global_options = global_options if global_options is not None else [] if self.editable: self.install_editable( install_options, global_options, prefix=prefix, ) return if self.is_wheel: version = wheel.wheel_version(self.source_dir) wheel.check_compatibility(version, self.name) self.move_wheel_files( self.source_dir, root=root, prefix=prefix, home=home, warn_script_location=warn_script_location, use_user_site=use_user_site, pycompile=pycompile, ) self.install_succeeded = True return # Extend the list of global and install options passed on to # the setup.py call with the ones from the requirements file. # Options specified in requirements file override those # specified on the command line, since the last option given # to setup.py is the one that is used. global_options = list(global_options) + \ self.options.get('global_options', []) install_options = list(install_options) + \ self.options.get('install_options', []) if self.isolated: # https://github.com/python/mypy/issues/1174 global_options = global_options + ["--no-user-cfg"] # type: ignore with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') install_args = self.get_install_args( global_options, record_filename, root, prefix, pycompile, ) msg = 'Running setup.py install for %s' % (self.name,) with open_spinner(msg) as spinner: with indent_log(): with self.build_env: call_subprocess( install_args + install_options, cwd=self.setup_py_dir, show_stdout=False, spinner=spinner, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True def prepend_root(path): if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: logger.warning( 'Could not find .egg-info directory in install record' ' for %s', self, ) # FIXME: put the record somewhere # FIXME: should this be an error? return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( os.path.relpath(prepend_root(filename), egg_info_dir) ) new_lines.sort() ensure_dir(egg_info_dir) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n') def get_install_args( self, global_options, # type: Sequence[str] record_filename, # type: str root, # type: Optional[str] prefix, # type: Optional[str] pycompile # type: bool ): # type: (...) -> List[str] sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) install_args = [sys_executable, "-u"] install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ ['install', '--record', record_filename] install_args += ['--single-version-externally-managed'] if root is not None: install_args += ['--root', root] if prefix is not None: install_args += ['--prefix', prefix] if pycompile: install_args += ["--compile"] else: install_args += ["--no-compile"] if running_under_virtualenv(): py_ver_str = 'python' + sysconfig.get_python_version() install_args += ['--install-headers', os.path.join(sys.prefix, 'include', 'site', py_ver_str, self.name)] return install_args
def run(self, options, args): cmdoptions.check_install_build_global(options) upgrade_strategy = "to-satisfy-only" if options.upgrade: upgrade_strategy = options.upgrade_strategy if options.build_dir: options.build_dir = os.path.abspath(options.build_dir) cmdoptions.check_dist_restriction(options, check_target=True) if options.python_version: python_versions = [options.python_version] else: python_versions = None options.src_dir = os.path.abspath(options.src_dir) install_options = options.install_options or [] if options.use_user_site: if options.prefix_path: raise CommandError( "Can not combine '--user' and '--prefix' as they imply " "different installation locations") if virtualenv_no_global(): raise InstallationError( "Can not perform a '--user' install. User site-packages " "are not visible in this virtualenv.") install_options.append('--user') install_options.append('--prefix=') target_temp_dir = TempDirectory(kind="target") if options.target_dir: options.ignore_installed = True options.target_dir = os.path.abspath(options.target_dir) if (os.path.exists(options.target_dir) and not os.path.isdir(options.target_dir)): raise CommandError( "Target path exists but is not a directory, will not " "continue.") # Create a target directory for using with the target option target_temp_dir.create() install_options.append('--home=' + target_temp_dir.path) global_options = options.global_options or [] with self._build_session(options) as session: finder = self._build_package_finder( options=options, session=session, platform=options.platform, python_versions=python_versions, abi=options.abi, implementation=options.implementation, ) build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) if options.cache_dir and not check_path_owner(options.cache_dir): logger.warning( "The directory '%s' or its parent directory is not owned " "by the current user and caching wheels has been " "disabled. check the permissions and owner of that " "directory. If executing pip with sudo, you may want " "sudo's -H flag.", options.cache_dir, ) options.cache_dir = None with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="install") as directory: requirement_set = RequirementSet( require_hashes=options.require_hashes, check_supported_wheels=not options.target_dir, ) try: self.populate_requirement_set(requirement_set, args, options, finder, session, self.name, wheel_cache) preparer = RequirementPreparer( build_dir=directory.path, src_dir=options.src_dir, download_dir=None, wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, req_tracker=req_tracker, ) resolver = Resolver( preparer=preparer, finder=finder, session=session, wheel_cache=wheel_cache, use_user_site=options.use_user_site, upgrade_strategy=upgrade_strategy, force_reinstall=options.force_reinstall, ignore_dependencies=options.ignore_dependencies, ignore_requires_python=options.ignore_requires_python, ignore_installed=options.ignore_installed, isolated=options.isolated_mode, use_pep517=options.use_pep517) resolver.resolve(requirement_set) protect_pip_from_modification_on_windows( modifying_pip=requirement_set.has_requirement("pip")) # Consider legacy and PEP517-using requirements separately legacy_requirements = [] pep517_requirements = [] for req in requirement_set.requirements.values(): if req.use_pep517: pep517_requirements.append(req) else: legacy_requirements.append(req) # We don't build wheels for legacy requirements if we # don't have wheel installed or we don't have a cache dir try: import wheel # noqa: F401 build_legacy = bool(options.cache_dir) except ImportError: build_legacy = False wb = WheelBuilder( finder, preparer, wheel_cache, build_options=[], global_options=[], ) # Always build PEP 517 requirements build_failures = wb.build(pep517_requirements, session=session, autobuilding=True) if build_legacy: # We don't care about failures building legacy # requirements, as we'll fall through to a direct # install for those. wb.build(legacy_requirements, session=session, autobuilding=True) # If we're using PEP 517, we cannot do a direct install # so we fail here. if build_failures: raise InstallationError( "Could not build wheels for {} which use" " PEP 517 and cannot be installed directly".format( ", ".join(r.name for r in build_failures))) to_install = resolver.get_installation_order( requirement_set) # Consistency Checking of the package set we're installing. should_warn_about_conflicts = ( not options.ignore_dependencies and options.warn_about_conflicts) if should_warn_about_conflicts: self._warn_about_conflicts(to_install) # Don't warn about script install locations if # --target has been specified warn_script_location = options.warn_script_location if options.target_dir: warn_script_location = False installed = install_given_reqs( to_install, install_options, global_options, root=options.root_path, home=target_temp_dir.path, prefix=options.prefix_path, pycompile=options.compile, warn_script_location=warn_script_location, use_user_site=options.use_user_site, ) lib_locations = get_lib_location_guesses( user=options.use_user_site, home=target_temp_dir.path, root=options.root_path, prefix=options.prefix_path, isolated=options.isolated_mode, ) working_set = pkg_resources.WorkingSet(lib_locations) reqs = sorted(installed, key=operator.attrgetter('name')) items = [] for req in reqs: item = req.name try: installed_version = get_installed_version( req.name, working_set=working_set) if installed_version: item += '-' + installed_version except Exception: pass items.append(item) installed = ' '.join(items) if installed: logger.info('Successfully installed %s', installed) except EnvironmentError as error: show_traceback = (self.verbosity >= 1) message = create_env_error_message( error, show_traceback, options.use_user_site, ) logger.error(message, exc_info=show_traceback) return ERROR except PreviousBuildDirError: options.no_clean = True raise finally: # Clean up if not options.no_clean: requirement_set.cleanup_files() wheel_cache.cleanup() if options.target_dir: self._handle_target_dir(options.target_dir, target_temp_dir, options.upgrade) return requirement_set
class BuildEnvironment(object): """Creates and manages an isolated environment to install build deps """ def __init__(self): self._temp_dir = TempDirectory(kind="build-env") self._temp_dir.create() @property def path(self): return self._temp_dir.path def __enter__(self): self.save_path = os.environ.get('PATH', None) self.save_pythonpath = os.environ.get('PYTHONPATH', None) self.save_nousersite = os.environ.get('PYTHONNOUSERSITE', None) install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' install_dirs = get_paths(install_scheme, vars={ 'base': self.path, 'platbase': self.path, }) scripts = install_dirs['scripts'] if self.save_path: os.environ['PATH'] = scripts + os.pathsep + self.save_path else: os.environ['PATH'] = scripts + os.pathsep + os.defpath # Note: prefer distutils' sysconfig to get the # library paths so PyPy is correctly supported. purelib = get_python_lib(plat_specific=0, prefix=self.path) platlib = get_python_lib(plat_specific=1, prefix=self.path) if purelib == platlib: lib_dirs = purelib else: lib_dirs = purelib + os.pathsep + platlib if self.save_pythonpath: os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ self.save_pythonpath else: os.environ['PYTHONPATH'] = lib_dirs os.environ['PYTHONNOUSERSITE'] = '1' return self.path def __exit__(self, exc_type, exc_val, exc_tb): def restore_var(varname, old_value): if old_value is None: os.environ.pop(varname, None) else: os.environ[varname] = old_value restore_var('PATH', self.save_path) restore_var('PYTHONPATH', self.save_pythonpath) restore_var('PYTHONNOUSERSITE', self.save_nousersite) def cleanup(self): self._temp_dir.cleanup() def install_requirements(self, finder, requirements, message): args = [ sys.executable, '-m', 'pip', 'install', '--ignore-installed', '--no-user', '--prefix', self.path, '--no-warn-script-location', ] if logger.getEffectiveLevel() <= logging.DEBUG: args.append('-v') for format_control in ('no_binary', 'only_binary'): formats = getattr(finder.format_control, format_control) args.extend(('--' + format_control.replace('_', '-'), ','.join(sorted(formats or {':none:'})))) if finder.index_urls: args.extend(['-i', finder.index_urls[0]]) for extra_index in finder.index_urls[1:]: args.extend(['--extra-index-url', extra_index]) else: args.append('--no-index') for link in finder.find_links: args.extend(['--find-links', link]) for _, host, _ in finder.secure_origins: args.extend(['--trusted-host', host]) if finder.allow_all_prereleases: args.append('--pre') if finder.process_dependency_links: args.append('--process-dependency-links') args.append('--') args.extend(requirements) with open_spinner(message) as spinner: call_subprocess(args, show_stdout=False, spinner=spinner)
class UninstallPathSet(object): """A set of file paths to be removed in the uninstallation of a requirement.""" def __init__(self, dist): self.paths = set() self._refuse = set() self.pth = {} self.dist = dist self.save_dir = TempDirectory(kind="uninstall") self._moved_paths = [] def _permitted(self, path): """ Return True if the given path is one we are permitted to remove/modify, False otherwise. """ return is_local(path) def add(self, path): head, tail = os.path.split(path) # we normalize the head to resolve parent directory symlinks, but not # the tail, since we only want to uninstall symlinks, not their targets path = os.path.join(normalize_path(head), os.path.normcase(tail)) if not os.path.exists(path): return if self._permitted(path): self.paths.add(path) else: self._refuse.add(path) # __pycache__ files can show up after 'installed-files.txt' is created, # due to imports if os.path.splitext(path)[1] == '.py' and uses_pycache: self.add(cache_from_source(path)) def add_pth(self, pth_file, entry): pth_file = normalize_path(pth_file) if self._permitted(pth_file): if pth_file not in self.pth: self.pth[pth_file] = UninstallPthEntries(pth_file) self.pth[pth_file].add(entry) else: self._refuse.add(pth_file) def _stash(self, path): return os.path.join(self.save_dir.path, os.path.splitdrive(path)[1].lstrip(os.path.sep)) def remove(self, auto_confirm=False, verbose=False): """Remove paths in ``self.paths`` with confirmation (unless ``auto_confirm`` is True).""" if not self.paths: logger.info( "Can't uninstall '%s'. No files were found to uninstall.", self.dist.project_name, ) return dist_name_version = (self.dist.project_name + "-" + self.dist.version) logger.info('Uninstalling %s:', dist_name_version) with indent_log(): if auto_confirm or self._allowed_to_proceed(verbose): self.save_dir.create() for path in sorted(compact(self.paths)): new_path = self._stash(path) logger.debug('Removing file or directory %s', path) self._moved_paths.append(path) renames(path, new_path) for pth in self.pth.values(): pth.remove() logger.info('Successfully uninstalled %s', dist_name_version) def _allowed_to_proceed(self, verbose): """Display which files would be deleted and prompt for confirmation """ def _display(msg, paths): if not paths: return logger.info(msg) with indent_log(): for path in sorted(compact(paths)): logger.info(path) if not verbose: will_remove, will_skip = compress_for_output_listing(self.paths) else: # In verbose mode, display all the files that are going to be # deleted. will_remove = list(self.paths) will_skip = set() _display('Would remove:', will_remove) _display('Would not remove (might be manually added):', will_skip) _display('Would not remove (outside of prefix):', self._refuse) return ask('Proceed (y/n)? ', ('y', 'n')) == 'y' def rollback(self): """Rollback the changes previously made by remove().""" if self.save_dir.path is None: logger.error( "Can't roll back %s; was not uninstalled", self.dist.project_name, ) return False logger.info('Rolling back uninstall of %s', self.dist.project_name) for path in self._moved_paths: tmp_path = self._stash(path) logger.debug('Replacing %s', path) renames(tmp_path, path) for pth in self.pth.values(): pth.rollback() def commit(self): """Remove temporary save dir: rollback will no longer be possible.""" self.save_dir.cleanup() self._moved_paths = [] @classmethod def from_dist(cls, dist): dist_path = normalize_path(dist.location) if not dist_is_local(dist): logger.info( "Not uninstalling %s at %s, outside environment %s", dist.key, dist_path, sys.prefix, ) return cls(dist) if dist_path in { p for p in {sysconfig.get_path("stdlib"), sysconfig.get_path("platstdlib")} if p }: logger.info( "Not uninstalling %s at %s, as it is in the standard library.", dist.key, dist_path, ) return cls(dist) paths_to_remove = cls(dist) develop_egg_link = egg_link_path(dist) develop_egg_link_egg_info = '{}.egg-info'.format( pkg_resources.to_filename(dist.project_name)) egg_info_exists = dist.egg_info and os.path.exists(dist.egg_info) # Special case for distutils installed package distutils_egg_info = getattr(dist._provider, 'path', None) # Uninstall cases order do matter as in the case of 2 installs of the # same package, pip needs to uninstall the currently detected version if (egg_info_exists and dist.egg_info.endswith('.egg-info') and not dist.egg_info.endswith(develop_egg_link_egg_info)): # if dist.egg_info.endswith(develop_egg_link_egg_info), we # are in fact in the develop_egg_link case paths_to_remove.add(dist.egg_info) if dist.has_metadata('installed-files.txt'): for installed_file in dist.get_metadata( 'installed-files.txt').splitlines(): path = os.path.normpath( os.path.join(dist.egg_info, installed_file)) paths_to_remove.add(path) # FIXME: need a test for this elif block # occurs with --single-version-externally-managed/--record outside # of pip elif dist.has_metadata('top_level.txt'): if dist.has_metadata('namespace_packages.txt'): namespaces = dist.get_metadata('namespace_packages.txt') else: namespaces = [] for top_level_pkg in [ p for p in dist.get_metadata( 'top_level.txt').splitlines() if p and p not in namespaces ]: path = os.path.join(dist.location, top_level_pkg) paths_to_remove.add(path) paths_to_remove.add(path + '.py') paths_to_remove.add(path + '.pyc') paths_to_remove.add(path + '.pyo') elif distutils_egg_info: raise UninstallationError( "Cannot uninstall {!r}. It is a distutils installed project " "and thus we cannot accurately determine which files belong " "to it which would lead to only a partial uninstall.".format( dist.project_name, )) elif dist.location.endswith('.egg'): # package installed by easy_install # We cannot match on dist.egg_name because it can slightly vary # i.e. setuptools-0.6c11-py2.6.egg vs setuptools-0.6rc11-py2.6.egg paths_to_remove.add(dist.location) easy_install_egg = os.path.split(dist.location)[1] easy_install_pth = os.path.join(os.path.dirname(dist.location), 'easy-install.pth') paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg) elif egg_info_exists and dist.egg_info.endswith('.dist-info'): for path in uninstallation_paths(dist): paths_to_remove.add(path) elif develop_egg_link: # develop egg with open(develop_egg_link, 'r') as fh: link_pointer = os.path.normcase(fh.readline().strip()) assert (link_pointer == dist.location), ( 'Egg-link %s does not match installed location of %s ' '(at %s)' % (link_pointer, dist.project_name, dist.location)) paths_to_remove.add(develop_egg_link) easy_install_pth = os.path.join(os.path.dirname(develop_egg_link), 'easy-install.pth') paths_to_remove.add_pth(easy_install_pth, dist.location) else: logger.debug( 'Not sure how to uninstall: %s - Check: %s', dist, dist.location, ) # find distutils scripts= scripts if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'): for script in dist.metadata_listdir('scripts'): if dist_in_usersite(dist): bin_dir = bin_user else: bin_dir = bin_py paths_to_remove.add(os.path.join(bin_dir, script)) if WINDOWS: paths_to_remove.add(os.path.join(bin_dir, script) + '.bat') # find console_scripts _scripts_to_remove = [] console_scripts = dist.get_entry_map(group='console_scripts') for name in console_scripts.keys(): _scripts_to_remove.extend(_script_names(dist, name, False)) # find gui_scripts gui_scripts = dist.get_entry_map(group='gui_scripts') for name in gui_scripts.keys(): _scripts_to_remove.extend(_script_names(dist, name, True)) for s in _scripts_to_remove: paths_to_remove.add(s) return paths_to_remove
class RequirementTracker(object): def __init__(self): self._root = os.environ.get('PIP_REQ_TRACKER') if self._root is None: self._temp_dir = TempDirectory(delete=False, kind='req-tracker') self._temp_dir.create() self._root = os.environ['PIP_REQ_TRACKER'] = self._temp_dir.path logger.debug('Created requirements tracker %r', self._root) else: self._temp_dir = None logger.debug('Re-using requirements tracker %r', self._root) self._entries = set() def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.cleanup() def _entry_path(self, link): hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest() return os.path.join(self._root, hashed) def add(self, req): link = req.link info = str(req) entry_path = self._entry_path(link) try: with open(entry_path) as fp: # Error, these's already a build in progress. raise LookupError('%s is already being built: %s' % (link, fp.read())) except IOError as e: if e.errno != errno.ENOENT: raise assert req not in self._entries with open(entry_path, 'w') as fp: fp.write(info) self._entries.add(req) logger.debug('Added %s to build tracker %r', req, self._root) def remove(self, req): link = req.link self._entries.remove(req) os.unlink(self._entry_path(link)) logger.debug('Removed %s from build tracker %r', req, self._root) def cleanup(self): for req in set(self._entries): self.remove(req) remove = self._temp_dir is not None if remove: self._temp_dir.cleanup() logger.debug('%s build tracker %r', 'Removed' if remove else 'Cleaned', self._root) @contextlib.contextmanager def track(self, req): self.add(req) yield self.remove(req)
class InstallRequirement(object): """ Represents something that may be installed later on, may have information about where to fetch the relavant requirement and also contains logic for installing the said requirement. """ def __init__(self, req, comes_from, source_dir=None, editable=False, link=None, update=True, markers=None, isolated=False, options=None, wheel_cache=None, constraint=False, extras=()): assert req is None or isinstance(req, Requirement), req self.req = req self.comes_from = comes_from self.constraint = constraint if source_dir is not None: self.source_dir = os.path.normpath(os.path.abspath(source_dir)) else: self.source_dir = None self.editable = editable self._wheel_cache = wheel_cache if link is not None: self.link = self.original_link = link else: from pipenv.patched.notpip._internal.index import Link self.link = self.original_link = req and req.url and Link(req.url) if extras: self.extras = extras elif req: self.extras = { pkg_resources.safe_extra(extra) for extra in req.extras } else: self.extras = set() if markers is not None: self.markers = markers else: self.markers = req and req.marker self._egg_info_path = None # This holds the pkg_resources.Distribution object if this requirement # is already available: self.satisfied_by = None # This hold the pkg_resources.Distribution object if this requirement # conflicts with another installed distribution: self.conflicts_with = None # Temporary build location self._temp_build_dir = TempDirectory(kind="req-build") # Used to store the global directory where the _temp_build_dir should # have been created. Cf _correct_build_location method. self._ideal_build_dir = None # True if the editable should be updated: self.update = update # Set to True after successful installation self.install_succeeded = None # UninstallPathSet of uninstalled distribution (for possible rollback) self.uninstalled_pathset = None self.options = options if options else {} # Set to True after successful preparation of this requirement self.prepared = False self.is_direct = False self.isolated = isolated self.build_env = BuildEnvironment(no_clean=True) @classmethod def from_editable(cls, editable_req, comes_from=None, isolated=False, options=None, wheel_cache=None, constraint=False): from pipenv.patched.notpip._internal.index import Link name, url, extras_override = parse_editable(editable_req) if url.startswith('file:'): source_dir = url_to_path(url) else: source_dir = None if name is not None: try: req = Requirement(name) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % name) else: req = None return cls( req, comes_from, source_dir=source_dir, editable=True, link=Link(url), constraint=constraint, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, extras=extras_override or (), ) @classmethod def from_req(cls, req, comes_from=None, isolated=False, wheel_cache=None): try: req = Requirement(req) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % req) if req.url: raise InstallationError( "Direct url requirement (like %s) are not allowed for " "dependencies" % req ) return cls(req, comes_from, isolated=isolated, wheel_cache=wheel_cache) @classmethod def from_line( cls, name, comes_from=None, isolated=False, options=None, wheel_cache=None, constraint=False): """Creates an InstallRequirement from a name, which might be a requirement, directory containing 'setup.py', filename, or URL. """ from pipenv.patched.notpip._internal.index import Link if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: name, markers = name.split(marker_sep, 1) markers = markers.strip() if not markers: markers = None else: markers = Marker(markers) else: markers = None name = name.strip() req = None path = os.path.normpath(os.path.abspath(name)) link = None extras = None if is_url(name): link = Link(name) else: p, extras = _strip_extras(path) looks_like_dir = os.path.isdir(p) and ( os.path.sep in name or (os.path.altsep is not None and os.path.altsep in name) or name.startswith('.') ) if looks_like_dir: if not is_installable_dir(p): raise InstallationError( "Directory %r is not installable. File 'setup.py' " "not found." % name ) link = Link(path_to_url(p)) elif is_archive_file(p): if not os.path.isfile(p): logger.warning( 'Requirement %r looks like a filename, but the ' 'file does not exist', name ) link = Link(path_to_url(p)) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == 'file' and re.search(r'\.\./', link.url): link = Link( path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req = link.egg_fragment # a requirement specifier else: req = name if extras: extras = Requirement("placeholder" + extras.lower()).extras else: extras = () if req is not None: try: req = Requirement(req) except InvalidRequirement: if os.path.sep in req: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req) elif '=' in req and not any(op in req for op in operators): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = traceback.format_exc() raise InstallationError( "Invalid requirement: '%s'\n%s" % (req, add_msg)) return cls( req, comes_from, link=link, markers=markers, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, constraint=constraint, extras=extras, ) def __str__(self): if self.req: s = str(self.req) if self.link: s += ' from %s' % self.link.url else: s = self.link.url if self.link else None if self.satisfied_by is not None: s += ' in %s' % display_path(self.satisfied_by.location) if self.comes_from: if isinstance(self.comes_from, six.string_types): comes_from = self.comes_from else: comes_from = self.comes_from.from_path() if comes_from: s += ' (from %s)' % comes_from return s def __repr__(self): return '<%s object: %s editable=%r>' % ( self.__class__.__name__, str(self), self.editable) def populate_link(self, finder, upgrade, require_hashes): """Ensure that if a link can be found for this, that it is found. Note that self.link may still be None - if Upgrade is False and the requirement is already installed. If require_hashes is True, don't use the wheel cache, because cached wheels, always built locally, have different hashes than the files downloaded from the index server and thus throw false hash mismatches. Furthermore, cached wheels at present have undeterministic contents due to file modification times. """ if self.link is None: self.link = finder.find_requirement(self, upgrade) if self._wheel_cache is not None and not require_hashes: old_link = self.link self.link = self._wheel_cache.get(self.link, self.name) if old_link != self.link: logger.debug('Using cached wheel link: %s', self.link) @property def specifier(self): return self.req.specifier @property def is_pinned(self): """Return whether I am pinned to an exact version. For example, some-package==1.2 is pinned; some-package>1.2 is not. """ specifiers = self.specifier return (len(specifiers) == 1 and next(iter(specifiers)).operator in {'==', '==='}) def from_path(self): if self.req is None: return None s = str(self.req) if self.comes_from: if isinstance(self.comes_from, six.string_types): comes_from = self.comes_from else: comes_from = self.comes_from.from_path() if comes_from: s += '->' + comes_from return s def build_location(self, build_dir): assert build_dir is not None if self._temp_build_dir.path is not None: return self._temp_build_dir.path if self.req is None: # for requirement via a path to a directory: the name of the # package is not available yet so we create a temp directory # Once run_egg_info will have run, we'll be able # to fix it via _correct_build_location # Some systems have /tmp as a symlink which confuses custom # builds (such as numpy). Thus, we ensure that the real path # is returned. self._temp_build_dir.create() self._ideal_build_dir = build_dir return self._temp_build_dir.path if self.editable: name = self.name.lower() else: name = self.name # FIXME: Is there a better place to create the build_dir? (hg and bzr # need this) if not os.path.exists(build_dir): logger.debug('Creating directory %s', build_dir) _make_build_dir(build_dir) return os.path.join(build_dir, name) def _correct_build_location(self): """Move self._temp_build_dir to self._ideal_build_dir/self.req.name For some requirements (e.g. a path to a directory), the name of the package is not available until we run egg_info, so the build_location will return a temporary directory and store the _ideal_build_dir. This is only called by self.egg_info_path to fix the temporary build directory. """ if self.source_dir is not None: return assert self.req is not None assert self._temp_build_dir.path assert self._ideal_build_dir.path old_location = self._temp_build_dir.path self._temp_build_dir.path = None new_location = self.build_location(self._ideal_build_dir) if os.path.exists(new_location): raise InstallationError( 'A package already exists in %s; please remove it to continue' % display_path(new_location)) logger.debug( 'Moving package %s from %s to new location %s', self, display_path(old_location), display_path(new_location), ) shutil.move(old_location, new_location) self._temp_build_dir.path = new_location self._ideal_build_dir = None self.source_dir = os.path.normpath(os.path.abspath(new_location)) self._egg_info_path = None @property def name(self): if self.req is None: return None return native_str(pkg_resources.safe_name(self.req.name)) @property def setup_py_dir(self): return os.path.join( self.source_dir, self.link and self.link.subdirectory_fragment or '') @property def setup_py(self): assert self.source_dir, "No source dir for %s" % self setup_py = os.path.join(self.setup_py_dir, 'setup.py') # Python2 __file__ should not be unicode if six.PY2 and isinstance(setup_py, six.text_type): setup_py = setup_py.encode(sys.getfilesystemencoding()) return setup_py @property def pyproject_toml(self): assert self.source_dir, "No source dir for %s" % self pp_toml = os.path.join(self.setup_py_dir, 'pyproject.toml') # Python2 __file__ should not be unicode if six.PY2 and isinstance(pp_toml, six.text_type): pp_toml = pp_toml.encode(sys.getfilesystemencoding()) return pp_toml def get_pep_518_info(self): """Get a list of the packages required to build the project, if any, and a flag indicating whether pyproject.toml is present, indicating that the build should be isolated. Build requirements can be specified in a pyproject.toml, as described in PEP 518. If this file exists but doesn't specify build requirements, pip will default to installing setuptools and wheel. """ if os.path.isfile(self.pyproject_toml): with open(self.pyproject_toml) as f: pp_toml = pytoml.load(f) build_sys = pp_toml.get('build-system', {}) return (build_sys.get('requires', ['setuptools', 'wheel']), True) return (['setuptools', 'wheel'], False) def run_egg_info(self): assert self.source_dir if self.name: logger.debug( 'Running setup.py (path:%s) egg_info for package %s', self.setup_py, self.name, ) else: logger.debug( 'Running setup.py (path:%s) egg_info for package from %s', self.setup_py, self.link, ) with indent_log(): script = SETUPTOOLS_SHIM % self.setup_py base_cmd = [os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', script] if self.isolated: base_cmd += ["--no-user-cfg"] egg_info_cmd = base_cmd + ['egg_info'] # We can't put the .egg-info files at the root, because then the # source code will be mistaken for an installed egg, causing # problems if self.editable: egg_base_option = [] else: egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info') ensure_dir(egg_info_dir) egg_base_option = ['--egg-base', 'pip-egg-info'] with self.build_env: call_subprocess( egg_info_cmd + egg_base_option, cwd=self.setup_py_dir, show_stdout=False, command_desc='python setup.py egg_info') if not self.req: if isinstance(parse_version(self.pkg_info()["Version"]), Version): op = "==" else: op = "===" self.req = Requirement( "".join([ self.pkg_info()["Name"], op, self.pkg_info()["Version"], ]) ) self._correct_build_location() else: metadata_name = canonicalize_name(self.pkg_info()["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( 'Running setup.py (path:%s) egg_info for package %s ' 'produced metadata for project name %s. Fix your ' '#egg=%s fragments.', self.setup_py, self.name, metadata_name, self.name ) self.req = Requirement(metadata_name) def egg_info_data(self, filename): if self.satisfied_by is not None: if not self.satisfied_by.has_metadata(filename): return None return self.satisfied_by.get_metadata(filename) assert self.source_dir filename = self.egg_info_path(filename) if not os.path.exists(filename): return None data = read_text_file(filename) return data def egg_info_path(self, filename): if self._egg_info_path is None: if self.editable: base = self.source_dir else: base = os.path.join(self.setup_py_dir, 'pip-egg-info') filenames = os.listdir(base) if self.editable: filenames = [] for root, dirs, files in os.walk(base): for dir in vcs.dirnames: if dir in dirs: dirs.remove(dir) # Iterate over a copy of ``dirs``, since mutating # a list while iterating over it can cause trouble. # (See https://github.com/pypa/pip/pull/462.) for dir in list(dirs): # Don't search in anything that looks like a virtualenv # environment if ( os.path.lexists( os.path.join(root, dir, 'bin', 'python') ) or os.path.exists( os.path.join( root, dir, 'Scripts', 'Python.exe' ) )): dirs.remove(dir) # Also don't search through tests elif dir == 'test' or dir == 'tests': dirs.remove(dir) filenames.extend([os.path.join(root, dir) for dir in dirs]) filenames = [f for f in filenames if f.endswith('.egg-info')] if not filenames: raise InstallationError( 'No files/directories in %s (from %s)' % (base, filename) ) assert filenames, \ "No files/directories in %s (from %s)" % (base, filename) # if we have more than one match, we pick the toplevel one. This # can easily be the case if there is a dist folder which contains # an extracted tarball for testing purposes. if len(filenames) > 1: filenames.sort( key=lambda x: x.count(os.path.sep) + (os.path.altsep and x.count(os.path.altsep) or 0) ) self._egg_info_path = os.path.join(base, filenames[0]) return os.path.join(self._egg_info_path, filename) def pkg_info(self): p = FeedParser() data = self.egg_info_data('PKG-INFO') if not data: logger.warning( 'No PKG-INFO file found in %s', display_path(self.egg_info_path('PKG-INFO')), ) p.feed(data or '') return p.close() _requirements_section_re = re.compile(r'\[(.*?)\]') @property def installed_version(self): return get_installed_version(self.name) def assert_source_matches_version(self): assert self.source_dir version = self.pkg_info()['version'] if self.req.specifier and version not in self.req.specifier: logger.warning( 'Requested %s, but installing version %s', self, version, ) else: logger.debug( 'Source in %s has version %s, which satisfies requirement %s', display_path(self.source_dir), version, self, ) def update_editable(self, obtain=True): if not self.link: logger.debug( "Cannot update repository at %s; repository location is " "unknown", self.source_dir, ) return assert self.editable assert self.source_dir if self.link.scheme == 'file': # Static paths don't get updated return assert '+' in self.link.url, "bad url: %r" % self.link.url if not self.update: return vc_type, url = self.link.url.split('+', 1) backend = vcs.get_backend(vc_type) if backend: vcs_backend = backend(self.link.url) if obtain: vcs_backend.obtain(self.source_dir) else: vcs_backend.export(self.source_dir) else: assert 0, ( 'Unexpected version control type (in %s): %s' % (self.link, vc_type)) def uninstall(self, auto_confirm=False, verbose=False, use_user_site=False): """ Uninstall the distribution currently satisfying this requirement. Prompts before removing or modifying files unless ``auto_confirm`` is True. Refuses to delete or modify files outside of ``sys.prefix`` - thus uninstallation within a virtual environment can only modify that virtual environment, even if the virtualenv is linked to global site-packages. """ if not self.check_if_exists(use_user_site): logger.warning("Skipping %s as it is not installed.", self.name) return dist = self.satisfied_by or self.conflicts_with uninstalled_pathset = UninstallPathSet.from_dist(dist) uninstalled_pathset.remove(auto_confirm, verbose) return uninstalled_pathset def archive(self, build_dir): assert self.source_dir create_archive = True archive_name = '%s-%s.zip' % (self.name, self.pkg_info()["version"]) archive_path = os.path.join(build_dir, archive_name) if os.path.exists(archive_path): response = ask_path_exists( 'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)bort ' % display_path(archive_path), ('i', 'w', 'b', 'a')) if response == 'i': create_archive = False elif response == 'w': logger.warning('Deleting %s', display_path(archive_path)) os.remove(archive_path) elif response == 'b': dest_file = backup_dir(archive_path) logger.warning( 'Backing up %s to %s', display_path(archive_path), display_path(dest_file), ) shutil.move(archive_path, dest_file) elif response == 'a': sys.exit(-1) if create_archive: zip = zipfile.ZipFile( archive_path, 'w', zipfile.ZIP_DEFLATED, allowZip64=True ) dir = os.path.normcase(os.path.abspath(self.setup_py_dir)) for dirpath, dirnames, filenames in os.walk(dir): if 'pip-egg-info' in dirnames: dirnames.remove('pip-egg-info') for dirname in dirnames: dirname = os.path.join(dirpath, dirname) name = self._clean_zip_name(dirname, dir) zipdir = zipfile.ZipInfo(self.name + '/' + name + '/') zipdir.external_attr = 0x1ED << 16 # 0o755 zip.writestr(zipdir, '') for filename in filenames: if filename == PIP_DELETE_MARKER_FILENAME: continue filename = os.path.join(dirpath, filename) name = self._clean_zip_name(filename, dir) zip.write(filename, self.name + '/' + name) zip.close() logger.info('Saved %s', display_path(archive_path)) def _clean_zip_name(self, name, prefix): assert name.startswith(prefix + os.path.sep), ( "name %r doesn't start with prefix %r" % (name, prefix) ) name = name[len(prefix) + 1:] name = name.replace(os.path.sep, '/') return name def match_markers(self, extras_requested=None): if not extras_requested: # Provide an extra to safely evaluate the markers # without matching any extra extras_requested = ('',) if self.markers is not None: return any( self.markers.evaluate({'extra': extra}) for extra in extras_requested) else: return True def install(self, install_options, global_options=None, root=None, home=None, prefix=None, warn_script_location=True, use_user_site=False, pycompile=True): global_options = global_options if global_options is not None else [] if self.editable: self.install_editable( install_options, global_options, prefix=prefix, ) return if self.is_wheel: version = wheel.wheel_version(self.source_dir) wheel.check_compatibility(version, self.name) self.move_wheel_files( self.source_dir, root=root, prefix=prefix, home=home, warn_script_location=warn_script_location, use_user_site=use_user_site, pycompile=pycompile, ) self.install_succeeded = True return # Extend the list of global and install options passed on to # the setup.py call with the ones from the requirements file. # Options specified in requirements file override those # specified on the command line, since the last option given # to setup.py is the one that is used. global_options = list(global_options) + \ self.options.get('global_options', []) install_options = list(install_options) + \ self.options.get('install_options', []) if self.isolated: global_options = global_options + ["--no-user-cfg"] with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') install_args = self.get_install_args( global_options, record_filename, root, prefix, pycompile, ) msg = 'Running setup.py install for %s' % (self.name,) with open_spinner(msg) as spinner: with indent_log(): with self.build_env: call_subprocess( install_args + install_options, cwd=self.setup_py_dir, show_stdout=False, spinner=spinner, ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) return self.install_succeeded = True def prepend_root(path): if root is None or not os.path.isabs(path): return path else: return change_root(root, path) with open(record_filename) as f: for line in f: directory = os.path.dirname(line) if directory.endswith('.egg-info'): egg_info_dir = prepend_root(directory) break else: logger.warning( 'Could not find .egg-info directory in install record' ' for %s', self, ) # FIXME: put the record somewhere # FIXME: should this be an error? return new_lines = [] with open(record_filename) as f: for line in f: filename = line.strip() if os.path.isdir(filename): filename += os.path.sep new_lines.append( os.path.relpath(prepend_root(filename), egg_info_dir) ) new_lines.sort() ensure_dir(egg_info_dir) inst_files_path = os.path.join(egg_info_dir, 'installed-files.txt') with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n') def ensure_has_source_dir(self, parent_dir): """Ensure that a source_dir is set. This will create a temporary build dir if the name of the requirement isn't known yet. :param parent_dir: The ideal pip parent_dir for the source_dir. Generally src_dir for editables and build_dir for sdists. :return: self.source_dir """ if self.source_dir is None: self.source_dir = self.build_location(parent_dir) return self.source_dir def get_install_args(self, global_options, record_filename, root, prefix, pycompile): install_args = [os.environ.get('PIP_PYTHON_PATH', sys.executable), "-u"] install_args.append('-c') install_args.append(SETUPTOOLS_SHIM % self.setup_py) install_args += list(global_options) + \ ['install', '--record', record_filename] install_args += ['--single-version-externally-managed'] if root is not None: install_args += ['--root', root] if prefix is not None: install_args += ['--prefix', prefix] if pycompile: install_args += ["--compile"] else: install_args += ["--no-compile"] if running_under_virtualenv(): py_ver_str = 'python' + sysconfig.get_python_version() install_args += ['--install-headers', os.path.join(sys.prefix, 'include', 'site', py_ver_str, self.name)] return install_args def remove_temporary_source(self): """Remove the source files from this requirement, if they are marked for deletion""" if self.source_dir and os.path.exists( os.path.join(self.source_dir, PIP_DELETE_MARKER_FILENAME)): logger.debug('Removing source in %s', self.source_dir) rmtree(self.source_dir) self.source_dir = None self._temp_build_dir.cleanup() self.build_env.cleanup() def install_editable(self, install_options, global_options=(), prefix=None): logger.info('Running setup.py develop for %s', self.name) if self.isolated: global_options = list(global_options) + ["--no-user-cfg"] if prefix: prefix_param = ['--prefix={}'.format(prefix)] install_options = list(install_options) + prefix_param with indent_log(): # FIXME: should we do --install-headers here too? with self.build_env: call_subprocess( [ os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', SETUPTOOLS_SHIM % self.setup_py ] + list(global_options) + ['develop', '--no-deps'] + list(install_options), cwd=self.setup_py_dir, show_stdout=False, ) self.install_succeeded = True def check_if_exists(self, use_user_site): """Find an installed distribution that satisfies or conflicts with this requirement, and set self.satisfied_by or self.conflicts_with appropriately. """ if self.req is None: return False try: # get_distribution() will resolve the entire list of requirements # anyway, and we've already determined that we need the requirement # in question, so strip the marker so that we don't try to # evaluate it. no_marker = Requirement(str(self.req)) no_marker.marker = None self.satisfied_by = pkg_resources.get_distribution(str(no_marker)) if self.editable and self.satisfied_by: self.conflicts_with = self.satisfied_by # when installing editables, nothing pre-existing should ever # satisfy self.satisfied_by = None return True except pkg_resources.DistributionNotFound: return False except pkg_resources.VersionConflict: existing_dist = pkg_resources.get_distribution( self.req.name ) if use_user_site: if dist_in_usersite(existing_dist): self.conflicts_with = existing_dist elif (running_under_virtualenv() and dist_in_site_packages(existing_dist)): raise InstallationError( "Will not install to the user site because it will " "lack sys.path precedence to %s in %s" % (existing_dist.project_name, existing_dist.location) ) else: self.conflicts_with = existing_dist return True @property def is_wheel(self): return self.link and self.link.is_wheel def move_wheel_files(self, wheeldir, root=None, home=None, prefix=None, warn_script_location=True, use_user_site=False, pycompile=True): move_wheel_files( self.name, self.req, wheeldir, user=use_user_site, home=home, root=root, prefix=prefix, pycompile=pycompile, isolated=self.isolated, warn_script_location=warn_script_location, ) def get_dist(self): """Return a pkg_resources.Distribution built from self.egg_info_path""" egg_info = self.egg_info_path('').rstrip(os.path.sep) base_dir = os.path.dirname(egg_info) metadata = pkg_resources.PathMetadata(base_dir, egg_info) dist_name = os.path.splitext(os.path.basename(egg_info))[0] return pkg_resources.Distribution( os.path.dirname(egg_info), project_name=dist_name, metadata=metadata, ) @property def has_hash_options(self): """Return whether any known-good hashes are specified as options. These activate --require-hashes mode; hashes specified as part of a URL do not. """ return bool(self.options.get('hashes', {})) def hashes(self, trust_internet=True): """Return a hash-comparer that considers my option- and URL-based hashes to be known-good. Hashes in URLs--ones embedded in the requirements file, not ones downloaded from an index server--are almost peers with ones from flags. They satisfy --require-hashes (whether it was implicitly or explicitly activated) but do not activate it. md5 and sha224 are not allowed in flags, which should nudge people toward good algos. We always OR all hashes together, even ones from URLs. :param trust_internet: Whether to trust URL-based (#md5=...) hashes downloaded from the internet, as by populate_link() """ good_hashes = self.options.get('hashes', {}).copy() link = self.link if trust_internet else self.original_link if link and link.hash: good_hashes.setdefault(link.hash_name, []).append(link.hash) return Hashes(good_hashes)
def run(self, options, args): cmdoptions.check_install_build_global(options) upgrade_strategy = "to-satisfy-only" if options.upgrade: upgrade_strategy = options.upgrade_strategy if options.build_dir: options.build_dir = os.path.abspath(options.build_dir) options.src_dir = os.path.abspath(options.src_dir) install_options = options.install_options or [] if options.use_user_site: if options.prefix_path: raise CommandError( "Can not combine '--user' and '--prefix' as they imply " "different installation locations") if virtualenv_no_global(): raise InstallationError( "Can not perform a '--user' install. User site-packages " "are not visible in this virtualenv.") install_options.append('--user') install_options.append('--prefix=') target_temp_dir = TempDirectory(kind="target") if options.target_dir: options.ignore_installed = True options.target_dir = os.path.abspath(options.target_dir) if (os.path.exists(options.target_dir) and not os.path.isdir(options.target_dir)): raise CommandError( "Target path exists but is not a directory, will not " "continue.") # Create a target directory for using with the target option target_temp_dir.create() install_options.append('--home=' + target_temp_dir.path) global_options = options.global_options or [] with self._build_session(options) as session: finder = self._build_package_finder(options, session) build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) if options.cache_dir and not check_path_owner(options.cache_dir): logger.warning( "The directory '%s' or its parent directory is not owned " "by the current user and caching wheels has been " "disabled. check the permissions and owner of that " "directory. If executing pip with sudo, you may want " "sudo's -H flag.", options.cache_dir, ) options.cache_dir = None with TempDirectory(options.build_dir, delete=build_delete, kind="install") as directory: requirement_set = RequirementSet( require_hashes=options.require_hashes, ) try: self.populate_requirement_set(requirement_set, args, options, finder, session, self.name, wheel_cache) preparer = RequirementPreparer( build_dir=directory.path, src_dir=options.src_dir, download_dir=None, wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, ) resolver = Resolver( preparer=preparer, finder=finder, session=session, wheel_cache=wheel_cache, use_user_site=options.use_user_site, upgrade_strategy=upgrade_strategy, force_reinstall=options.force_reinstall, ignore_dependencies=options.ignore_dependencies, ignore_requires_python=options.ignore_requires_python, ignore_installed=options.ignore_installed, isolated=options.isolated_mode, ) resolver.resolve(requirement_set) # If caching is disabled or wheel is not installed don't # try to build wheels. if wheel and options.cache_dir: # build wheels before install. wb = WheelBuilder( finder, preparer, wheel_cache, build_options=[], global_options=[], ) # Ignore the result: a failed wheel will be # installed from the sdist/vcs whatever. wb.build(requirement_set.requirements.values(), session=session, autobuilding=True) to_install = resolver.get_installation_order( requirement_set) # Consistency Checking of the package set we're installing. should_warn_about_conflicts = ( not options.ignore_dependencies and options.warn_about_conflicts) if should_warn_about_conflicts: self._warn_about_conflicts(to_install) # Don't warn about script install locations if # --target has been specified warn_script_location = options.warn_script_location if options.target_dir: warn_script_location = False installed = install_given_reqs( to_install, install_options, global_options, root=options.root_path, home=target_temp_dir.path, prefix=options.prefix_path, pycompile=options.compile, warn_script_location=warn_script_location, use_user_site=options.use_user_site, ) possible_lib_locations = get_lib_location_guesses( user=options.use_user_site, home=target_temp_dir.path, root=options.root_path, prefix=options.prefix_path, isolated=options.isolated_mode, ) reqs = sorted(installed, key=operator.attrgetter('name')) items = [] for req in reqs: item = req.name try: installed_version = get_installed_version( req.name, possible_lib_locations) if installed_version: item += '-' + installed_version except Exception: pass items.append(item) installed = ' '.join(items) if installed: logger.info('Successfully installed %s', installed) except EnvironmentError as error: show_traceback = (self.verbosity >= 1) message = create_env_error_message( error, show_traceback, options.use_user_site, ) logger.error(message, exc_info=show_traceback) return ERROR except PreviousBuildDirError: options.no_clean = True raise finally: # Clean up if not options.no_clean: requirement_set.cleanup_files() wheel_cache.cleanup() if options.target_dir: self._handle_target_dir(options.target_dir, target_temp_dir, options.upgrade) return requirement_set
class BuildEnvironment(object): """Creates and manages an isolated environment to install build deps """ def __init__(self, no_clean): self._temp_dir = TempDirectory(kind="build-env") self._no_clean = no_clean @property def path(self): return self._temp_dir.path def __enter__(self): self._temp_dir.create() self.save_path = os.environ.get('PATH', None) self.save_pythonpath = os.environ.get('PYTHONPATH', None) self.save_nousersite = os.environ.get('PYTHONNOUSERSITE', None) install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' install_dirs = get_paths(install_scheme, vars={ 'base': self.path, 'platbase': self.path, }) scripts = install_dirs['scripts'] if self.save_path: os.environ['PATH'] = scripts + os.pathsep + self.save_path else: os.environ['PATH'] = scripts + os.pathsep + os.defpath # Note: prefer distutils' sysconfig to get the # library paths so PyPy is correctly supported. purelib = get_python_lib(plat_specific=0, prefix=self.path) platlib = get_python_lib(plat_specific=1, prefix=self.path) if purelib == platlib: lib_dirs = purelib else: lib_dirs = purelib + os.pathsep + platlib if self.save_pythonpath: os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ self.save_pythonpath else: os.environ['PYTHONPATH'] = lib_dirs os.environ['PYTHONNOUSERSITE'] = '1' return self.path def __exit__(self, exc_type, exc_val, exc_tb): if not self._no_clean: self._temp_dir.cleanup() def restore_var(varname, old_value): if old_value is None: os.environ.pop(varname, None) else: os.environ[varname] = old_value restore_var('PATH', self.save_path) restore_var('PYTHONPATH', self.save_pythonpath) restore_var('PYTHONNOUSERSITE', self.save_nousersite) def cleanup(self): self._temp_dir.cleanup()
def run(self, options, args): cmdoptions.check_install_build_global(options) upgrade_strategy = "to-satisfy-only" if options.upgrade: upgrade_strategy = options.upgrade_strategy if options.build_dir: options.build_dir = os.path.abspath(options.build_dir) options.src_dir = os.path.abspath(options.src_dir) install_options = options.install_options or [] if options.use_user_site: if options.prefix_path: raise CommandError( "Can not combine '--user' and '--prefix' as they imply " "different installation locations" ) if virtualenv_no_global(): raise InstallationError( "Can not perform a '--user' install. User site-packages " "are not visible in this virtualenv." ) install_options.append('--user') install_options.append('--prefix=') target_temp_dir = TempDirectory(kind="target") if options.target_dir: options.ignore_installed = True options.target_dir = os.path.abspath(options.target_dir) if (os.path.exists(options.target_dir) and not os.path.isdir(options.target_dir)): raise CommandError( "Target path exists but is not a directory, will not " "continue." ) # Create a target directory for using with the target option target_temp_dir.create() install_options.append('--home=' + target_temp_dir.path) global_options = options.global_options or [] with self._build_session(options) as session: finder = self._build_package_finder(options, session) build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) if options.cache_dir and not check_path_owner(options.cache_dir): logger.warning( "The directory '%s' or its parent directory is not owned " "by the current user and caching wheels has been " "disabled. check the permissions and owner of that " "directory. If executing pip with sudo, you may want " "sudo's -H flag.", options.cache_dir, ) options.cache_dir = None with TempDirectory( options.build_dir, delete=build_delete, kind="install" ) as directory: requirement_set = RequirementSet( require_hashes=options.require_hashes, ) try: self.populate_requirement_set( requirement_set, args, options, finder, session, self.name, wheel_cache ) preparer = RequirementPreparer( build_dir=directory.path, src_dir=options.src_dir, download_dir=None, wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, ) resolver = Resolver( preparer=preparer, finder=finder, session=session, wheel_cache=wheel_cache, use_user_site=options.use_user_site, upgrade_strategy=upgrade_strategy, force_reinstall=options.force_reinstall, ignore_dependencies=options.ignore_dependencies, ignore_requires_python=options.ignore_requires_python, ignore_installed=options.ignore_installed, isolated=options.isolated_mode, ) resolver.resolve(requirement_set) # If caching is disabled or wheel is not installed don't # try to build wheels. if wheel and options.cache_dir: # build wheels before install. wb = WheelBuilder( finder, preparer, wheel_cache, build_options=[], global_options=[], ) # Ignore the result: a failed wheel will be # installed from the sdist/vcs whatever. wb.build( requirement_set.requirements.values(), session=session, autobuilding=True ) to_install = resolver.get_installation_order( requirement_set ) # Consistency Checking of the package set we're installing. should_warn_about_conflicts = ( not options.ignore_dependencies and options.warn_about_conflicts ) if should_warn_about_conflicts: self._warn_about_conflicts(to_install) # Don't warn about script install locations if # --target has been specified warn_script_location = options.warn_script_location if options.target_dir: warn_script_location = False installed = install_given_reqs( to_install, install_options, global_options, root=options.root_path, home=target_temp_dir.path, prefix=options.prefix_path, pycompile=options.compile, warn_script_location=warn_script_location, use_user_site=options.use_user_site, ) possible_lib_locations = get_lib_location_guesses( user=options.use_user_site, home=target_temp_dir.path, root=options.root_path, prefix=options.prefix_path, isolated=options.isolated_mode, ) reqs = sorted(installed, key=operator.attrgetter('name')) items = [] for req in reqs: item = req.name try: installed_version = get_installed_version( req.name, possible_lib_locations ) if installed_version: item += '-' + installed_version except Exception: pass items.append(item) installed = ' '.join(items) if installed: logger.info('Successfully installed %s', installed) except EnvironmentError as error: show_traceback = (self.verbosity >= 1) message = create_env_error_message( error, show_traceback, options.use_user_site, ) logger.error(message, exc_info=show_traceback) return ERROR except PreviousBuildDirError: options.no_clean = True raise finally: # Clean up if not options.no_clean: requirement_set.cleanup_files() wheel_cache.cleanup() if options.target_dir: self._handle_target_dir( options.target_dir, target_temp_dir, options.upgrade ) return requirement_set
class BuildEnvironment(object): """Creates and manages an isolated environment to install build deps """ def __init__(self): # type: () -> None self._temp_dir = TempDirectory(kind="build-env") self._temp_dir.create() self._prefixes = OrderedDict( ((name, _Prefix(os.path.join(self._temp_dir.path, name))) for name in ('normal', 'overlay'))) self._bin_dirs = [] # type: List[str] self._lib_dirs = [] # type: List[str] for prefix in reversed(list(self._prefixes.values())): self._bin_dirs.append(prefix.bin_dir) self._lib_dirs.extend(prefix.lib_dirs) # Customize site to: # - ensure .pth files are honored # - prevent access to system site packages system_sites = { os.path.normcase(site) for site in ( get_python_lib(plat_specific=False), get_python_lib(plat_specific=True), ) } self._site_dir = os.path.join(self._temp_dir.path, 'site') if not os.path.exists(self._site_dir): os.mkdir(self._site_dir) with open(os.path.join(self._site_dir, 'sitecustomize.py'), 'w') as fp: fp.write( textwrap.dedent(''' import os, site, sys # First, drop system-sites related paths. original_sys_path = sys.path[:] known_paths = set() for path in {system_sites!r}: site.addsitedir(path, known_paths=known_paths) system_paths = set( os.path.normcase(path) for path in sys.path[len(original_sys_path):] ) original_sys_path = [ path for path in original_sys_path if os.path.normcase(path) not in system_paths ] sys.path = original_sys_path # Second, add lib directories. # ensuring .pth file are processed. for path in {lib_dirs!r}: assert not path in sys.path site.addsitedir(path) ''').format(system_sites=system_sites, lib_dirs=self._lib_dirs)) def __enter__(self): self._save_env = { name: os.environ.get(name, None) for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH') } path = self._bin_dirs[:] old_path = self._save_env['PATH'] if old_path: path.extend(old_path.split(os.pathsep)) pythonpath = [self._site_dir] os.environ.update({ 'PATH': os.pathsep.join(path), 'PYTHONNOUSERSITE': '1', 'PYTHONPATH': os.pathsep.join(pythonpath), }) def __exit__(self, exc_type, exc_val, exc_tb): for varname, old_value in self._save_env.items(): if old_value is None: os.environ.pop(varname, None) else: os.environ[varname] = old_value def cleanup(self): # type: () -> None self._temp_dir.cleanup() def check_requirements(self, reqs): # type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]] """Return 2 sets: - conflicting requirements: set of (installed, wanted) reqs tuples - missing requirements: set of reqs """ missing = set() conflicting = set() if reqs: ws = WorkingSet(self._lib_dirs) for req in reqs: try: if ws.find(Requirement.parse(req)) is None: missing.add(req) except VersionConflict as e: conflicting.add( (str(e.args[0].as_requirement()), str(e.args[1]))) return conflicting, missing def install_requirements( self, finder, # type: PackageFinder requirements, # type: Iterable[str] prefix_as_string, # type: str message # type: Optional[str] ): # type: (...) -> None prefix = self._prefixes[prefix_as_string] assert not prefix.setup prefix.setup = True if not requirements: return args = [ sys.executable, os.path.dirname(pip_location), 'install', '--ignore-installed', '--no-user', '--prefix', prefix.path, '--no-warn-script-location', ] # type: List[str] if logger.getEffectiveLevel() <= logging.DEBUG: args.append('-v') for format_control in ('no_binary', 'only_binary'): formats = getattr(finder.format_control, format_control) args.extend(('--' + format_control.replace('_', '-'), ','.join(sorted(formats or {':none:'})))) if finder.index_urls: args.extend(['-i', finder.index_urls[0]]) for extra_index in finder.index_urls[1:]: args.extend(['--extra-index-url', extra_index]) else: args.append('--no-index') for link in finder.find_links: args.extend(['--find-links', link]) for _, host, _ in finder.secure_origins: args.extend(['--trusted-host', host]) if finder.allow_all_prereleases: args.append('--pre') args.append('--') args.extend(requirements) with open_spinner(message) as spinner: call_subprocess(args, show_stdout=False, spinner=spinner)
class UninstallPathSet(object): """A set of file paths to be removed in the uninstallation of a requirement.""" def __init__(self, dist): self.paths = set() self._refuse = set() self.pth = {} self.dist = dist self.save_dir = TempDirectory(kind="uninstall") self._moved_paths = [] def _permitted(self, path): """ Return True if the given path is one we are permitted to remove/modify, False otherwise. """ return is_local(path) def add(self, path): head, tail = os.path.split(path) # we normalize the head to resolve parent directory symlinks, but not # the tail, since we only want to uninstall symlinks, not their targets path = os.path.join(normalize_path(head), os.path.normcase(tail)) if not os.path.exists(path): return if self._permitted(path): self.paths.add(path) else: self._refuse.add(path) # __pycache__ files can show up after 'installed-files.txt' is created, # due to imports if os.path.splitext(path)[1] == '.py' and uses_pycache: self.add(cache_from_source(path)) def add_pth(self, pth_file, entry): pth_file = normalize_path(pth_file) if self._permitted(pth_file): if pth_file not in self.pth: self.pth[pth_file] = UninstallPthEntries(pth_file) self.pth[pth_file].add(entry) else: self._refuse.add(pth_file) def _stash(self, path): return os.path.join( self.save_dir.path, os.path.splitdrive(path)[1].lstrip(os.path.sep) ) def remove(self, auto_confirm=False, verbose=False): """Remove paths in ``self.paths`` with confirmation (unless ``auto_confirm`` is True).""" if not self.paths: logger.info( "Can't uninstall '%s'. No files were found to uninstall.", self.dist.project_name, ) return dist_name_version = ( self.dist.project_name + "-" + self.dist.version ) logger.info('Uninstalling %s:', dist_name_version) with indent_log(): if auto_confirm or self._allowed_to_proceed(verbose): self.save_dir.create() for path in sorted(compact(self.paths)): new_path = self._stash(path) logger.debug('Removing file or directory %s', path) self._moved_paths.append(path) renames(path, new_path) for pth in self.pth.values(): pth.remove() logger.info('Successfully uninstalled %s', dist_name_version) def _allowed_to_proceed(self, verbose): """Display which files would be deleted and prompt for confirmation """ def _display(msg, paths): if not paths: return logger.info(msg) with indent_log(): for path in sorted(compact(paths)): logger.info(path) if not verbose: will_remove, will_skip = compress_for_output_listing(self.paths) else: # In verbose mode, display all the files that are going to be # deleted. will_remove = list(self.paths) will_skip = set() _display('Would remove:', will_remove) _display('Would not remove (might be manually added):', will_skip) _display('Would not remove (outside of prefix):', self._refuse) return ask('Proceed (y/n)? ', ('y', 'n')) == 'y' def rollback(self): """Rollback the changes previously made by remove().""" if self.save_dir.path is None: logger.error( "Can't roll back %s; was not uninstalled", self.dist.project_name, ) return False logger.info('Rolling back uninstall of %s', self.dist.project_name) for path in self._moved_paths: tmp_path = self._stash(path) logger.debug('Replacing %s', path) renames(tmp_path, path) for pth in self.pth.values(): pth.rollback() def commit(self): """Remove temporary save dir: rollback will no longer be possible.""" self.save_dir.cleanup() self._moved_paths = [] @classmethod def from_dist(cls, dist): dist_path = normalize_path(dist.location) if not dist_is_local(dist): logger.info( "Not uninstalling %s at %s, outside environment %s", dist.key, dist_path, sys.prefix, ) return cls(dist) if dist_path in {p for p in {sysconfig.get_path("stdlib"), sysconfig.get_path("platstdlib")} if p}: logger.info( "Not uninstalling %s at %s, as it is in the standard library.", dist.key, dist_path, ) return cls(dist) paths_to_remove = cls(dist) develop_egg_link = egg_link_path(dist) develop_egg_link_egg_info = '{}.egg-info'.format( pkg_resources.to_filename(dist.project_name)) egg_info_exists = dist.egg_info and os.path.exists(dist.egg_info) # Special case for distutils installed package distutils_egg_info = getattr(dist._provider, 'path', None) # Uninstall cases order do matter as in the case of 2 installs of the # same package, pip needs to uninstall the currently detected version if (egg_info_exists and dist.egg_info.endswith('.egg-info') and not dist.egg_info.endswith(develop_egg_link_egg_info)): # if dist.egg_info.endswith(develop_egg_link_egg_info), we # are in fact in the develop_egg_link case paths_to_remove.add(dist.egg_info) if dist.has_metadata('installed-files.txt'): for installed_file in dist.get_metadata( 'installed-files.txt').splitlines(): path = os.path.normpath( os.path.join(dist.egg_info, installed_file) ) paths_to_remove.add(path) # FIXME: need a test for this elif block # occurs with --single-version-externally-managed/--record outside # of pip elif dist.has_metadata('top_level.txt'): if dist.has_metadata('namespace_packages.txt'): namespaces = dist.get_metadata('namespace_packages.txt') else: namespaces = [] for top_level_pkg in [ p for p in dist.get_metadata('top_level.txt').splitlines() if p and p not in namespaces]: path = os.path.join(dist.location, top_level_pkg) paths_to_remove.add(path) paths_to_remove.add(path + '.py') paths_to_remove.add(path + '.pyc') paths_to_remove.add(path + '.pyo') elif distutils_egg_info: raise UninstallationError( "Cannot uninstall {!r}. It is a distutils installed project " "and thus we cannot accurately determine which files belong " "to it which would lead to only a partial uninstall.".format( dist.project_name, ) ) elif dist.location.endswith('.egg'): # package installed by easy_install # We cannot match on dist.egg_name because it can slightly vary # i.e. setuptools-0.6c11-py2.6.egg vs setuptools-0.6rc11-py2.6.egg paths_to_remove.add(dist.location) easy_install_egg = os.path.split(dist.location)[1] easy_install_pth = os.path.join(os.path.dirname(dist.location), 'easy-install.pth') paths_to_remove.add_pth(easy_install_pth, './' + easy_install_egg) elif egg_info_exists and dist.egg_info.endswith('.dist-info'): for path in uninstallation_paths(dist): paths_to_remove.add(path) elif develop_egg_link: # develop egg with open(develop_egg_link, 'r') as fh: link_pointer = os.path.normcase(fh.readline().strip()) assert (link_pointer == dist.location), ( 'Egg-link %s does not match installed location of %s ' '(at %s)' % (link_pointer, dist.project_name, dist.location) ) paths_to_remove.add(develop_egg_link) easy_install_pth = os.path.join(os.path.dirname(develop_egg_link), 'easy-install.pth') paths_to_remove.add_pth(easy_install_pth, dist.location) else: logger.debug( 'Not sure how to uninstall: %s - Check: %s', dist, dist.location, ) # find distutils scripts= scripts if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'): for script in dist.metadata_listdir('scripts'): if dist_in_usersite(dist): bin_dir = bin_user else: bin_dir = bin_py paths_to_remove.add(os.path.join(bin_dir, script)) if WINDOWS: paths_to_remove.add(os.path.join(bin_dir, script) + '.bat') # find console_scripts _scripts_to_remove = [] console_scripts = dist.get_entry_map(group='console_scripts') for name in console_scripts.keys(): _scripts_to_remove.extend(_script_names(dist, name, False)) # find gui_scripts gui_scripts = dist.get_entry_map(group='gui_scripts') for name in gui_scripts.keys(): _scripts_to_remove.extend(_script_names(dist, name, True)) for s in _scripts_to_remove: paths_to_remove.add(s) return paths_to_remove