def from_editable(cls, editable_req, comes_from=None, default_vcs=None, isolated=False, options=None, wheel_cache=None, constraint=False): from pip.index import Link name, url, extras_override = parse_editable( editable_req, default_vcs) if url.startswith('file:'): source_dir = url_to_path(url) else: source_dir = None if name is not None: try: req = Requirement(name) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % req) else: req = None return cls( req, comes_from, source_dir=source_dir, editable=True, link=Link(url), constraint=constraint, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, extras=extras_override or (), )
def from_editable(cls, editable_req, comes_from=None, default_vcs=None, isolated=False, options=None, wheel_cache=None, constraint=False): from pip.index import Link name, url, extras_override, editable_options = parse_editable( editable_req, default_vcs) if url.startswith('file:'): source_dir = url_to_path(url) else: source_dir = None res = cls(name, comes_from, source_dir=source_dir, editable=True, link=Link(url), constraint=constraint, editable_options=editable_options, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache) if extras_override is not None: res.extras = extras_override return res
def _find_all_versions(self, project_name): """Find all available versions for project_name This checks index_urls, find_links and dependency_links All versions found are returned See _link_package_versions for details on which files are accepted """ index_locations = self._get_index_urls_locations(project_name) file_locations, url_locations = self._sort_locations(index_locations) fl_file_loc, fl_url_loc = self._sort_locations(self.find_links) file_locations.extend(fl_file_loc) url_locations.extend(fl_url_loc) _flocations, _ulocations = self._sort_locations(self.dependency_links) file_locations.extend(_flocations) # We trust every url that the user has given us whether it was given # via --index-url or --find-links locations = [Link(url, trusted=True) for url in url_locations] # We explicitly do not trust links that came from dependency_links locations.extend([Link(url) for url in _ulocations]) logger.debug("%d location(s) to search for versions of %s:", len(locations), project_name) for location in locations: logger.debug("* %s", location) self._validate_secure_origin(logger, location) find_links_versions = list( self._package_versions( # We trust every directly linked archive in find_links (Link(url, "-f", trusted=True) for url in self.find_links), project_name.lower(), ) ) page_versions = [] for page in self._get_pages(locations, project_name): logger.debug("Analyzing links from page %s", page.url) with indent_log(): page_versions.extend(self._package_versions(page.links, project_name.lower())) dependency_versions = list( self._package_versions((Link(url) for url in self.dependency_links), project_name.lower()) ) if dependency_versions: logger.debug( "dependency_links found: %s", ", ".join([version.location.url for version in dependency_versions]) ) file_versions = list(self._package_versions((Link(url) for url in file_locations), project_name.lower())) if file_versions: file_versions.sort(reverse=True) logger.debug( "Local files found: %s", ", ".join([url_to_path(candidate.location.url) for candidate in file_versions]) ) # This is an intentional priority ordering return file_versions + find_links_versions + page_versions + dependency_versions
def _sort_locations(locations): """ Sort locations into "files" (archives) and "urls", and return a pair of lists (files,urls) """ files = [] urls = [] # puts the url for the given file path into the appropriate # list def sort_path(path): url = path_to_url2(path) if mimetypes.guess_type(url, strict=False)[0] == 'text/html': urls.append(url) else: files.append(url) for url in locations: if url.startswith('file:'): path = url_to_path(url) if os.path.isdir(path): path = os.path.realpath(path) for item in os.listdir(path): sort_path(os.path.join(path, item)) elif os.path.isfile(path): sort_path(path) else: urls.append(url) return files, urls
def open_local_or_remote_file(link, session): """ Open local or remote file for reading. :type link: pip.index.Link :type session: requests.Session :raises ValueError: If link points to a local directory. :return: a context manager to the opened file-like object """ url = link.url_without_fragment if is_file_url(link): # Local URL local_path = url_to_path(url) if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(url)) else: with open(local_path, 'rb') as local_file: yield local_file else: # Remote URL headers = {"Accept-Encoding": "identity"} response = session.get(url, headers=headers, stream=True) try: yield response.raw finally: response.close()
def from_editable(cls, editable_req, comes_from=None, default_vcs=None, isolated=False, options=None, wheel_cache=None, constraint=False): from pip.index import Link name, url, extras_override = parse_editable(editable_req, default_vcs) if url.startswith('file:'): source_dir = url_to_path(url) else: source_dir = None if name is not None: try: req = Requirement(name) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % req) else: req = None return cls( req, comes_from, source_dir=source_dir, editable=True, link=Link(url), constraint=constraint, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, extras=extras_override or (), )
def _downloaded_filename(self): """Download the package's archive if necessary, and return its filename. --no-deps is implied, as we have reimplemented the bits that would ordinarily do dependency resolution. """ # Peep doesn't support requirements that don't come down as a single # file, because it can't hash them. Thus, it doesn't support editable # requirements, because pip itself doesn't support editable # requirements except for "local projects or a VCS url". Nor does it # support VCS requirements yet, because we haven't yet come up with a # portable, deterministic way to hash them. In summary, all we support # is == requirements and tarballs/zips/etc. # TODO: Stop on reqs that are editable or aren't ==. finder = package_finder(self._argv) # If the requirement isn't already specified as a URL, get a URL # from an index: link = (finder.find_requirement(self._req, upgrade=False) if self._req.url is None else Link(self._req.url)) if link: lower_scheme = link.scheme.lower() # pip lower()s it for some reason. if lower_scheme == 'http' or lower_scheme == 'https': file_path = self._download(link) return basename(file_path) elif lower_scheme == 'file': # The following is inspired by pip's unpack_file_url(): link_path = url_to_path(link.url_without_fragment) if isdir(link_path): raise UnsupportedRequirementError( "%s: %s is a directory. So that it can compute " "a hash, peep supports only filesystem paths which " "point to files" % (self._req, link.url_without_fragment)) else: copy(link_path, self._temp_path) return basename(link_path) else: raise UnsupportedRequirementError( "%s: The download link, %s, would not result in a file " "that can be hashed. Peep supports only == requirements, " "file:// URLs pointing to files (not folders), and " "http:// and https:// URLs pointing to tarballs, zips, " "etc." % (self._req, link.url)) else: raise UnsupportedRequirementError( "%s: couldn't determine where to download this requirement from." % (self._req,))
def test_sync_with_editable_uses_abspath(from_editable, small_fake_package_dir): ireq = from_editable(small_fake_package_dir) rel_path = os.path.relpath(url_to_path(ireq.link.url)) ireq.link.url = 'file:{}'.format(rel_path.replace(os.path.sep, '/')) with mock.patch('prequ.sync.check_call') as check_call: sync({ireq}, set()) check_call.assert_called_once_with([ 'pip', 'install', '-q', '-e', path_to_url(os.path.abspath(small_fake_package_dir)) ])
def _downloaded_filename(self): """Download the package's archive if necessary, and return its filename. --no-deps is implied, as we have reimplemented the bits that would ordinarily do dependency resolution. """ # Peep doesn't support requirements that don't come down as a single # file, because it can't hash them. Thus, it doesn't support editable # requirements, because pip itself doesn't support editable # requirements except for "local projects or a VCS url". Nor does it # support VCS requirements yet, because we haven't yet come up with a # portable, deterministic way to hash them. In summary, all we support # is == requirements and tarballs/zips/etc. # TODO: Stop on reqs that are editable or aren't ==. finder = package_finder(self._argv) # If the requirement isn't already specified as a URL, get a URL # from an index: link = (finder.find_requirement(self._req, upgrade=False) if self._req.url is None else Link(self._req.url)) if link: lower_scheme = link.scheme.lower( ) # pip lower()s it for some reason. if lower_scheme == 'http' or lower_scheme == 'https': file_path = self._download(link) return basename(file_path) elif lower_scheme == 'file': # The following is inspired by pip's unpack_file_url(): link_path = url_to_path(link.url_without_fragment) if isdir(link_path): raise UnsupportedRequirementError( "%s: %s is a directory. So that it can compute " "a hash, peep supports only filesystem paths which " "point to files" % (self._req, link.url_without_fragment)) else: copy(link_path, self._temp_path) return basename(link_path) else: raise UnsupportedRequirementError( "%s: The download link, %s, would not result in a file " "that can be hashed. Peep supports only == requirements, " "file:// URLs pointing to files (not folders), and " "http:// and https:// URLs pointing to tarballs, zips, " "etc." % (self._req, link.url)) else: raise UnsupportedRequirementError( "%s: couldn't determine where to download this requirement from." % (self._req, ))
def _sort_locations(locations, expand_dir=False): """ Sort locations into "files" (archives) and "urls", and return a pair of lists (files,urls) """ files = [] urls = [] # puts the url for the given file path into the appropriate list def sort_path(path): url = path_to_url(path) if mimetypes.guess_type(url, strict=False)[0] == 'text/html': urls.append(url) else: files.append(url) for url in locations: is_local_path = os.path.exists(url) is_file_url = url.startswith('file:') if is_local_path or is_file_url: if is_local_path: path = url else: path = url_to_path(url) if os.path.isdir(path): if expand_dir: path = os.path.realpath(path) for item in os.listdir(path): sort_path(os.path.join(path, item)) elif is_file_url: urls.append(url) elif os.path.isfile(path): sort_path(path) else: logger.warning( "Url '%s' is ignored: it is neither a file " "nor a directory.", url) elif is_url(url): # Only add url with clear scheme urls.append(url) else: logger.warning( "Url '%s' is ignored. It is either a non-existing " "path or lacks a specific scheme.", url) return files, urls
def from_editable(cls, editable_req, comes_from=None, default_vcs=None): name, url, extras_override = parse_editable(editable_req, default_vcs) if url.startswith('file:'): source_dir = url_to_path(url) else: source_dir = None res = cls(name, comes_from, source_dir=source_dir, editable=True, url=url, editable_options=extras_override, prereleases=True) if extras_override is not None: res.extras = extras_override return res
def _format_link(link, root_dir): """ Format link as URL or path. :type link: pip.index.Link :type root_dir: str|None :rtype: str """ if link.scheme != 'file': return link.url path = url_to_path(link.url) if root_dir is not None and is_subdirectory(root_dir, path): relpath = os.path.relpath(path, start=root_dir) if relpath == '.': return '.' return './' + relpath.replace(os.path.sep, '/') # Make sure it's absolute return path_to_url(path)
def _sort_locations(self, locations): """ Sort locations into "files" (archives) and "urls", and return a pair of lists (files,urls) """ files = [] urls = [] # puts the url for the given file path into the appropriate list def sort_path(path): url = path_to_url(path) if mimetypes.guess_type(url, strict=False)[0] == "text/html": urls.append(url) else: files.append(url) for url in locations: is_local_path = os.path.exists(url) is_file_url = url.startswith("file:") is_find_link = url in self.find_links if is_local_path or is_file_url: if is_local_path: path = url else: path = url_to_path(url) if is_find_link and os.path.isdir(path): path = os.path.realpath(path) for item in os.listdir(path): sort_path(os.path.join(path, item)) elif is_file_url and os.path.isdir(path): urls.append(url) elif os.path.isfile(path): sort_path(path) else: urls.append(url) return files, urls
def test_url_to_path_path_to_url_symmetry_win(): path = r'C:\tmp\file' assert url_to_path(path_to_url(path)) == path unc_path = r'\\unc\share\path' assert url_to_path(path_to_url(unc_path)) == unc_path
def find_requirement(self, req, upgrade): def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's # behavior. if not loc.endswith('/'): loc = loc + '/' return loc url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link( mkurl_pypi_url(self.index_urls[0]), trusted=True, ) page = self._get_page(main_index_url, req) if page is None and PyPI.netloc not in str(main_index_url): warnings.warn( "Failed to find %r at %s. It is suggested to upgrade " "your index to support normalized names as the name in " "/simple/{name}." % (req.name, main_index_url), RemovedInPip8Warning, ) url_name = self._find_url_name( Link(self.index_urls[0], trusted=True), url_name, req) or req.url_name if url_name is not None: locations = [mkurl_pypi_url(url) for url in self.index_urls] + self.find_links else: locations = list(self.find_links) file_locations, url_locations = self._sort_locations(locations) _flocations, _ulocations = self._sort_locations(self.dependency_links) file_locations.extend(_flocations) # We trust every url that the user has given us whether it was given # via --index-url or --find-links locations = [Link(url, trusted=True) for url in url_locations] # We explicitly do not trust links that came from dependency_links locations.extend([Link(url) for url in _ulocations]) logger.debug('URLs to search for versions for %s:', req) for location in locations: logger.debug('* %s', location) self._validate_secure_origin(logger, location) found_versions = [] found_versions.extend( self._package_versions( # We trust every directly linked archive in find_links [Link(url, '-f', trusted=True) for url in self.find_links], req.name.lower())) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s', page.url) with indent_log(): page_versions.extend( self._package_versions(page.links, req.name.lower())) dependency_versions = list( self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.debug( 'dependency_links found: %s', ', '.join( [link.url for p, link, version in dependency_versions])) file_versions = list( self._package_versions([Link(url) for url in file_locations], req.name.lower())) if (not found_versions and not page_versions and not dependency_versions and not file_versions): logger.critical( 'Could not find any downloads that satisfy the requirement %s', req, ) if self.need_warn_external: logger.warning( "Some externally hosted files were ignored as access to " "them may be unreliable (use --allow-external %s to " "allow).", req.name, ) if self.need_warn_unverified: logger.warning( "Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow).", req.name, ) raise DistributionNotFound('No distributions at all found for %s' % req) installed_version = [] if req.satisfied_by is not None: installed_version = [ InstallationCandidate( req.name, req.satisfied_by.version, INSTALLED_VERSION, ), ] if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.location.url) for candidate in file_versions ])) # This is an intentional priority ordering all_versions = (file_versions + found_versions + page_versions + dependency_versions) # Filter out anything which doesn't match our specifier _versions = set( req.specifier.filter( [x.version for x in all_versions], prereleases=(self.allow_all_prereleases if self.allow_all_prereleases else None), )) all_versions = [x for x in all_versions if x.version in _versions] # Finally add our existing versions to the front of our versions. applicable_versions = installed_version + all_versions applicable_versions = self._sort_versions(applicable_versions) existing_applicable = any(i.location is INSTALLED_VERSION for i in applicable_versions) if not upgrade and existing_applicable: if applicable_versions[0].location is INSTALLED_VERSION: logger.debug( 'Existing installed version (%s) is most up-to-date and ' 'satisfies requirement', req.satisfied_by.version, ) else: logger.debug( 'Existing installed version (%s) satisfies requirement ' '(most up-to-date version is %s)', req.satisfied_by.version, applicable_versions[0][2], ) return None if not applicable_versions: logger.critical( 'Could not find a version that satisfies the requirement %s ' '(from versions: %s)', req, ', '.join( sorted( set(str(i.version) for i in all_versions), key=parse_version, ))) if self.need_warn_external: logger.warning( "Some externally hosted files were ignored as access to " "them may be unreliable (use --allow-external to allow).") if self.need_warn_unverified: logger.warning( "Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow).", req.name, ) raise DistributionNotFound( 'No distributions matching the version for %s' % req) if applicable_versions[0].location is INSTALLED_VERSION: # We have an existing version, and its the best version logger.debug( 'Installed version (%s) is most up-to-date (past versions: ', '%s)', req.satisfied_by.version, ', '.join(str(i.version) for i in applicable_versions[1:]) or "none", ) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.debug( 'Using version %s (newest of versions: %s)', applicable_versions[0].version, ', '.join(str(i.version) for i in applicable_versions)) selected_version = applicable_versions[0].location if (selected_version.verifiable is not None and not selected_version.verifiable): logger.warning( "%s is potentially insecure and unverifiable.", req.name, ) if selected_version._deprecated_regex: warnings.warn( "%s discovered using a deprecated method of parsing, in the " "future it will no longer be discovered." % req.name, RemovedInPip7Warning, ) return selected_version
def find_requirement(self, req, upgrade): def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's behavior. if not loc.endswith('/'): loc = loc + '/' return loc url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link(mkurl_pypi_url(self.index_urls[0]), trusted=True) # This will also cache the page, so it's okay that we get it again later: page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name( Link(self.index_urls[0], trusted=True), url_name, req) or req.url_name if url_name is not None: locations = [mkurl_pypi_url(url) for url in self.index_urls] + self.find_links else: locations = list(self.find_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: locations = [posixpath.join(main_index_url.url, version) ] + locations file_locations, url_locations = self._sort_locations(locations) _flocations, _ulocations = self._sort_locations(self.dependency_links) file_locations.extend(_flocations) # We trust every url that the user has given us whether it was given # via --index-url or --find-links locations = [Link(url, trusted=True) for url in url_locations] # We explicitly do not trust links that came from dependency_links locations.extend([Link(url) for url in _ulocations]) logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) # Determine if this url used a secure transport mechanism parsed = urlparse.urlparse(str(location)) if parsed.scheme in INSECURE_SCHEMES: secure_schemes = INSECURE_SCHEMES[parsed.scheme] if len(secure_schemes) == 1: ctx = (location, parsed.scheme, secure_schemes[0], parsed.netloc) logger.warn("%s uses an insecure transport scheme (%s). " "Consider using %s if %s has it available" % ctx) elif len(secure_schemes) > 1: ctx = (location, parsed.scheme, ", ".join(secure_schemes), parsed.netloc) logger.warn("%s uses an insecure transport scheme (%s). " "Consider using one of %s if %s has any of " "them available" % ctx) else: ctx = (location, parsed.scheme) logger.warn("%s uses an insecure transport scheme (%s)." % ctx) found_versions = [] found_versions.extend( self._package_versions( # We trust every directly linked archive in find_links [Link(url, '-f', trusted=True) for url in self.find_links], req.name.lower())) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: page_versions.extend( self._package_versions(page.links, req.name.lower())) finally: logger.indent -= 2 dependency_versions = list( self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.info('dependency_links found: %s' % ', '.join( [link.url for parsed, link, version in dependency_versions])) file_versions = list( self._package_versions([Link(url) for url in file_locations], req.name.lower())) if not found_versions and not page_versions and not dependency_versions and not file_versions: logger.fatal( 'Could not find any downloads that satisfy the requirement %s' % req) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external %s to allow)." % req.name) if self.need_warn_unverified: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow)." % req.name) raise DistributionNotFound('No distributions at all found for %s' % req) installed_version = [] if req.satisfied_by is not None: installed_version = [(req.satisfied_by.parsed_version, INSTALLED_VERSION, req.satisfied_by.version)] if file_versions: file_versions.sort(reverse=True) logger.info('Local files found: %s' % ', '.join([ url_to_path(link.url) for parsed, link, version in file_versions ])) #this is an intentional priority ordering all_versions = installed_version + file_versions + found_versions + page_versions + dependency_versions applicable_versions = [] for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info("Ignoring link %s, version %s doesn't match %s" % (link, version, ','.join( [''.join(s) for s in req.req.specs]))) continue elif is_prerelease(version) and not (self.allow_all_prereleases or req.prereleases): # If this version isn't the already installed one, then # ignore it if it's a pre-release. if link is not INSTALLED_VERSION: logger.info( "Ignoring link %s, version %s is a pre-release (use --pre to allow)." % (link, version)) continue applicable_versions.append((parsed_version, link, version)) applicable_versions = self._sort_versions(applicable_versions) existing_applicable = bool([ link for parsed_version, link, version in applicable_versions if link is INSTALLED_VERSION ]) if not upgrade and existing_applicable: if applicable_versions[0][1] is INSTALLED_VERSION: logger.info( 'Existing installed version (%s) is most up-to-date and satisfies requirement' % req.satisfied_by.version) else: logger.info( 'Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' % (req.satisfied_by.version, applicable_versions[0][2])) return None if not applicable_versions: logger.fatal( 'Could not find a version that satisfies the requirement %s (from versions: %s)' % (req, ', '.join([ version for parsed_version, link, version in all_versions ]))) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external to allow).") if self.need_warn_unverified: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow)." % req.name) raise DistributionNotFound( 'No distributions matching the version for %s' % req) if applicable_versions[0][1] is INSTALLED_VERSION: # We have an existing version, and its the best version logger.info( 'Installed version (%s) is most up-to-date (past versions: %s)' % (req.satisfied_by.version, ', '.join([ version for parsed_version, link, version in applicable_versions[1:] ]) or 'none')) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.info( 'Using version %s (newest of versions: %s)' % (applicable_versions[0][2], ', '.join([ version for parsed_version, link, version in applicable_versions ]))) selected_version = applicable_versions[0][1] # TODO: Remove after 1.4 has been released # if (selected_version.internal is not None # and not selected_version.internal): # logger.warn("You are installing an externally hosted file. Future " # "versions of pip will default to disallowing " # "externally hosted files.") # if (selected_version.verifiable is not None # and not selected_version.verifiable): # logger.warn("You are installing a potentially insecure and " # "unverifiable file. Future versions of pip will " # "default to disallowing insecure files.") if selected_version._deprecated_regex: logger.deprecated( "1.7", "%s discovered using a deprecated method of parsing, " "in the future it will no longer be discovered" % req.name) return selected_version
def _prepare_file(self, finder, req_to_install): """Prepare a single requirements files. :return: A list of addition InstallRequirements to also install. """ # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) if req_to_install.constraint or req_to_install.prepared: return [] req_to_install.prepared = True if req_to_install.editable: logger.info('Obtaining %s', req_to_install) else: # satisfied_by is only evaluated by calling _check_skip_installed, # so it must be None here. assert req_to_install.satisfied_by is None if not self.ignore_installed: skip_reason = self._check_skip_installed( req_to_install, finder) if req_to_install.satisfied_by: assert skip_reason is not None, ( '_check_skip_installed returned None but ' 'req_to_install.satisfied_by is set to %r' % (req_to_install.satisfied_by,)) logger.info( 'Requirement already %s: %s', skip_reason, req_to_install) else: if (req_to_install.link and req_to_install.link.scheme == 'file'): path = url_to_path(req_to_install.link.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req_to_install) with indent_log(): # ################################ # # # vcs update or unpack archive # # # ################################ # if req_to_install.editable: req_to_install.ensure_has_source_dir(self.src_dir) req_to_install.update_editable(not self.is_download) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if self.is_download: req_to_install.archive(self.download_dir) elif req_to_install.satisfied_by: abstract_dist = Installed(req_to_install) else: # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory req_to_install.ensure_has_source_dir(self.build_dir) # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req_to_install.source_dir` if os.path.exists( os.path.join(req_to_install.source_dir, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing build directory (%s). This is " "likely due to a previous installation that failed" ". pip is being responsible and not assuming it " "can delete this. Please delete it and try again." % (req_to_install, req_to_install.source_dir) ) req_to_install.populate_link(finder, self.upgrade) # We can't hit this spot and have populate_link return None. # req_to_install.satisfied_by is None here (because we're # guarded) and upgrade has no impact except when satisfied_by # is not None. # Then inside find_requirement existing_applicable -> False # If no new versions are found, DistributionNotFound is raised, # otherwise a result is guaranteed. assert req_to_install.link try: download_dir = self.download_dir # We always delete unpacked sdists after pip ran. autodelete_unpacked = True if req_to_install.link.is_wheel \ and self.wheel_download_dir: # when doing 'pip wheel` we download wheels to a # dedicated dir. download_dir = self.wheel_download_dir if req_to_install.link.is_wheel: if download_dir: # When downloading, we only unpack wheels to get # metadata. autodelete_unpacked = True else: # When installing a wheel, we use the unpacked # wheel. autodelete_unpacked = False unpack_url( req_to_install.link, req_to_install.source_dir, download_dir, autodelete_unpacked, session=self.session) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because ' 'of error %s', req_to_install, exc, ) raise InstallationError( 'Could not install requirement %s because ' 'of HTTP error %s for URL %s' % (req_to_install, exc, req_to_install.link) ) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if self.is_download: # Make a .zip of the source_dir we already created. if req_to_install.link.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) # req_to_install.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not self.ignore_installed: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: logger.info( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s', req_to_install, ) # ###################### # # # parse dependencies # # # ###################### # dist = abstract_dist.dist(finder) more_reqs = [] def add_req(subreq): sub_install_req = InstallRequirement( str(subreq), req_to_install, isolated=self.isolated, wheel_cache=self._wheel_cache, ) more_reqs.extend(self.add_requirement( sub_install_req, req_to_install.name)) # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not self.has_requirement(req_to_install.name): # 'unnamed' requirements will get added here self.add_requirement(req_to_install, None) if not self.ignore_dependencies: if (req_to_install.extras): logger.debug( "Installing extra requirements: %r", ','.join(req_to_install.extras), ) missing_requested = sorted( set(req_to_install.extras) - set(dist.extras) ) for missing in missing_requested: logger.warning( '%s does not provide the extra \'%s\'', dist, missing ) available_requested = sorted( set(dist.extras) & set(req_to_install.extras) ) for subreq in dist.requires(available_requested): add_req(subreq) # cleanup tmp src self.reqs_to_cleanup.append(req_to_install) if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took # action on them. self.successfully_downloaded.append(req_to_install) return more_reqs
def prepare_files(self, finder): """ Prepare process. Create temp directories, download and/or unpack files. """ from pip.index import Link unnamed = list(self.unnamed_requirements) reqs = list(self.requirements.values()) while reqs or unnamed: if unnamed: req_to_install = unnamed.pop(0) else: req_to_install = reqs.pop(0) install = True best_installed = False not_found = None # ############################################# # # # Search for archive to fulfill requirement # # # ############################################# # if not self.ignore_installed and not req_to_install.editable: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade: if not self.force_reinstall and not req_to_install.url: try: url = finder.find_requirement( req_to_install, self.upgrade) except BestVersionAlreadyInstalled: best_installed = True install = False except DistributionNotFound as exc: not_found = exc else: # Avoid the need to call find_requirement again req_to_install.url = url.url if not best_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by )): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if req_to_install.satisfied_by: if best_installed: logger.info( 'Requirement already up-to-date: %s', req_to_install, ) else: logger.info( 'Requirement already satisfied (use --upgrade to ' 'upgrade): %s', req_to_install, ) if req_to_install.editable: logger.info('Obtaining %s', req_to_install) elif install: if (req_to_install.url and req_to_install.url.lower().startswith('file:')): path = url_to_path(req_to_install.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req_to_install) with indent_log(): # ################################ # # # vcs update or unpack archive # # # ################################ # is_wheel = False if req_to_install.editable: if req_to_install.source_dir is None: location = req_to_install.build_location(self.src_dir) req_to_install.source_dir = location else: location = req_to_install.source_dir if not os.path.exists(self.build_dir): _make_build_dir(self.build_dir) req_to_install.update_editable(not self.is_download) if self.is_download: req_to_install.run_egg_info() req_to_install.archive(self.download_dir) else: req_to_install.run_egg_info() elif install: # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory # NB: This call can result in the creation of a temporary # build directory location = req_to_install.build_location( self.build_dir, ) unpack = True url = None # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. if os.path.exists(os.path.join(location, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing build directory (%s). This is " "likely due to a previous installation that failed" ". pip is being responsible and not assuming it " "can delete this. Please delete it and try again." % (req_to_install, location) ) else: # FIXME: this won't upgrade when there's an existing # package unpacked in `location` if req_to_install.url is None: if not_found: raise not_found url = finder.find_requirement( req_to_install, upgrade=self.upgrade, ) else: # FIXME: should req_to_install.url already be a # link? url = Link(req_to_install.url) assert url if url: try: if ( url.filename.endswith(wheel_ext) and self.wheel_download_dir ): # when doing 'pip wheel` download_dir = self.wheel_download_dir do_download = True else: download_dir = self.download_dir do_download = self.is_download unpack_url( url, location, download_dir, do_download, session=self.session, ) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because ' 'of error %s', req_to_install, exc, ) raise InstallationError( 'Could not install requirement %s because ' 'of HTTP error %s for URL %s' % (req_to_install, exc, url) ) else: unpack = False if unpack: is_wheel = url and url.filename.endswith(wheel_ext) if self.is_download: req_to_install.source_dir = location if not is_wheel: # FIXME:https://github.com/pypa/pip/issues/1112 req_to_install.run_egg_info() if url and url.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) elif is_wheel: req_to_install.source_dir = location req_to_install.url = url.url else: req_to_install.source_dir = location req_to_install.run_egg_info() req_to_install.assert_source_matches_version() # req_to_install.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not self.ignore_installed: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: logger.info( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s', req_to_install, ) install = False # ###################### # # # parse dependencies # # # ###################### # if (req_to_install.extras): logger.debug( "Installing extra requirements: %r", ','.join(req_to_install.extras), ) if is_wheel: dist = list( pkg_resources.find_distributions(location) )[0] else: # sdists if req_to_install.satisfied_by: dist = req_to_install.satisfied_by else: dist = req_to_install.get_dist() # FIXME: shouldn't be globally added: if dist.has_metadata('dependency_links.txt'): finder.add_dependency_links( dist.get_metadata_lines('dependency_links.txt') ) if not self.ignore_dependencies: for subreq in dist.requires( req_to_install.extras): if self.has_requirement( subreq.project_name): # FIXME: check for conflict continue subreq = InstallRequirement( str(subreq), req_to_install, isolated=self.isolated, ) reqs.append(subreq) self.add_requirement(subreq) if not self.has_requirement(req_to_install.name): # 'unnamed' requirements will get added here self.add_requirement(req_to_install) # cleanup tmp src if (self.is_download or req_to_install._temp_build_dir is not None): self.reqs_to_cleanup.append(req_to_install) if install: self.successfully_downloaded.append(req_to_install)
def test_url_to_path_unix(): assert url_to_path('file:///tmp/file') == '/tmp/file'
def test_url_to_path_win(): assert url_to_path('file:///c:/tmp/file') == 'C:\\tmp\\file' assert url_to_path('file://unc/as/path') == r'\\unc\as\path'
def find_all_candidates(self, project_name): """Find all available InstallationCandidate for project_name This checks index_urls, find_links and dependency_links. All versions found are returned as an InstallationCandidate list. See _link_package_versions for details on which files are accepted """ index_locations = self._get_index_urls_locations(project_name) index_file_loc, index_url_loc = self._sort_locations(index_locations) fl_file_loc, fl_url_loc = self._sort_locations( self.find_links, expand_dir=True) dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links) file_locations = ( Link(url) for url in itertools.chain( index_file_loc, fl_file_loc, dep_file_loc) ) # We trust every url that the user has given us whether it was given # via --index-url or --find-links # We explicitly do not trust links that came from dependency_links # We want to filter out any thing which does not have a secure origin. url_locations = [ link for link in itertools.chain( (Link(url) for url in index_url_loc), (Link(url) for url in fl_url_loc), (Link(url) for url in dep_url_loc), ) if self._validate_secure_origin(logger, link) ] logger.debug('%d location(s) to search for versions of %s:', len(url_locations), project_name) for location in url_locations: logger.debug('* %s', location) canonical_name = canonicalize_name(project_name) formats = fmt_ctl_formats(self.format_control, canonical_name) search = Search(project_name, canonical_name, formats) find_links_versions = self._package_versions( # We trust every directly linked archive in find_links (Link(url, '-f') for url in self.find_links), search ) page_versions = [] for page in self._get_pages(url_locations, project_name): logger.debug('Analyzing links from page %s', page.url) with indent_log(): page_versions.extend( self._package_versions(page.links, search) ) dependency_versions = self._package_versions( (Link(url) for url in self.dependency_links), search ) if dependency_versions: logger.debug( 'dependency_links found: %s', ', '.join([ version.location.url for version in dependency_versions ]) ) file_versions = self._package_versions(file_locations, search) if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.location.url) for candidate in file_versions ]) ) # This is an intentional priority ordering return ( file_versions + find_links_versions + page_versions + dependency_versions )
def get_local_directory( self, cuppa_env, location, sub_dir, branch, full_url ): offline = cuppa_env['offline'] local_directory = None base = cuppa_env['download_root'] if not os.path.isabs( base ): base = os.path.join( cuppa_env['working_dir'], base ) if location.startswith( 'file:' ): location = pip_download.url_to_path( location ) if not pip_download.is_url( location ): if pip_download.is_archive_file( location ): self._local_folder = self.folder_name_from_path( location, cuppa_env ) local_directory = os.path.join( base, self._local_folder ) local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "" ) if os.path.exists( local_dir_with_sub_dir ): try: os.rmdir( local_dir_with_sub_dir ) except: return local_directory self.extract( location, local_dir_with_sub_dir ) logger.debug( "(local archive) Location = [{}]".format( as_info( location ) ) ) logger.debug( "(local archive) Local folder = [{}]".format( as_info( self._local_folder ) ) ) else: local_directory = branch and os.path.join( location, branch ) or location self._local_folder = self.folder_name_from_path( location, cuppa_env ) logger.debug( "(local file) Location = [{}]".format( as_info( location ) ) ) logger.debug( "(local file) Local folder = [{}]".format( as_info( self._local_folder ) ) ) return local_directory else: self._local_folder = self.folder_name_from_path( full_url, cuppa_env ) local_directory = os.path.join( base, self._local_folder ) if full_url.scheme.startswith( 'http' ) and self.url_is_download_archive_url( full_url.path ): logger.debug( "[{}] is an archive download".format( as_info( location ) ) ) local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "" ) # First we check to see if we already downloaded and extracted this archive before if os.path.exists( local_dir_with_sub_dir ): try: # If not empty this will fail os.rmdir( local_dir_with_sub_dir ) except: # Not empty so we'll return this as the local_directory logger.debug( "(already present) Location = [{}]".format( as_info( location ) ) ) logger.debug( "(already present) Local folder = [{}]".format( as_info( str(self._local_folder) ) ) ) return local_directory if cuppa_env['dump'] or cuppa_env['clean']: return local_directory # If not we then check to see if we cached the download cached_archive = self.get_cached_archive( cuppa_env['cache_root'], self._local_folder ) if cached_archive: logger.debug( "Cached archive [{}] found for [{}]".format( as_info( cached_archive ), as_info( location ) ) ) self.extract( cached_archive, local_dir_with_sub_dir ) else: logger.info( "Downloading [{}]...".format( as_info( location ) ) ) try: report_hook = None if logger.isEnabledFor( logging.INFO ): report_hook = ReportDownloadProgress() filename, headers = urllib.urlretrieve( location, reporthook=report_hook ) name, extension = os.path.splitext( filename ) logger.info( "[{}] successfully downloaded to [{}]".format( as_info( location ), as_info( filename ) ) ) self.extract( filename, local_dir_with_sub_dir ) if cuppa_env['cache_root']: cached_archive = os.path.join( cuppa_env['cache_root'], self._local_folder ) logger.debug( "Caching downloaded file as [{}]".format( as_info( cached_archive ) ) ) shutil.copyfile( filename, cached_archive ) except urllib.ContentTooShortError as error: logger.error( "Download of [{}] failed with error [{}]".format( as_error( location ), as_error( str(error) ) ) ) raise LocationException( error ) elif '+' in full_url.scheme: vc_type = location.split('+', 1)[0] backend = pip_vcs.vcs.get_backend( vc_type ) if backend: vcs_backend = backend( self.expand_secret( location ) ) local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "" ) if cuppa_env['dump'] or cuppa_env['clean']: return local_directory if os.path.exists( local_directory ): url, repository, branch, remote, revision = self.get_info( location, local_dir_with_sub_dir, full_url, vc_type ) rev_options = self.get_rev_options( vc_type, vcs_backend, local_remote=remote ) version = self.ver_rev_summary( branch, revision, self._full_url.path )[0] if not offline: logger.info( "Updating [{}] in [{}]{} at [{}]".format( as_info( location ), as_notice( local_dir_with_sub_dir ), ( rev_options and " on {}".format( as_notice( str(rev_options) ) ) or "" ), as_info( version ) ) ) try: update( vcs_backend, local_dir_with_sub_dir, rev_options ) logger.debug( "Successfully updated [{}]".format( as_info( location ) ) ) except pip_exceptions.PipError as error: logger.warn( "Could not update [{}] in [{}]{} due to error [{}]".format( as_warning( location ), as_warning( local_dir_with_sub_dir ), ( rev_options and " at {}".format( as_warning( str(rev_options) ) ) or "" ), as_warning( str(error) ) ) ) else: logger.debug( "Skipping update for [{}] as running in offline mode".format( as_info( location ) ) ) else: rev_options = self.get_rev_options( vc_type, vcs_backend ) action = "Cloning" if vc_type == "svn": action = "Checking out" max_attempts = 2 attempt = 1 while attempt <= max_attempts: logger.info( "{} [{}] into [{}]{}".format( action, as_info( location ), as_info( local_dir_with_sub_dir ), attempt > 1 and "(attempt {})".format( str(attempt) ) or "" ) ) try: vcs_backend.obtain( local_dir_with_sub_dir ) logger.debug( "Successfully retrieved [{}]".format( as_info( location ) ) ) break except pip_exceptions.PipError as error: attempt = attempt + 1 log_as = logger.warn if attempt > max_attempts: log_as = logger.error log_as( "Could not retrieve [{}] into [{}]{} due to error [{}]".format( as_info( location ), as_notice( local_dir_with_sub_dir ), ( rev_options and " to {}".format( as_notice( str(rev_options) ) ) or ""), as_error( str(error) ) ) ) if attempt > max_attempts: raise LocationException( str(error) ) logger.debug( "(url path) Location = [{}]".format( as_info( location ) ) ) logger.debug( "(url path) Local folder = [{}]".format( as_info( self._local_folder ) ) ) return local_directory
def find_requirement(self, req, upgrade): def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's behavior. if not loc.endswith('/'): loc = loc + '/' return loc url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link(mkurl_pypi_url(self.index_urls[0]), trusted=True) # This will also cache the page, so it's okay that we get it again later: page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name(Link(self.index_urls[0], trusted=True), url_name, req) or req.url_name # Combine index URLs with mirror URLs here to allow # adding more index URLs from requirements files all_index_urls = self.index_urls + self.mirror_urls if url_name is not None: locations = [ mkurl_pypi_url(url) for url in all_index_urls] + self.find_links else: locations = list(self.find_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: locations = [ posixpath.join(main_index_url.url, version)] + locations file_locations, url_locations = self._sort_locations(locations) _flocations, _ulocations = self._sort_locations(self.dependency_links) file_locations.extend(_flocations) # We trust every url that the user has given us whether it was given # via --index-url, --user-mirrors/--mirror, or --find-links or a # default option thereof locations = [Link(url, trusted=True) for url in url_locations] # We explicitly do not trust links that came from dependency_links locations.extend([Link(url) for url in _ulocations]) logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) found_versions = [] found_versions.extend( self._package_versions( # We trust every directly linked archive in find_links [Link(url, '-f', trusted=True) for url in self.find_links], req.name.lower())) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: page_versions.extend(self._package_versions(page.links, req.name.lower())) finally: logger.indent -= 2 dependency_versions = list(self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions])) file_versions = list(self._package_versions( [Link(url) for url in file_locations], req.name.lower())) if not found_versions and not page_versions and not dependency_versions and not file_versions: logger.fatal('Could not find any downloads that satisfy the requirement %s' % req) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external %s to allow)." % req.name) if self.need_warn_insecure: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-insecure %s to allow)." % req.name) raise DistributionNotFound('No distributions at all found for %s' % req) installed_version = [] if req.satisfied_by is not None: installed_version = [(req.satisfied_by.parsed_version, InfLink, req.satisfied_by.version)] if file_versions: file_versions.sort(reverse=True) logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions])) #this is an intentional priority ordering all_versions = installed_version + file_versions + found_versions + page_versions + dependency_versions applicable_versions = [] for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info("Ignoring link %s, version %s doesn't match %s" % (link, version, ','.join([''.join(s) for s in req.req.specs]))) continue elif is_prerelease(version) and not (self.allow_all_prereleases or req.prereleases): # If this version isn't the already installed one, then # ignore it if it's a pre-release. if link is not InfLink: logger.info("Ignoring link %s, version %s is a pre-release (use --pre to allow)." % (link, version)) continue applicable_versions.append((parsed_version, link, version)) applicable_versions = self._sort_versions(applicable_versions) existing_applicable = bool([link for parsed_version, link, version in applicable_versions if link is InfLink]) if not upgrade and existing_applicable: if applicable_versions[0][1] is InfLink: logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement' % req.satisfied_by.version) else: logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' % (req.satisfied_by.version, applicable_versions[0][2])) return None if not applicable_versions: logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)' % (req, ', '.join([version for parsed_version, link, version in all_versions]))) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external to allow).") if self.need_warn_insecure: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-insecure %s to allow)." % req.name) raise DistributionNotFound('No distributions matching the version for %s' % req) if applicable_versions[0][1] is InfLink: # We have an existing version, and its the best version logger.info('Installed version (%s) is most up-to-date (past versions: %s)' % (req.satisfied_by.version, ', '.join([version for parsed_version, link, version in applicable_versions[1:]]) or 'none')) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.info('Using version %s (newest of versions: %s)' % (applicable_versions[0][2], ', '.join([version for parsed_version, link, version in applicable_versions]))) selected_version = applicable_versions[0][1] # TODO: Remove after 1.4 has been released if (selected_version.internal is not None and not selected_version.internal): logger.warn("You are installing an externally hosted file. Future " "versions of pip will default to disallowing " "externally hosted files.") if (selected_version.verifiable is not None and not selected_version.verifiable): logger.warn("You are installing a potentially insecure and " "unverifiable file. Future versions of pip will " "default to disallowing insecure files.") return selected_version
def _prepare_file(self, finder, req_to_install): """Prepare a single requirements files. :return: A list of addition InstallRequirements to also install. """ # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) if req_to_install.constraint or req_to_install.prepared: return [] req_to_install.prepared = True if req_to_install.editable: logger.info('Obtaining %s', req_to_install) else: # satisfied_by is only evaluated by calling _check_skip_installed, # so it must be None here. assert req_to_install.satisfied_by is None if not self.ignore_installed: skip_reason = self._check_skip_installed( req_to_install, finder) if req_to_install.satisfied_by: assert skip_reason is not None, ( '_check_skip_installed returned None but ' 'req_to_install.satisfied_by is set to %r' % (req_to_install.satisfied_by, )) logger.info('Requirement already %s: %s', skip_reason, req_to_install) else: if (req_to_install.link and req_to_install.link.scheme == 'file'): path = url_to_path(req_to_install.link.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req_to_install) with indent_log(): # ################################ # # # vcs update or unpack archive # # # ################################ # if req_to_install.editable: req_to_install.ensure_has_source_dir(self.src_dir) req_to_install.update_editable(not self.is_download) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if self.is_download: req_to_install.archive(self.download_dir) elif req_to_install.satisfied_by: abstract_dist = Installed(req_to_install) else: # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory req_to_install.ensure_has_source_dir(self.build_dir) # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req_to_install.source_dir` if os.path.exists( os.path.join(req_to_install.source_dir, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing build directory (%s). This is " "likely due to a previous installation that failed" ". pip is being responsible and not assuming it " "can delete this. Please delete it and try again." % (req_to_install, req_to_install.source_dir)) req_to_install.populate_link(finder, self.upgrade) # We can't hit this spot and have populate_link return None. # req_to_install.satisfied_by is None here (because we're # guarded) and upgrade has no impact except when satisfied_by # is not None. # Then inside find_requirement existing_applicable -> False # If no new versions are found, DistributionNotFound is raised, # otherwise a result is guaranteed. assert req_to_install.link try: download_dir = self.download_dir # We always delete unpacked sdists after pip ran. autodelete_unpacked = True if req_to_install.link.is_wheel \ and self.wheel_download_dir: # when doing 'pip wheel` we download wheels to a # dedicated dir. download_dir = self.wheel_download_dir if req_to_install.link.is_wheel: if download_dir: # When downloading, we only unpack wheels to get # metadata. autodelete_unpacked = True else: # When installing a wheel, we use the unpacked # wheel. autodelete_unpacked = False unpack_url(req_to_install.link, req_to_install.source_dir, download_dir, autodelete_unpacked, session=self.session) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because ' 'of error %s', req_to_install, exc, ) raise InstallationError( 'Could not install requirement %s because ' 'of HTTP error %s for URL %s' % (req_to_install, exc, req_to_install.link)) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if self.is_download: # Make a .zip of the source_dir we already created. if req_to_install.link.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) # req_to_install.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not self.ignore_installed: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: logger.info( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s', req_to_install, ) # ###################### # # # parse dependencies # # # ###################### # dist = abstract_dist.dist(finder) more_reqs = [] def add_req(subreq): sub_install_req = InstallRequirement( str(subreq), req_to_install, isolated=self.isolated, wheel_cache=self._wheel_cache, ) more_reqs.extend( self.add_requirement(sub_install_req, req_to_install.name)) # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not self.has_requirement(req_to_install.name): # 'unnamed' requirements will get added here self.add_requirement(req_to_install, None) if not self.ignore_dependencies: if (req_to_install.extras): logger.debug( "Installing extra requirements: %r", ','.join(req_to_install.extras), ) missing_requested = sorted( set(req_to_install.extras) - set(dist.extras)) for missing in missing_requested: logger.warning('%s does not provide the extra \'%s\'', dist, missing) available_requested = sorted( set(dist.extras) & set(req_to_install.extras)) for subreq in dist.requires(available_requested): add_req(subreq) # cleanup tmp src self.reqs_to_cleanup.append(req_to_install) if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took # action on them. self.successfully_downloaded.append(req_to_install) return more_reqs
def find_requirement(self, req, upgrade): def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's # behavior. if not loc.endswith('/'): loc = loc + '/' return loc url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link( mkurl_pypi_url(self.index_urls[0]), trusted=True, ) # This will also cache the page, so it's okay that we get it again # later: page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name( Link(self.index_urls[0], trusted=True), url_name, req ) or req.url_name if url_name is not None: locations = [ mkurl_pypi_url(url) for url in self.index_urls] + self.find_links else: locations = list(self.find_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: locations = [ posixpath.join(main_index_url.url, version)] + locations file_locations, url_locations = self._sort_locations(locations) # We trust every url that the user has given us whether it was given # via --index-url or --find-links locations = [Link(url, trusted=True) for url in url_locations] logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) # Determine if this url used a secure transport mechanism parsed = urlparse.urlparse(str(location)) if parsed.scheme in INSECURE_SCHEMES: secure_schemes = INSECURE_SCHEMES[parsed.scheme] if len(secure_schemes) == 1: ctx = (location, parsed.scheme, secure_schemes[0], parsed.netloc) logger.warn("%s uses an insecure transport scheme (%s). " "Consider using %s if %s has it available" % ctx) elif len(secure_schemes) > 1: ctx = ( location, parsed.scheme, ", ".join(secure_schemes), parsed.netloc, ) logger.warn("%s uses an insecure transport scheme (%s). " "Consider using one of %s if %s has any of " "them available" % ctx) else: ctx = (location, parsed.scheme) logger.warn("%s uses an insecure transport scheme (%s)." % ctx) found_versions = [] found_versions.extend( self._package_versions( # We trust every directly linked archive in find_links [Link(url, '-f', trusted=True) for url in self.find_links], req.name.lower() ) ) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: page_versions.extend( self._package_versions(page.links, req.name.lower()) ) finally: logger.indent -= 2 file_versions = list( self._package_versions( [Link(url) for url in file_locations], req.name.lower() ) ) if (not found_versions and not page_versions and not file_versions): logger.fatal( 'Could not find any downloads that satisfy the requirement' ' %s' % req ) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external %s to allow)." % req.name) if self.need_warn_unverified: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow)." % req.name) raise DistributionNotFound( 'No distributions at all found for %s' % req ) installed_version = [] if req.satisfied_by is not None: installed_version = [( req.satisfied_by.parsed_version, INSTALLED_VERSION, req.satisfied_by.version, )] if file_versions: file_versions.sort(reverse=True) logger.info( 'Local files found: %s' % ', '.join([ url_to_path(link.url) for parsed, link, version in file_versions ]) ) # this is an intentional priority ordering all_versions = installed_version + file_versions + found_versions \ + page_versions applicable_versions = [] for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info( "Ignoring link %s, version %s doesn't match %s" % ( link, version, ','.join([''.join(s) for s in req.req.specs]) ) ) continue elif (is_prerelease(version) and not (self.allow_all_prereleases or req.prereleases)): # If this version isn't the already installed one, then # ignore it if it's a pre-release. if link is not INSTALLED_VERSION: logger.info( "Ignoring link %s, version %s is a pre-release (use " "--pre to allow)." % (link, version) ) continue applicable_versions.append((parsed_version, link, version)) applicable_versions = self._sort_versions(applicable_versions) existing_applicable = bool([ link for parsed_version, link, version in applicable_versions if link is INSTALLED_VERSION ]) if not upgrade and existing_applicable: if applicable_versions[0][1] is INSTALLED_VERSION: logger.info( 'Existing installed version (%s) is most up-to-date and ' 'satisfies requirement' % req.satisfied_by.version ) else: logger.info( 'Existing installed version (%s) satisfies requirement ' '(most up-to-date version is %s)' % (req.satisfied_by.version, applicable_versions[0][2]) ) return None if not applicable_versions: logger.fatal( 'Could not find a version that satisfies the requirement %s ' '(from versions: %s)' % ( req, ', '.join([ version for parsed_version, link, version in all_versions ]) ) ) if self.need_warn_external: logger.warn("Some externally hosted files were ignored (use " "--allow-external to allow).") if self.need_warn_unverified: logger.warn("Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow)." % req.name) raise DistributionNotFound( 'No distributions matching the version for %s' % req ) if applicable_versions[0][1] is INSTALLED_VERSION: # We have an existing version, and its the best version logger.info( 'Installed version (%s) is most up-to-date (past versions: ' '%s)' % ( req.satisfied_by.version, ', '.join([ version for parsed_version, link, version in applicable_versions[1:] ]) or 'none')) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.info( 'Using version %s (newest of versions: %s)' % ( applicable_versions[0][2], ', '.join([ version for parsed_version, link, version in applicable_versions ]) ) ) selected_version = applicable_versions[0][1] if (selected_version.internal is not None and not selected_version.internal): logger.warn("%s an externally hosted file and may be " "unreliable" % req.name) if (selected_version.verifiable is not None and not selected_version.verifiable): logger.warn("%s is potentially insecure and " "unverifiable." % req.name) if selected_version._deprecated_regex: logger.deprecated( "1.7", "%s discovered using a deprecated method of parsing, " "in the future it will no longer be discovered" % req.name ) return selected_version
def prepare_files(self, finder, force_root_egg_info=False, bundle=False): """Prepare process. Create temp directories, download and/or unpack files.""" unnamed = list(self.unnamed_requirements) reqs = list(self.requirements.values()) while reqs or unnamed: if unnamed: req_to_install = unnamed.pop(0) else: req_to_install = reqs.pop(0) install = True best_installed = False if not self.ignore_installed and not req_to_install.editable: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade: if not self.force_reinstall: try: url = finder.find_requirement( req_to_install, self.upgrade) except BestVersionAlreadyInstalled: best_installed = True install = False else: # Avoid the need to call find_requirement again req_to_install.url = url.url if not best_installed: req_to_install.conflicts_with = req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if req_to_install.satisfied_by: if best_installed: logger.notify('Requirement already up-to-date: %s' % req_to_install) else: logger.notify('Requirement already satisfied ' '(use --upgrade to upgrade): %s' % req_to_install) if req_to_install.editable: logger.notify('Obtaining %s' % req_to_install) elif install: if req_to_install.url and req_to_install.url.lower().startswith('file:'): logger.notify('Unpacking %s' % display_path(url_to_path(req_to_install.url))) else: logger.notify('Downloading/unpacking %s' % req_to_install) logger.indent += 2 try: is_bundle = False if req_to_install.editable: if req_to_install.source_dir is None: location = req_to_install.build_location(self.src_dir) req_to_install.source_dir = location else: location = req_to_install.source_dir if not os.path.exists(self.build_dir): _make_build_dir(self.build_dir) req_to_install.update_editable(not self.is_download) if self.is_download: req_to_install.run_egg_info() req_to_install.archive(self.download_dir) else: req_to_install.run_egg_info() elif install: ##@@ if filesystem packages are not marked ##editable in a req, a non deterministic error ##occurs when the script attempts to unpack the ##build directory location = req_to_install.build_location(self.build_dir, not self.is_download) ## FIXME: is the existance of the checkout good enough to use it? I don't think so. unpack = True url = None if not os.path.exists(os.path.join(location, 'setup.py')): ## FIXME: this won't upgrade when there's an existing package unpacked in `location` if req_to_install.url is None: url = finder.find_requirement(req_to_install, upgrade=self.upgrade) else: ## FIXME: should req_to_install.url already be a link? url = Link(req_to_install.url) assert url if url: try: self.unpack_url(url, location, self.is_download) except HTTPError: e = sys.exc_info()[1] logger.fatal('Could not install requirement %s because of error %s' % (req_to_install, e)) raise InstallationError( 'Could not install requirement %s because of HTTP error %s for URL %s' % (req_to_install, e, url)) else: unpack = False if unpack: is_bundle = req_to_install.is_bundle if is_bundle: req_to_install.move_bundle_files(self.build_dir, self.src_dir) for subreq in req_to_install.bundle_requirements(): reqs.append(subreq) self.add_requirement(subreq) elif self.is_download: req_to_install.source_dir = location req_to_install.run_egg_info() if url and url.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) else: req_to_install.source_dir = location req_to_install.run_egg_info() if force_root_egg_info: # We need to run this to make sure that the .egg-info/ # directory is created for packing in the bundle req_to_install.run_egg_info(force_root_egg_info=True) req_to_install.assert_source_matches_version() #@@ sketchy way of identifying packages not grabbed from an index if bundle and req_to_install.url: self.copy_to_build_dir(req_to_install) install = False # req_to_install.req is only avail after unpack for URL pkgs # repeat check_if_exists to uninstall-on-upgrade (#14) req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: req_to_install.conflicts_with = req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if not is_bundle: ## FIXME: shouldn't be globally added: finder.add_dependency_links(req_to_install.dependency_links) if (req_to_install.extras): logger.notify("Installing extra requirements: %r" % ','.join(req_to_install.extras)) if not self.ignore_dependencies: for req in req_to_install.requirements(req_to_install.extras): try: name = pkg_resources.Requirement.parse(req).project_name except ValueError: e = sys.exc_info()[1] ## FIXME: proper warning logger.error('Invalid requirement: %r (%s) in requirement %s' % (req, e, req_to_install)) continue subreq = InstallRequirement(req, req_to_install) if self.has_requirement(name): investigate.append([ self.get_requirement(name), subreq ]) continue reqs.append(subreq) self.add_requirement(subreq) if req_to_install.name not in self.requirements: self.requirements[req_to_install.name] = req_to_install if self.is_download: self.reqs_to_cleanup.append(req_to_install) else: self.reqs_to_cleanup.append(req_to_install) if install: self.successfully_downloaded.append(req_to_install) if bundle and (req_to_install.url and req_to_install.url.startswith('file:///')): self.copy_to_build_dir(req_to_install) finally: logger.indent -= 2
def test_url_to_path_win(): assert url_to_path('file:///c:/tmp/file') == 'c:/tmp/file'
def test_url_to_path_win(): assert url_to_path("file:///c:/tmp/file") == "C:\\tmp\\file" assert url_to_path("file://unc/as/path") == r"\\unc\as\path"
def _prepare_linked_requirement(self, req, resolver): """Prepare a requirement that would be obtained from req.link """ # TODO: Breakup into smaller functions if req.link and req.link.scheme == 'file': path = url_to_path(req.link.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req) with indent_log(): # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory req.ensure_has_source_dir(self.build_dir) # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req.source_dir` # package unpacked in `req.source_dir` if os.path.exists(os.path.join(req.source_dir, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing build directory (%s). This is " "likely due to a previous installation that failed" ". pip is being responsible and not assuming it " "can delete this. Please delete it and try again." % (req, req.source_dir)) req.populate_link(resolver.finder, resolver._is_upgrade_allowed(req), resolver.require_hashes) # We can't hit this spot and have populate_link return None. # req.satisfied_by is None here (because we're # guarded) and upgrade has no impact except when satisfied_by # is not None. # Then inside find_requirement existing_applicable -> False # If no new versions are found, DistributionNotFound is raised, # otherwise a result is guaranteed. assert req.link link = req.link # Now that we have the real link, we can tell what kind of # requirements we have and raise some more informative errors # than otherwise. (For example, we can raise VcsHashUnsupported # for a VCS URL rather than HashMissing.) if resolver.require_hashes: # We could check these first 2 conditions inside # unpack_url and save repetition of conditions, but then # we would report less-useful error messages for # unhashable requirements, complaining that there's no # hash provided. if is_vcs_url(link): raise VcsHashUnsupported() elif is_file_url(link) and is_dir_url(link): raise DirectoryUrlHashUnsupported() if not req.original_link and not req.is_pinned: # Unpinned packages are asking for trouble when a new # version is uploaded. This isn't a security check, but # it saves users a surprising hash mismatch in the # future. # # file:/// URLs aren't pinnable, so don't complain # about them not being pinned. raise HashUnpinned() hashes = req.hashes(trust_internet=not resolver.require_hashes) if resolver.require_hashes and not hashes: # Known-good hashes are missing for this requirement, so # shim it with a facade object that will provoke hash # computation and then raise a HashMissing exception # showing the user what the hash should be. hashes = MissingHashes() try: download_dir = self.download_dir # We always delete unpacked sdists after pip ran. autodelete_unpacked = True if req.link.is_wheel and self.wheel_download_dir: # when doing 'pip wheel` we download wheels to a # dedicated dir. download_dir = self.wheel_download_dir if req.link.is_wheel: if download_dir: # When downloading, we only unpack wheels to get # metadata. autodelete_unpacked = True else: # When installing a wheel, we use the unpacked # wheel. autodelete_unpacked = False unpack_url(req.link, req.source_dir, download_dir, autodelete_unpacked, session=resolver.session, hashes=hashes, progress_bar=self.progress_bar) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because of error %s', req, exc, ) raise InstallationError( 'Could not install requirement %s because of HTTP ' 'error %s for URL %s' % (req, exc, req.link)) abstract_dist = make_abstract_dist(req) abstract_dist.prep_for_dist() if self._download_should_save: # Make a .zip of the source_dir we already created. if req.link.scheme in vcs.all_schemes: req.archive(self.download_dir) # req.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not resolver.ignore_installed: req.check_if_exists() if req.satisfied_by: should_modify = (resolver.upgrade_strategy != "to-satisfy-only" or resolver.ignore_installed) if should_modify: resolver._set_req_to_reinstall(req) else: logger.info( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s', req, ) return abstract_dist
def _find_all_versions(self, project_name): """Find all available versions for project_name This checks index_urls, find_links and dependency_links All versions found are returned See _link_package_versions for details on which files are accepted """ index_locations = self._get_index_urls_locations(project_name) file_locations, url_locations = self._sort_locations(index_locations) fl_file_loc, fl_url_loc = self._sort_locations(self.find_links) file_locations.extend(fl_file_loc) url_locations.extend(fl_url_loc) _flocations, _ulocations = self._sort_locations(self.dependency_links) file_locations.extend(_flocations) # We trust every url that the user has given us whether it was given # via --index-url or --find-links locations = [Link(url, trusted=True) for url in url_locations] # We explicitly do not trust links that came from dependency_links locations.extend([Link(url) for url in _ulocations]) logger.debug('%d location(s) to search for versions of %s:', len(locations), project_name) for location in locations: logger.debug('* %s', location) self._validate_secure_origin(logger, location) find_links_versions = list( self._package_versions( # We trust every directly linked archive in find_links (Link(url, '-f', trusted=True) for url in self.find_links), project_name.lower())) page_versions = [] for page in self._get_pages(locations, project_name): logger.debug('Analyzing links from page %s', page.url) with indent_log(): page_versions.extend( self._package_versions(page.links, project_name.lower())) dependency_versions = list( self._package_versions( (Link(url) for url in self.dependency_links), project_name.lower())) if dependency_versions: logger.debug( 'dependency_links found: %s', ', '.join( [version.location.url for version in dependency_versions])) file_versions = list( self._package_versions((Link(url) for url in file_locations), project_name.lower())) if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.location.url) for candidate in file_versions ])) # This is an intentional priority ordering return (file_versions + find_links_versions + page_versions + dependency_versions)
def prepare_requirement(self, req_to_install, resolver, requirement_set): # ###################### # # # print log messages # # # ###################### # if req_to_install.editable: logger.info('Obtaining %s', req_to_install) else: # satisfied_by is only evaluated by calling _check_skip_installed, # so it must be None here. assert req_to_install.satisfied_by is None if not resolver.ignore_installed: skip_reason = resolver._check_skip_installed(req_to_install) if req_to_install.satisfied_by: assert skip_reason is not None, ( '_check_skip_installed returned None but ' 'req_to_install.satisfied_by is set to %r' % (req_to_install.satisfied_by,)) logger.info( 'Requirement %s: %s (%s)', skip_reason, req_to_install, req_to_install.satisfied_by.version) else: if (req_to_install.link and req_to_install.link.scheme == 'file'): path = url_to_path(req_to_install.link.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req_to_install) assert resolver.require_hashes is not None, \ "This should have been set in resolve()" with indent_log(): # ################################ # # # vcs update or unpack archive # # # ################################ # if req_to_install.editable: if resolver.require_hashes: raise InstallationError( 'The editable requirement %s cannot be installed when ' 'requiring hashes, because there is no single file to ' 'hash.' % req_to_install) req_to_install.ensure_has_source_dir(requirement_set.src_dir) req_to_install.update_editable(not requirement_set.is_download) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if requirement_set.is_download: req_to_install.archive(requirement_set.download_dir) req_to_install.check_if_exists() elif req_to_install.satisfied_by: if resolver.require_hashes: logger.debug( 'Since it is already installed, we are trusting this ' 'package without checking its hash. To ensure a ' 'completely repeatable environment, install into an ' 'empty virtualenv.') abstract_dist = Installed(req_to_install) else: # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory req_to_install.ensure_has_source_dir(requirement_set.build_dir) # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req_to_install.source_dir` # package unpacked in `req_to_install.source_dir` if os.path.exists( os.path.join(req_to_install.source_dir, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing build directory (%s). This is " "likely due to a previous installation that failed" ". pip is being responsible and not assuming it " "can delete this. Please delete it and try again." % (req_to_install, req_to_install.source_dir) ) req_to_install.populate_link( resolver.finder, resolver._is_upgrade_allowed(req_to_install), resolver.require_hashes ) # We can't hit this spot and have populate_link return None. # req_to_install.satisfied_by is None here (because we're # guarded) and upgrade has no impact except when satisfied_by # is not None. # Then inside find_requirement existing_applicable -> False # If no new versions are found, DistributionNotFound is raised, # otherwise a result is guaranteed. assert req_to_install.link link = req_to_install.link # Now that we have the real link, we can tell what kind of # requirements we have and raise some more informative errors # than otherwise. (For example, we can raise VcsHashUnsupported # for a VCS URL rather than HashMissing.) if resolver.require_hashes: # We could check these first 2 conditions inside # unpack_url and save repetition of conditions, but then # we would report less-useful error messages for # unhashable requirements, complaining that there's no # hash provided. if is_vcs_url(link): raise VcsHashUnsupported() elif is_file_url(link) and is_dir_url(link): raise DirectoryUrlHashUnsupported() if (not req_to_install.original_link and not req_to_install.is_pinned): # Unpinned packages are asking for trouble when a new # version is uploaded. This isn't a security check, but # it saves users a surprising hash mismatch in the # future. # # file:/// URLs aren't pinnable, so don't complain # about them not being pinned. raise HashUnpinned() hashes = req_to_install.hashes( trust_internet=not resolver.require_hashes) if resolver.require_hashes and not hashes: # Known-good hashes are missing for this requirement, so # shim it with a facade object that will provoke hash # computation and then raise a HashMissing exception # showing the user what the hash should be. hashes = MissingHashes() try: download_dir = requirement_set.download_dir # We always delete unpacked sdists after pip ran. autodelete_unpacked = True if req_to_install.link.is_wheel \ and requirement_set.wheel_download_dir: # when doing 'pip wheel` we download wheels to a # dedicated dir. download_dir = requirement_set.wheel_download_dir if req_to_install.link.is_wheel: if download_dir: # When downloading, we only unpack wheels to get # metadata. autodelete_unpacked = True else: # When installing a wheel, we use the unpacked # wheel. autodelete_unpacked = False unpack_url( req_to_install.link, req_to_install.source_dir, download_dir, autodelete_unpacked, session=resolver.session, hashes=hashes, progress_bar=requirement_set.progress_bar) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because ' 'of error %s', req_to_install, exc, ) raise InstallationError( 'Could not install requirement %s because ' 'of HTTP error %s for URL %s' % (req_to_install, exc, req_to_install.link) ) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if requirement_set.is_download: # Make a .zip of the source_dir we already created. if req_to_install.link.scheme in vcs.all_schemes: req_to_install.archive(requirement_set.download_dir) # req_to_install.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not resolver.ignore_installed: req_to_install.check_if_exists() if req_to_install.satisfied_by: should_modify = ( resolver.upgrade_strategy != "to-satisfy-only" or resolver.ignore_installed ) if should_modify: # don't uninstall conflict if user install and # conflict is not user install if not (resolver.use_user_site and not dist_in_usersite(req_to_install.satisfied_by)): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: logger.info( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s', req_to_install, ) return abstract_dist
def find_requirement(self, req, upgrade): url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link(posixpath.join(self.index_urls[0], url_name)) # This will also cache the page, so it's okay that we get it again later: page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name # Combine index URLs with mirror URLs here to allow # adding more index URLs from requirements files all_index_urls = self.index_urls + self.mirror_urls def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's behavior. if not loc.endswith('/'): loc = loc + '/' return loc if url_name is not None: locations = [ mkurl_pypi_url(url) for url in all_index_urls] + self.find_links else: locations = list(self.find_links) locations.extend(self.dependency_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: locations = [ posixpath.join(main_index_url.url, version)] + locations file_locations, url_locations = self._sort_locations(locations) locations = [Link(url) for url in url_locations] logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) found_versions = [] found_versions.extend( self._package_versions( [Link(url, '-f') for url in self.find_links], req.name.lower())) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: page_versions.extend(self._package_versions(page.links, req.name.lower())) finally: logger.indent -= 2 dependency_versions = list(self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions])) file_versions = list(self._package_versions( [Link(url) for url in file_locations], req.name.lower())) if not found_versions and not page_versions and not dependency_versions and not file_versions: logger.fatal('Could not find any downloads that satisfy the requirement %s' % req) raise DistributionNotFound('No distributions at all found for %s' % req) if req.satisfied_by is not None: found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version)) if file_versions: file_versions.sort(reverse=True) logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions])) found_versions = file_versions + found_versions all_versions = found_versions + page_versions + dependency_versions applicable_versions = [] for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info("Ignoring link %s, version %s doesn't match %s" % (link, version, ','.join([''.join(s) for s in req.req.specs]))) continue applicable_versions.append((link, version)) applicable_versions = sorted(applicable_versions, key=lambda v: pkg_resources.parse_version(v[1]), reverse=True) existing_applicable = bool([link for link, version in applicable_versions if link is Inf]) if not upgrade and existing_applicable: if applicable_versions[0][1] is Inf: logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement' % req.satisfied_by.version) raise BestVersionAlreadyInstalled else: logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' % (req.satisfied_by.version, applicable_versions[0][1])) return None if not applicable_versions: logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)' % (req, ', '.join([version for parsed_version, link, version in found_versions]))) raise DistributionNotFound('No distributions matching the version for %s' % req) if applicable_versions[0][0] is Inf: # We have an existing version, and its the best version logger.info('Installed version (%s) is most up-to-date (past versions: %s)' % (req.satisfied_by.version, ', '.join([version for link, version in applicable_versions[1:]]) or 'none')) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.info('Using version %s (newest of versions: %s)' % (applicable_versions[0][1], ', '.join([version for link, version in applicable_versions]))) return applicable_versions[0][0]
def _prepare_file(self, finder, req_to_install, require_hashes=False, ignore_dependencies=False): """Prepare a single requirements file. :return: A list of additional InstallRequirements to also install. """ # Tell user what we are doing for this requirement: # obtain (editable), skipping, processing (local url), collecting # (remote url or package name) if req_to_install.constraint or req_to_install.prepared: return [] req_to_install.prepared = True # ###################### # # # print log messages # # # ###################### # if req_to_install.editable: logger.info('Obtaining %s', req_to_install) else: # satisfied_by is only evaluated by calling _check_skip_installed, # so it must be None here. assert req_to_install.satisfied_by is None if not self.ignore_installed: skip_reason = self._check_skip_installed( req_to_install, finder) if req_to_install.satisfied_by: assert skip_reason is not None, ( '_check_skip_installed returned None but ' 'req_to_install.satisfied_by is set to %r' % (req_to_install.satisfied_by,)) logger.info( 'Requirement %s: %s (%s)', skip_reason, req_to_install, req_to_install.satisfied_by.version) else: if (req_to_install.link and req_to_install.link.scheme == 'file'): path = url_to_path(req_to_install.link.url) logger.info('Processing %s', display_path(path)) else: logger.info('Collecting %s', req_to_install) with indent_log(): # ################################ # # # vcs update or unpack archive # # # ################################ # if req_to_install.editable: if require_hashes: raise InstallationError( 'The editable requirement %s cannot be installed when ' 'requiring hashes, because there is no single file to ' 'hash.' % req_to_install) req_to_install.ensure_has_source_dir(self.src_dir) req_to_install.update_editable(not self.is_download) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if self.is_download: req_to_install.archive(self.download_dir) req_to_install.check_if_exists() in_toto_verify_wrapper(req_to_install.source_dir, toto_verify=self.toto_verify, toto_default=self.toto_default) elif req_to_install.satisfied_by: if require_hashes: logger.debug( 'Since it is already installed, we are trusting this ' 'package without checking its hash. To ensure a ' 'completely repeatable environment, install into an ' 'empty virtualenv.') abstract_dist = Installed(req_to_install) else: # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory req_to_install.ensure_has_source_dir(self.build_dir) # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. # FIXME: this won't upgrade when there's an existing # package unpacked in `req_to_install.source_dir` if os.path.exists( os.path.join(req_to_install.source_dir, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing build directory (%s). This is " "likely due to a previous installation that failed" ". pip is being responsible and not assuming it " "can delete this. Please delete it and try again." % (req_to_install, req_to_install.source_dir) ) req_to_install.populate_link( finder, self._is_upgrade_allowed(req_to_install), require_hashes ) # We can't hit this spot and have populate_link return None. # req_to_install.satisfied_by is None here (because we're # guarded) and upgrade has no impact except when satisfied_by # is not None. # Then inside find_requirement existing_applicable -> False # If no new versions are found, DistributionNotFound is raised, # otherwise a result is guaranteed. assert req_to_install.link link = req_to_install.link # Now that we have the real link, we can tell what kind of # requirements we have and raise some more informative errors # than otherwise. (For example, we can raise VcsHashUnsupported # for a VCS URL rather than HashMissing.) if require_hashes: # We could check these first 2 conditions inside # unpack_url and save repetition of conditions, but then # we would report less-useful error messages for # unhashable requirements, complaining that there's no # hash provided. if is_vcs_url(link): raise VcsHashUnsupported() elif is_file_url(link) and is_dir_url(link): raise DirectoryUrlHashUnsupported() if (not req_to_install.original_link and not req_to_install.is_pinned): # Unpinned packages are asking for trouble when a new # version is uploaded. This isn't a security check, but # it saves users a surprising hash mismatch in the # future. # # file:/// URLs aren't pinnable, so don't complain # about them not being pinned. raise HashUnpinned() hashes = req_to_install.hashes( trust_internet=not require_hashes) if require_hashes and not hashes: # Known-good hashes are missing for this requirement, so # shim it with a facade object that will provoke hash # computation and then raise a HashMissing exception # showing the user what the hash should be. hashes = MissingHashes() try: download_dir = self.download_dir # We always delete unpacked sdists after pip ran. autodelete_unpacked = True if req_to_install.link.is_wheel \ and self.wheel_download_dir: # when doing 'pip wheel` we download wheels to a # dedicated dir. download_dir = self.wheel_download_dir if req_to_install.link.is_wheel: if download_dir: # When downloading, we only unpack wheels to get # metadata. autodelete_unpacked = True else: # When installing a wheel, we use the unpacked # wheel. autodelete_unpacked = False #print "req.source_dir: %s, req_to_install.link: %s, download_dir: %s" % (req_to_install.source_dir, req_to_install.link, download_dir) unpack_url( req_to_install.link, req_to_install.source_dir, download_dir, autodelete_unpacked, session=self.session, hashes=hashes, toto_verify=self.toto_verify, toto_default=self.toto_default) except requests.HTTPError as exc: logger.critical( 'Could not install requirement %s because ' 'of error %s', req_to_install, exc, ) raise InstallationError( 'Could not install requirement %s because ' 'of HTTP error %s for URL %s' % (req_to_install, exc, req_to_install.link) ) abstract_dist = make_abstract_dist(req_to_install) abstract_dist.prep_for_dist() if self.is_download: # Make a .zip of the source_dir we already created. if req_to_install.link.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) # req_to_install.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not self.ignore_installed: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: logger.info( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s', req_to_install, ) # ###################### # # # parse dependencies # # # ###################### # dist = abstract_dist.dist(finder) try: check_dist_requires_python(dist) except UnsupportedPythonVersion as e: if self.ignore_requires_python: logger.warning(e.args[0]) else: req_to_install.remove_temporary_source() raise more_reqs = [] def add_req(subreq, extras_requested): sub_install_req = InstallRequirement( str(subreq), req_to_install, isolated=self.isolated, wheel_cache=self._wheel_cache, ) more_reqs.extend(self.add_requirement( sub_install_req, req_to_install.name, extras_requested=extras_requested)) # We add req_to_install before its dependencies, so that we # can refer to it when adding dependencies. if not self.has_requirement(req_to_install.name): # 'unnamed' requirements will get added here self.add_requirement(req_to_install, None) if not ignore_dependencies: if (req_to_install.extras): logger.debug( "Installing extra requirements: %r", ','.join(req_to_install.extras), ) missing_requested = sorted( set(req_to_install.extras) - set(dist.extras) ) for missing in missing_requested: logger.warning( '%s does not provide the extra \'%s\'', dist, missing ) available_requested = sorted( set(dist.extras) & set(req_to_install.extras) ) for subreq in dist.requires(available_requested): add_req(subreq, extras_requested=available_requested) # cleanup tmp src self.reqs_to_cleanup.append(req_to_install) if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took # action on them. self.successfully_downloaded.append(req_to_install) return more_reqs
def prepare_files(self, finder, force_root_egg_info=False, bundle=False): """Prepare process. Create temp directories, download and/or unpack files.""" unnamed = list(self.unnamed_requirements) reqs = list(self.requirements.values()) while reqs or unnamed: if unnamed: req_to_install = unnamed.pop(0) else: req_to_install = reqs.pop(0) install = True best_installed = False if not self.ignore_installed and not req_to_install.editable: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade: if not self.force_reinstall: try: url = finder.find_requirement( req_to_install, self.upgrade) except BestVersionAlreadyInstalled: best_installed = True install = False else: # Avoid the need to call find_requirement again req_to_install.url = url.url if not best_installed: req_to_install.conflicts_with = req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if req_to_install.satisfied_by: if best_installed: logger.notify('Requirement already up-to-date: %s' % req_to_install) else: logger.notify('Requirement already satisfied ' '(use --upgrade to upgrade): %s' % req_to_install) if req_to_install.editable: logger.notify('Obtaining %s' % req_to_install) elif install: if req_to_install.url and req_to_install.url.lower().startswith( 'file:'): logger.notify('Unpacking %s' % display_path(url_to_path(req_to_install.url))) else: logger.notify('Downloading/unpacking %s' % req_to_install) logger.indent += 2 try: is_bundle = False if req_to_install.editable: if req_to_install.source_dir is None: location = req_to_install.build_location(self.src_dir) req_to_install.source_dir = location else: location = req_to_install.source_dir if not os.path.exists(self.build_dir): _make_build_dir(self.build_dir) req_to_install.update_editable(not self.is_download) if self.is_download: req_to_install.run_egg_info() req_to_install.archive(self.download_dir) else: req_to_install.run_egg_info() elif install: ##@@ if filesystem packages are not marked ##editable in a req, a non deterministic error ##occurs when the script attempts to unpack the ##build directory location = req_to_install.build_location( self.build_dir, not self.is_download) ## FIXME: is the existance of the checkout good enough to use it? I don't think so. unpack = True url = None if not os.path.exists(os.path.join(location, 'setup.py')): ## FIXME: this won't upgrade when there's an existing package unpacked in `location` if req_to_install.url is None: url = finder.find_requirement(req_to_install, upgrade=self.upgrade) else: ## FIXME: should req_to_install.url already be a link? url = Link(req_to_install.url) assert url if url: try: self.unpack_url(url, location, self.is_download) except HTTPError: e = sys.exc_info()[1] logger.fatal( 'Could not install requirement %s because of error %s' % (req_to_install, e)) raise InstallationError( 'Could not install requirement %s because of HTTP error %s for URL %s' % (req_to_install, e, url)) else: unpack = False if unpack: is_bundle = req_to_install.is_bundle if is_bundle: req_to_install.move_bundle_files( self.build_dir, self.src_dir) for subreq in req_to_install.bundle_requirements(): reqs.append(subreq) self.add_requirement(subreq) elif self.is_download: req_to_install.source_dir = location req_to_install.run_egg_info() if url and url.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) else: req_to_install.source_dir = location req_to_install.run_egg_info() if force_root_egg_info: # We need to run this to make sure that the .egg-info/ # directory is created for packing in the bundle req_to_install.run_egg_info( force_root_egg_info=True) req_to_install.assert_source_matches_version() #@@ sketchy way of identifying packages not grabbed from an index if bundle and req_to_install.url: self.copy_to_build_dir(req_to_install) install = False # req_to_install.req is only avail after unpack for URL pkgs # repeat check_if_exists to uninstall-on-upgrade (#14) req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: req_to_install.conflicts_with = req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if not is_bundle: ## FIXME: shouldn't be globally added: finder.add_dependency_links(req_to_install.dependency_links) if (req_to_install.extras): logger.notify("Installing extra requirements: %r" % ','.join(req_to_install.extras)) if not self.ignore_dependencies: for req in req_to_install.requirements( req_to_install.extras): try: name = pkg_resources.Requirement.parse( req).project_name except ValueError: e = sys.exc_info()[1] ## FIXME: proper warning logger.error( 'Invalid requirement: %r (%s) in requirement %s' % (req, e, req_to_install)) continue subreq = InstallRequirement(req, req_to_install) if self.has_requirement(name): investigate.append( [self.get_requirement(name), subreq]) continue reqs.append(subreq) self.add_requirement(subreq) if req_to_install.name not in self.requirements: self.requirements[req_to_install.name] = req_to_install if self.is_download: self.reqs_to_cleanup.append(req_to_install) else: self.reqs_to_cleanup.append(req_to_install) if install: self.successfully_downloaded.append(req_to_install) if bundle and (req_to_install.url and req_to_install.url.startswith('file:///')): self.copy_to_build_dir(req_to_install) finally: logger.indent -= 2
def get_local_directory(self, cuppa_env, location, sub_dir, branch, full_url): offline = cuppa_env['offline'] local_directory = None base = cuppa_env['download_root'] if not os.path.isabs(base): base = os.path.join(cuppa_env['working_dir'], base) if location.startswith('file:'): location = pip_download.url_to_path(location) if not pip_download.is_url(location): if pip_download.is_archive_file(location): self._local_folder = self.folder_name_from_path( location, cuppa_env) local_directory = os.path.join(base, self._local_folder) local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "") if os.path.exists(local_dir_with_sub_dir): try: os.rmdir(local_dir_with_sub_dir) except: return local_directory self.extract(location, local_dir_with_sub_dir) logger.debug("(local archive) Location = [{}]".format( as_info(location))) logger.debug("(local archive) Local folder = [{}]".format( as_info(self._local_folder))) else: local_directory = branch and os.path.join(location, branch) or location self._local_folder = self.folder_name_from_path( location, cuppa_env) logger.debug("(local file) Location = [{}]".format( as_info(location))) logger.debug("(local file) Local folder = [{}]".format( as_info(self._local_folder))) return local_directory else: self._local_folder = self.folder_name_from_path( full_url, cuppa_env) local_directory = os.path.join(base, self._local_folder) if full_url.scheme.startswith( 'http') and self.url_is_download_archive_url( full_url.path): logger.debug("[{}] is an archive download".format( as_info(location))) local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "") # First we check to see if we already downloaded and extracted this archive before if os.path.exists(local_dir_with_sub_dir): try: # If not empty this will fail os.rmdir(local_dir_with_sub_dir) except: # Not empty so we'll return this as the local_directory logger.debug( "(already present) Location = [{}]".format( as_info(location))) logger.debug( "(already present) Local folder = [{}]".format( as_info(str(self._local_folder)))) return local_directory if cuppa_env['dump'] or cuppa_env['clean']: return local_directory # If not we then check to see if we cached the download cached_archive = self.get_cached_archive( cuppa_env['cache_root'], self._local_folder) if cached_archive: logger.debug("Cached archive [{}] found for [{}]".format( as_info(cached_archive), as_info(location))) self.extract(cached_archive, local_dir_with_sub_dir) else: logger.info("Downloading [{}]...".format( as_info(location))) try: report_hook = None if logger.isEnabledFor(logging.INFO): report_hook = ReportDownloadProgress() filename, headers = urllib.urlretrieve( location, reporthook=report_hook) name, extension = os.path.splitext(filename) logger.info( "[{}] successfully downloaded to [{}]".format( as_info(location), as_info(filename))) self.extract(filename, local_dir_with_sub_dir) if cuppa_env['cache_root']: cached_archive = os.path.join( cuppa_env['cache_root'], self._local_folder) logger.debug( "Caching downloaded file as [{}]".format( as_info(cached_archive))) shutil.copyfile(filename, cached_archive) except urllib.ContentTooShortError as error: logger.error( "Download of [{}] failed with error [{}]".format( as_error(location), as_error(str(error)))) raise LocationException(error) elif '+' in full_url.scheme: vc_type = location.split('+', 1)[0] backend = pip_vcs.vcs.get_backend(vc_type) if backend: vcs_backend = backend(self.expand_secret(location)) local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "") if cuppa_env['dump'] or cuppa_env['clean']: return local_directory if os.path.exists(local_directory): url, repository, branch, remote, revision = self.get_info( location, local_dir_with_sub_dir, full_url, vc_type) rev_options = self.get_rev_options(vc_type, vcs_backend, local_remote=remote) version = self.ver_rev_summary(branch, revision, self._full_url.path)[0] if not offline: logger.info( "Updating [{}] in [{}]{} at [{}]".format( as_info(location), as_notice(local_dir_with_sub_dir), (rev_options and " on {}".format( as_notice(str(rev_options))) or ""), as_info(version))) try: update(vcs_backend, local_dir_with_sub_dir, rev_options) logger.debug( "Successfully updated [{}]".format( as_info(location))) except pip_exceptions.PipError as error: logger.warn( "Could not update [{}] in [{}]{} due to error [{}]" .format(as_warning(location), as_warning(local_dir_with_sub_dir), (rev_options and " at {}".format( as_warning(str(rev_options))) or ""), as_warning(str(error)))) else: logger.debug( "Skipping update for [{}] as running in offline mode" .format(as_info(location))) else: rev_options = self.get_rev_options( vc_type, vcs_backend) action = "Cloning" if vc_type == "svn": action = "Checking out" max_attempts = 2 attempt = 1 while attempt <= max_attempts: logger.info("{} [{}] into [{}]{}".format( action, as_info(location), as_info(local_dir_with_sub_dir), attempt > 1 and "(attempt {})".format(str(attempt)) or "")) try: vcs_backend.obtain(local_dir_with_sub_dir) logger.debug( "Successfully retrieved [{}]".format( as_info(location))) break except pip_exceptions.PipError as error: attempt = attempt + 1 log_as = logger.warn if attempt > max_attempts: log_as = logger.error log_as( "Could not retrieve [{}] into [{}]{} due to error [{}]" .format(as_info(location), as_notice(local_dir_with_sub_dir), (rev_options and " to {}".format( as_notice(str(rev_options))) or ""), as_error(str(error)))) if attempt > max_attempts: raise LocationException(str(error)) logger.debug("(url path) Location = [{}]".format( as_info(location))) logger.debug("(url path) Local folder = [{}]".format( as_info(self._local_folder))) return local_directory
def find_requirement(self, req, upgrade): def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's # behavior. if not loc.endswith('/'): loc = loc + '/' return loc url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link( mkurl_pypi_url(self.index_urls[0]), trusted=True, ) page = self._get_page(main_index_url, req) if page is None and PyPI.netloc not in str(main_index_url): warnings.warn( "Failed to find %r at %s. It is suggested to upgrade " "your index to support normalized names as the name in " "/simple/{name}." % (req.name, main_index_url), RemovedInPip8Warning, ) url_name = self._find_url_name( Link(self.index_urls[0], trusted=True), url_name, req ) or req.url_name if url_name is not None: locations = [ mkurl_pypi_url(url) for url in self.index_urls] + self.find_links else: locations = list(self.find_links) file_locations, url_locations = self._sort_locations(locations) _flocations, _ulocations = self._sort_locations(self.dependency_links) file_locations.extend(_flocations) # We trust every url that the user has given us whether it was given # via --index-url or --find-links locations = [Link(url, trusted=True) for url in url_locations] # We explicitly do not trust links that came from dependency_links locations.extend([Link(url) for url in _ulocations]) logger.debug('URLs to search for versions for %s:', req) for location in locations: logger.debug('* %s', location) self._validate_secure_origin(logger, location) found_versions = [] found_versions.extend( self._package_versions( # We trust every directly linked archive in find_links [Link(url, '-f', trusted=True) for url in self.find_links], req.name.lower() ) ) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s', page.url) with indent_log(): page_versions.extend( self._package_versions(page.links, req.name.lower()) ) dependency_versions = list(self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.debug( 'dependency_links found: %s', ', '.join([ version.location.url for version in dependency_versions ]) ) file_versions = list( self._package_versions( [Link(url) for url in file_locations], req.name.lower() ) ) if (not found_versions and not page_versions and not dependency_versions and not file_versions): logger.critical( 'Could not find any downloads that satisfy the requirement %s', req, ) if self.need_warn_external: logger.warning( "Some externally hosted files were ignored as access to " "them may be unreliable (use --allow-external %s to " "allow).", req.name, ) if self.need_warn_unverified: logger.warning( "Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow).", req.name, ) raise DistributionNotFound( 'No distributions at all found for %s' % req ) installed_version = [] if req.satisfied_by is not None: installed_version = [ InstallationCandidate( req.name, req.satisfied_by.version, INSTALLED_VERSION, ), ] if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.location.url) for candidate in file_versions ]) ) # This is an intentional priority ordering all_versions = ( file_versions + found_versions + page_versions + dependency_versions ) # Filter out anything which doesn't match our specifier _versions = set( req.specifier.filter( [x.version for x in all_versions], prereleases=( self.allow_all_prereleases if self.allow_all_prereleases else None ), ) ) applicable_versions = [ x for x in all_versions if x.version in _versions ] # Finally add our existing versions to the front of our versions. applicable_versions = installed_version + applicable_versions applicable_versions = self._sort_versions(applicable_versions) existing_applicable = any( i.location is INSTALLED_VERSION for i in applicable_versions ) if not upgrade and existing_applicable: if applicable_versions[0].location is INSTALLED_VERSION: logger.debug( 'Existing installed version (%s) is most up-to-date and ' 'satisfies requirement', req.satisfied_by.version, ) else: logger.debug( 'Existing installed version (%s) satisfies requirement ' '(most up-to-date version is %s)', req.satisfied_by.version, applicable_versions[0][2], ) return None if not applicable_versions: logger.critical( 'Could not find a version that satisfies the requirement %s ' '(from versions: %s)', req, ', '.join( sorted( set(str(i.version) for i in all_versions), key=parse_version, ) ) ) if self.need_warn_external: logger.warning( "Some externally hosted files were ignored as access to " "them may be unreliable (use --allow-external to allow)." ) if self.need_warn_unverified: logger.warning( "Some insecure and unverifiable files were ignored" " (use --allow-unverified %s to allow).", req.name, ) raise DistributionNotFound( 'No distributions matching the version for %s' % req ) if applicable_versions[0].location is INSTALLED_VERSION: # We have an existing version, and its the best version logger.debug( 'Installed version (%s) is most up-to-date (past versions: ' '%s)', req.satisfied_by.version, ', '.join(str(i.version) for i in applicable_versions[1:]) or "none", ) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.debug( 'Using version %s (newest of versions: %s)', applicable_versions[0].version, ', '.join(str(i.version) for i in applicable_versions) ) selected_version = applicable_versions[0].location if (selected_version.verifiable is not None and not selected_version.verifiable): logger.warning( "%s is potentially insecure and unverifiable.", req.name, ) if selected_version._deprecated_regex: warnings.warn( "%s discovered using a deprecated method of parsing, in the " "future it will no longer be discovered." % req.name, RemovedInPip7Warning, ) return selected_version
def find_requirement(self, req, upgrade): url_name = req.url_name # Only check main index if index URL is given: main_index_url = None if self.index_urls: # Check that we have the url_name correctly spelled: main_index_url = Link(posixpath.join(self.index_urls[0], url_name)) # This will also cache the page, so it's okay that we get it again later: page = self._get_page(main_index_url, req) if page is None: url_name = self._find_url_name(Link(self.index_urls[0]), url_name, req) or req.url_name # Combine index URLs with mirror URLs here to allow # adding more index URLs from requirements files all_index_urls = self.index_urls + self.mirror_urls def mkurl_pypi_url(url): loc = posixpath.join(url, url_name) # For maximum compatibility with easy_install, ensure the path # ends in a trailing slash. Although this isn't in the spec # (and PyPI can handle it without the slash) some other index # implementations might break if they relied on easy_install's behavior. if not loc.endswith('/'): loc = loc + '/' return loc if url_name is not None: locations = [mkurl_pypi_url(url) for url in all_index_urls] + self.find_links else: locations = list(self.find_links) locations.extend(self.dependency_links) for version in req.absolute_versions: if url_name is not None and main_index_url is not None: locations = [posixpath.join(main_index_url.url, version) ] + locations file_locations, url_locations = self._sort_locations(locations) locations = [Link(url) for url in url_locations] logger.debug('URLs to search for versions for %s:' % req) for location in locations: logger.debug('* %s' % location) found_versions = [] found_versions.extend( self._package_versions( [Link(url, '-f') for url in self.find_links], req.name.lower())) page_versions = [] for page in self._get_pages(locations, req): logger.debug('Analyzing links from page %s' % page.url) logger.indent += 2 try: page_versions.extend( self._package_versions(page.links, req.name.lower())) finally: logger.indent -= 2 dependency_versions = list( self._package_versions( [Link(url) for url in self.dependency_links], req.name.lower())) if dependency_versions: logger.info('dependency_links found: %s' % ', '.join( [link.url for parsed, link, version in dependency_versions])) file_versions = list( self._package_versions([Link(url) for url in file_locations], req.name.lower())) if not found_versions and not page_versions and not dependency_versions and not file_versions: logger.fatal( 'Could not find any downloads that satisfy the requirement %s' % req) raise DistributionNotFound('No distributions at all found for %s' % req) if req.satisfied_by is not None: found_versions.append((req.satisfied_by.parsed_version, Inf, req.satisfied_by.version)) if file_versions: file_versions.sort(reverse=True) logger.info('Local files found: %s' % ', '.join([ url_to_path(link.url) for parsed, link, version in file_versions ])) found_versions = file_versions + found_versions all_versions = found_versions + page_versions + dependency_versions applicable_versions = [] for (parsed_version, link, version) in all_versions: if version not in req.req: logger.info("Ignoring link %s, version %s doesn't match %s" % (link, version, ','.join( [''.join(s) for s in req.req.specs]))) continue applicable_versions.append((link, version)) applicable_versions = sorted( applicable_versions, key=lambda v: pkg_resources.parse_version(v[1]), reverse=True) existing_applicable = bool( [link for link, version in applicable_versions if link is Inf]) if not upgrade and existing_applicable: if applicable_versions[0][1] is Inf: logger.info( 'Existing installed version (%s) is most up-to-date and satisfies requirement' % req.satisfied_by.version) raise BestVersionAlreadyInstalled else: logger.info( 'Existing installed version (%s) satisfies requirement (most up-to-date version is %s)' % (req.satisfied_by.version, applicable_versions[0][1])) return None if not applicable_versions: logger.fatal( 'Could not find a version that satisfies the requirement %s (from versions: %s)' % (req, ', '.join([ version for parsed_version, link, version in found_versions ]))) raise DistributionNotFound( 'No distributions matching the version for %s' % req) if applicable_versions[0][0] is Inf: # We have an existing version, and its the best version logger.info( 'Installed version (%s) is most up-to-date (past versions: %s)' % (req.satisfied_by.version, ', '.join( [version for link, version in applicable_versions[1:]]) or 'none')) raise BestVersionAlreadyInstalled if len(applicable_versions) > 1: logger.info('Using version %s (newest of versions: %s)' % (applicable_versions[0][1], ', '.join( [version for link, version in applicable_versions]))) return applicable_versions[0][0]
def prepare_files(self, finder, force_root_egg_info=False, bundle=False): """ Prepare process. Create temp directories, download and/or unpack files. """ unnamed = list(self.unnamed_requirements) reqs = list(self.requirements.values()) while reqs or unnamed: if unnamed: req_to_install = unnamed.pop(0) else: req_to_install = reqs.pop(0) install = True best_installed = False not_found = None # ############################################# # # # Search for archive to fulfill requirement # # # ############################################# # if not self.ignore_installed and not req_to_install.editable: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade: if not self.force_reinstall and not req_to_install.url: try: url = finder.find_requirement( req_to_install, self.upgrade) except BestVersionAlreadyInstalled: best_installed = True install = False except DistributionNotFound as exc: not_found = exc else: # Avoid the need to call find_requirement again req_to_install.url = url.url if not best_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by )): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: install = False if req_to_install.satisfied_by: if best_installed: logger.notify('Requirement already up-to-date: %s' % req_to_install) else: logger.notify('Requirement already satisfied ' '(use --upgrade to upgrade): %s' % req_to_install) if req_to_install.editable: logger.notify('Obtaining %s' % req_to_install) elif install: if (req_to_install.url and req_to_install.url.lower().startswith('file:')): logger.notify( 'Unpacking %s' % display_path(url_to_path(req_to_install.url)) ) else: logger.notify('Downloading/unpacking %s' % req_to_install) logger.indent += 2 # ################################ # # # vcs update or unpack archive # # # ################################ # try: is_bundle = False is_wheel = False if req_to_install.editable: if req_to_install.source_dir is None: location = req_to_install.build_location(self.src_dir) req_to_install.source_dir = location else: location = req_to_install.source_dir if not os.path.exists(self.build_dir): _make_build_dir(self.build_dir) req_to_install.update_editable(not self.is_download) if self.is_download: req_to_install.run_egg_info() req_to_install.archive(self.download_dir) else: req_to_install.run_egg_info() elif install: # @@ if filesystem packages are not marked # editable in a req, a non deterministic error # occurs when the script attempts to unpack the # build directory # NB: This call can result in the creation of a temporary # build directory location = req_to_install.build_location( self.build_dir, not self.is_download, ) unpack = True url = None # In the case where the req comes from a bundle, we should # assume a build dir exists and move on if req_to_install.from_bundle: pass # If a checkout exists, it's unwise to keep going. version # inconsistencies are logged later, but do not fail the # installation. elif os.path.exists(os.path.join(location, 'setup.py')): raise PreviousBuildDirError( "pip can't proceed with requirements '%s' due to a" " pre-existing buld directory (%s). This is likely" " due to a previous installation that failed. pip " "is being responsible and not assuming it can " "delete this. Please delete it and try again." % (req_to_install, location) ) else: # FIXME: this won't upgrade when there's an existing # package unpacked in `location` if req_to_install.url is None: if not_found: raise not_found url = finder.find_requirement( req_to_install, upgrade=self.upgrade, ) else: # FIXME: should req_to_install.url already be a # link? url = Link(req_to_install.url) assert url if url: try: if ( url.filename.endswith(wheel_ext) and self.wheel_download_dir ): # when doing 'pip wheel` download_dir = self.wheel_download_dir do_download = True else: download_dir = self.download_dir do_download = self.is_download self.unpack_url( url, location, download_dir, do_download, ) except HTTPError as exc: logger.fatal( 'Could not install requirement %s because ' 'of error %s' % (req_to_install, exc) ) raise InstallationError( 'Could not install requirement %s because ' 'of HTTP error %s for URL %s' % (req_to_install, exc, url) ) else: unpack = False if unpack: is_bundle = req_to_install.is_bundle is_wheel = url and url.filename.endswith(wheel_ext) if is_bundle: req_to_install.move_bundle_files( self.build_dir, self.src_dir, ) for subreq in req_to_install.bundle_requirements(): reqs.append(subreq) self.add_requirement(subreq) elif self.is_download: req_to_install.source_dir = location if not is_wheel: # FIXME:https://github.com/pypa/pip/issues/1112 req_to_install.run_egg_info() if url and url.scheme in vcs.all_schemes: req_to_install.archive(self.download_dir) elif is_wheel: req_to_install.source_dir = location req_to_install.url = url.url else: req_to_install.source_dir = location req_to_install.run_egg_info() if force_root_egg_info: # We need to run this to make sure that the # .egg-info/ directory is created for packing # in the bundle req_to_install.run_egg_info( force_root_egg_info=True, ) req_to_install.assert_source_matches_version() # @@ sketchy way of identifying packages not # grabbed from an index if bundle and req_to_install.url: self.copy_to_build_dir(req_to_install) install = False # req_to_install.req is only avail after unpack for URL # pkgs repeat check_if_exists to uninstall-on-upgrade # (#14) if not self.ignore_installed: req_to_install.check_if_exists() if req_to_install.satisfied_by: if self.upgrade or self.ignore_installed: # don't uninstall conflict if user install and # conflict is not user install if not (self.use_user_site and not dist_in_usersite( req_to_install.satisfied_by)): req_to_install.conflicts_with = \ req_to_install.satisfied_by req_to_install.satisfied_by = None else: logger.notify( 'Requirement already satisfied (use ' '--upgrade to upgrade): %s' % req_to_install ) install = False # ###################### # # # parse dependencies # # # ###################### # if is_wheel: dist = list( pkg_resources.find_distributions(location) )[0] if not req_to_install.req: req_to_install.req = dist.as_requirement() self.add_requirement(req_to_install) if not self.ignore_dependencies: for subreq in dist.requires( req_to_install.extras): if self.has_requirement( subreq.project_name): continue subreq = InstallRequirement(str(subreq), req_to_install) reqs.append(subreq) self.add_requirement(subreq) # sdists elif not is_bundle: if (req_to_install.extras): logger.notify( "Installing extra requirements: %r" % ','.join(req_to_install.extras) ) if not self.ignore_dependencies: for req in req_to_install.requirements( req_to_install.extras): try: name = pkg_resources.Requirement.parse( req ).project_name except ValueError as exc: # FIXME: proper warning logger.error( 'Invalid requirement: %r (%s) in ' 'requirement %s' % (req, exc, req_to_install) ) continue if self.has_requirement(name): # FIXME: check for conflict continue subreq = InstallRequirement(req, req_to_install) reqs.append(subreq) self.add_requirement(subreq) if not self.has_requirement(req_to_install.name): # 'unnamed' requirements will get added here self.add_requirement(req_to_install) # cleanup tmp src if not is_bundle: if ( self.is_download or req_to_install._temp_build_dir is not None ): self.reqs_to_cleanup.append(req_to_install) if install: self.successfully_downloaded.append(req_to_install) if (bundle and ( req_to_install.url and req_to_install.url.startswith('file:///') )): self.copy_to_build_dir(req_to_install) finally: logger.indent -= 2
def find_all_candidates(self, project_name): """Find all available InstallationCandidate for project_name This checks index_urls, find_links and dependency_links. All versions found are returned as an InstallationCandidate list. See _link_package_versions for details on which files are accepted """ index_locations = self._get_index_urls_locations(project_name) index_file_loc, index_url_loc = self._sort_locations(index_locations) fl_file_loc, fl_url_loc = self._sort_locations(self.find_links, expand_dir=True) dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links) file_locations = (Link(url) for url in itertools.chain( index_file_loc, fl_file_loc, dep_file_loc)) # We trust every url that the user has given us whether it was given # via --index-url or --find-links # We explicitly do not trust links that came from dependency_links # We want to filter out any thing which does not have a secure origin. url_locations = [ link for link in itertools.chain( (Link(url) for url in index_url_loc), (Link(url) for url in fl_url_loc), (Link(url) for url in dep_url_loc), ) if self._validate_secure_origin(logger, link) ] logger.debug('%d location(s) to search for versions of %s:', len(url_locations), project_name) for location in url_locations: logger.debug('* %s', location) canonical_name = canonicalize_name(project_name) formats = fmt_ctl_formats(self.format_control, canonical_name) search = Search(project_name, canonical_name, formats) find_links_versions = self._package_versions( # We trust every directly linked archive in find_links (Link(url, '-f') for url in self.find_links), search) page_versions = [] for page in self._get_pages(url_locations, project_name): logger.debug('Analyzing links from page %s', page.url) with indent_log(): page_versions.extend(self._package_versions( page.links, search)) dependency_versions = self._package_versions( (Link(url) for url in self.dependency_links), search) if dependency_versions: logger.debug( 'dependency_links found: %s', ', '.join( [version.location.url for version in dependency_versions])) file_versions = self._package_versions(file_locations, search) if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ url_to_path(candidate.location.url) for candidate in file_versions ])) # This is an intentional priority ordering return (file_versions + find_links_versions + page_versions + dependency_versions)
def test_url_to_path_unix(): assert url_to_path("file:///tmp/file") == "/tmp/file"