def _is_url_like_archive(url): # type: (str) -> bool """Return whether the URL looks like an archive. """ filename = Link(url).filename for bad_ext in ARCHIVE_EXTENSIONS: if filename.endswith(bad_ext): return True return False
def collect_links(self, project_name): # type: (str) -> CollectedLinks """Find all available links for the given project name. :return: All the Link objects (unfiltered), as a CollectedLinks object. """ search_scope = self.search_scope index_locations = search_scope.get_index_urls_locations(project_name) index_file_loc, index_url_loc = group_locations(index_locations) fl_file_loc, fl_url_loc = group_locations( self.find_links, expand_dir=True, ) file_links = [ Link(url) for url in itertools.chain(index_file_loc, fl_file_loc) ] # We trust every directly linked archive in find_links find_link_links = [Link(url, '-f') for url in self.find_links] # We trust every url that the user has given us whether it was given # via --index-url or --find-links. # We want to filter out anything that does not have a secure origin. url_locations = [ link for link in itertools.chain( # Mark PyPI indices as "cache_link_parsing == False" -- this # will avoid caching the result of parsing the page for links. (Link(url, cache_link_parsing=False) for url in index_url_loc), (Link(url) for url in fl_url_loc), ) if self.session.is_secure_origin(link) ] url_locations = _remove_duplicate_links(url_locations) lines = [ '{} location(s) to search for versions of {}:'.format( len(url_locations), project_name, ), ] for link in url_locations: lines.append('* {}'.format(link)) logger.debug('\n'.join(lines)) return CollectedLinks( files=file_links, find_links=find_link_links, project_urls=url_locations, )
def _create_link_from_element( anchor, # type: HTMLElement page_url, # type: str base_url, # type: str ): # type: (...) -> Optional[Link] """ Convert an anchor element in a simple repository page to a Link. """ href = anchor.get("href") if not href: return None url = _clean_link(urllib_parse.urljoin(base_url, href)) pyrequire = anchor.get('data-requires-python') pyrequire = unescape(pyrequire) if pyrequire else None yanked_reason = anchor.get('data-yanked') if yanked_reason: # This is a unicode string in Python 2 (and 3). yanked_reason = unescape(yanked_reason) link = Link( url, comes_from=page_url, requires_python=pyrequire, yanked_reason=yanked_reason, ) return link
def build( requirements, # type: Iterable[InstallRequirement] wheel_cache, # type: WheelCache build_options, # type: List[str] global_options, # type: List[str] ): # type: (...) -> BuildResult """Build wheels. :return: The list of InstallRequirement that succeeded to build and the list of InstallRequirement that failed to build. """ if not requirements: return [], [] # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join(req.name for req in requirements), ) with indent_log(): build_successes, build_failures = [], [] for req in requirements: cache_dir = _get_cache_dir(req, wheel_cache) wheel_file = _build_one( req, cache_dir, build_options, global_options ) if wheel_file: # Update the link for this. req.link = Link(path_to_url(wheel_file)) req.local_file_path = req.link.file_path assert req.link.is_wheel build_successes.append(req) else: build_failures.append(req) # notify success/failure if build_successes: logger.info( 'Successfully built %s', ' '.join([req.name for req in build_successes]), ) if build_failures: logger.info( 'Failed to build %s', ' '.join([req.name for req in build_failures]), ) # Return a list of requirements that failed to build return build_successes, build_failures
def parse_req_from_editable(editable_req): # type: (str) -> RequirementParts name, url, extras_override = parse_editable(editable_req) if name is not None: try: req = Requirement(name) except InvalidRequirement: raise InstallationError("Invalid requirement: '{}'".format(name)) else: req = None link = Link(url) return RequirementParts(req, link, None, extras_override)
def get( self, link, # type: Link package_name, # type: Optional[str] supported_tags, # type: List[Tag] ): # type: (...) -> Link candidates = [] if not package_name: return link canonical_package_name = canonicalize_name(package_name) for wheel_name, wheel_dir in self._get_candidates( link, canonical_package_name): try: wheel = Wheel(wheel_name) except InvalidWheelFilename: continue if canonicalize_name(wheel.name) != canonical_package_name: logger.debug( "Ignoring cached wheel {} for {} as it " "does not match the expected distribution name {}.".format( wheel_name, link, package_name)) continue if not wheel.supported(supported_tags): # Built for a different python/arch/etc continue candidates.append(( wheel.support_index_min(supported_tags), wheel_name, wheel_dir, )) if not candidates: return link _, wheel_name, wheel_dir = min(candidates) return Link(path_to_url(os.path.join(wheel_dir, wheel_name)))
def __init__( self, req, # type: Optional[Requirement] comes_from, # type: Optional[Union[str, InstallRequirement]] editable=False, # type: bool link=None, # type: Optional[Link] markers=None, # type: Optional[Marker] use_pep517=None, # type: Optional[bool] isolated=False, # type: bool install_options=None, # type: Optional[List[str]] global_options=None, # type: Optional[List[str]] hash_options=None, # type: Optional[Dict[str, List[str]]] constraint=False, # type: bool extras=() # type: Iterable[str] ): # type: (...) -> None assert req is None or isinstance(req, Requirement), req self.req = req self.comes_from = comes_from self.constraint = constraint self.editable = editable # source_dir is the local directory where the linked requirement is # located, or unpacked. In case unpacking is needed, creating and # populating source_dir is done by the RequirementPreparer. Note this # is not necessarily the directory where pyproject.toml or setup.py is # located - that one is obtained via unpacked_source_directory. self.source_dir = None # type: Optional[str] if self.editable: assert link if link.is_file: self.source_dir = os.path.normpath( os.path.abspath(link.file_path) ) if link is None and req and req.url: # PEP 508 URL requirement link = Link(req.url) self.link = self.original_link = link self.original_link_is_in_wheel_cache = False # Path to any downloaded or already-existing package. self.local_file_path = None # type: Optional[str] if self.link and self.link.is_file: self.local_file_path = self.link.file_path if extras: self.extras = extras elif req: self.extras = { pkg_resources.safe_extra(extra) for extra in req.extras } else: self.extras = set() if markers is None and req: markers = req.marker self.markers = markers # This holds the pkg_resources.Distribution object if this requirement # is already available: self.satisfied_by = None # type: Optional[Distribution] # Whether the installation process should try to uninstall an existing # distribution before installing this requirement. self.should_reinstall = False # Temporary build location self._temp_build_dir = None # type: Optional[TempDirectory] # Set to True after successful installation self.install_succeeded = None # type: Optional[bool] # Supplied options self.install_options = install_options if install_options else [] self.global_options = global_options if global_options else [] self.hash_options = hash_options if hash_options else {} # Set to True after successful preparation of this requirement self.prepared = False self.is_direct = False # Set by the legacy resolver when the requirement has been downloaded # TODO: This introduces a strong coupling between the resolver and the # requirement (the coupling was previously between the resolver # and the requirement set). This should be refactored to allow # the requirement to decide for itself when it has been # successfully downloaded - but that is more tricky to get right, # se we are making the change in stages. self.successfully_downloaded = False self.isolated = isolated self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment # For PEP 517, the directory where we request the project metadata # gets stored. We need this to pass to build_wheel, so the backend # can ensure that the wheel matches the metadata (see the PEP for # details). self.metadata_directory = None # type: Optional[str] # The static build requirements (from pyproject.toml) self.pyproject_requires = None # type: Optional[List[str]] # Build requirements that we will check are available self.requirements_to_check = [] # type: List[str] # The PEP 517 backend we should use to build the project self.pep517_backend = None # type: Optional[Pep517HookCaller] # Are we using PEP 517 for this requirement? # After pyproject.toml has been loaded, the only valid values are True # and False. Before loading, None is valid (meaning "use the default"). # Setting an explicit value before loading pyproject.toml is supported, # but after loading this flag should be treated as read only. self.use_pep517 = use_pep517
def parse_editable(editable_req): # type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]] """Parses an editable requirement into: - a requirement name - an URL - extras - editable options Accepted requirements: svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir .[some_extra] """ url = editable_req # If a file path is specified with extras, strip off the extras. url_no_extras, extras = _strip_extras(url) if os.path.isdir(url_no_extras): if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): msg = ( 'File "setup.py" not found. Directory cannot be installed ' 'in editable mode: {}'.format(os.path.abspath(url_no_extras)) ) pyproject_path = make_pyproject_path(url_no_extras) if os.path.isfile(pyproject_path): msg += ( '\n(A "pyproject.toml" file was found, but editable ' 'mode currently requires a setup.py based build.)' ) raise InstallationError(msg) # Treating it as code that has already been checked out url_no_extras = path_to_url(url_no_extras) if url_no_extras.lower().startswith('file:'): package_name = Link(url_no_extras).egg_fragment if extras: return ( package_name, url_no_extras, Requirement("placeholder" + extras.lower()).extras, ) else: return package_name, url_no_extras, None for version_control in vcs: if url.lower().startswith('{}:'.format(version_control)): url = '{}+{}'.format(version_control, url) break if '+' not in url: raise InstallationError( '{} is not a valid editable requirement. ' 'It should either be a path to a local project or a VCS URL ' '(beginning with svn+, git+, hg+, or bzr+).'.format(editable_req) ) vc_type = url.split('+', 1)[0].lower() if not vcs.get_backend(vc_type): backends = ", ".join([bends.name + '+URL' for bends in vcs.backends]) error_message = "For --editable={}, " \ "only {} are currently supported".format( editable_req, backends) raise InstallationError(error_message) package_name = Link(url).egg_fragment if not package_name: raise InstallationError( "Could not detect requirement name for '{}', please specify one " "with #egg=your_package_name".format(editable_req) ) return package_name, url, None
def parse_req_from_line(name, line_source): # type: (str, Optional[str]) -> RequirementParts if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: name, markers_as_string = name.split(marker_sep, 1) markers_as_string = markers_as_string.strip() if not markers_as_string: markers = None else: markers = Marker(markers_as_string) else: markers = None name = name.strip() req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None extras_as_string = None if is_url(name): link = Link(name) else: p, extras_as_string = _strip_extras(path) url = _get_url_from_path(p, name) if url is not None: link = Link(url) # it's a local file, dir, or url if link: # Handle relative file URLs if link.scheme == 'file' and re.search(r'\.\./', link.url): link = Link( path_to_url(os.path.normpath(os.path.abspath(link.path)))) # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename req_as_string = "{wheel.name}=={wheel.version}".format(**locals()) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement req_as_string = link.egg_fragment # a requirement specifier else: req_as_string = name extras = convert_extras(extras_as_string) def with_source(text): # type: (str) -> str if not line_source: return text return '{} (from {})'.format(text, line_source) if req_as_string is not None: try: req = Requirement(req_as_string) except InvalidRequirement: if os.path.sep in req_as_string: add_msg = "It looks like a path." add_msg += deduce_helpful_msg(req_as_string) elif ('=' in req_as_string and not any(op in req_as_string for op in operators)): add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = '' msg = with_source( 'Invalid requirement: {!r}'.format(req_as_string) ) if add_msg: msg += '\nHint: {}'.format(add_msg) raise InstallationError(msg) else: req = None return RequirementParts(req, link, markers, extras)