def _is_url_like_archive(url):
# type: (str) -> bool
"""Return whether the URL looks like an archive.
"""
filename = Link(url).filename
for bad_ext in ARCHIVE_EXTENSIONS:
if filename.endswith(bad_ext):
return True
return False
def collect_links(self, project_name):
# type: (str) -> CollectedLinks
"""Find all available links for the given project name.
:return: All the Link objects (unfiltered), as a CollectedLinks object.
"""
search_scope = self.search_scope
index_locations = search_scope.get_index_urls_locations(project_name)
index_file_loc, index_url_loc = group_locations(index_locations)
fl_file_loc, fl_url_loc = group_locations(
self.find_links,
expand_dir=True,
)
file_links = [
Link(url) for url in itertools.chain(index_file_loc, fl_file_loc)
]
# We trust every directly linked archive in find_links
find_link_links = [Link(url, '-f') for url in self.find_links]
# We trust every url that the user has given us whether it was given
# via --index-url or --find-links.
# We want to filter out anything that does not have a secure origin.
url_locations = [
link for link in itertools.chain(
# Mark PyPI indices as "cache_link_parsing == False" -- this
# will avoid caching the result of parsing the page for links.
(Link(url, cache_link_parsing=False) for url in index_url_loc),
(Link(url) for url in fl_url_loc),
) if self.session.is_secure_origin(link)
]
url_locations = _remove_duplicate_links(url_locations)
lines = [
'{} location(s) to search for versions of {}:'.format(
len(url_locations),
project_name,
),
]
for link in url_locations:
lines.append('* {}'.format(link))
logger.debug('\n'.join(lines))
return CollectedLinks(
files=file_links,
find_links=find_link_links,
project_urls=url_locations,
)
def _create_link_from_element(
anchor, # type: HTMLElement
page_url, # type: str
base_url, # type: str
):
# type: (...) -> Optional[Link]
"""
Convert an anchor element in a simple repository page to a Link.
"""
href = anchor.get("href")
if not href:
return None
url = _clean_link(urllib_parse.urljoin(base_url, href))
pyrequire = anchor.get('data-requires-python')
pyrequire = unescape(pyrequire) if pyrequire else None
yanked_reason = anchor.get('data-yanked')
if yanked_reason:
# This is a unicode string in Python 2 (and 3).
yanked_reason = unescape(yanked_reason)
link = Link(
url,
comes_from=page_url,
requires_python=pyrequire,
yanked_reason=yanked_reason,
)
return link
def build(
requirements, # type: Iterable[InstallRequirement]
wheel_cache, # type: WheelCache
build_options, # type: List[str]
global_options, # type: List[str]
):
# type: (...) -> BuildResult
"""Build wheels.
:return: The list of InstallRequirement that succeeded to build and
the list of InstallRequirement that failed to build.
"""
if not requirements:
return [], []
# Build the wheels.
logger.info(
'Building wheels for collected packages: %s',
', '.join(req.name for req in requirements),
)
with indent_log():
build_successes, build_failures = [], []
for req in requirements:
cache_dir = _get_cache_dir(req, wheel_cache)
wheel_file = _build_one(
req, cache_dir, build_options, global_options
)
if wheel_file:
# Update the link for this.
req.link = Link(path_to_url(wheel_file))
req.local_file_path = req.link.file_path
assert req.link.is_wheel
build_successes.append(req)
else:
build_failures.append(req)
# notify success/failure
if build_successes:
logger.info(
'Successfully built %s',
' '.join([req.name for req in build_successes]),
)
if build_failures:
logger.info(
'Failed to build %s',
' '.join([req.name for req in build_failures]),
)
# Return a list of requirements that failed to build
return build_successes, build_failures
def parse_req_from_editable(editable_req):
# type: (str) -> RequirementParts
name, url, extras_override = parse_editable(editable_req)
if name is not None:
try:
req = Requirement(name)
except InvalidRequirement:
raise InstallationError("Invalid requirement: '{}'".format(name))
else:
req = None
link = Link(url)
return RequirementParts(req, link, None, extras_override)
def get(
self,
link, # type: Link
package_name, # type: Optional[str]
supported_tags, # type: List[Tag]
):
# type: (...) -> Link
candidates = []
if not package_name:
return link
canonical_package_name = canonicalize_name(package_name)
for wheel_name, wheel_dir in self._get_candidates(
link, canonical_package_name):
try:
wheel = Wheel(wheel_name)
except InvalidWheelFilename:
continue
if canonicalize_name(wheel.name) != canonical_package_name:
logger.debug(
"Ignoring cached wheel {} for {} as it "
"does not match the expected distribution name {}.".format(
wheel_name, link, package_name))
continue
if not wheel.supported(supported_tags):
# Built for a different python/arch/etc
continue
candidates.append((
wheel.support_index_min(supported_tags),
wheel_name,
wheel_dir,
))
if not candidates:
return link
_, wheel_name, wheel_dir = min(candidates)
return Link(path_to_url(os.path.join(wheel_dir, wheel_name)))
def __init__(
self,
req, # type: Optional[Requirement]
comes_from, # type: Optional[Union[str, InstallRequirement]]
editable=False, # type: bool
link=None, # type: Optional[Link]
markers=None, # type: Optional[Marker]
use_pep517=None, # type: Optional[bool]
isolated=False, # type: bool
install_options=None, # type: Optional[List[str]]
global_options=None, # type: Optional[List[str]]
hash_options=None, # type: Optional[Dict[str, List[str]]]
constraint=False, # type: bool
extras=() # type: Iterable[str]
):
# type: (...) -> None
assert req is None or isinstance(req, Requirement), req
self.req = req
self.comes_from = comes_from
self.constraint = constraint
self.editable = editable
# source_dir is the local directory where the linked requirement is
# located, or unpacked. In case unpacking is needed, creating and
# populating source_dir is done by the RequirementPreparer. Note this
# is not necessarily the directory where pyproject.toml or setup.py is
# located - that one is obtained via unpacked_source_directory.
self.source_dir = None # type: Optional[str]
if self.editable:
assert link
if link.is_file:
self.source_dir = os.path.normpath(
os.path.abspath(link.file_path)
)
if link is None and req and req.url:
# PEP 508 URL requirement
link = Link(req.url)
self.link = self.original_link = link
self.original_link_is_in_wheel_cache = False
# Path to any downloaded or already-existing package.
self.local_file_path = None # type: Optional[str]
if self.link and self.link.is_file:
self.local_file_path = self.link.file_path
if extras:
self.extras = extras
elif req:
self.extras = {
pkg_resources.safe_extra(extra) for extra in req.extras
}
else:
self.extras = set()
if markers is None and req:
markers = req.marker
self.markers = markers
# This holds the pkg_resources.Distribution object if this requirement
# is already available:
self.satisfied_by = None # type: Optional[Distribution]
# Whether the installation process should try to uninstall an existing
# distribution before installing this requirement.
self.should_reinstall = False
# Temporary build location
self._temp_build_dir = None # type: Optional[TempDirectory]
# Set to True after successful installation
self.install_succeeded = None # type: Optional[bool]
# Supplied options
self.install_options = install_options if install_options else []
self.global_options = global_options if global_options else []
self.hash_options = hash_options if hash_options else {}
# Set to True after successful preparation of this requirement
self.prepared = False
self.is_direct = False
# Set by the legacy resolver when the requirement has been downloaded
# TODO: This introduces a strong coupling between the resolver and the
# requirement (the coupling was previously between the resolver
# and the requirement set). This should be refactored to allow
# the requirement to decide for itself when it has been
# successfully downloaded - but that is more tricky to get right,
# se we are making the change in stages.
self.successfully_downloaded = False
self.isolated = isolated
self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment
# For PEP 517, the directory where we request the project metadata
# gets stored. We need this to pass to build_wheel, so the backend
# can ensure that the wheel matches the metadata (see the PEP for
# details).
self.metadata_directory = None # type: Optional[str]
# The static build requirements (from pyproject.toml)
self.pyproject_requires = None # type: Optional[List[str]]
# Build requirements that we will check are available
self.requirements_to_check = [] # type: List[str]
# The PEP 517 backend we should use to build the project
self.pep517_backend = None # type: Optional[Pep517HookCaller]
# Are we using PEP 517 for this requirement?
# After pyproject.toml has been loaded, the only valid values are True
# and False. Before loading, None is valid (meaning "use the default").
# Setting an explicit value before loading pyproject.toml is supported,
# but after loading this flag should be treated as read only.
self.use_pep517 = use_pep517
def parse_editable(editable_req):
# type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]]
"""Parses an editable requirement into:
- a requirement name
- an URL
- extras
- editable options
Accepted requirements:
svn+http://blahblah@rev#egg=Foobar[baz]&subdirectory=version_subdir
.[some_extra]
"""
url = editable_req
# If a file path is specified with extras, strip off the extras.
url_no_extras, extras = _strip_extras(url)
if os.path.isdir(url_no_extras):
if not os.path.exists(os.path.join(url_no_extras, 'setup.py')):
msg = (
'File "setup.py" not found. Directory cannot be installed '
'in editable mode: {}'.format(os.path.abspath(url_no_extras))
)
pyproject_path = make_pyproject_path(url_no_extras)
if os.path.isfile(pyproject_path):
msg += (
'\n(A "pyproject.toml" file was found, but editable '
'mode currently requires a setup.py based build.)'
)
raise InstallationError(msg)
# Treating it as code that has already been checked out
url_no_extras = path_to_url(url_no_extras)
if url_no_extras.lower().startswith('file:'):
package_name = Link(url_no_extras).egg_fragment
if extras:
return (
package_name,
url_no_extras,
Requirement("placeholder" + extras.lower()).extras,
)
else:
return package_name, url_no_extras, None
for version_control in vcs:
if url.lower().startswith('{}:'.format(version_control)):
url = '{}+{}'.format(version_control, url)
break
if '+' not in url:
raise InstallationError(
'{} is not a valid editable requirement. '
'It should either be a path to a local project or a VCS URL '
'(beginning with svn+, git+, hg+, or bzr+).'.format(editable_req)
)
vc_type = url.split('+', 1)[0].lower()
if not vcs.get_backend(vc_type):
backends = ", ".join([bends.name + '+URL' for bends in vcs.backends])
error_message = "For --editable={}, " \
"only {} are currently supported".format(
editable_req, backends)
raise InstallationError(error_message)
package_name = Link(url).egg_fragment
if not package_name:
raise InstallationError(
"Could not detect requirement name for '{}', please specify one "
"with #egg=your_package_name".format(editable_req)
)
return package_name, url, None
def parse_req_from_line(name, line_source):
# type: (str, Optional[str]) -> RequirementParts
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers_as_string = name.split(marker_sep, 1)
markers_as_string = markers_as_string.strip()
if not markers_as_string:
markers = None
else:
markers = Marker(markers_as_string)
else:
markers = None
name = name.strip()
req_as_string = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras_as_string = None
if is_url(name):
link = Link(name)
else:
p, extras_as_string = _strip_extras(path)
url = _get_url_from_path(p, name)
if url is not None:
link = Link(url)
# it's a local file, dir, or url
if link:
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', link.url):
link = Link(
path_to_url(os.path.normpath(os.path.abspath(link.path))))
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
req_as_string = "{wheel.name}=={wheel.version}".format(**locals())
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req_as_string = link.egg_fragment
# a requirement specifier
else:
req_as_string = name
extras = convert_extras(extras_as_string)
def with_source(text):
# type: (str) -> str
if not line_source:
return text
return '{} (from {})'.format(text, line_source)
if req_as_string is not None:
try:
req = Requirement(req_as_string)
except InvalidRequirement:
if os.path.sep in req_as_string:
add_msg = "It looks like a path."
add_msg += deduce_helpful_msg(req_as_string)
elif ('=' in req_as_string and
not any(op in req_as_string for op in operators)):
add_msg = "= is not a valid operator. Did you mean == ?"
else:
add_msg = ''
msg = with_source(
'Invalid requirement: {!r}'.format(req_as_string)
)
if add_msg:
msg += '\nHint: {}'.format(add_msg)
raise InstallationError(msg)
else:
req = None
return RequirementParts(req, link, markers, extras)