def install(candidate):
pip_logging._log_state.indentation = 0
dependencies = repository.get_dependencies(candidate)[0]
key = safe_name(candidate.name).lower()
dist = Distribution(key, candidate.version, candidate.req.editable)
dist.dependencies = dependencies
rv.add_distribution(dist)
def add_dependencies(self,
requirements: Dict[str, Requirement],
show_message: bool = True) -> None:
for name, dep in requirements.items():
if dep.from_section == "default":
deps = self.tool_settings["dependencies"]
elif dep.from_section == "dev":
deps = self.tool_settings["dev-dependencies"]
else:
section = f"{dep.from_section}-dependencies"
if section not in self.tool_settings:
self.tool_settings[section] = tomlkit.table()
deps = self.tool_settings[section]
matched_name = next(
filter(lambda k: strip_extras(name)[0] == safe_name(k).lower(),
deps.keys()),
None,
)
name_to_save = dep.name if matched_name is None else matched_name
_, req_dict = dep.as_req_dict()
if isinstance(req_dict, dict):
req = tomlkit.inline_table()
req.update(req_dict)
req_dict = req
deps[name_to_save] = req_dict
self.write_pyproject(show_message)
def cached_wheel(cache_dir, link, format_control, package_name):
if not cache_dir:
return link
if not link:
return link
if link.is_wheel:
return link
if not package_name:
return link
canonical_name = pkg_resources.safe_name(package_name).lower()
formats = pip.index.fmt_ctl_formats(format_control, canonical_name)
if "binary" not in formats:
return link
root = _cache_for_link(cache_dir, link)
try:
wheel_names = os.listdir(root)
except OSError as e:
if e.errno == errno.ENOENT:
return link
raise
candidates = []
for wheel_name in wheel_names:
try:
wheel = Wheel(wheel_name)
except InvalidWheelFilename:
continue
if not wheel.supported():
# Built for a different python/arch/etc
continue
candidates.append((wheel.support_index_min(), wheel_name))
if not candidates:
return link
candidates.sort()
path = os.path.join(root, candidates[0][1])
return pip.index.Link(path_to_url(path), trusted=True)
def cached_wheel(cache_dir, link, format_control, package_name):
if not cache_dir:
return link
if not link:
return link
if link.is_wheel:
return link
if not package_name:
return link
canonical_name = pkg_resources.safe_name(package_name).lower()
formats = pip.index.fmt_ctl_formats(format_control, canonical_name)
if "binary" not in formats:
return link
root = _cache_for_link(cache_dir, link)
try:
wheel_names = os.listdir(root)
except OSError as e:
if e.errno == errno.ENOENT:
return link
raise
candidates = []
for wheel_name in wheel_names:
try:
wheel = Wheel(wheel_name)
except InvalidWheelFilename:
continue
if not wheel.supported():
# Built for a different python/arch/etc
continue
candidates.append((wheel.support_index_min(), wheel_name))
if not candidates:
return link
candidates.sort()
path = os.path.join(root, candidates[0][1])
return pip.index.Link(path_to_url(path), trusted=True)
def get_locked_candidates(self,
section: Optional[str] = None
) -> Dict[str, Candidate]:
if not self.lockfile_file.is_file():
return {}
section = section or "default"
result = {}
for package in [dict(p) for p in self.lockfile.get("package", [])]:
if section != "__all__" and section not in package["sections"]:
continue
version = package.get("version")
if version:
package["version"] = f"=={version}"
package_name = package.pop("name")
req = Requirement.from_req_dict(package_name, dict(package))
can = Candidate(req,
self.environment,
name=package_name,
version=version)
can.marker = req.marker
can.hashes = {
item["file"]: item["hash"]
for item in self.lockfile["metadata"].get(
f"{req.key} {version}", [])
} or None
result[req.identify()] = can
if section in ("default",
"__all__") and self.meta.name and self.meta.version:
result[safe_name(
self.meta.name).lower()] = self.make_self_candidate(True)
return result
def update_candidate(self, key: str) -> Tuple[Distribution, Candidate]:
"""Update candidate"""
can = self.candidates[key]
dist = self.working_set[safe_name(can.name).lower()]
installer = self.get_installer()
installer.uninstall(dist)
installer.install(can)
return dist, can
def name(self):
# type: () -> Optional[str]
if self.req is None:
return None
return six.ensure_str(pkg_resources.safe_name(self.req.name))
def find_packages_latest_versions(self, options):
index_urls = [options.index_url] + options.extra_index_urls
if options.no_index:
logger.info('Ignoring indexes: %s', ','.join(index_urls))
index_urls = []
dependency_links = []
for dist in get_installed_distributions(local_only=options.local,
user_only=options.user):
if dist.has_metadata('dependency_links.txt'):
dependency_links.extend(
dist.get_metadata_lines('dependency_links.txt'),
)
with self._build_session(options) as session:
finder = self._build_package_finder(options, index_urls, session)
finder.add_dependency_links(dependency_links)
installed_packages = get_installed_distributions(
local_only=options.local,
user_only=options.user,
include_editables=False,
)
format_control = FormatControl(set(), set())
wheel_cache = WheelCache(options.cache_dir, format_control)
for dist in installed_packages:
req = InstallRequirement.from_line(
dist.key, None, isolated=options.isolated_mode,
wheel_cache=wheel_cache
)
typ = 'unknown'
try:
link = finder.find_requirement(req, True)
# If link is None, means installed version is most
# up-to-date
if link is None:
continue
except DistributionNotFound:
continue
else:
canonical_name = pkg_resources.safe_name(req.name).lower()
formats = fmt_ctl_formats(format_control, canonical_name)
search = Search(
req.name,
canonical_name,
formats)
remote_version = finder._link_package_versions(
link, search).version
if link.is_wheel:
typ = 'wheel'
else:
typ = 'sdist'
yield dist, remote_version, typ
def print_results(
ui: termui.UI,
hits: SearchResult,
working_set: WorkingSet,
terminal_width: Optional[int] = None,
) -> None:
if not hits:
return
name_column_width = (
max([len(hit.name) + len(hit.version or "") for hit in hits]) + 4
)
for hit in hits:
name = hit.name
summary = hit.summary or ""
latest = hit.version or ""
if terminal_width is not None:
target_width = terminal_width - name_column_width - 5
if target_width > 10:
# wrap and indent summary to fit terminal
summary = textwrap.wrap(summary, target_width)
summary = ("\n" + " " * (name_column_width + 2)).join(summary)
current_width = len(name) + len(latest) + 4
spaces = " " * (name_column_width - current_width)
line = "{name} ({latest}){spaces} - {summary}".format(
name=termui.green(name, bold=True),
latest=termui.yellow(latest),
spaces=spaces,
summary=summary,
)
try:
ui.echo(line)
if safe_name(name).lower() in working_set:
dist = working_set[safe_name(name).lower()]
if dist.version == latest:
ui.echo(" INSTALLED: %s (latest)" % dist.version)
else:
ui.echo(" INSTALLED: %s" % dist.version)
ui.echo(" LATEST: %s" % latest)
except UnicodeEncodeError:
pass
def __init__(self, **kwargs):
self._marker = None
self.from_section = "default"
self.marker_no_python = None # type: Optional[Marker]
self.requires_python = PySpecSet() # type: PySpecSet
for k, v in kwargs.items():
if k == "specifier":
v = get_specifier(v)
setattr(self, k, v)
if self.name and not self.project_name:
self.project_name = safe_name(self.name)
self.key = self.project_name.lower()
def find_packages_latest_versions(self, options):
index_urls = [options.index_url] + options.extra_index_urls
if options.no_index:
logger.info('Ignoring indexes: %s', ','.join(index_urls))
index_urls = []
dependency_links = []
for dist in get_installed_distributions(local_only=options.local,
user_only=options.user):
if dist.has_metadata('dependency_links.txt'):
dependency_links.extend(
dist.get_metadata_lines('dependency_links.txt'), )
with self._build_session(options) as session:
finder = self._build_package_finder(options, index_urls, session)
finder.add_dependency_links(dependency_links)
installed_packages = get_installed_distributions(
local_only=options.local,
user_only=options.user,
include_editables=False,
)
format_control = FormatControl(set(), set())
wheel_cache = WheelCache(options.cache_dir, format_control)
for dist in installed_packages:
req = InstallRequirement.from_line(
dist.key,
None,
isolated=options.isolated_mode,
wheel_cache=wheel_cache)
typ = 'unknown'
try:
link = finder.find_requirement(req, True)
# If link is None, means installed version is most
# up-to-date
if link is None:
continue
except DistributionNotFound:
continue
else:
canonical_name = pkg_resources.safe_name(req.name).lower()
formats = fmt_ctl_formats(format_control, canonical_name)
search = Search(req.name, canonical_name, formats)
remote_version = finder._link_package_versions(
link, search).version
if link.is_wheel:
typ = 'wheel'
else:
typ = 'sdist'
yield dist, remote_version, typ
def fmt_ctl_handle_mutual_exclude(value, target, other):
new = value.split(',')
while ':all:' in new:
other.clear()
target.clear()
target.add(':all:')
del new[:new.index(':all:') + 1]
if ':none:' not in new:
# Without a none, we want to discard everything as :all: covers it
return
for name in new:
if name == ':none:':
target.clear()
continue
name = pkg_resources.safe_name(name).lower()
other.discard(name)
target.add(name)
def fmt_ctl_handle_mutual_exclude(value, target, other):
new = value.split(',')
while ':all:' in new:
other.clear()
target.clear()
target.add(':all:')
del new[:new.index(':all:') + 1]
if ':none:' not in new:
# Without a none, we want to discard everything as :all: covers it
return
for name in new:
if name == ':none:':
target.clear()
continue
name = pkg_resources.safe_name(name).lower()
other.discard(name)
target.add(name)
def update_candidate(self, key: str) -> Tuple[Distribution, Candidate]:
"""Update candidate"""
can = self.candidates[key]
dist = self.working_set[safe_name(can.name).lower()]
installer = self.get_installer()
with stream.open_spinner(
f"Updating {stream.green(key, bold=True)} {stream.yellow(dist.version)} "
f"-> {stream.yellow(can.version)}...") as spinner:
try:
installer.uninstall(dist)
installer.install(can)
except Exception:
spinner.fail(f"Update {stream.green(key, bold=True)} "
f"-> {stream.yellow(can.version)} failed")
raise
else:
spinner.succeed(f"Update {stream.green(key, bold=True)} "
f"-> {stream.yellow(can.version)} successful")
return dist, can
def install_candidates(self,
candidates: List[Candidate],
update: bool = False) -> None:
"""Install candidates.
:param candidates: a list of candidates to be installed.
:param update: whether to remove existed packages.
"""
installer = self.get_installer()
working_set = self.environment.get_working_set()
for can in candidates:
if update:
dist = working_set[safe_name(can.name).lower()]
context.io.echo(
f"Updating {context.io.green(can.name, bold=True)} "
f"{context.io.yellow(dist.version)} -> "
f"{context.io.yellow(can.version)}")
installer.uninstall(dist)
else:
context.io.echo(f"Installing {can.format()}...")
installer.install(can)
def project_name(self) -> Optional[str]:
return safe_name(self.name) if self.name else None
def canonicalize_name(name):
"""Convert an arbitrary string to a canonical name used for comparison"""
return pkg_resources.safe_name(name).lower()
def do_update(
project: Project,
dev: bool = False,
sections: Sequence[str] = (),
default: bool = True,
strategy: str = "reuse",
save: str = "compatible",
unconstrained: bool = False,
top: bool = False,
dry_run: bool = False,
packages: Sequence[str] = (),
) -> None:
"""Update specified packages or all packages"""
check_project_file(project)
if len(packages) > 0 and (top or len(sections) > 1 or not default):
raise PdmUsageError(
"packages argument can't be used together with multiple -s or "
"--no-default and --top.")
all_dependencies = project.all_dependencies
updated_deps = {}
if not packages:
sections = translate_sections(project, default, dev, sections or ())
for section in sections:
updated_deps.update(all_dependencies[section])
else:
section = sections[0] if sections else ("dev" if dev else "default")
dependencies = all_dependencies[section]
for name in packages:
matched_name = next(
filter(
lambda k: safe_name(strip_extras(k)[0]).lower() ==
safe_name(name).lower(),
dependencies.keys(),
),
None,
)
if not matched_name:
raise ProjectError(
"{} does not exist in {} {}dependencies.".format(
termui.green(name, bold=True), section,
"dev-" if dev else ""))
updated_deps[matched_name] = dependencies[matched_name]
project.core.ui.echo("Updating packages: {}.".format(", ".join(
termui.green(v, bold=True) for v in updated_deps)))
if unconstrained:
for _, dep in updated_deps.items():
dep.specifier = get_specifier("")
reqs = [r for deps in all_dependencies.values() for r in deps.values()]
resolved = do_lock(
project,
strategy if top or packages else "all",
updated_deps.keys(),
reqs,
dry_run=dry_run,
)
do_sync(
project,
sections=sections,
dev=dev,
default=default,
clean=False,
dry_run=dry_run,
tracked_names=updated_deps.keys() if top else None,
)
if unconstrained and not dry_run:
# Need to update version constraints
save_version_specifiers(updated_deps, resolved, save)
project.add_dependencies(updated_deps, section, dev)
lockfile = project.lockfile
project.write_lockfile(lockfile, False)
def _link_package_versions(self, link, search_name):
"""
Return an iterable of triples (pkg_resources_version_key,
link, python_version) that can be extracted from the given
link.
Meant to be overridden by subclasses, not called by clients.
"""
platform = get_platform()
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
else:
egg_info, ext = link.splitext()
if not ext:
if link not in self.logged_links:
logger.debug('Skipping link %s; not a file', link)
self.logged_links.add(link)
return
if egg_info.endswith('.tar'):
# Special double-extension case:
egg_info = egg_info[:-4]
ext = '.tar' + ext
if ext not in self._known_extensions():
if link not in self.logged_links:
logger.debug(
'Skipping link %s; unknown archive format: %s',
link,
ext,
)
self.logged_links.add(link)
return
if "macosx10" in link.path and ext == '.zip':
if link not in self.logged_links:
logger.debug('Skipping link %s; macosx10 one', link)
self.logged_links.add(link)
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
logger.debug(
'Skipping %s because the wheel filename is invalid',
link
)
return
if (pkg_resources.safe_name(wheel.name).lower()
!= pkg_resources.safe_name(search_name).lower()):
logger.debug(
'Skipping link %s; wrong project name (not %s)',
link,
search_name,
)
return
if not wheel.supported():
logger.debug(
'Skipping %s because it is not compatible with this '
'Python',
link,
)
return
# This is a dirty hack to prevent installing Binary Wheels from
# PyPI unless it is a Windows or Mac Binary Wheel. This is
# paired with a change to PyPI disabling uploads for the
# same. Once we have a mechanism for enabling support for
# binary wheels on linux that deals with the inherent problems
# of binary distribution this can be removed.
comes_from = getattr(link, "comes_from", None)
if (
(
not platform.startswith('win')
and not platform.startswith('macosx')
and not platform == 'cli'
)
and comes_from is not None
and urllib_parse.urlparse(
comes_from.url
).netloc.endswith(PyPI.netloc)):
if not wheel.supported(tags=supported_tags_noarch):
logger.debug(
"Skipping %s because it is a pypi-hosted binary "
"Wheel on an unsupported platform",
link,
)
return
version = wheel.version
if not version:
version = self._egg_info_matches(egg_info, search_name, link)
if version is None:
logger.debug(
'Skipping link %s; wrong project name (not %s)',
link,
search_name,
)
return
if (link.internal is not None
and not link.internal
and not normalize_name(search_name).lower()
in self.allow_external
and not self.allow_all_external):
# We have a link that we are sure is external, so we should skip
# it unless we are allowing externals
logger.debug("Skipping %s because it is externally hosted.", link)
self.need_warn_external = True
return
if (link.verifiable is not None
and not link.verifiable
and not (normalize_name(search_name).lower()
in self.allow_unverified)):
# We have a link that we are sure we cannot verify its integrity,
# so we should skip it unless we are allowing unsafe installs
# for this requirement.
logger.debug(
"Skipping %s because it is an insecure and unverifiable file.",
link,
)
self.need_warn_unverified = True
return
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
logger.debug(
'Skipping %s because Python version is incorrect', link
)
return
logger.debug('Found link %s, version: %s', link, version)
return InstallationCandidate(search_name, version, link)
def project_name(self) -> str | None:
return safe_name(self.name) if self.name else None # type: ignore
def _link_package_versions(self, link, search):
"""Return an InstallationCandidate or None"""
platform = get_platform()
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
self._log_skipped_link(link, 'not a file')
return
if ext not in SUPPORTED_EXTENSIONS:
self._log_skipped_link(link,
'unsupported archive format: %s' % ext)
return
if "binary" not in search.formats and ext == wheel_ext:
self._log_skipped_link(
link, 'No binaries permitted for %s' % search.supplied)
return
if "macosx10" in link.path and ext == '.zip':
self._log_skipped_link(link, 'macosx10 one')
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
self._log_skipped_link(link, 'invalid wheel filename')
return
if (pkg_resources.safe_name(wheel.name).lower() !=
search.canonical):
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if not wheel.supported():
self._log_skipped_link(
link, 'it is not compatible with this Python')
return
# This is a dirty hack to prevent installing Binary Wheels from
# PyPI unless it is a Windows or Mac Binary Wheel. This is
# paired with a change to PyPI disabling uploads for the
# same. Once we have a mechanism for enabling support for
# binary wheels on linux that deals with the inherent problems
# of binary distribution this can be removed.
comes_from = getattr(link, "comes_from", None)
if ((not platform.startswith('win')
and not platform.startswith('macosx')
and not platform == 'cli') and comes_from is not None
and urllib_parse.urlparse(
comes_from.url).netloc.endswith(PyPI.netloc)):
if not wheel.supported(tags=supported_tags_noarch):
self._log_skipped_link(
link,
"it is a pypi-hosted binary "
"Wheel on an unsupported platform",
)
return
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != wheel_ext:
self._log_skipped_link(
link, 'No sources permitted for %s' % search.supplied)
return
if not version:
version = egg_info_matches(egg_info, search.supplied, link)
if version is None:
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if (link.internal is not None and not link.internal
and not normalize_name(search.supplied).lower()
in self.allow_external and not self.allow_all_external):
# We have a link that we are sure is external, so we should skip
# it unless we are allowing externals
self._log_skipped_link(link, 'it is externally hosted')
self.need_warn_external = True
return
if (link.verifiable is not None and not link.verifiable
and not (normalize_name(search.supplied).lower()
in self.allow_unverified)):
# We have a link that we are sure we cannot verify its integrity,
# so we should skip it unless we are allowing unsafe installs
# for this requirement.
self._log_skipped_link(link,
'it is an insecure and unverifiable file')
self.need_warn_unverified = True
return
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
self._log_skipped_link(link, 'Python version is incorrect')
return
logger.debug('Found link %s, version: %s', link, version)
return InstallationCandidate(search.supplied, version, link)
def _find_all_versions(self, project_name):
"""Find all available versions for project_name
This checks index_urls, find_links and dependency_links
All versions found are returned
See _link_package_versions for details on which files are accepted
"""
index_locations = self._get_index_urls_locations(project_name)
index_file_loc, index_url_loc = self._sort_locations(index_locations)
fl_file_loc, fl_url_loc = self._sort_locations(
self.find_links, expand_dir=True)
dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links)
file_locations = (
Link(url) for url in itertools.chain(
index_file_loc, fl_file_loc, dep_file_loc)
)
# We trust every url that the user has given us whether it was given
# via --index-url or --find-links
# We explicitly do not trust links that came from dependency_links
# We want to filter out any thing which does not have a secure origin.
url_locations = [
link for link in itertools.chain(
(Link(url, trusted=True) for url in index_url_loc),
(Link(url, trusted=True) for url in fl_url_loc),
(Link(url) for url in dep_url_loc),
)
if self._validate_secure_origin(logger, link)
]
logger.debug('%d location(s) to search for versions of %s:',
len(url_locations), project_name)
for location in url_locations:
logger.debug('* %s', location)
canonical_name = pkg_resources.safe_name(project_name).lower()
formats = fmt_ctl_formats(self.format_control, canonical_name)
search = Search(project_name.lower(), canonical_name, formats)
find_links_versions = self._package_versions(
# We trust every directly linked archive in find_links
(Link(url, '-f', trusted=True) for url in self.find_links),
search
)
page_versions = []
for page in self._get_pages(url_locations, project_name):
logger.debug('Analyzing links from page %s', page.url)
with indent_log():
page_versions.extend(
self._package_versions(page.links, search)
)
dependency_versions = self._package_versions(
(Link(url) for url in self.dependency_links), search
)
if dependency_versions:
logger.debug(
'dependency_links found: %s',
', '.join([
version.location.url for version in dependency_versions
])
)
file_versions = self._package_versions(file_locations, search)
if file_versions:
file_versions.sort(reverse=True)
logger.debug(
'Local files found: %s',
', '.join([
url_to_path(candidate.location.url)
for candidate in file_versions
])
)
# This is an intentional priority ordering
return (
file_versions + find_links_versions + page_versions +
dependency_versions
)
def canonicalize_name(name):
"""Convert an arbitrary string to a canonical name used for comparison"""
return pkg_resources.safe_name(name).lower()
def normalize_package(name):
return safe_name(name).lower()
def do_update(
project: Project,
dev: bool = False,
sections: Sequence[str] = (),
default: bool = True,
strategy: str = "reuse",
save: str = "compatible",
unconstrained: bool = False,
packages: Sequence[str] = (),
) -> None:
"""Update specified packages or all packages
:param project: The project instance
:param dev: whether to update dev dependencies
:param sections: update speicified sections
:param default: update default
:param strategy: update strategy (reuse/eager)
:param save: save strategy (compatible/exact/wildcard)
:param unconstrained: ignore version constraint
:param packages: specified packages to update
:return: None
"""
check_project_file(project)
if len(packages) > 0 and (len(sections) > 1 or not default):
raise PdmUsageError(
"packages argument can't be used together with multple -s or --no-default."
)
if not packages:
if unconstrained:
raise PdmUsageError(
"--unconstrained must be used with package names given.")
# pdm update with no packages given, same as 'lock' + 'sync'
do_lock(project)
do_sync(project, sections, dev, default, clean=False)
return
section = sections[0] if sections else ("dev" if dev else "default")
all_dependencies = project.all_dependencies
dependencies = all_dependencies[section]
updated_deps = {}
tracked_names = set()
for name in packages:
matched_name = next(
filter(
lambda k: safe_name(strip_extras(k)[0]).lower() == safe_name(
name).lower(),
dependencies.keys(),
),
None,
)
if not matched_name:
raise ProjectError("{} does not exist in {} dependencies.".format(
stream.green(name, bold=True), section))
if unconstrained:
dependencies[matched_name].specifier = get_specifier("")
tracked_names.add(matched_name)
updated_deps[matched_name] = dependencies[matched_name]
stream.echo("Updating packages: {}.".format(", ".join(
stream.green(v, bold=True) for v in tracked_names)))
reqs = [r for deps in all_dependencies.values() for r in deps.values()]
resolved = do_lock(project, strategy, tracked_names, reqs)
do_sync(project, sections=(section, ), default=False, clean=False)
if unconstrained:
# Need to update version constraints
save_version_specifiers(updated_deps, resolved, save)
project.add_dependencies(updated_deps)
lockfile = project.lockfile
project.write_lockfile(lockfile, False)
def normalize_package(name: str) -> str:
return safe_name(name).lower()
def name(self) -> Optional[str]:
if self.req is None:
return None
return pkg_resources.safe_name(self.req.name)
def _link_package_versions(self, link, search):
"""Return an InstallationCandidate or None"""
platform = get_platform()
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
self._log_skipped_link(link, 'not a file')
return
if ext not in SUPPORTED_EXTENSIONS:
self._log_skipped_link(
link, 'unsupported archive format: %s' % ext)
return
if "binary" not in search.formats and ext == wheel_ext:
self._log_skipped_link(
link, 'No binaries permitted for %s' % search.supplied)
return
if "macosx10" in link.path and ext == '.zip':
self._log_skipped_link(link, 'macosx10 one')
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
self._log_skipped_link(link, 'invalid wheel filename')
return
if (pkg_resources.safe_name(wheel.name).lower() !=
search.canonical):
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if not wheel.supported():
self._log_skipped_link(
link, 'it is not compatible with this Python')
return
# This is a dirty hack to prevent installing Binary Wheels from
# PyPI unless it is a Windows or Mac Binary Wheel. This is
# paired with a change to PyPI disabling uploads for the
# same. Once we have a mechanism for enabling support for
# binary wheels on linux that deals with the inherent problems
# of binary distribution this can be removed.
comes_from = getattr(link, "comes_from", None)
if (
(
not platform.startswith('win') and not
platform.startswith('macosx') and not
platform == 'cli'
) and
comes_from is not None and
urllib_parse.urlparse(
comes_from.url
).netloc.endswith(PyPI.netloc)):
if not wheel.supported(tags=supported_tags_noarch):
self._log_skipped_link(
link,
"it is a pypi-hosted binary "
"Wheel on an unsupported platform",
)
return
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != wheel_ext:
self._log_skipped_link(
link, 'No sources permitted for %s' % search.supplied)
return
if not version:
version = egg_info_matches(egg_info, search.supplied, link)
if version is None:
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if (link.internal is not None and not
link.internal and not
normalize_name(search.supplied).lower()
in self.allow_external and not
self.allow_all_external):
# We have a link that we are sure is external, so we should skip
# it unless we are allowing externals
self._log_skipped_link(link, 'it is externally hosted')
self.need_warn_external = True
return
if (link.verifiable is not None and not
link.verifiable and not
(normalize_name(search.supplied).lower()
in self.allow_unverified)):
# We have a link that we are sure we cannot verify its integrity,
# so we should skip it unless we are allowing unsafe installs
# for this requirement.
self._log_skipped_link(
link, 'it is an insecure and unverifiable file')
self.need_warn_unverified = True
return
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
self._log_skipped_link(
link, 'Python version is incorrect')
return
logger.debug('Found link %s, version: %s', link, version)
return InstallationCandidate(search.supplied, version, link)
def _link_package_versions(self, link, search_name):
"""
Return an iterable of triples (pkg_resources_version_key,
link, python_version) that can be extracted from the given
link.
Meant to be overridden by subclasses, not called by clients.
"""
platform = get_platform()
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
else:
egg_info, ext = link.splitext()
if not ext:
if link not in self.logged_links:
logger.debug('Skipping link %s; not a file', link)
self.logged_links.add(link)
return
if egg_info.endswith('.tar'):
# Special double-extension case:
egg_info = egg_info[:-4]
ext = '.tar' + ext
if ext not in self._known_extensions():
if link not in self.logged_links:
logger.debug(
'Skipping link %s; unknown archive format: %s',
link,
ext,
)
self.logged_links.add(link)
return
if "macosx10" in link.path and ext == '.zip':
if link not in self.logged_links:
logger.debug('Skipping link %s; macosx10 one', link)
self.logged_links.add(link)
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
logger.debug(
'Skipping %s because the wheel filename is invalid',
link)
return
if (pkg_resources.safe_name(wheel.name).lower() !=
pkg_resources.safe_name(search_name).lower()):
logger.debug(
'Skipping link %s; wrong project name (not %s)',
link,
search_name,
)
return
if not wheel.supported():
logger.debug(
'Skipping %s because it is not compatible with this '
'Python',
link,
)
return
# This is a dirty hack to prevent installing Binary Wheels from
# PyPI unless it is a Windows or Mac Binary Wheel. This is
# paired with a change to PyPI disabling uploads for the
# same. Once we have a mechanism for enabling support for
# binary wheels on linux that deals with the inherent problems
# of binary distribution this can be removed.
comes_from = getattr(link, "comes_from", None)
if ((not platform.startswith('win')
and not platform.startswith('macosx')
and not platform == 'cli') and comes_from is not None
and urllib_parse.urlparse(
comes_from.url).netloc.endswith(PyPI.netloc)):
if not wheel.supported(tags=supported_tags_noarch):
logger.debug(
"Skipping %s because it is a pypi-hosted binary "
"Wheel on an unsupported platform",
link,
)
return
version = wheel.version
if not version:
version = self._egg_info_matches(egg_info, search_name, link)
if version is None:
logger.debug(
'Skipping link %s; wrong project name (not %s)',
link,
search_name,
)
return
if (link.internal is not None and not link.internal and
not normalize_name(search_name).lower() in self.allow_external
and not self.allow_all_external):
# We have a link that we are sure is external, so we should skip
# it unless we are allowing externals
logger.debug("Skipping %s because it is externally hosted.", link)
self.need_warn_external = True
return
if (link.verifiable is not None and not link.verifiable
and not (normalize_name(search_name).lower()
in self.allow_unverified)):
# We have a link that we are sure we cannot verify its integrity,
# so we should skip it unless we are allowing unsafe installs
# for this requirement.
logger.debug(
"Skipping %s because it is an insecure and unverifiable file.",
link,
)
self.need_warn_unverified = True
return
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
logger.debug('Skipping %s because Python version is incorrect',
link)
return
logger.debug('Found link %s, version: %s', link, version)
return InstallationCandidate(search_name, version, link)
def _find_all_versions(self, project_name):
"""Find all available versions for project_name
This checks index_urls, find_links and dependency_links
All versions found are returned
See _link_package_versions for details on which files are accepted
"""
index_locations = self._get_index_urls_locations(project_name)
index_file_loc, index_url_loc = self._sort_locations(index_locations)
fl_file_loc, fl_url_loc = self._sort_locations(self.find_links,
expand_dir=True)
dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links)
file_locations = (Link(url) for url in itertools.chain(
index_file_loc, fl_file_loc, dep_file_loc))
# We trust every url that the user has given us whether it was given
# via --index-url or --find-links
# We explicitly do not trust links that came from dependency_links
# We want to filter out any thing which does not have a secure origin.
url_locations = [
link for link in itertools.chain(
(Link(url, trusted=True) for url in index_url_loc),
(Link(url, trusted=True) for url in fl_url_loc),
(Link(url) for url in dep_url_loc),
) if self._validate_secure_origin(logger, link)
]
logger.debug('%d location(s) to search for versions of %s:',
len(url_locations), project_name)
for location in url_locations:
logger.debug('* %s', location)
canonical_name = pkg_resources.safe_name(project_name).lower()
formats = fmt_ctl_formats(self.format_control, canonical_name)
search = Search(project_name.lower(), canonical_name, formats)
find_links_versions = self._package_versions(
# We trust every directly linked archive in find_links
(Link(url, '-f', trusted=True) for url in self.find_links),
search)
page_versions = []
for page in self._get_pages(url_locations, project_name):
logger.debug('Analyzing links from page %s', page.url)
with indent_log():
page_versions.extend(self._package_versions(
page.links, search))
dependency_versions = self._package_versions(
(Link(url) for url in self.dependency_links), search)
if dependency_versions:
logger.debug(
'dependency_links found: %s', ', '.join(
[version.location.url for version in dependency_versions]))
file_versions = self._package_versions(file_locations, search)
if file_versions:
file_versions.sort(reverse=True)
logger.debug(
'Local files found: %s', ', '.join([
url_to_path(candidate.location.url)
for candidate in file_versions
]))
# This is an intentional priority ordering
return (file_versions + find_links_versions + page_versions +
dependency_versions)
def add_dependencies(self, name, version, requirements):
pypi_data = self._pypi_data[safe_name(name)][version]
pypi_data.setdefault("dependencies", []).extend(requirements)
def name(self):
# type: () -> Optional[str]
if self.req is None:
return None
return native_str(pkg_resources.safe_name(self.req.name))
def build(self, autobuilding=False):
"""Build wheels.
:param unpack: If True, replace the sdist we built from the with the
newly built wheel, in preparation for installation.
:return: True if all the wheels built correctly.
"""
assert self._wheel_dir or (autobuilding and self._cache_root)
# unpack sdists and constructs req set
self.requirement_set.prepare_files(self.finder)
reqset = self.requirement_set.requirements.values()
buildset = []
for req in reqset:
if req.is_wheel:
if not autobuilding:
logger.info(
'Skipping %s, due to already being wheel.', req.name)
elif req.editable:
if not autobuilding:
logger.info(
'Skipping bdist_wheel for %s, due to being editable',
req.name)
elif autobuilding and req.link and not req.link.is_artifact:
pass
elif autobuilding and not req.source_dir:
pass
else:
if autobuilding:
link = req.link
base, ext = link.splitext()
if pip.index.egg_info_matches(base, None, link) is None:
# Doesn't look like a package - don't autobuild a wheel
# because we'll have no way to lookup the result sanely
continue
if "binary" not in pip.index.fmt_ctl_formats(
self.finder.format_control,
pkg_resources.safe_name(req.name).lower()):
logger.info(
"Skipping bdist_wheel for %s, due to binaries "
"being disabled for it.", req.name)
continue
buildset.append(req)
if not buildset:
return True
# Build the wheels.
logger.info(
'Building wheels for collected packages: %s',
', '.join([req.name for req in buildset]),
)
with indent_log():
build_success, build_failure = [], []
for req in buildset:
if autobuilding:
output_dir = _cache_for_link(self._cache_root, req.link)
ensure_dir(output_dir)
else:
output_dir = self._wheel_dir
wheel_file = self._build_one(req, output_dir)
if wheel_file:
build_success.append(req)
if autobuilding:
# XXX: This is mildly duplicative with prepare_files,
# but not close enough to pull out to a single common
# method.
# The code below assumes temporary source dirs -
# prevent it doing bad things.
if req.source_dir and not os.path.exists(os.path.join(
req.source_dir, PIP_DELETE_MARKER_FILENAME)):
raise AssertionError(
"bad source dir - missing marker")
# Delete the source we built the wheel from
req.remove_temporary_source()
# set the build directory again - name is known from
# the work prepare_files did.
req.source_dir = req.build_location(
self.requirement_set.build_dir)
# Update the link for this.
req.link = pip.index.Link(
path_to_url(wheel_file), trusted=True)
assert req.link.is_wheel
# extract the wheel into the dir
unpack_url(
req.link, req.source_dir, None, False,
session=self.requirement_set.session)
else:
build_failure.append(req)
# notify success/failure
if build_success:
logger.info(
'Successfully built %s',
' '.join([req.name for req in build_success]),
)
if build_failure:
logger.info(
'Failed to build %s',
' '.join([req.name for req in build_failure]),
)
# Return True if all builds were successful
return len(build_failure) == 0
def normalize_name(name: str) -> str:
return safe_name(name).lower() # type: ignore
def name(self):
if self.req is None:
return None
return native_str(pkg_resources.safe_name(self.req.name))
def add_candidate(self, candidate):
key = safe_name(candidate.name).lower()
self._data[key] = Distribution(key, candidate.version)
def build(self, autobuilding=False):
"""Build wheels.
:param unpack: If True, replace the sdist we built from the with the
newly built wheel, in preparation for installation.
:return: True if all the wheels built correctly.
"""
assert self._wheel_dir or (autobuilding and self._cache_root)
# unpack sdists and constructs req set
self.requirement_set.prepare_files(self.finder)
reqset = self.requirement_set.requirements.values()
buildset = []
for req in reqset:
if req.constraint:
continue
if req.is_wheel:
if not autobuilding:
logger.info('Skipping %s, due to already being wheel.',
req.name)
elif req.editable:
if not autobuilding:
logger.info(
'Skipping bdist_wheel for %s, due to being editable',
req.name)
elif autobuilding and req.link and not req.link.is_artifact:
pass
elif autobuilding and not req.source_dir:
pass
else:
if autobuilding:
link = req.link
base, ext = link.splitext()
if pip.index.egg_info_matches(base, None, link) is None:
# Doesn't look like a package - don't autobuild a wheel
# because we'll have no way to lookup the result sanely
continue
if "binary" not in pip.index.fmt_ctl_formats(
self.finder.format_control,
pkg_resources.safe_name(req.name).lower()):
logger.info(
"Skipping bdist_wheel for %s, due to binaries "
"being disabled for it.", req.name)
continue
buildset.append(req)
if not buildset:
return True
# Build the wheels.
logger.info(
'Building wheels for collected packages: %s',
', '.join([req.name for req in buildset]),
)
with indent_log():
build_success, build_failure = [], []
for req in buildset:
if autobuilding:
output_dir = _cache_for_link(self._cache_root, req.link)
ensure_dir(output_dir)
else:
output_dir = self._wheel_dir
wheel_file = self._build_one(req, output_dir)
if wheel_file:
build_success.append(req)
if autobuilding:
# XXX: This is mildly duplicative with prepare_files,
# but not close enough to pull out to a single common
# method.
# The code below assumes temporary source dirs -
# prevent it doing bad things.
if req.source_dir and not os.path.exists(
os.path.join(req.source_dir,
PIP_DELETE_MARKER_FILENAME)):
raise AssertionError(
"bad source dir - missing marker")
# Delete the source we built the wheel from
req.remove_temporary_source()
# set the build directory again - name is known from
# the work prepare_files did.
req.source_dir = req.build_location(
self.requirement_set.build_dir)
# Update the link for this.
req.link = pip.index.Link(path_to_url(wheel_file),
trusted=True)
assert req.link.is_wheel
# extract the wheel into the dir
unpack_url(req.link,
req.source_dir,
None,
False,
session=self.requirement_set.session)
else:
build_failure.append(req)
# notify success/failure
if build_success:
logger.info(
'Successfully built %s',
' '.join([req.name for req in build_success]),
)
if build_failure:
logger.info(
'Failed to build %s',
' '.join([req.name for req in build_failure]),
)
# Return True if all builds were successful
return len(build_failure) == 0
def add_candidate(self, name, version, requires_python=""):
pypi_data = self._pypi_data.setdefault(safe_name(name),
{}).setdefault(version, {})
pypi_data["requires_python"] = requires_python
def name(self):
if self.req is None:
return None
return native_str(pkg_resources.safe_name(self.req.name))
def install(candidate):
dependencies = repository.get_dependencies(candidate)[0]
key = safe_name(candidate.name).lower()
dist = Distribution(key, candidate.version)
dist.dependencies = dependencies
rv.add_distribution(dist)
