def run_egg_info(self):
assert self.source_dir
if self.name:
logger.debug(
'Running setup.py (path:%s) egg_info for package %s',
self.setup_py, self.name,
)
else:
logger.debug(
'Running setup.py (path:%s) egg_info for package from %s',
self.setup_py, self.link,
)
with indent_log():
script = SETUPTOOLS_SHIM % self.setup_py
base_cmd = [sys.executable, '-c', script]
if self.isolated:
base_cmd += ["--no-user-cfg"]
egg_info_cmd = base_cmd + ['egg_info']
# We can't put the .egg-info files at the root, because then the
# source code will be mistaken for an installed egg, causing
# problems
if self.editable:
egg_base_option = []
else:
egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info')
ensure_dir(egg_info_dir)
egg_base_option = ['--egg-base', 'pip-egg-info']
call_subprocess(
egg_info_cmd + egg_base_option,
cwd=self.setup_py_dir,
show_stdout=False,
command_level=logging.DEBUG,
command_desc='python setup.py egg_info')
if not self.req:
if isinstance(
pkg_resources.parse_version(self.pkg_info()["Version"]),
Version):
op = "=="
else:
op = "==="
self.req = Requirement(
"".join([
self.pkg_info()["Name"],
op,
self.pkg_info()["Version"],
])
)
self._correct_build_location()
else:
metadata_name = canonicalize_name(self.pkg_info()["Name"])
if canonicalize_name(self.req.name) != metadata_name:
logger.warning(
'Running setup.py (path:%s) egg_info for package %s '
'produced metadata for project name %s. Fix your '
'#egg=%s fragments.',
self.setup_py, self.name, metadata_name, self.name
)
self.req = Requirement(metadata_name)
def clobber(source, dest, is_base, fixer=None, filter=None):
ensure_dir(dest) # common for the 'include' path
for dir, subdirs, files in os.walk(source):
basedir = dir[len(source):].lstrip(os.path.sep)
destdir = os.path.join(dest, basedir)
if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'):
continue
for s in subdirs:
destsubdir = os.path.join(dest, basedir, s)
if is_base and basedir == '' and destsubdir.endswith('.data'):
data_dirs.append(s)
continue
elif (is_base and
s.endswith('.dist-info') and
canonicalize_name(s).startswith(
canonicalize_name(req.name))):
assert not info_dir, ('Multiple .dist-info directories: ' +
destsubdir + ', ' +
', '.join(info_dir))
info_dir.append(destsubdir)
for f in files:
# Skip unwanted files
if filter and filter(f):
continue
srcfile = os.path.join(dir, f)
destfile = os.path.join(dest, basedir, f)
# directory creation is lazy and after the file filtering above
# to ensure we don't install empty dirs; empty dirs can't be
# uninstalled.
ensure_dir(destdir)
# We use copyfile (not move, copy, or copy2) to be extra sure
# that we are not moving directories over (copyfile fails for
# directories) as well as to ensure that we are not copying
# over any metadata because we want more control over what
# metadata we actually copy over.
shutil.copyfile(srcfile, destfile)
# Copy over the metadata for the file, currently this only
# includes the atime and mtime.
st = os.stat(srcfile)
if hasattr(os, "utime"):
os.utime(destfile, (st.st_atime, st.st_mtime))
# If our file is executable, then make our destination file
# executable.
if os.access(srcfile, os.X_OK):
st = os.stat(srcfile)
permissions = (
st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
)
os.chmod(destfile, permissions)
changed = False
if fixer:
changed = fixer(destfile)
record_installed(srcfile, destfile, changed)
def _link_package_versions(self, link, search):
"""Return an InstallationCandidate or None"""
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
self._log_skipped_link(link, "not a file")
return
if ext not in SUPPORTED_EXTENSIONS:
self._log_skipped_link(link, "unsupported archive format: %s" % ext)
return
if "binary" not in search.formats and ext == wheel_ext:
self._log_skipped_link(link, "No binaries permitted for %s" % search.supplied)
return
if "macosx10" in link.path and ext == ".zip":
self._log_skipped_link(link, "macosx10 one")
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
self._log_skipped_link(link, "invalid wheel filename")
return
if canonicalize_name(wheel.name) != search.canonical:
self._log_skipped_link(link, "wrong project name (not %s)" % search.supplied)
return
if not wheel.supported():
self._log_skipped_link(link, "it is not compatible with this Python")
return
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != wheel_ext:
self._log_skipped_link(link, "No sources permitted for %s" % search.supplied)
return
if not version:
version = egg_info_matches(egg_info, search.supplied, link)
if version is None:
self._log_skipped_link(link, "wrong project name (not %s)" % search.supplied)
return
match = self._py_version_re.search(version)
if match:
version = version[: match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
self._log_skipped_link(link, "Python version is incorrect")
return
logger.debug("Found link %s, version: %s", link, version)
return InstallationCandidate(search.supplied, version, link)
def _simulate_installation_of(to_install, state):
# type: (List[InstallRequirement], PackageSet) -> None
"""Computes the version of packages after installing to_install.
"""
# Modify it as installing requirement_set would (assuming no errors)
for inst_req in to_install:
dist = make_abstract_dist(inst_req).dist(finder=None)
name = canonicalize_name(dist.key)
state[name] = PackageDetails(dist.version, dist.requires())
def mkurl_pypi_url(url):
loc = posixpath.join(url, urllib_parse.quote(canonicalize_name(project_name)))
# For maximum compatibility with easy_install, ensure the path
# ends in a trailing slash. Although this isn't in the spec
# (and PyPI can handle it without the slash) some other index
# implementations might break if they relied on easy_install's
# behavior.
if not loc.endswith("/"):
loc = loc + "/"
return loc
def create_package_set_from_installed(**kwargs):
# type: (**Any) -> PackageSet
"""Converts a list of distributions into a PackageSet.
"""
# Default to using all packages installed on the system
if kwargs == {}:
kwargs = {"local_only": False, "skip": ()}
retval = {}
for dist in get_installed_distributions(**kwargs):
name = canonicalize_name(dist.project_name)
retval[name] = PackageDetails(dist.version, dist.requires())
return retval
def should_use_ephemeral_cache(
req, # type: InstallRequirement
format_control, # type: FormatControl
autobuilding, # type: bool
cache_available # type: bool
):
# type: (...) -> Optional[bool]
"""
Return whether to build an InstallRequirement object using the
ephemeral cache.
:param cache_available: whether a cache directory is available for the
autobuilding=True case.
:return: True or False to build the requirement with ephem_cache=True
or False, respectively; or None not to build the requirement.
"""
if req.constraint:
return None
if req.is_wheel:
if not autobuilding:
logger.info(
'Skipping %s, due to already being wheel.', req.name,
)
return None
if not autobuilding:
return False
if req.editable or not req.source_dir:
return None
if req.link and not req.link.is_artifact:
# VCS checkout. Build wheel just for this run.
return True
if "binary" not in format_control.get_allowed_formats(
canonicalize_name(req.name)):
logger.info(
"Skipping bdist_wheel for %s, due to binaries "
"being disabled for it.", req.name,
)
return None
link = req.link
base, ext = link.splitext()
if cache_available and _contains_egg_info(base):
return False
# Otherwise, build the wheel just for this run using the ephemeral
# cache since we are either in the case of e.g. a local directory, or
# no cache directory is available to use.
return True
def prepare_metadata(self):
# type: () -> None
"""Ensure that project metadata is available.
Under PEP 517, call the backend hook to prepare the metadata.
Under legacy processing, call setup.py egg-info.
"""
assert self.source_dir
with indent_log():
if self.use_pep517:
self.prepare_pep517_metadata()
else:
self.run_egg_info()
if not self.req:
if isinstance(parse_version(self.metadata["Version"]), Version):
op = "=="
else:
op = "==="
self.req = Requirement(
"".join([
self.metadata["Name"],
op,
self.metadata["Version"],
])
)
self._correct_build_location()
else:
metadata_name = canonicalize_name(self.metadata["Name"])
if canonicalize_name(self.req.name) != metadata_name:
logger.warning(
'Generating metadata for package %s '
'produced metadata for project name %s. Fix your '
'#egg=%s fragments.',
self.name, metadata_name, self.name
)
self.req = Requirement(metadata_name)
def _create_whitelist(would_be_installed, package_set):
# type: (Set[str], PackageSet) -> Set[str]
packages_affected = set(would_be_installed)
for package_name in package_set:
if package_name in packages_affected:
continue
for req in package_set[package_name].requires:
if canonicalize_name(req.name) in packages_affected:
packages_affected.add(package_name)
break
return packages_affected
def fmt_ctl_handle_mutual_exclude(value, target, other):
new = value.split(',')
while ':all:' in new:
other.clear()
target.clear()
target.add(':all:')
del new[:new.index(':all:') + 1]
if ':none:' not in new:
# Without a none, we want to discard everything as :all: covers it
return
for name in new:
if name == ':none:':
target.clear()
continue
name = canonicalize_name(name)
other.discard(name)
target.add(name)
def _simulate_installation_of(to_install, package_set):
# type: (List[InstallRequirement], PackageSet) -> Set[str]
"""Computes the version of packages after installing to_install.
"""
# Keep track of packages that were installed
installed = set()
# Modify it as installing requirement_set would (assuming no errors)
for inst_req in to_install:
dist = make_abstract_dist(inst_req).dist(finder=None)
name = canonicalize_name(dist.key)
package_set[name] = PackageDetails(dist.version, dist.requires())
installed.add(name)
return installed
def check_package_set(package_set, should_ignore=None):
# type: (PackageSet, Optional[Callable[[str], bool]]) -> CheckResult
"""Check if a package set is consistent
If should_ignore is passed, it should be a callable that takes a
package name and returns a boolean.
"""
if should_ignore is None:
def should_ignore(name):
return False
missing = dict()
conflicting = dict()
for package_name in package_set:
# Info about dependencies of package_name
missing_deps = set() # type: Set[Missing]
conflicting_deps = set() # type: Set[Conflicting]
if should_ignore(package_name):
continue
for req in package_set[package_name].requires:
name = canonicalize_name(req.project_name) # type: str
# Check if it's missing
if name not in package_set:
missed = True
if req.marker is not None:
missed = req.marker.evaluate()
if missed:
missing_deps.add((name, req))
continue
# Check if there's a conflict
version = package_set[name].version # type: str
if not req.specifier.contains(version, prereleases=True):
conflicting_deps.add((name, version, req))
if missing_deps:
missing[package_name] = sorted(missing_deps, key=str)
if conflicting_deps:
conflicting[package_name] = sorted(conflicting_deps, key=str)
return missing, conflicting
def handle_mutual_excludes(value, target, other):
# type: (str, Optional[Set], Optional[Set]) -> None
new = value.split(',')
while ':all:' in new:
other.clear()
target.clear()
target.add(':all:')
del new[:new.index(':all:') + 1]
# Without a none, we want to discard everything as :all: covers it
if ':none:' not in new:
return
for name in new:
if name == ':none:':
target.clear()
continue
name = canonicalize_name(name)
other.discard(name)
target.add(name)
def create_package_set_from_installed(**kwargs):
# type: (**Any) -> Tuple[PackageSet, bool]
"""Converts a list of distributions into a PackageSet.
"""
# Default to using all packages installed on the system
if kwargs == {}:
kwargs = {"local_only": False, "skip": ()}
package_set = {}
problems = False
for dist in get_installed_distributions(**kwargs):
name = canonicalize_name(dist.project_name)
try:
package_set[name] = PackageDetails(dist.version, dist.requires())
except RequirementParseError as e:
# Don't crash on broken metadata
logging.warning("Error parsing requirements for %s: %s", name, e)
problems = True
return package_set, problems
def check_package_set(package_set):
# type: (PackageSet) -> CheckResult
"""Check if a package set is consistent
"""
missing = dict()
conflicting = dict()
for package_name in package_set:
# Info about dependencies of package_name
missing_deps = set() # type: Set[Missing]
conflicting_deps = set() # type: Set[Conflicting]
for req in package_set[package_name].requires:
name = canonicalize_name(req.project_name) # type: str
# Check if it's missing
if name not in package_set:
missed = True
if req.marker is not None:
missed = req.marker.evaluate()
if missed:
missing_deps.add((name, req))
continue
# Check if there's a conflict
version = package_set[name].version # type: str
if not req.specifier.contains(version, prereleases=True):
conflicting_deps.add((name, version, req))
def str_key(x):
return str(x)
if missing_deps:
missing[package_name] = sorted(missing_deps, key=str_key)
if conflicting_deps:
conflicting[package_name] = sorted(conflicting_deps, key=str_key)
return missing, conflicting
def _find_name_version_sep(egg_info, canonical_name):
"""Find the separator's index based on the package's canonical name.
`egg_info` must be an egg info string for the given package, and
`canonical_name` must be the package's canonical name.
This function is needed since the canonicalized name does not necessarily
have the same length as the egg info's name part. An example::
>>> egg_info = 'foo__bar-1.0'
>>> canonical_name = 'foo-bar'
>>> _find_name_version_sep(egg_info, canonical_name)
8
"""
# Project name and version must be separated by one single dash. Find all
# occurrences of dashes; if the string in front of it matches the canonical
# name, this is the one separating the name and version parts.
for i, c in enumerate(egg_info):
if c != "-":
continue
if canonicalize_name(egg_info[:i]) == canonical_name:
return i
raise ValueError("{} does not match {}".format(egg_info, canonical_name))
def _get_candidates(self, link, package_name):
can_not_cache = (
not self.cache_dir or
not package_name or
not link
)
if can_not_cache:
return []
canonical_name = canonicalize_name(package_name)
formats = self.format_control.get_allowed_formats(
canonical_name
)
if not self.allowed_formats.intersection(formats):
return []
root = self.get_path_for_link(link)
try:
return os.listdir(root)
except OSError as err:
if err.errno in {errno.ENOENT, errno.ENOTDIR}:
return []
raise
def cached_wheel(cache_dir, link, format_control, package_name):
if not cache_dir:
return link
if not link:
return link
if link.is_wheel:
return link
if not link.is_artifact:
return link
if not package_name:
return link
canonical_name = canonicalize_name(package_name)
formats = pip.index.fmt_ctl_formats(format_control, canonical_name)
if "binary" not in formats:
return link
root = _cache_for_link(cache_dir, link)
try:
wheel_names = os.listdir(root)
except OSError as e:
if e.errno in {errno.ENOENT, errno.ENOTDIR}:
return link
raise
candidates = []
for wheel_name in wheel_names:
try:
wheel = Wheel(wheel_name)
except InvalidWheelFilename:
continue
if not wheel.supported():
# Built for a different python/arch/etc
continue
candidates.append((wheel.support_index_min(), wheel_name))
if not candidates:
return link
candidates.sort()
path = os.path.join(root, candidates[0][1])
return pip.index.Link(path_to_url(path))
def project_name(self) -> NormalizedName:
"""The normalised name of the project the candidate refers to"""
if self._name is None:
self._name = canonicalize_name(self.dist.project_name)
return self._name
def evaluate_link(self, link, search):
# type: (Link, Search) -> Tuple[bool, Optional[str]]
"""
Determine whether a link is a candidate for installation.
:return: A tuple (is_candidate, result), where `result` is (1) a
version string if `is_candidate` is True, and (2) if
`is_candidate` is False, an optional string to log the reason
the link fails to qualify.
"""
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
return (False, 'not a file')
if ext not in SUPPORTED_EXTENSIONS:
return (False, 'unsupported archive format: %s' % ext)
if "binary" not in search.formats and ext == WHEEL_EXTENSION:
reason = 'No binaries permitted for %s' % search.supplied
return (False, reason)
if "macosx10" in link.path and ext == '.zip':
return (False, 'macosx10 one')
if ext == WHEEL_EXTENSION:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
return (False, 'invalid wheel filename')
if canonicalize_name(wheel.name) != search.canonical:
reason = 'wrong project name (not %s)' % search.supplied
return (False, reason)
if not self._is_wheel_supported(wheel):
# Include the wheel's tags in the reason string to
# simplify troubleshooting compatibility issues.
file_tags = wheel.get_formatted_file_tags()
reason = ("none of the wheel's tags match: {}".format(
', '.join(file_tags)))
return (False, reason)
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != WHEEL_EXTENSION:
return (False, 'No sources permitted for %s' % search.supplied)
if not version:
version = _egg_info_matches(egg_info, search.canonical)
if not version:
return (False, 'Missing project version for %s' % search.supplied)
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != self._py_version:
return (False, 'Python version is incorrect')
supports_python = _check_link_requires_python(
link,
version_info=self._py_version_info,
ignore_requires_python=self._ignore_requires_python,
)
if not supports_python:
# Return None for the reason text to suppress calling
# _log_skipped_link().
return (False, None)
logger.debug('Found link %s, version: %s', link, version)
return (True, version)
def name(self):
# type: () -> str
canonical_name = canonicalize_name(self._ireq.req.name)
return format_name(canonical_name, self.extras)
def project_name(self) -> NormalizedName:
return canonicalize_name(self.dist.project_name)
def freeze(
requirement=None,
find_links=None,
local_only=None,
user_only=None,
skip_regex=None,
default_vcs=None,
isolated=False,
wheel_cache=None,
skip=(),
):
find_links = find_links or []
skip_match = None
if skip_regex:
skip_match = re.compile(skip_regex).search
dependency_links = []
for dist in pkg_resources.working_set:
if dist.has_metadata("dependency_links.txt"):
dependency_links.extend(dist.get_metadata_lines("dependency_links.txt"))
for link in find_links:
if "#egg=" in link:
dependency_links.append(link)
for link in find_links:
yield "-f %s" % link
installations = {}
for dist in get_installed_distributions(local_only=local_only, skip=(), user_only=user_only):
try:
req = pip.FrozenRequirement.from_dist(dist, dependency_links)
except RequirementParseError:
logger.warning("Could not parse requirement: %s", dist.project_name)
continue
installations[req.name] = req
if requirement:
with open(requirement) as req_file:
for line in req_file:
if (
not line.strip()
or line.strip().startswith("#")
or (skip_match and skip_match(line))
or line.startswith(
(
"-r",
"--requirement",
"-Z",
"--always-unzip",
"-f",
"--find-links",
"-i",
"--index-url",
"--pre",
"--trusted-host",
"--process-dependency-links",
"--extra-index-url",
)
)
):
yield line.rstrip()
continue
if line.startswith("-e") or line.startswith("--editable"):
if line.startswith("-e"):
line = line[2:].strip()
else:
line = line[len("--editable") :].strip().lstrip("=")
line_req = InstallRequirement.from_editable(
line, default_vcs=default_vcs, isolated=isolated, wheel_cache=wheel_cache
)
else:
line_req = InstallRequirement.from_line(line, isolated=isolated, wheel_cache=wheel_cache)
if not line_req.name:
logger.info("Skipping line because it's not clear what it " "would install: %s", line.strip())
logger.info(" (add #egg=PackageName to the URL to avoid" " this warning)")
elif line_req.name not in installations:
logger.warning("Requirement file contains %s, but that package is" " not installed", line.strip())
else:
yield str(installations[line_req.name]).rstrip()
del installations[line_req.name]
yield ("## The following requirements were added by " "pip freeze:")
for installation in sorted(installations.values(), key=lambda x: x.name.lower()):
if canonicalize_name(installation.name) not in skip:
yield str(installation).rstrip()
def freeze(
requirement=None, # type: Optional[List[str]]
find_links=None, # type: Optional[List[str]]
local_only=False, # type: bool
user_only=False, # type: bool
paths=None, # type: Optional[List[str]]
isolated=False, # type: bool
wheel_cache=None, # type: Optional[WheelCache]
exclude_editable=False, # type: bool
skip=() # type: Container[str]
):
# type: (...) -> Iterator[str]
find_links = find_links or []
for link in find_links:
yield '-f {}'.format(link)
installations = {} # type: Dict[str, FrozenRequirement]
for dist in get_installed_distributions(local_only=local_only,
skip=(),
user_only=user_only,
paths=paths):
try:
req = FrozenRequirement.from_dist(dist)
except RequirementParseError as exc:
# We include dist rather than dist.project_name because the
# dist string includes more information, like the version and
# location. We also include the exception message to aid
# troubleshooting.
logger.warning(
'Could not generate requirement for distribution %r: %s', dist,
exc)
continue
if exclude_editable and req.editable:
continue
installations[req.canonical_name] = req
if requirement:
# the options that don't get turned into an InstallRequirement
# should only be emitted once, even if the same option is in multiple
# requirements files, so we need to keep track of what has been emitted
# so that we don't emit it again if it's seen again
emitted_options = set() # type: Set[str]
# keep track of which files a requirement is in so that we can
# give an accurate warning if a requirement appears multiple times.
req_files = collections.defaultdict(list) # type: Dict[str, List[str]]
for req_file_path in requirement:
with open(req_file_path) as req_file:
for line in req_file:
if (not line.strip() or line.strip().startswith('#')
or line.startswith(
('-r', '--requirement', '-f', '--find-links',
'-i', '--index-url', '--pre',
'--trusted-host',
'--process-dependency-links',
'--extra-index-url', '--use-feature'))):
line = line.rstrip()
if line not in emitted_options:
emitted_options.add(line)
yield line
continue
if line.startswith('-e') or line.startswith('--editable'):
if line.startswith('-e'):
line = line[2:].strip()
else:
line = line[len('--editable'):].strip().lstrip('=')
line_req = install_req_from_editable(
line,
isolated=isolated,
)
else:
line_req = install_req_from_line(
COMMENT_RE.sub('', line).strip(),
isolated=isolated,
)
if not line_req.name:
logger.info(
"Skipping line in requirement file [%s] because "
"it's not clear what it would install: %s",
req_file_path,
line.strip(),
)
logger.info(
" (add #egg=PackageName to the URL to avoid"
" this warning)")
else:
line_req_canonical_name = canonicalize_name(
line_req.name)
if line_req_canonical_name not in installations:
# either it's not installed, or it is installed
# but has been processed already
if not req_files[line_req.name]:
logger.warning(
"Requirement file [%s] contains %s, but "
"package %r is not installed",
req_file_path,
COMMENT_RE.sub('', line).strip(),
line_req.name)
else:
req_files[line_req.name].append(req_file_path)
else:
yield str(installations[line_req_canonical_name]
).rstrip()
del installations[line_req_canonical_name]
req_files[line_req.name].append(req_file_path)
# Warn about requirements that were included multiple times (in a
# single requirements file or in different requirements files).
for name, files in six.iteritems(req_files):
if len(files) > 1:
logger.warning("Requirement %s included multiple times [%s]",
name, ', '.join(sorted(set(files))))
yield ('## The following requirements were added by ' 'pip freeze:')
for installation in sorted(installations.values(),
key=lambda x: x.name.lower()):
if installation.canonical_name not in skip:
yield str(installation).rstrip()
def find_all_candidates(self, project_name):
"""Find all available InstallationCandidate for project_name
This checks index_urls, find_links and dependency_links.
All versions found are returned as an InstallationCandidate list.
See _link_package_versions for details on which files are accepted
"""
index_locations = self._get_index_urls_locations(project_name)
index_file_loc, index_url_loc = self._sort_locations(index_locations)
fl_file_loc, fl_url_loc = self._sort_locations(self.find_links,
expand_dir=True)
dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links)
file_locations = (Link(url) for url in itertools.chain(
index_file_loc, fl_file_loc, dep_file_loc))
# We trust every url that the user has given us whether it was given
# via --index-url or --find-links
# We explicitly do not trust links that came from dependency_links
# We want to filter out any thing which does not have a secure origin.
url_locations = [
link for link in itertools.chain(
(Link(url) for url in index_url_loc),
(Link(url) for url in fl_url_loc),
(Link(url) for url in dep_url_loc),
) if self._validate_secure_origin(logger, link)
]
logger.debug('%d location(s) to search for versions of %s:',
len(url_locations), project_name)
for location in url_locations:
logger.debug('* %s', location)
canonical_name = canonicalize_name(project_name)
formats = fmt_ctl_formats(self.format_control, canonical_name)
search = Search(project_name, canonical_name, formats)
find_links_versions = self._package_versions(
# We trust every directly linked archive in find_links
(Link(url, '-f') for url in self.find_links),
search)
page_versions = []
for page in self._get_pages(url_locations, project_name):
logger.debug('Analyzing links from page %s', page.url)
with indent_log():
page_versions.extend(self._package_versions(
page.links, search))
dependency_versions = self._package_versions(
(Link(url) for url in self.dependency_links), search)
if dependency_versions:
logger.debug(
'dependency_links found: %s', ', '.join(
[version.location.url for version in dependency_versions]))
file_versions = self._package_versions(file_locations, search)
if file_versions:
file_versions.sort(reverse=True)
logger.debug(
'Local files found: %s', ', '.join([
url_to_path(candidate.location.url)
for candidate in file_versions
]))
# This is an intentional priority ordering
return (file_versions + find_links_versions + page_versions +
dependency_versions)
def freeze(
requirement=None,
find_links=None, local_only=None, user_only=None, skip_regex=None,
default_vcs=None,
isolated=False,
wheel_cache=None,
skip=()):
find_links = find_links or []
skip_match = None
if skip_regex:
skip_match = re.compile(skip_regex).search
dependency_links = []
for dist in pkg_resources.working_set:
if dist.has_metadata('dependency_links.txt'):
dependency_links.extend(
dist.get_metadata_lines('dependency_links.txt')
)
for link in find_links:
if '#egg=' in link:
dependency_links.append(link)
for link in find_links:
yield '-f %s' % link
installations = {}
for dist in get_installed_distributions(local_only=local_only,
skip=(),
user_only=user_only):
try:
req = pip.FrozenRequirement.from_dist(
dist,
dependency_links
)
except RequirementParseError:
logger.warning(
"Could not parse requirement: %s",
dist.project_name
)
continue
installations[req.name] = req
if requirement:
# the options that don't get turned into an InstallRequirement
# should only be emitted once, even if the same option is in multiple
# requirements files, so we need to keep track of what has been emitted
# so that we don't emit it again if it's seen again
emitted_options = set()
for req_file_path in requirement:
with open(req_file_path) as req_file:
for line in req_file:
if (not line.strip() or
line.strip().startswith('#') or
(skip_match and skip_match(line)) or
line.startswith((
'-r', '--requirement',
'-Z', '--always-unzip',
'-f', '--find-links',
'-i', '--index-url',
'--pre',
'--trusted-host',
'--process-dependency-links',
'--extra-index-url'))):
line = line.rstrip()
if line not in emitted_options:
emitted_options.add(line)
yield line
continue
if line.startswith('-e') or line.startswith('--editable'):
if line.startswith('-e'):
line = line[2:].strip()
else:
line = line[len('--editable'):].strip().lstrip('=')
line_req = InstallRequirement.from_editable(
line,
default_vcs=default_vcs,
isolated=isolated,
wheel_cache=wheel_cache,
)
else:
line_req = InstallRequirement.from_line(
COMMENT_RE.sub('', line).strip(),
isolated=isolated,
wheel_cache=wheel_cache,
)
if not line_req.name:
logger.info(
"Skipping line in requirement file [%s] because "
"it's not clear what it would install: %s",
req_file_path, line.strip(),
)
logger.info(
" (add #egg=PackageName to the URL to avoid"
" this warning)"
)
elif line_req.name not in installations:
logger.warning(
"Requirement file [%s] contains %s, but that "
"package is not installed",
req_file_path, COMMENT_RE.sub('', line).strip(),
)
else:
yield str(installations[line_req.name]).rstrip()
del installations[line_req.name]
yield(
'## The following requirements were added by '
'pip freeze:'
)
for installation in sorted(
installations.values(), key=lambda x: x.name.lower()):
if canonicalize_name(installation.name) not in skip:
yield str(installation).rstrip()
def name(self):
# type: () -> str
"""The normalised name of the project the candidate refers to"""
if self._name is None:
self._name = canonicalize_name(self.dist.project_name)
return self._name
def name(self):
# type: () -> str
return canonicalize_name(self.dist.project_name)
def _iter_found_candidates(
self,
ireqs, # type: Sequence[InstallRequirement]
specifier, # type: SpecifierSet
):
# type: (...) -> Iterable[Candidate]
if not ireqs:
return ()
# The InstallRequirement implementation requires us to give it a
# "template". Here we just choose the first requirement to represent
# all of them.
# Hopefully the Project model can correct this mismatch in the future.
template = ireqs[0]
name = canonicalize_name(template.req.name)
hashes = Hashes()
extras = frozenset() # type: FrozenSet[str]
for ireq in ireqs:
specifier &= ireq.req.specifier
hashes |= ireq.hashes(trust_internet=False)
extras |= frozenset(ireq.extras)
# We use this to ensure that we only yield a single candidate for
# each version (the finder's preferred one for that version). The
# requirement needs to return only one candidate per version, so we
# implement that logic here so that requirements using this helper
# don't all have to do the same thing later.
candidates = collections.OrderedDict() # type: VersionCandidates
# Get the installed version, if it matches, unless the user
# specified `--force-reinstall`, when we want the version from
# the index instead.
installed_version = None
installed_candidate = None
if not self._force_reinstall and name in self._installed_dists:
installed_dist = self._installed_dists[name]
installed_version = installed_dist.parsed_version
if specifier.contains(installed_version, prereleases=True):
installed_candidate = self._make_candidate_from_dist(
dist=installed_dist,
extras=extras,
template=template,
)
found = self._finder.find_best_candidate(
project_name=name,
specifier=specifier,
hashes=hashes,
)
for ican in found.iter_applicable():
if ican.version == installed_version and installed_candidate:
candidate = installed_candidate
else:
candidate = self._make_candidate_from_link(
link=ican.link,
extras=extras,
template=template,
name=name,
version=ican.version,
)
candidates[ican.version] = candidate
# Yield the installed version even if it is not found on the index.
if installed_version and installed_candidate:
candidates[installed_version] = installed_candidate
return six.itervalues(candidates)
def check_binary_allowed(req):
# type: (InstallRequirement) -> bool
canonical_name = canonicalize_name(req.name)
allowed_formats = format_control.get_allowed_formats(canonical_name)
return "binary" in allowed_formats
def evaluate_link(self, link):
# type: (Link) -> Tuple[bool, Optional[Text]]
"""
Determine whether a link is a candidate for installation.
:return: A tuple (is_candidate, result), where `result` is (1) a
version string if `is_candidate` is True, and (2) if
`is_candidate` is False, an optional string to log the reason
the link fails to qualify.
"""
version = None
if link.is_yanked and not self._allow_yanked:
reason = link.yanked_reason or '<none given>'
# Mark this as a unicode string to prevent "UnicodeEncodeError:
# 'ascii' codec can't encode character" in Python 2 when
# the reason contains non-ascii characters.
return (False, u'yanked for reason: {}'.format(reason))
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
return (False, 'not a file')
if ext not in SUPPORTED_EXTENSIONS:
return (False, 'unsupported archive format: {}'.format(ext))
if "binary" not in self._formats and ext == WHEEL_EXTENSION:
reason = 'No binaries permitted for {}'.format(
self.project_name)
return (False, reason)
if "macosx10" in link.path and ext == '.zip':
return (False, 'macosx10 one')
if ext == WHEEL_EXTENSION:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
return (False, 'invalid wheel filename')
if canonicalize_name(wheel.name) != self._canonical_name:
reason = 'wrong project name (not {})'.format(
self.project_name)
return (False, reason)
supported_tags = self._target_python.get_tags()
if not wheel.supported(supported_tags):
# Include the wheel's tags in the reason string to
# simplify troubleshooting compatibility issues.
file_tags = wheel.get_formatted_file_tags()
reason = ("none of the wheel's tags match: {}".format(
', '.join(file_tags)))
return (False, reason)
version = wheel.version
# This should be up by the self.ok_binary check, but see issue 2700.
if "source" not in self._formats and ext != WHEEL_EXTENSION:
reason = 'No sources permitted for {}'.format(self.project_name)
return (False, reason)
if not version:
version = _extract_version_from_fragment(
egg_info,
self._canonical_name,
)
if not version:
reason = 'Missing project version for {}'.format(self.project_name)
return (False, reason)
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != self._target_python.py_version:
return (False, 'Python version is incorrect')
supports_python = _check_link_requires_python(
link,
version_info=self._target_python.py_version_info,
ignore_requires_python=self._ignore_requires_python,
)
if not supports_python:
# Return None for the reason text to suppress calling
# _log_skipped_link().
return (False, None)
logger.debug('Found link %s, version: %s', link, version)
return (True, version)
def freeze(
requirement=None, # type: Optional[List[str]]
find_links=None, # type: Optional[List[str]]
local_only=None, # type: Optional[bool]
user_only=None, # type: Optional[bool]
skip_regex=None, # type: Optional[str]
isolated=False, # type: bool
wheel_cache=None, # type: Optional[WheelCache]
exclude_editable=False, # type: bool
skip=() # type: Container[str]
):
# type: (...) -> Iterator[str]
find_links = find_links or []
skip_match = None
if skip_regex:
skip_match = re.compile(skip_regex).search
for link in find_links:
yield '-f %s' % link
installations = {} # type: Dict[str, FrozenRequirement]
for dist in get_installed_distributions(local_only=local_only,
skip=(),
user_only=user_only):
try:
req = FrozenRequirement.from_dist(dist)
except RequirementParseError:
logger.warning(
"Could not parse requirement: %s",
dist.project_name
)
continue
if exclude_editable and req.editable:
continue
installations[req.name] = req
if requirement:
# the options that don't get turned into an InstallRequirement
# should only be emitted once, even if the same option is in multiple
# requirements files, so we need to keep track of what has been emitted
# so that we don't emit it again if it's seen again
emitted_options = set() # type: Set[str]
# keep track of which files a requirement is in so that we can
# give an accurate warning if a requirement appears multiple times.
req_files = collections.defaultdict(list) # type: Dict[str, List[str]]
for req_file_path in requirement:
with open(req_file_path) as req_file:
for line in req_file:
if (not line.strip() or
line.strip().startswith('#') or
(skip_match and skip_match(line)) or
line.startswith((
'-r', '--requirement',
'-Z', '--always-unzip',
'-f', '--find-links',
'-i', '--index-url',
'--pre',
'--trusted-host',
'--process-dependency-links',
'--extra-index-url'))):
line = line.rstrip()
if line not in emitted_options:
emitted_options.add(line)
yield line
continue
if line.startswith('-e') or line.startswith('--editable'):
if line.startswith('-e'):
line = line[2:].strip()
else:
line = line[len('--editable'):].strip().lstrip('=')
line_req = install_req_from_editable(
line,
isolated=isolated,
wheel_cache=wheel_cache,
)
else:
line_req = install_req_from_line(
COMMENT_RE.sub('', line).strip(),
isolated=isolated,
wheel_cache=wheel_cache,
)
if not line_req.name:
logger.info(
"Skipping line in requirement file [%s] because "
"it's not clear what it would install: %s",
req_file_path, line.strip(),
)
logger.info(
" (add #egg=PackageName to the URL to avoid"
" this warning)"
)
elif line_req.name not in installations:
# either it's not installed, or it is installed
# but has been processed already
if not req_files[line_req.name]:
logger.warning(
"Requirement file [%s] contains %s, but "
"package %r is not installed",
req_file_path,
COMMENT_RE.sub('', line).strip(), line_req.name
)
else:
req_files[line_req.name].append(req_file_path)
else:
yield str(installations[line_req.name]).rstrip()
del installations[line_req.name]
req_files[line_req.name].append(req_file_path)
# Warn about requirements that were included multiple times (in a
# single requirements file or in different requirements files).
for name, files in six.iteritems(req_files):
if len(files) > 1:
logger.warning("Requirement %s included multiple times [%s]",
name, ', '.join(sorted(set(files))))
yield(
'## The following requirements were added by '
'pip freeze:'
)
for installation in sorted(
installations.values(), key=lambda x: x.name.lower()):
if canonicalize_name(installation.name) not in skip:
yield str(installation).rstrip()
def _link_package_versions(self, link, search):
"""Return an InstallationCandidate or None"""
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
self._log_skipped_link(link, 'not a file')
return
if ext not in SUPPORTED_EXTENSIONS:
self._log_skipped_link(link,
'unsupported archive format: %s' % ext)
return
if "binary" not in search.formats and ext == wheel_ext:
self._log_skipped_link(
link, 'No binaries permitted for %s' % search.supplied)
return
if "macosx10" in link.path and ext == '.zip':
self._log_skipped_link(link, 'macosx10 one')
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
self._log_skipped_link(link, 'invalid wheel filename')
return
if canonicalize_name(wheel.name) != search.canonical:
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if not wheel.supported():
self._log_skipped_link(
link, 'it is not compatible with this Python')
return
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != wheel_ext:
self._log_skipped_link(
link, 'No sources permitted for %s' % search.supplied)
return
if not version:
version = egg_info_matches(egg_info, search.supplied, link)
if version is None:
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
self._log_skipped_link(link, 'Python version is incorrect')
return
logger.debug('Found link %s, version: %s', link, version)
return InstallationCandidate(search.supplied, version, link)
def search_packages_info(query):
"""
Gather details from installed distributions. Print distribution name,
version, location, and installed files. Installed files requires a
pip generated 'installed-files.txt' in the distributions '.egg-info'
directory.
"""
installed = {}
for p in pkg_resources.working_set:
installed[canonicalize_name(p.project_name)] = p
query_names = [canonicalize_name(name) for name in query]
missing = sorted([
name for name, pkg in zip(query, query_names) if pkg not in installed
])
if missing:
logger.warning("Package(s) not found: %s", ", ".join(missing))
def get_requiring_packages(package_name):
canonical_name = canonicalize_name(package_name)
return [
pkg.project_name for pkg in pkg_resources.working_set
if canonical_name in
[canonicalize_name(required.name) for required in pkg.requires()]
]
for dist in [installed[pkg] for pkg in query_names if pkg in installed]:
package = {
"name": dist.project_name,
"version": dist.version,
"location": dist.location,
"requires": [dep.project_name for dep in dist.requires()],
"required_by": get_requiring_packages(dist.project_name),
}
file_list = None
metadata = None
if isinstance(dist, pkg_resources.DistInfoDistribution):
# RECORDs should be part of .dist-info metadatas
if dist.has_metadata("RECORD"):
lines = dist.get_metadata_lines("RECORD")
paths = [l.split(",")[0] for l in lines]
paths = [os.path.join(dist.location, p) for p in paths]
file_list = [os.path.relpath(p, dist.location) for p in paths]
if dist.has_metadata("METADATA"):
metadata = dist.get_metadata("METADATA")
else:
# Otherwise use pip's log for .egg-info's
if dist.has_metadata("installed-files.txt"):
paths = dist.get_metadata_lines("installed-files.txt")
paths = [os.path.join(dist.egg_info, p) for p in paths]
file_list = [os.path.relpath(p, dist.location) for p in paths]
if dist.has_metadata("PKG-INFO"):
metadata = dist.get_metadata("PKG-INFO")
if dist.has_metadata("entry_points.txt"):
entry_points = dist.get_metadata_lines("entry_points.txt")
package["entry_points"] = entry_points
if dist.has_metadata("INSTALLER"):
for line in dist.get_metadata_lines("INSTALLER"):
if line.strip():
package["installer"] = line.strip()
break
# @todo: Should pkg_resources.Distribution have a
# `get_pkg_info` method?
feed_parser = FeedParser()
feed_parser.feed(metadata)
pkg_info_dict = feed_parser.close()
for key in (
"metadata-version",
"summary",
"home-page",
"author",
"author-email",
"license",
):
package[key] = pkg_info_dict.get(key)
# It looks like FeedParser cannot deal with repeated headers
classifiers = []
for line in metadata.splitlines():
if line.startswith("Classifier: "):
classifiers.append(line[len("Classifier: "):])
package["classifiers"] = classifiers
if file_list:
package["files"] = sorted(file_list)
yield package
def has_requirement(self, name):
# type: (str) -> bool
project_name = canonicalize_name(name)
return (project_name in self.requirements
and not self.requirements[project_name].constraint)
def resolve(self, root_reqs, check_supported_wheels):
# type: (List[InstallRequirement], bool) -> RequirementSet
constraints = {} # type: Dict[str, SpecifierSet]
user_requested = set() # type: Set[str]
requirements = []
for req in root_reqs:
if req.constraint:
# Ensure we only accept valid constraints
reject_invalid_constraint_types(req)
name = canonicalize_name(req.name)
if name in constraints:
constraints[name] = constraints[name] & req.specifier
else:
constraints[name] = req.specifier
else:
if req.is_direct and req.name:
user_requested.add(canonicalize_name(req.name))
requirements.append(
self.factory.make_requirement_from_install_req(req))
provider = PipProvider(
factory=self.factory,
constraints=constraints,
ignore_dependencies=self.ignore_dependencies,
upgrade_strategy=self.upgrade_strategy,
user_requested=user_requested,
)
reporter = BaseReporter()
resolver = RLResolver(provider, reporter)
try:
self._result = resolver.resolve(requirements)
except ResolutionImpossible as e:
error = self.factory.get_installation_error(e)
if not error:
# TODO: This needs fixing, we need to look at the
# factory.get_installation_error infrastructure, as that
# doesn't really allow for the logger.critical calls I'm
# using here.
for req, parent in e.causes:
logger.critical(
"Could not find a version that satisfies " +
"the requirement " + str(req) +
("" if parent is None else " (from {})".format(parent.
name)))
raise InstallationError(
"No matching distribution found for " +
", ".join([r.name for r, _ in e.causes]))
raise
six.raise_from(error, e)
req_set = RequirementSet(check_supported_wheels=check_supported_wheels)
for candidate in self._result.mapping.values():
ireq = candidate.get_install_requirement()
if ireq is None:
continue
ireq.should_reinstall = self.factory.should_reinstall(candidate)
req_set.add_named_requirement(ireq)
return req_set
def add_named_requirement(self, install_req):
# type: (InstallRequirement) -> None
assert install_req.name
project_name = canonicalize_name(install_req.name)
self.requirements[project_name] = install_req
def freeze(
requirement=None,
find_links=None, local_only=None, user_only=None, skip_regex=None,
default_vcs=None,
isolated=False,
wheel_cache=None,
skip=()):
find_links = find_links or []
skip_match = None
if skip_regex:
skip_match = re.compile(skip_regex).search
dependency_links = []
for dist in pkg_resources.working_set:
if dist.has_metadata('dependency_links.txt'):
dependency_links.extend(
dist.get_metadata_lines('dependency_links.txt')
)
for link in find_links:
if '#egg=' in link:
dependency_links.append(link)
for link in find_links:
yield '-f %s' % link
installations = {}
for dist in get_installed_distributions(local_only=local_only,
skip=(),
user_only=user_only):
req = pip.FrozenRequirement.from_dist(
dist,
dependency_links
)
installations[req.name] = req
if requirement:
with open(requirement) as req_file:
for line in req_file:
if (not line.strip() or
line.strip().startswith('#') or
(skip_match and skip_match(line)) or
line.startswith((
'-r', '--requirement',
'-Z', '--always-unzip',
'-f', '--find-links',
'-i', '--index-url',
'--pre',
'--trusted-host',
'--process-dependency-links',
'--extra-index-url'))):
yield line.rstrip()
continue
if line.startswith('-e') or line.startswith('--editable'):
if line.startswith('-e'):
line = line[2:].strip()
else:
line = line[len('--editable'):].strip().lstrip('=')
line_req = InstallRequirement.from_editable(
line,
default_vcs=default_vcs,
isolated=isolated,
wheel_cache=wheel_cache,
)
else:
line_req = InstallRequirement.from_line(
line,
isolated=isolated,
wheel_cache=wheel_cache,
)
if not line_req.name:
logger.info(
"Skipping line because it's not clear what it "
"would install: %s",
line.strip(),
)
logger.info(
" (add #egg=PackageName to the URL to avoid"
" this warning)"
)
elif line_req.name not in installations:
logger.warning(
"Requirement file contains %s, but that package is"
" not installed",
line.strip(),
)
else:
yield str(installations[line_req.name]).rstrip()
del installations[line_req.name]
yield(
'## The following requirements were added by '
'pip freeze:'
)
for installation in sorted(
installations.values(), key=lambda x: x.name.lower()):
if canonicalize_name(installation.name) not in skip:
yield str(installation).rstrip()
def search_packages_info(query):
"""
Gather details from installed distributions. Print distribution name,
version, location, and installed files. Installed files requires a
pip generated 'installed-files.txt' in the distributions '.egg-info'
directory.
"""
installed = {}
for p in pkg_resources.working_set:
installed[canonicalize_name(p.project_name)] = p
query_names = [canonicalize_name(name) for name in query]
for dist in [installed[pkg] for pkg in query_names if pkg in installed]:
package = {
'name': dist.project_name,
'version': dist.version,
'location': dist.location,
'requires': [dep.project_name for dep in dist.requires()],
}
file_list = None
metadata = None
if isinstance(dist, pkg_resources.DistInfoDistribution):
# RECORDs should be part of .dist-info metadatas
if dist.has_metadata('RECORD'):
lines = dist.get_metadata_lines('RECORD')
paths = [l.split(',')[0] for l in lines]
paths = [os.path.join(dist.location, p) for p in paths]
file_list = [os.path.relpath(p, dist.location) for p in paths]
if dist.has_metadata('METADATA'):
metadata = dist.get_metadata('METADATA')
else:
# Otherwise use pip's log for .egg-info's
if dist.has_metadata('installed-files.txt'):
paths = dist.get_metadata_lines('installed-files.txt')
paths = [os.path.join(dist.egg_info, p) for p in paths]
file_list = [os.path.relpath(p, dist.location) for p in paths]
if dist.has_metadata('PKG-INFO'):
metadata = dist.get_metadata('PKG-INFO')
if dist.has_metadata('entry_points.txt'):
entry_points = dist.get_metadata_lines('entry_points.txt')
package['entry_points'] = entry_points
if dist.has_metadata('INSTALLER'):
for line in dist.get_metadata_lines('INSTALLER'):
if line.strip():
package['installer'] = line.strip()
break
# @todo: Should pkg_resources.Distribution have a
# `get_pkg_info` method?
feed_parser = FeedParser()
feed_parser.feed(metadata)
pkg_info_dict = feed_parser.close()
for key in ('metadata-version', 'summary',
'home-functions', 'author', 'author-email', 'license'):
package[key] = pkg_info_dict.get(key)
# It looks like FeedParser cannot deal with repeated headers
classifiers = []
for line in metadata.splitlines():
if line.startswith('Classifier: '):
classifiers.append(line[len('Classifier: '):])
package['classifiers'] = classifiers
if file_list:
package['files'] = sorted(file_list)
yield package
def freeze(
requirement=None,
find_links=None, local_only=None, user_only=None, skip_regex=None,
isolated=False,
wheel_cache=None,
exclude_editable=False,
skip=()):
find_links = find_links or []
skip_match = None
if skip_regex:
skip_match = re.compile(skip_regex).search
dependency_links = []
for dist in pkg_resources.working_set:
if dist.has_metadata('dependency_links.txt'):
dependency_links.extend(
dist.get_metadata_lines('dependency_links.txt')
)
for link in find_links:
if '#egg=' in link:
dependency_links.append(link)
for link in find_links:
yield '-f %s' % link
installations = {}
for dist in get_installed_distributions(local_only=local_only,
skip=(),
user_only=user_only):
try:
req = FrozenRequirement.from_dist(
dist,
dependency_links
)
except RequirementParseError:
logger.warning(
"Could not parse requirement: %s",
dist.project_name
)
continue
if exclude_editable and req.editable:
continue
installations[req.name] = req
if requirement:
# the options that don't get turned into an InstallRequirement
# should only be emitted once, even if the same option is in multiple
# requirements files, so we need to keep track of what has been emitted
# so that we don't emit it again if it's seen again
emitted_options = set()
# keep track of which files a requirement is in so that we can
# give an accurate warning if a requirement appears multiple times.
req_files = collections.defaultdict(list)
for req_file_path in requirement:
with open(req_file_path) as req_file:
for line in req_file:
if (not line.strip() or
line.strip().startswith('#') or
(skip_match and skip_match(line)) or
line.startswith((
'-r', '--requirement',
'-Z', '--always-unzip',
'-f', '--find-links',
'-i', '--index-url',
'--pre',
'--trusted-host',
'--process-dependency-links',
'--extra-index-url'))):
line = line.rstrip()
if line not in emitted_options:
emitted_options.add(line)
yield line
continue
if line.startswith('-e') or line.startswith('--editable'):
if line.startswith('-e'):
line = line[2:].strip()
else:
line = line[len('--editable'):].strip().lstrip('=')
line_req = InstallRequirement.from_editable(
line,
isolated=isolated,
wheel_cache=wheel_cache,
)
else:
line_req = InstallRequirement.from_line(
COMMENT_RE.sub('', line).strip(),
isolated=isolated,
wheel_cache=wheel_cache,
)
if not line_req.name:
logger.info(
"Skipping line in requirement file [%s] because "
"it's not clear what it would install: %s",
req_file_path, line.strip(),
)
logger.info(
" (add #egg=PackageName to the URL to avoid"
" this warning)"
)
elif line_req.name not in installations:
# either it's not installed, or it is installed
# but has been processed already
if not req_files[line_req.name]:
logger.warning(
"Requirement file [%s] contains %s, but that "
"package is not installed",
req_file_path,
COMMENT_RE.sub('', line).strip(),
)
else:
req_files[line_req.name].append(req_file_path)
else:
yield str(installations[line_req.name]).rstrip()
del installations[line_req.name]
req_files[line_req.name].append(req_file_path)
# Warn about requirements that were included multiple times (in a
# single requirements file or in different requirements files).
for name, files in six.iteritems(req_files):
if len(files) > 1:
logger.warning("Requirement %s included multiple times [%s]",
name, ', '.join(sorted(set(files))))
yield(
'## The following requirements were added by '
'pip freeze:'
)
for installation in sorted(
installations.values(), key=lambda x: x.name.lower()):
if canonicalize_name(installation.name) not in skip:
yield str(installation).rstrip()
def format_name(project, extras):
# type: (str, FrozenSet[str]) -> str
if not extras:
return project
canonical_extras = sorted(canonicalize_name(e) for e in extras)
return "{}[{}]".format(project, ",".join(canonical_extras))
def _link_package_versions(self, link, search):
# type: (Link, Search) -> Optional[InstallationCandidate]
"""Return an InstallationCandidate or None"""
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
self._log_skipped_link(link, 'not a file')
return None
if ext not in SUPPORTED_EXTENSIONS:
self._log_skipped_link(
link,
'unsupported archive format: %s' % ext,
)
return None
if "binary" not in search.formats and ext == WHEEL_EXTENSION:
self._log_skipped_link(
link,
'No binaries permitted for %s' % search.supplied,
)
return None
if "macosx10" in link.path and ext == '.zip':
self._log_skipped_link(link, 'macosx10 one')
return None
if ext == WHEEL_EXTENSION:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
self._log_skipped_link(link, 'invalid wheel filename')
return None
if canonicalize_name(wheel.name) != search.canonical:
self._log_skipped_link(
link, 'wrong pokemon name (not %s)' % search.supplied)
return None
if not wheel.supported(self.valid_tags):
self._log_skipped_link(
link, 'it is not compatible with this Python')
return None
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != WHEEL_EXTENSION:
self._log_skipped_link(
link,
'No sources permitted for %s' % search.supplied,
)
return None
if not version:
version = _egg_info_matches(egg_info, search.canonical)
if not version:
self._log_skipped_link(
link, 'Missing pokemon version for %s' % search.supplied)
return None
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
self._log_skipped_link(link, 'Python version is incorrect')
return None
try:
support_this_python = check_requires_python(link.requires_python)
except specifiers.InvalidSpecifier:
logger.debug("Package %s has an invalid Requires-Python entry: %s",
link.filename, link.requires_python)
support_this_python = True
if not support_this_python:
logger.debug(
"The package %s is incompatible with the python "
"version in use. Acceptable python versions are: %s", link,
link.requires_python)
return None
logger.debug('Found link %s, version: %s', link, version)
return InstallationCandidate(search.supplied, version, link)
def resolve(self, root_reqs, check_supported_wheels):
# type: (List[InstallRequirement], bool) -> RequirementSet
constraints = {} # type: Dict[str, SpecifierSet]
user_requested = set() # type: Set[str]
requirements = []
for req in root_reqs:
if req.constraint:
# Ensure we only accept valid constraints
problem = check_invalid_constraint_type(req)
if problem:
raise InstallationError(problem)
name = canonicalize_name(req.name)
if name in constraints:
constraints[name] = constraints[name] & req.specifier
else:
constraints[name] = req.specifier
else:
if req.user_supplied and req.name:
user_requested.add(canonicalize_name(req.name))
r = self.factory.make_requirement_from_install_req(
req, requested_extras=(),
)
if r is not None:
requirements.append(r)
provider = PipProvider(
factory=self.factory,
constraints=constraints,
ignore_dependencies=self.ignore_dependencies,
upgrade_strategy=self.upgrade_strategy,
user_requested=user_requested,
)
reporter = BaseReporter()
resolver = RLResolver(provider, reporter)
try:
try_to_avoid_resolution_too_deep = 2000000
self._result = resolver.resolve(
requirements, max_rounds=try_to_avoid_resolution_too_deep,
)
except ResolutionImpossible as e:
error = self.factory.get_installation_error(e)
six.raise_from(error, e)
req_set = RequirementSet(check_supported_wheels=check_supported_wheels)
for candidate in self._result.mapping.values():
ireq = candidate.get_install_requirement()
if ireq is None:
continue
link = candidate.source_link
if link and link.is_yanked:
# The reason can contain non-ASCII characters, Unicode
# is required for Python 2.
msg = (
u'The candidate selected for download or install is a '
u'yanked version: {name!r} candidate (version {version} '
u'at {link})\nReason for being yanked: {reason}'
).format(
name=candidate.name,
version=candidate.version,
link=link,
reason=link.yanked_reason or u'<none given>',
)
logger.warning(msg)
ireq.should_reinstall = self.factory.should_reinstall(candidate)
req_set.add_named_requirement(ireq)
return req_set
def clobber(source, dest, is_base, fixer=None, filter=None):
ensure_dir(dest) # common for the 'include' path
for dir, subdirs, files in os.walk(source):
basedir = dir[len(source):].lstrip(os.path.sep)
destdir = os.path.join(dest, basedir)
if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'):
continue
for s in subdirs:
destsubdir = os.path.join(dest, basedir, s)
if is_base and basedir == '' and destsubdir.endswith('.data'):
data_dirs.append(s)
continue
elif (is_base and
s.endswith('.dist-info') and
canonicalize_name(s).startswith(
canonicalize_name(req.name))):
assert not info_dir, ('Multiple .dist-info directories: ' +
destsubdir + ', ' +
', '.join(info_dir))
info_dir.append(destsubdir)
for f in files:
# Skip unwanted files
if filter and filter(f):
continue
srcfile = os.path.join(dir, f)
destfile = os.path.join(dest, basedir, f)
# directory creation is lazy and after the file filtering above
# to ensure we don't install empty dirs; empty dirs can't be
# uninstalled.
ensure_dir(destdir)
# copyfile (called below) truncates the destination if it
# exists and then writes the new contents. This is fine in most
# cases, but can cause a segfault if pip has loaded a shared
# object (e.g. from pyopenssl through its vendored urllib3)
# Since the shared object is mmap'd an attempt to call a
# symbol in it will then cause a segfault. Unlinking the file
# allows writing of new contents while allowing the process to
# continue to use the old copy.
if os.path.exists(destfile):
os.unlink(destfile)
# We use copyfile (not move, copy, or copy2) to be extra sure
# that we are not moving directories over (copyfile fails for
# directories) as well as to ensure that we are not copying
# over any metadata because we want more control over what
# metadata we actually copy over.
shutil.copyfile(srcfile, destfile)
# Copy over the metadata for the file, currently this only
# includes the atime and mtime.
st = os.stat(srcfile)
if hasattr(os, "utime"):
os.utime(destfile, (st.st_atime, st.st_mtime))
# If our file is executable, then make our destination file
# executable.
if os.access(srcfile, os.X_OK):
st = os.stat(srcfile)
permissions = (
st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
)
os.chmod(destfile, permissions)
changed = False
if fixer:
changed = fixer(destfile)
record_installed(srcfile, destfile, changed)
def assert_installed(script: PipTestEnvironment, names: str) -> None:
list_output = json.loads(script.pip("list", "--format=json").stdout)
installed = {canonicalize_name(item["name"]) for item in list_output}
assert installed.issuperset(map(canonicalize_name, names))
def find_all_candidates(self, project_name):
"""Find all available InstallationCandidate for project_name
This checks index_urls, find_links and dependency_links.
All versions found are returned as an InstallationCandidate list.
See _link_package_versions for details on which files are accepted
"""
index_locations = self._get_index_urls_locations(project_name)
index_file_loc, index_url_loc = self._sort_locations(index_locations)
fl_file_loc, fl_url_loc = self._sort_locations(
self.find_links, expand_dir=True,
)
dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links)
file_locations = (Link(url) for url in itertools.chain(
index_file_loc, fl_file_loc, dep_file_loc,
))
# We trust every url that the user has given us whether it was given
# via --index-url or --find-links
# We explicitly do not trust links that came from dependency_links
# We want to filter out any thing which does not have a secure origin.
url_locations = [
link for link in itertools.chain(
(Link(url) for url in index_url_loc),
(Link(url) for url in fl_url_loc),
(Link(url) for url in dep_url_loc),
)
if self._validate_secure_origin(logger, link)
]
logger.debug('%d location(s) to search for versions of %s:',
len(url_locations), project_name)
for location in url_locations:
logger.debug('* %s', location)
canonical_name = canonicalize_name(project_name)
formats = fmt_ctl_formats(self.format_control, canonical_name)
search = Search(project_name, canonical_name, formats)
find_links_versions = self._package_versions(
# We trust every directly linked archive in find_links
(Link(url, '-f') for url in self.find_links),
search
)
page_versions = []
for page in self._get_pages(url_locations, project_name):
logger.debug('Analyzing links from page %s', page.url)
with indent_log():
page_versions.extend(
self._package_versions(page.links, search)
)
dependency_versions = self._package_versions(
(Link(url) for url in self.dependency_links), search
)
if dependency_versions:
logger.debug(
'dependency_links found: %s',
', '.join([
version.location.url for version in dependency_versions
])
)
file_versions = self._package_versions(file_locations, search)
if file_versions:
file_versions.sort(reverse=True)
logger.debug(
'Local files found: %s',
', '.join([
url_to_path(candidate.location.url)
for candidate in file_versions
])
)
# This is an intentional priority ordering
return (
file_versions + find_links_versions + page_versions +
dependency_versions
)
def check_package_set(package_set, should_ignore=None):
# type: (PackageSet, Optional[Callable[[str], bool]]) -> CheckResult
"""Check if a package set is consistent
If should_ignore is passed, it should be a callable that takes a
package name and returns a boolean.
"""
missing = {}
conflicting = {}
for package_name in package_set:
# Info about dependencies of package_name
missing_deps = set() # type: Set[Missing]
conflicting_deps = set() # type: Set[Conflicting]
if should_ignore and should_ignore(package_name):
continue
for req in package_set[package_name].requires:
name = canonicalize_name(req.project_name) # type: str
# Check if it's missing
if name not in package_set:
missed = True
if req.marker is not None:
missed = req.marker.evaluate()
if missed:
missing_deps.add((name, req))
continue
# Check if there's a conflict
version = package_set[name].version # type: str
if not req.specifier.contains(version, prereleases=True):
conflicting_deps.add((name, version, req))
if missing_deps:
missing[package_name] = sorted(missing_deps, key=str)
if conflicting_deps:
conflicting[package_name] = sorted(conflicting_deps, key=str)
return missing, conflicting
def build(self, autobuilding=False):
"""Build wheels.
:param unpack: If True, replace the sdist we built from with the
newly built wheel, in preparation for installation.
:return: True if all the wheels built correctly.
"""
assert self._wheel_dir or (autobuilding and self._cache_root)
# unpack sdists and constructs req set
self.requirement_set.prepare_files(self.finder)
reqset = self.requirement_set.requirements.values()
buildset = []
for req in reqset:
if req.constraint:
continue
if req.is_wheel:
if not autobuilding:
logger.info(
'Skipping %s, due to already being wheel.', req.name)
elif autobuilding and req.editable:
pass
elif autobuilding and req.link and not req.link.is_artifact:
pass
elif autobuilding and not req.source_dir:
pass
else:
if autobuilding:
link = req.link
base, ext = link.splitext()
if pip.index.egg_info_matches(base, None, link) is None:
# Doesn't look like a package - don't autobuild a wheel
# because we'll have no way to lookup the result sanely
continue
if "binary" not in pip.index.fmt_ctl_formats(
self.finder.format_control,
canonicalize_name(req.name)):
logger.info(
"Skipping bdist_wheel for %s, due to binaries "
"being disabled for it.", req.name)
continue
buildset.append(req)
if not buildset:
return True
# Build the wheels.
logger.info(
'Building wheels for collected packages: %s',
', '.join([req.name for req in buildset]),
)
with indent_log():
build_success, build_failure = [], []
for req in buildset:
python_tag = None
if autobuilding:
python_tag = pep425tags.implementation_tag
output_dir = _cache_for_link(self._cache_root, req.link)
try:
ensure_dir(output_dir)
except OSError as e:
logger.warning("Building wheel for %s failed: %s",
req.name, e)
build_failure.append(req)
continue
else:
output_dir = self._wheel_dir
wheel_file = self._build_one(
req, output_dir,
python_tag=python_tag,
)
if wheel_file:
build_success.append(req)
if autobuilding:
# XXX: This is mildly duplicative with prepare_files,
# but not close enough to pull out to a single common
# method.
# The code below assumes temporary source dirs -
# prevent it doing bad things.
if req.source_dir and not os.path.exists(os.path.join(
req.source_dir, PIP_DELETE_MARKER_FILENAME)):
raise AssertionError(
"bad source dir - missing marker")
# Delete the source we built the wheel from
req.remove_temporary_source()
# set the build directory again - name is known from
# the work prepare_files did.
req.source_dir = req.build_location(
self.requirement_set.build_dir)
# Update the link for this.
req.link = pip.index.Link(
path_to_url(wheel_file))
assert req.link.is_wheel
# extract the wheel into the dir
unpack_url(
req.link, req.source_dir, None, False,
session=self.requirement_set.session)
else:
build_failure.append(req)
# notify success/failure
if build_success:
logger.info(
'Successfully built %s',
' '.join([req.name for req in build_success]),
)
if build_failure:
logger.info(
'Failed to build %s',
' '.join([req.name for req in build_failure]),
)
# Return True if all builds were successful
return len(build_failure) == 0
def _iter_found_candidates(
self,
ireqs: Sequence[InstallRequirement],
specifier: SpecifierSet,
hashes: Hashes,
prefers_installed: bool,
incompatible_ids: Set[int],
) -> Iterable[Candidate]:
if not ireqs:
return ()
# The InstallRequirement implementation requires us to give it a
# "template". Here we just choose the first requirement to represent
# all of them.
# Hopefully the Project model can correct this mismatch in the future.
template = ireqs[0]
assert template.req, "Candidates found on index must be PEP 508"
name = canonicalize_name(template.req.name)
extras: FrozenSet[str] = frozenset()
for ireq in ireqs:
assert ireq.req, "Candidates found on index must be PEP 508"
specifier &= ireq.req.specifier
hashes &= ireq.hashes(trust_internet=False)
extras |= frozenset(ireq.extras)
def _get_installed_candidate() -> Optional[Candidate]:
"""Get the candidate for the currently-installed version."""
# If --force-reinstall is set, we want the version from the index
# instead, so we "pretend" there is nothing installed.
if self._force_reinstall:
return None
try:
installed_dist = self._installed_dists[name]
except KeyError:
return None
# Don't use the installed distribution if its version does not fit
# the current dependency graph.
if not specifier.contains(installed_dist.version, prereleases=True):
return None
candidate = self._make_candidate_from_dist(
dist=installed_dist,
extras=extras,
template=template,
)
# The candidate is a known incompatiblity. Don't use it.
if id(candidate) in incompatible_ids:
return None
return candidate
def iter_index_candidate_infos() -> Iterator[IndexCandidateInfo]:
result = self._finder.find_best_candidate(
project_name=name,
specifier=specifier,
hashes=hashes,
)
icans = list(result.iter_applicable())
# PEP 592: Yanked releases must be ignored unless only yanked
# releases can satisfy the version range. So if this is false,
# all yanked icans need to be skipped.
all_yanked = all(ican.link.is_yanked for ican in icans)
# PackageFinder returns earlier versions first, so we reverse.
for ican in reversed(icans):
if not all_yanked and ican.link.is_yanked:
continue
func = functools.partial(
self._make_candidate_from_link,
link=ican.link,
extras=extras,
template=template,
name=name,
version=ican.version,
)
yield ican.version, func
return FoundCandidates(
iter_index_candidate_infos,
_get_installed_candidate(),
prefers_installed,
incompatible_ids,
)
def search_packages_info(query):
"""
Gather details from installed distributions. Print distribution name,
version, location, and installed files. Installed files requires a
pip generated 'installed-files.txt' in the distributions '.egg-info'
directory.
"""
installed = {}
for p in pkg_resources.working_set:
installed[canonicalize_name(p.project_name)] = p
query_names = [canonicalize_name(name) for name in query]
for dist in [installed[pkg] for pkg in query_names if pkg in installed]:
package = {
'name': dist.project_name,
'version': dist.version,
'location': dist.location,
'requires': [dep.project_name for dep in dist.requires()],
}
file_list = None
metadata = None
if isinstance(dist, pkg_resources.DistInfoDistribution):
# RECORDs should be part of .dist-info metadatas
if dist.has_metadata('RECORD'):
lines = dist.get_metadata_lines('RECORD')
paths = [l.split(',')[0] for l in lines]
paths = [os.path.join(dist.location, p) for p in paths]
file_list = [os.path.relpath(p, dist.location) for p in paths]
if dist.has_metadata('METADATA'):
metadata = dist.get_metadata('METADATA')
else:
# Otherwise use pip's log for .egg-info's
if dist.has_metadata('installed-files.txt'):
paths = dist.get_metadata_lines('installed-files.txt')
paths = [os.path.join(dist.egg_info, p) for p in paths]
file_list = [os.path.relpath(p, dist.location) for p in paths]
if dist.has_metadata('PKG-INFO'):
metadata = dist.get_metadata('PKG-INFO')
if dist.has_metadata('entry_points.txt'):
entry_points = dist.get_metadata_lines('entry_points.txt')
package['entry_points'] = entry_points
if dist.has_metadata('INSTALLER'):
for line in dist.get_metadata_lines('INSTALLER'):
if line.strip():
package['installer'] = line.strip()
break
# @todo: Should pkg_resources.Distribution have a
# `get_pkg_info` method?
feed_parser = FeedParser()
feed_parser.feed(metadata)
pkg_info_dict = feed_parser.close()
for key in ('metadata-version', 'summary',
'home-page', 'author', 'author-email', 'license'):
package[key] = pkg_info_dict.get(key)
# It looks like FeedParser cannot deal with repeated headers
classifiers = []
for line in metadata.splitlines():
if line.startswith('Classifier: '):
classifiers.append(line[len('Classifier: '):])
package['classifiers'] = classifiers
if file_list:
package['files'] = sorted(file_list)
yield package
def _package_name_option_check(option, opt, value):
# type: (Option, str, str) -> str
return canonicalize_name(value)
def _iter_found_candidates(
self,
ireqs, # type: Sequence[InstallRequirement]
specifier, # type: SpecifierSet
hashes, # type: Hashes
prefers_installed, # type: bool
):
# type: (...) -> Iterable[Candidate]
if not ireqs:
return ()
# The InstallRequirement implementation requires us to give it a
# "template". Here we just choose the first requirement to represent
# all of them.
# Hopefully the Project model can correct this mismatch in the future.
template = ireqs[0]
name = canonicalize_name(template.req.name)
extras = frozenset() # type: FrozenSet[str]
for ireq in ireqs:
specifier &= ireq.req.specifier
hashes &= ireq.hashes(trust_internet=False)
extras |= frozenset(ireq.extras)
# Get the installed version, if it matches, unless the user
# specified `--force-reinstall`, when we want the version from
# the index instead.
installed_candidate = None
if not self._force_reinstall and name in self._installed_dists:
installed_dist = self._installed_dists[name]
if specifier.contains(installed_dist.version, prereleases=True):
installed_candidate = self._make_candidate_from_dist(
dist=installed_dist,
extras=extras,
template=template,
)
def iter_index_candidate_infos():
# type: () -> Iterator[IndexCandidateInfo]
result = self._finder.find_best_candidate(
project_name=name,
specifier=specifier,
hashes=hashes,
)
icans = list(result.iter_applicable())
# PEP 592: Yanked releases must be ignored unless only yanked
# releases can satisfy the version range. So if this is false,
# all yanked icans need to be skipped.
all_yanked = all(ican.link.is_yanked for ican in icans)
# PackageFinder returns earlier versions first, so we reverse.
for ican in reversed(icans):
if not all_yanked and ican.link.is_yanked:
continue
func = functools.partial(
self._make_candidate_from_link,
link=ican.link,
extras=extras,
template=template,
name=name,
version=ican.version,
)
yield ican.version, func
return FoundCandidates(
iter_index_candidate_infos,
installed_candidate,
prefers_installed,
)
def clobber(source, dest, is_base, fixer=None, filter=None):
ensure_dir(dest) # common for the 'include' path
for dir, subdirs, files in os.walk(source):
basedir = dir[len(source):].lstrip(os.path.sep)
destdir = os.path.join(dest, basedir)
if is_base and basedir.split(os.path.sep, 1)[0].endswith('.data'):
continue
for s in subdirs:
destsubdir = os.path.join(dest, basedir, s)
if is_base and basedir == '' and destsubdir.endswith('.data'):
data_dirs.append(s)
continue
elif (is_base and s.endswith('.dist-info')
and canonicalize_name(s).startswith(
canonicalize_name(req.name))):
assert not info_dir, ('Multiple .dist-info directories: ' +
destsubdir + ', ' +
', '.join(info_dir))
info_dir.append(destsubdir)
for f in files:
# Skip unwanted files
if filter and filter(f):
continue
srcfile = os.path.join(dir, f)
destfile = os.path.join(dest, basedir, f)
# directory creation is lazy and after the file filtering above
# to ensure we don't install empty dirs; empty dirs can't be
# uninstalled.
ensure_dir(destdir)
# copyfile (called below) truncates the destination if it
# exists and then writes the new contents. This is fine in most
# cases, but can cause a segfault if pip has loaded a shared
# object (e.g. from pyopenssl through its vendored urllib3)
# Since the shared object is mmap'd an attempt to call a
# symbol in it will then cause a segfault. Unlinking the file
# allows writing of new contents while allowing the process to
# continue to use the old copy.
if os.path.exists(destfile):
os.unlink(destfile)
# We use copyfile (not move, copy, or copy2) to be extra sure
# that we are not moving directories over (copyfile fails for
# directories) as well as to ensure that we are not copying
# over any metadata because we want more control over what
# metadata we actually copy over.
shutil.copyfile(srcfile, destfile)
# Copy over the metadata for the file, currently this only
# includes the atime and mtime.
st = os.stat(srcfile)
if hasattr(os, "utime"):
os.utime(destfile, (st.st_atime, st.st_mtime))
# If our file is executable, then make our destination file
# executable.
if os.access(srcfile, os.X_OK):
st = os.stat(srcfile)
permissions = (st.st_mode | stat.S_IXUSR | stat.S_IXGRP
| stat.S_IXOTH)
os.chmod(destfile, permissions)
changed = False
if fixer:
changed = fixer(destfile)
record_installed(srcfile, destfile, changed)
def _link_package_versions(self, link, search):
"""Return an InstallationCandidate or None"""
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
self._log_skipped_link(link, 'not a file')
return
if ext not in SUPPORTED_EXTENSIONS:
self._log_skipped_link(
link, 'unsupported archive format: %s' % ext,
)
return
if "binary" not in search.formats and ext == wheel_ext:
self._log_skipped_link(
link, 'No binaries permitted for %s' % search.supplied,
)
return
if "macosx10" in link.path and ext == '.zip':
self._log_skipped_link(link, 'macosx10 one')
return
if ext == wheel_ext:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
self._log_skipped_link(link, 'invalid wheel filename')
return
if canonicalize_name(wheel.name) != search.canonical:
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
if not wheel.supported(self.valid_tags):
self._log_skipped_link(
link, 'it is not compatible with this Python')
return
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != wheel_ext:
self._log_skipped_link(
link, 'No sources permitted for %s' % search.supplied,
)
return
if not version:
version = egg_info_matches(egg_info, search.supplied, link)
if version is None:
self._log_skipped_link(
link, 'wrong project name (not %s)' % search.supplied)
return
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
self._log_skipped_link(
link, 'Python version is incorrect')
return
try:
support_this_python = check_requires_python(link.requires_python)
except specifiers.InvalidSpecifier:
logger.debug("Package %s has an invalid Requires-Python entry: %s",
link.filename, link.requires_python)
support_this_python = True
if not support_this_python:
logger.debug("The package %s is incompatible with the python"
"version in use. Acceptable python versions are:%s",
link, link.requires_python)
return
logger.debug('Found link %s, version: %s', link, version)
return InstallationCandidate(search.supplied, version, link)
def build(
self,
requirements, # type: Iterable[InstallRequirement]
session, # type: PipSession
autobuilding=False # type: bool
):
# type: (...) -> List[InstallRequirement]
"""Build wheels.
:param unpack: If True, replace the sdist we built from with the
newly built wheel, in preparation for installation.
:return: True if all the wheels built correctly.
"""
buildset = []
format_control = self.finder.format_control
for req in requirements:
if req.constraint:
continue
if req.is_wheel:
if not autobuilding:
logger.info(
'Skipping %s, due to already being wheel.', req.name,
)
elif autobuilding and req.editable:
pass
elif autobuilding and not req.source_dir:
pass
elif autobuilding and req.link and not req.link.is_artifact:
# VCS checkout. Build wheel just for this run.
buildset.append((req, True))
else:
ephem_cache = False
if autobuilding:
link = req.link
base, ext = link.splitext()
if not _contains_egg_info(base):
# E.g. local directory. Build wheel just for this run.
ephem_cache = True
if "binary" not in format_control.get_allowed_formats(
canonicalize_name(req.name)):
logger.info(
"Skipping bdist_wheel for %s, due to binaries "
"being disabled for it.", req.name,
)
continue
buildset.append((req, ephem_cache))
if not buildset:
return []
# Is any wheel build not using the ephemeral cache?
if any(not ephem_cache for _, ephem_cache in buildset):
have_directory_for_build = self._wheel_dir or (
autobuilding and self.wheel_cache.cache_dir
)
assert have_directory_for_build
# TODO by @pradyunsg
# Should break up this method into 2 separate methods.
# Build the wheels.
logger.info(
'Building wheels for collected packages: %s',
', '.join([req.name for (req, _) in buildset]),
)
_cache = self.wheel_cache # shorter name
with indent_log():
build_success, build_failure = [], []
for req, ephem in buildset:
python_tag = None
if autobuilding:
python_tag = pep425tags.implementation_tag
if ephem:
output_dir = _cache.get_ephem_path_for_link(req.link)
else:
output_dir = _cache.get_path_for_link(req.link)
try:
ensure_dir(output_dir)
except OSError as e:
logger.warning("Building wheel for %s failed: %s",
req.name, e)
build_failure.append(req)
continue
else:
output_dir = self._wheel_dir
wheel_file = self._build_one(
req, output_dir,
python_tag=python_tag,
)
if wheel_file:
build_success.append(req)
if autobuilding:
# XXX: This is mildly duplicative with prepare_files,
# but not close enough to pull out to a single common
# method.
# The code below assumes temporary source dirs -
# prevent it doing bad things.
if req.source_dir and not os.path.exists(os.path.join(
req.source_dir, PIP_DELETE_MARKER_FILENAME)):
raise AssertionError(
"bad source dir - missing marker")
# Delete the source we built the wheel from
req.remove_temporary_source()
# set the build directory again - name is known from
# the work prepare_files did.
req.source_dir = req.build_location(
self.preparer.build_dir
)
# Update the link for this.
req.link = Link(path_to_url(wheel_file))
assert req.link.is_wheel
# extract the wheel into the dir
unpack_url(
req.link, req.source_dir, None, False,
session=session,
)
else:
build_failure.append(req)
# notify success/failure
if build_success:
logger.info(
'Successfully built %s',
' '.join([req.name for req in build_success]),
)
if build_failure:
logger.info(
'Failed to build %s',
' '.join([req.name for req in build_failure]),
)
# Return a list of requirements that failed to build
return build_failure
def resolve(self, root_reqs, check_supported_wheels):
# type: (List[InstallRequirement], bool) -> RequirementSet
constraints = {} # type: Dict[str, Constraint]
user_requested = {} # type: Dict[str, int]
requirements = []
for i, req in enumerate(root_reqs):
if req.constraint:
# Ensure we only accept valid constraints
problem = check_invalid_constraint_type(req)
if problem:
raise InstallationError(problem)
if not req.match_markers():
continue
name = canonicalize_name(req.name)
if name in constraints:
constraints[name] &= req
else:
constraints[name] = Constraint.from_ireq(req)
else:
if req.user_supplied and req.name:
canonical_name = canonicalize_name(req.name)
if canonical_name not in user_requested:
user_requested[canonical_name] = i
r = self.factory.make_requirement_from_install_req(
req, requested_extras=())
if r is not None:
requirements.append(r)
provider = PipProvider(
factory=self.factory,
constraints=constraints,
ignore_dependencies=self.ignore_dependencies,
upgrade_strategy=self.upgrade_strategy,
user_requested=user_requested,
)
if "PIP_RESOLVER_DEBUG" in os.environ:
reporter = PipDebuggingReporter()
else:
reporter = PipReporter()
resolver = RLResolver(provider, reporter)
try:
try_to_avoid_resolution_too_deep = 2000000
self._result = resolver.resolve(
requirements, max_rounds=try_to_avoid_resolution_too_deep)
except ResolutionImpossible as e:
error = self.factory.get_installation_error(e, constraints)
raise error from e
req_set = RequirementSet(check_supported_wheels=check_supported_wheels)
for candidate in self._result.mapping.values():
ireq = candidate.get_install_requirement()
if ireq is None:
continue
# Check if there is already an installation under the same name,
# and set a flag for later stages to uninstall it, if needed.
installed_dist = self.factory.get_dist_to_uninstall(candidate)
if installed_dist is None:
# There is no existing installation -- nothing to uninstall.
ireq.should_reinstall = False
elif self.factory.force_reinstall:
# The --force-reinstall flag is set -- reinstall.
ireq.should_reinstall = True
elif parse_version(installed_dist.version) != candidate.version:
# The installation is different in version -- reinstall.
ireq.should_reinstall = True
elif candidate.is_editable or dist_is_editable(installed_dist):
# The incoming distribution is editable, or different in
# editable-ness to installation -- reinstall.
ireq.should_reinstall = True
elif candidate.source_link.is_file:
# The incoming distribution is under file://
if candidate.source_link.is_wheel:
# is a local wheel -- do nothing.
logger.info(
"%s is already installed with the same version as the "
"provided wheel. Use --force-reinstall to force an "
"installation of the wheel.",
ireq.name,
)
continue
looks_like_sdist = (is_archive_file(
candidate.source_link.file_path)
and candidate.source_link.ext != ".zip")
if looks_like_sdist:
# is a local sdist -- show a deprecation warning!
reason = (
"Source distribution is being reinstalled despite an "
"installed package having the same name and version as "
"the installed package.")
replacement = "use --force-reinstall"
deprecated(
reason=reason,
replacement=replacement,
gone_in="21.1",
issue=8711,
)
# is a local sdist or path -- reinstall
ireq.should_reinstall = True
else:
continue
link = candidate.source_link
if link and link.is_yanked:
# The reason can contain non-ASCII characters, Unicode
# is required for Python 2.
msg = ("The candidate selected for download or install is a "
"yanked version: {name!r} candidate (version {version} "
"at {link})\nReason for being yanked: {reason}").format(
name=candidate.name,
version=candidate.version,
link=link,
reason=link.yanked_reason or "<none given>",
)
logger.warning(msg)
req_set.add_named_requirement(ireq)
reqs = req_set.all_requirements
self.factory.preparer.prepare_linked_requirements_more(reqs)
return req_set
def _evaluate_link(self, link, search):
# type: (Link, Search) -> Tuple[bool, Optional[str]]
"""
Determine whether a link is a candidate for installation.
:return: A tuple (is_candidate, result), where `result` is (1) a
version string if `is_candidate` is True, and (2) if
`is_candidate` is False, an optional string to log the reason
the link fails to qualify.
"""
version = None
if link.egg_fragment:
egg_info = link.egg_fragment
ext = link.ext
else:
egg_info, ext = link.splitext()
if not ext:
return (False, 'not a file')
if ext not in SUPPORTED_EXTENSIONS:
return (False, 'unsupported archive format: %s' % ext)
if "binary" not in search.formats and ext == WHEEL_EXTENSION:
reason = 'No binaries permitted for %s' % search.supplied
return (False, reason)
if "macosx10" in link.path and ext == '.zip':
return (False, 'macosx10 one')
if ext == WHEEL_EXTENSION:
try:
wheel = Wheel(link.filename)
except InvalidWheelFilename:
return (False, 'invalid wheel filename')
if canonicalize_name(wheel.name) != search.canonical:
reason = 'wrong project name (not %s)' % search.supplied
return (False, reason)
if not self._is_wheel_supported(wheel):
return (False, 'it is not compatible with this Python')
version = wheel.version
# This should be up by the search.ok_binary check, but see issue 2700.
if "source" not in search.formats and ext != WHEEL_EXTENSION:
return (False, 'No sources permitted for %s' % search.supplied)
if not version:
version = _egg_info_matches(egg_info, search.canonical)
if not version:
return (False, 'Missing project version for %s' % search.supplied)
match = self._py_version_re.search(version)
if match:
version = version[:match.start()]
py_version = match.group(1)
if py_version != sys.version[:3]:
return (False, 'Python version is incorrect')
try:
support_this_python = check_requires_python(
link.requires_python, version_info=sys.version_info[:3],
)
except specifiers.InvalidSpecifier:
logger.debug("Package %s has an invalid Requires-Python entry: %s",
link.filename, link.requires_python)
support_this_python = True
if not support_this_python:
logger.debug("The package %s is incompatible with the python "
"version in use. Acceptable python versions are: %s",
link, link.requires_python)
# Return None for the reason text to suppress calling
# _log_skipped_link().
return (False, None)
logger.debug('Found link %s, version: %s', link, version)
return (True, version)
def build(self, requirements, session, autobuilding=False):
"""Build wheels.
:param unpack: If True, replace the sdist we built from with the
newly built wheel, in preparation for installation.
:return: True if all the wheels built correctly.
"""
from pip._internal import index
building_is_possible = self._wheel_dir or (autobuilding and
self.wheel_cache.cache_dir)
assert building_is_possible
buildset = []
for req in requirements:
if req.constraint:
continue
if req.is_wheel:
if not autobuilding:
logger.info(
'Skipping %s, due to already being wheel.',
req.name,
)
elif autobuilding and req.editable:
pass
elif autobuilding and not req.source_dir:
pass
elif autobuilding and req.link and not req.link.is_artifact:
# VCS checkout. Build wheel just for this run.
buildset.append((req, True))
else:
ephem_cache = False
if autobuilding:
link = req.link
base, ext = link.splitext()
if index.egg_info_matches(base, None, link) is None:
# E.g. local directory. Build wheel just for this run.
ephem_cache = True
if "binary" not in index.fmt_ctl_formats(
self.finder.format_control,
canonicalize_name(req.name)):
logger.info(
"Skipping bdist_wheel for %s, due to binaries "
"being disabled for it.",
req.name,
)
continue
buildset.append((req, ephem_cache))
if not buildset:
return True
# Build the wheels.
logger.info(
'Building wheels for collected packages: %s',
', '.join([req.name for (req, _) in buildset]),
)
_cache = self.wheel_cache # shorter name
with indent_log():
build_success, build_failure = [], []
for req, ephem in buildset:
python_tag = None
if autobuilding:
python_tag = pep425tags.implementation_tag
if ephem:
output_dir = _cache.get_ephem_path_for_link(req.link)
else:
output_dir = _cache.get_path_for_link(req.link)
try:
ensure_dir(output_dir)
except OSError as e:
logger.warning("Building wheel for %s failed: %s",
req.name, e)
build_failure.append(req)
continue
else:
output_dir = self._wheel_dir
wheel_file = self._build_one(
req,
output_dir,
python_tag=python_tag,
)
if wheel_file:
build_success.append(req)
if autobuilding:
# XXX: This is mildly duplicative with prepare_files,
# but not close enough to pull out to a single common
# method.
# The code below assumes temporary source dirs -
# prevent it doing bad things.
if req.source_dir and not os.path.exists(
os.path.join(req.source_dir,
PIP_DELETE_MARKER_FILENAME)):
raise AssertionError(
"bad source dir - missing marker")
# Delete the source we built the wheel from
req.remove_temporary_source()
# set the build directory again - name is known from
# the work prepare_files did.
req.source_dir = req.build_location(
self.preparer.build_dir)
# Update the link for this.
req.link = index.Link(path_to_url(wheel_file))
assert req.link.is_wheel
# extract the wheel into the dir
unpack_url(
req.link,
req.source_dir,
None,
False,
session=session,
)
else:
build_failure.append(req)
# notify success/failure
if build_success:
logger.info(
'Successfully built %s',
' '.join([req.name for req in build_success]),
)
if build_failure:
logger.info(
'Failed to build %s',
' '.join([req.name for req in build_failure]),
)
# Return True if all builds were successful
return len(build_failure) == 0
def _package_name_option_check(option: Option, opt: str, value: str) -> str:
return canonicalize_name(value)
def run(self, options, args):
with self._build_session(options) as session:
reqs_to_uninstall = {}
for name in args:
req = InstallRequirement.from_line(
name, isolated=options.isolated_mode,
)
if req.name:
reqs_to_uninstall[canonicalize_name(req.name)] = req
for filename in options.requirements:
for req in parse_requirements(
filename,
options=options,
session=session):
if req.name:
reqs_to_uninstall[canonicalize_name(req.name)] = req
if not reqs_to_uninstall:
raise InstallationError(
'You must give at least one requirement to %(name)s (see '
'"pip help %(name)s")' % dict(name=self.name)
)
protect_pip_from_modification_on_windows(
modifying_pip="pip" in reqs_to_uninstall
)
for req in reqs_to_uninstall.values():
uninstall_pathset = req.uninstall(
auto_confirm=options.yes, verbose=self.verbosity > 0,
)
if uninstall_pathset:
uninstall_pathset.commit()
