def show_actual_vendor_versions(vendor_txt_versions: Dict[str, str]) -> None:
"""Log the actual version and print extra info if there is
a conflict or if the actual version could not be imported.
"""
for module_name, expected_version in vendor_txt_versions.items():
extra_message = ''
actual_version = get_vendor_version_from_module(module_name)
if not actual_version:
extra_message = ' (Unable to locate actual module version, using'\
' vendor.txt specified version)'
actual_version = expected_version
elif parse_version(actual_version) != parse_version(expected_version):
extra_message = ' (CONFLICT: vendor.txt suggests version should'\
' be {})'.format(expected_version)
logger.info('%s==%s%s', module_name, actual_version, extra_message)
def find_best_match(self, ireq, prereleases=None):
"""
Returns a Version object that indicates the best match for the given
InstallRequirement according to the external repository.
"""
if ireq.editable:
return ireq # return itself as the best match
py_version = parse_version(
os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3])))
all_candidates = []
for c in self.find_all_candidates(ireq.name):
if c.requires_python:
# Old specifications had people setting this to single digits
# which is effectively the same as '>=digit,<digit+1'
if c.requires_python.isdigit():
c.requires_python = '>={0},<{1}'.format(
c.requires_python,
int(c.requires_python) + 1)
try:
specifier_set = SpecifierSet(c.requires_python)
except InvalidSpecifier:
pass
else:
if not specifier_set.contains(py_version):
continue
all_candidates.append(c)
candidates_by_version = lookup_table(all_candidates,
key=lambda c: c.version,
unique=True)
try:
matching_versions = ireq.specifier.filter(
(candidate.version for candidate in all_candidates),
prereleases=prereleases)
except TypeError:
matching_versions = [
candidate.version for candidate in all_candidates
]
# Reuses pip's internal candidate sort key to sort
matching_candidates = [
candidates_by_version[ver] for ver in matching_versions
]
if not matching_candidates:
raise NoCandidateFound(ireq, all_candidates, self.finder)
best_candidate = max(matching_candidates,
key=self.finder._candidate_sort_key)
# Turn the candidate into a pinned InstallRequirement
new_req = make_install_requirement(best_candidate.project,
best_candidate.version,
ireq.extras,
ireq.markers,
constraint=ireq.constraint)
# KR TODO: Marker here?
return new_req
def run_egg_info(self):
assert self.source_dir
if self.name:
logger.debug(
'Running setup.py (path:%s) egg_info for package %s',
self.setup_py, self.name,
)
else:
logger.debug(
'Running setup.py (path:%s) egg_info for package from %s',
self.setup_py, self.link,
)
with indent_log():
script = SETUPTOOLS_SHIM % self.setup_py
base_cmd = [os.environ.get('PIP_PYTHON_PATH', sys.executable), '-c', script]
if self.isolated:
base_cmd += ["--no-user-cfg"]
egg_info_cmd = base_cmd + ['egg_info']
# We can't put the .egg-info files at the root, because then the
# source code will be mistaken for an installed egg, causing
# problems
if self.editable:
egg_base_option = []
else:
egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info')
ensure_dir(egg_info_dir)
egg_base_option = ['--egg-base', 'pip-egg-info']
with self.build_env:
call_subprocess(
egg_info_cmd + egg_base_option,
cwd=self.setup_py_dir,
show_stdout=False,
command_desc='python setup.py egg_info')
if not self.req:
if isinstance(parse_version(self.pkg_info()["Version"]), Version):
op = "=="
else:
op = "==="
self.req = Requirement(
"".join([
self.pkg_info()["Name"],
op,
self.pkg_info()["Version"],
])
)
self._correct_build_location()
else:
metadata_name = canonicalize_name(self.pkg_info()["Name"])
if canonicalize_name(self.req.name) != metadata_name:
logger.warning(
'Running setup.py (path:%s) egg_info for package %s '
'produced metadata for project name %s. Fix your '
'#egg=%s fragments.',
self.setup_py, self.name, metadata_name, self.name
)
self.req = Requirement(metadata_name)
def __init__(self, name: str, version: str, link: Link) -> None:
self.name = name
self.version = parse_version(version)
self.link = link
super().__init__(
key=(self.name, self.version, self.link),
defining_class=InstallationCandidate,
)
def __init__(self, project, version, link, requires_python=None):
# type: (Any, str, Link, Any) -> None
self.project = project
self.version = parse_version(version) # type: _BaseVersion
self.link = link
self.requires_python = requires_python
super(InstallationCandidate,
self).__init__(key=(self.project, self.version, self.link),
defining_class=InstallationCandidate)
def get_git_version(self):
VERSION_PFX = 'git version '
version = self.run_command(['version'], show_stdout=False)
if version.startswith(VERSION_PFX):
version = version[len(VERSION_PFX):].split()[0]
else:
version = ''
# get first 3 positions of the git version becasue
# on windows it is x.y.z.windows.t, and this parses as
# LegacyVersion which always smaller than a Version.
version = '.'.join(version.split('.')[:3])
return parse_version(version)
def get_git_version(self):
VERSION_PFX = 'git version '
version = self.run_command(['version'], show_stdout=False)
if version.startswith(VERSION_PFX):
version = version[len(VERSION_PFX):].split()[0]
else:
version = ''
# get first 3 positions of the git version becasue
# on windows it is x.y.z.windows.t, and this parses as
# LegacyVersion which always smaller than a Version.
version = '.'.join(version.split('.')[:3])
return parse_version(version)
def update(self, dest, rev_options):
# First fetch changes from the default remote
if self.get_git_version() >= parse_version('1.9.0'):
# fetch tags in addition to everything else
self.run_command(['fetch', '-q', '--tags'], cwd=dest)
else:
self.run_command(['fetch', '-q'], cwd=dest)
# Then reset to wanted revision (maybe even origin/master)
rev_options = self.check_rev_options(dest, rev_options)
cmd_args = ['reset', '--hard', '-q'] + rev_options.to_args()
self.run_command(cmd_args, cwd=dest)
#: update submodules
self.update_submodules(dest)
def update(self, dest, rev_options):
# First fetch changes from the default remote
if self.get_git_version() >= parse_version('1.9.0'):
# fetch tags in addition to everything else
self.run_command(['fetch', '-q', '--tags'], cwd=dest)
else:
self.run_command(['fetch', '-q'], cwd=dest)
# Then reset to wanted revision (maybe even origin/master)
rev_options = self.check_rev_options(dest, rev_options)
cmd_args = ['reset', '--hard', '-q'] + rev_options.to_args()
self.run_command(cmd_args, cwd=dest)
#: update submodules
self.update_submodules(dest)
def update(self, dest, url, rev_options):
# type: (str, HiddenText, RevOptions) -> None
# First fetch changes from the default remote
if self.get_git_version() >= parse_version('1.9.0'):
# fetch tags in addition to everything else
self.run_command(['fetch', '-q', '--tags'], cwd=dest)
else:
self.run_command(['fetch', '-q'], cwd=dest)
# Then reset to wanted revision (maybe even origin/master)
rev_options = self.resolve_revision(dest, url, rev_options)
cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args())
self.run_command(cmd_args, cwd=dest)
#: update submodules
self.update_submodules(dest)
def print_dist_installation_info(name: str, latest: str) -> None:
env = get_default_environment()
dist = env.get_distribution(name)
if dist is not None:
with indent_log():
if dist.version == latest:
write_output('INSTALLED: %s (latest)', dist.version)
else:
write_output('INSTALLED: %s', dist.version)
if parse_version(latest).pre:
write_output(
'LATEST: %s (pre-release; install'
' with "pip install --pre")', latest)
else:
write_output('LATEST: %s', latest)
def _set_requirement(self) -> None:
"""Set requirement after generating metadata."""
assert self.req is None
assert self.metadata is not None
assert self.source_dir is not None
# Construct a Requirement object from the generated metadata
if isinstance(parse_version(self.metadata["Version"]), Version):
op = "=="
else:
op = "==="
self.req = Requirement("".join([
self.metadata["Name"],
op,
self.metadata["Version"],
]))
def _check_metadata_consistency(self, dist: Distribution) -> None:
"""Check for consistency of project name and version of dist."""
canonical_name = canonicalize_name(dist.project_name)
if self._name is not None and self._name != canonical_name:
raise MetadataInconsistent(
self._ireq,
"name",
self._name,
dist.project_name,
)
parsed_version = parse_version(dist.version)
if self._version is not None and self._version != parsed_version:
raise MetadataInconsistent(
self._ireq,
"version",
str(self._version),
dist.version,
)
def clean_requires_python(candidates):
"""Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes."""
all_candidates = []
py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3]))))
for c in candidates:
if c.requires_python:
# Old specifications had people setting this to single digits
# which is effectively the same as '>=digit,<digit+1'
if c.requires_python.isdigit():
c.requires_python = '>={0},<{1}'.format(c.requires_python, int(c.requires_python) + 1)
try:
specifierset = SpecifierSet(c.requires_python)
except InvalidSpecifier:
continue
else:
if not specifierset.contains(py_version):
continue
all_candidates.append(c)
return all_candidates
def clean_requires_python(candidates):
"""Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes."""
all_candidates = []
py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3]))))
for c in candidates:
if c.requires_python:
# Old specifications had people setting this to single digits
# which is effectively the same as '>=digit,<digit+1'
if c.requires_python.isdigit():
c.requires_python = '>={0},<{1}'.format(c.requires_python, int(c.requires_python) + 1)
try:
specifierset = SpecifierSet(c.requires_python)
except InvalidSpecifier:
continue
else:
if not specifierset.contains(py_version):
continue
all_candidates.append(c)
return all_candidates
def print_results(hits, name_column_width=None, terminal_width=None):
if not hits:
return
if name_column_width is None:
name_column_width = max([
len(hit['name']) + len(highest_version(hit.get('versions', ['-'])))
for hit in hits
]) + 4
installed_packages = [p.project_name for p in pkg_resources.working_set]
for hit in hits:
name = hit['name']
summary = hit['summary'] or ''
latest = highest_version(hit.get('versions', ['-']))
if terminal_width is not None:
target_width = terminal_width - name_column_width - 5
if target_width > 10:
# wrap and indent summary to fit terminal
summary = textwrap.wrap(summary, target_width)
summary = ('\n' + ' ' * (name_column_width + 3)).join(summary)
line = '%-*s - %s' % (name_column_width,
'%s (%s)' % (name, latest), summary)
try:
write_output(line)
if name in installed_packages:
dist = pkg_resources.get_distribution(name)
with indent_log():
if dist.version == latest:
write_output('INSTALLED: %s (latest)', dist.version)
else:
write_output('INSTALLED: %s', dist.version)
if parse_version(latest).pre:
write_output('LATEST: %s (pre-release; install'
' with "pip install --pre")', latest)
else:
write_output('LATEST: %s', latest)
except UnicodeEncodeError:
pass
def prepare_metadata(self):
# type: () -> None
"""Ensure that project metadata is available.
Under PEP 517, call the backend hook to prepare the metadata.
Under legacy processing, call setup.py egg-info.
"""
assert self.source_dir
with indent_log():
if self.use_pep517:
self.prepare_pep517_metadata()
else:
self.run_egg_info()
if not self.req:
if isinstance(parse_version(self.metadata["Version"]), Version):
op = "=="
else:
op = "==="
self.req = Requirement(
"".join([
self.metadata["Name"],
op,
self.metadata["Version"],
])
)
self._correct_build_location()
else:
metadata_name = canonicalize_name(self.metadata["Name"])
if canonicalize_name(self.req.name) != metadata_name:
logger.warning(
'Generating metadata for package %s '
'produced metadata for project name %s. Fix your '
'#egg=%s fragments.',
self.name, metadata_name, self.name
)
self.req = Requirement(metadata_name)
def __init__(self, project, version, location, requires_python=''):
self.project = project
self.version = parse_version(version)
self.location = location
self._key = (self.project, self.version, self.location)
self.requires_python = requires_python
def find_requirement(self, req, upgrade, ignore_compatibility=False):
"""Try to find a Link matching req
Expects req, an InstallRequirement and upgrade, a boolean
Returns a Link if found,
Raises DistributionNotFound or BestVersionAlreadyInstalled otherwise
"""
all_candidates = self.find_all_candidates(req.name)
# Filter out anything which doesn't match our specifier
if not ignore_compatibility:
compatible_versions = set(
req.specifier.filter(
# We turn the version object into a str here because otherwise
# when we're debundled but setuptools isn't, Python will see
# packaging.version.Version and
# pkg_resources._vendor.packaging.version.Version as different
# types. This way we'll use a str as a common data interchange
# format. If we stop using the pkg_resources provided specifier
# and start using our own, we can drop the cast to str().
[str(c.version) for c in all_candidates],
prereleases=(
self.allow_all_prereleases
if self.allow_all_prereleases else None
),
)
)
else:
compatible_versions = [str(c.version) for c in all_candidates]
applicable_candidates = [
# Again, converting to str to deal with debundling.
c for c in all_candidates if str(c.version) in compatible_versions
]
if applicable_candidates:
best_candidate = max(applicable_candidates,
key=self._candidate_sort_key)
else:
best_candidate = None
if req.satisfied_by is not None:
installed_version = parse_version(req.satisfied_by.version)
else:
installed_version = None
if installed_version is None and best_candidate is None:
logger.critical(
'Could not find a version that satisfies the requirement %s '
'(from versions: %s)',
req,
', '.join(
sorted(
{str(c.version) for c in all_candidates},
key=parse_version,
)
)
)
raise DistributionNotFound(
'No matching distribution found for %s' % req
)
best_installed = False
if installed_version and (
best_candidate is None or
best_candidate.version <= installed_version):
best_installed = True
if not upgrade and installed_version is not None:
if best_installed:
logger.debug(
'Existing installed version (%s) is most up-to-date and '
'satisfies requirement',
installed_version,
)
else:
logger.debug(
'Existing installed version (%s) satisfies requirement '
'(most up-to-date version is %s)',
installed_version,
best_candidate.version,
)
return None
if best_installed:
# We have an existing version, and its the best version
logger.debug(
'Installed version (%s) is most up-to-date (past versions: '
'%s)',
installed_version,
', '.join(sorted(compatible_versions, key=parse_version)) or
"none",
)
raise BestVersionAlreadyInstalled
logger.debug(
'Using version %s (newest of versions: %s)',
best_candidate.version,
', '.join(sorted(compatible_versions, key=parse_version))
)
return best_candidate.location
def version(self) -> DistributionVersion:
return parse_version(self._dist.version)
def find_requirement(self, req, upgrade, ignore_compatibility=False):
"""Try to find a Link matching req
Expects req, an InstallRequirement and upgrade, a boolean
Returns a Link if found,
Raises DistributionNotFound or BestVersionAlreadyInstalled otherwise
"""
all_candidates = self.find_all_candidates(req.name)
# Filter out anything which doesn't match our specifier
compatible_versions = set(
req.specifier.filter(
# We turn the version object into a str here because otherwise
# when we're debundled but setuptools isn't, Python will see
# packaging.version.Version and
# pkg_resources._vendor.packaging.version.Version as different
# types. This way we'll use a str as a common data interchange
# format. If we stop using the pkg_resources provided specifier
# and start using our own, we can drop the cast to str().
[str(c.version) for c in all_candidates],
prereleases=(self.allow_all_prereleases
if self.allow_all_prereleases else None),
))
applicable_candidates = [
# Again, converting to str to deal with debundling.
c for c in all_candidates if str(c.version) in compatible_versions
]
if applicable_candidates:
best_candidate = max(applicable_candidates,
key=self._candidate_sort_key)
else:
best_candidate = None
if req.satisfied_by is not None:
installed_version = parse_version(req.satisfied_by.version)
else:
installed_version = None
if installed_version is None and best_candidate is None:
logger.critical(
'Could not find a version that satisfies the requirement %s '
'(from versions: %s)', req, ', '.join(
sorted(
{str(c.version)
for c in all_candidates},
key=parse_version,
)))
raise DistributionNotFound(
'No matching distribution found for %s' % req)
best_installed = False
if installed_version and (best_candidate is None or
best_candidate.version <= installed_version):
best_installed = True
if not upgrade and installed_version is not None:
if best_installed:
logger.debug(
'Existing installed version (%s) is most up-to-date and '
'satisfies requirement',
installed_version,
)
else:
logger.debug(
'Existing installed version (%s) satisfies requirement '
'(most up-to-date version is %s)',
installed_version,
best_candidate.version,
)
return None
if best_installed:
# We have an existing version, and its the best version
logger.debug(
'Installed version (%s) is most up-to-date (past versions: '
'%s)',
installed_version,
', '.join(sorted(compatible_versions, key=parse_version))
or "none",
)
raise BestVersionAlreadyInstalled
logger.debug('Using version %s (newest of versions: %s)',
best_candidate.version,
', '.join(sorted(compatible_versions, key=parse_version)))
return best_candidate.location
def pip_self_version_check(session: PipSession, options: optparse.Values) -> None:
"""Check for an update for pip.
Limit the frequency of checks to once per week. State is stored either in
the active virtualenv or in the user's USER_CACHE_DIR keyed off the prefix
of the pip script path.
"""
installed_dist = get_default_environment().get_distribution("pip")
if not installed_dist:
return
pip_version = installed_dist.version
pypi_version = None
try:
state = SelfCheckState(cache_dir=options.cache_dir)
current_time = datetime.datetime.utcnow()
# Determine if we need to refresh the state
if "last_check" in state.state and "pypi_version" in state.state:
last_check = datetime.datetime.strptime(
state.state["last_check"], SELFCHECK_DATE_FMT
)
if (current_time - last_check).total_seconds() < 7 * 24 * 60 * 60:
pypi_version = state.state["pypi_version"]
# Refresh the version if we need to or just see if we need to warn
if pypi_version is None:
# Lets use PackageFinder to see what the latest pip version is
link_collector = LinkCollector.create(
session,
options=options,
suppress_no_index=True,
)
# Pass allow_yanked=False so we don't suggest upgrading to a
# yanked version.
selection_prefs = SelectionPreferences(
allow_yanked=False,
allow_all_prereleases=False, # Explicitly set to False
)
finder = PackageFinder.create(
link_collector=link_collector,
selection_prefs=selection_prefs,
use_deprecated_html5lib=(
"html5lib" in options.deprecated_features_enabled
),
)
best_candidate = finder.find_best_candidate("pip").best_candidate
if best_candidate is None:
return
pypi_version = str(best_candidate.version)
# save that we've performed a check
state.save(pypi_version, current_time)
remote_version = parse_version(pypi_version)
local_version_is_older = (
pip_version < remote_version
and pip_version.base_version != remote_version.base_version
and was_installed_by_pip("pip")
)
# Determine if our pypi_version is older
if not local_version_is_older:
return
# We cannot tell how the current pip is available in the current
# command context, so be pragmatic here and suggest the command
# that's always available. This does not accommodate spaces in
# `sys.executable` on purpose as it is not possible to do it
# correctly without knowing the user's shell. Thus,
# it won't be done until possible through the standard library.
# Do not be tempted to use the undocumented subprocess.list2cmdline.
# It is considered an internal implementation detail for a reason.
pip_cmd = f"{sys.executable} -m pip"
logger.warning(
"You are using pip version %s; however, version %s is "
"available.\nYou should consider upgrading via the "
"'%s install --upgrade pip' command.",
pip_version,
pypi_version,
pip_cmd,
)
except Exception:
logger.debug(
"There was an error checking the latest version of pip",
exc_info=True,
)
def version(self) -> CandidateVersion:
if self._version is None:
self._version = parse_version(self.dist.version)
return self._version
def version(self) -> CandidateVersion:
return parse_version(self.dist.version)
def find_requirement(self, req: InstallRequirement,
upgrade: bool) -> Optional[InstallationCandidate]:
"""Try to find a Link matching req
Expects req, an InstallRequirement and upgrade, a boolean
Returns a InstallationCandidate if found,
Raises DistributionNotFound or BestVersionAlreadyInstalled otherwise
"""
hashes = req.hashes(trust_internet=False)
best_candidate_result = self.find_best_candidate(
req.name,
specifier=req.specifier,
hashes=hashes,
)
best_candidate = best_candidate_result.best_candidate
installed_version: Optional[_BaseVersion] = None
if req.satisfied_by is not None:
installed_version = parse_version(req.satisfied_by.version)
def _format_versions(
cand_iter: Iterable[InstallationCandidate]) -> str:
# This repeated parse_version and str() conversion is needed to
# handle different vendoring sources from pip and pkg_resources.
# If we stop using the pkg_resources provided specifier and start
# using our own, we can drop the cast to str().
return ", ".join(
sorted(
{str(c.version)
for c in cand_iter},
key=parse_version,
)) or "none"
if installed_version is None and best_candidate is None:
logger.critical(
'Could not find a version that satisfies the requirement %s '
'(from versions: %s)',
req,
_format_versions(best_candidate_result.iter_all()),
)
raise DistributionNotFound(
'No matching distribution found for {}'.format(req))
best_installed = False
if installed_version and (best_candidate is None or
best_candidate.version <= installed_version):
best_installed = True
if not upgrade and installed_version is not None:
if best_installed:
logger.debug(
'Existing installed version (%s) is most up-to-date and '
'satisfies requirement',
installed_version,
)
else:
logger.debug(
'Existing installed version (%s) satisfies requirement '
'(most up-to-date version is %s)',
installed_version,
best_candidate.version,
)
return None
if best_installed:
# We have an existing version, and its the best version
logger.debug(
'Installed version (%s) is most up-to-date (past versions: '
'%s)',
installed_version,
_format_versions(best_candidate_result.iter_applicable()),
)
raise BestVersionAlreadyInstalled
logger.debug(
'Using version %s (newest of versions: %s)',
best_candidate.version,
_format_versions(best_candidate_result.iter_applicable()),
)
return best_candidate
def __init__(self, project, version, location, requires_python=None):
self.project = project
self.version = parse_version(version)
self.location = location
self._key = (self.project, self.version, self.location)
self.requires_python = requires_python
