def get_legacy_dependencies(self, ireq):
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
They indicate the secondary dependencies for the given requirement.
"""
if not (ireq.editable or is_pinned_requirement(ireq)):
raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq))
if ireq not in self._dependencies_cache:
if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)):
# No download_dir for locally available editable requirements.
# If a download_dir is passed, pip will unnecessarely
# archive the entire source directory
download_dir = None
elif ireq.link and not ireq.link.is_artifact:
# No download_dir for VCS sources. This also works around pip
# using git-checkout-index, which gets rid of the .git dir.
download_dir = None
else:
download_dir = self._download_dir
if not os.path.isdir(download_dir):
os.makedirs(download_dir)
if not os.path.isdir(self._wheel_download_dir):
os.makedirs(self._wheel_download_dir)
# Collect setup_requires info from local eggs.
# Do this after we call the preparer on these reqs to make sure their
# egg info has been created
setup_requires = {}
dist = None
if ireq.editable:
try:
dist = ireq.get_dist()
except InstallationError:
ireq.run_egg_info()
dist = ireq.get_dist()
except (TypeError, ValueError, AttributeError):
pass
else:
if dist.has_metadata('requires.txt'):
setup_requires = self.finder.get_extras_links(
dist.get_metadata_lines('requires.txt')
)
try:
# Pip 9 and below
reqset = RequirementSet(
self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
ignore_installed=True,
ignore_compatibility=False,
wheel_cache=self.wheel_cache,
)
result = reqset._prepare_file(
self.finder,
ireq,
ignore_requires_python=True
)
except TypeError:
# Pip >= 10 (new resolver!)
preparer = RequirementPreparer(
build_dir=self.build_dir,
src_dir=self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
progress_bar='off',
build_isolation=False
)
reqset = RequirementSet()
ireq.is_direct = True
reqset.add_requirement(ireq)
self.resolver = PipResolver(
preparer=preparer,
finder=self.finder,
session=self.session,
upgrade_strategy="to-satisfy-only",
force_reinstall=True,
ignore_dependencies=False,
ignore_requires_python=True,
ignore_installed=True,
isolated=False,
wheel_cache=self.wheel_cache,
use_user_site=False,
ignore_compatibility=False
)
self.resolver.resolve(reqset)
result = set(reqset.requirements.values())
# HACK: Sometimes the InstallRequirement doesn't properly get
# these values set on it during the resolution process. It's
# difficult to pin down what is going wrong. This fixes things.
if not getattr(ireq, 'version', None):
try:
dist = ireq.get_dist() if not dist else None
ireq.version = ireq.get_dist().version
except (ValueError, OSError, TypeError, AttributeError) as e:
pass
if not getattr(ireq, 'project_name', None):
try:
ireq.project_name = dist.project_name if dist else None
except (ValueError, TypeError) as e:
pass
if not getattr(ireq, 'req', None):
try:
ireq.req = dist.as_requirement() if dist else None
except (ValueError, TypeError) as e:
pass
# Convert setup_requires dict into a somewhat usable form.
if setup_requires:
for section in setup_requires:
python_version = section
not_python = not (section.startswith('[') and ':' in section)
# This is for cleaning up :extras: formatted markers
# by adding them to the results of the resolver
# since any such extra would have been returned as a result anyway
for value in setup_requires[section]:
# This is a marker.
if value.startswith('[') and ':' in value:
python_version = value[1:-1]
not_python = False
# Strip out other extras.
if value.startswith('[') and ':' not in value:
not_python = True
if ':' not in value:
try:
if not not_python:
result = result + [InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';'))]
# Anything could go wrong here -- can't be too careful.
except Exception:
pass
# this section properly creates 'python_version' markers for cross-python
# virtualenv creation and for multi-python compatibility.
requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else self.resolver.requires_python
if requires_python:
marker_str = ''
# This corrects a logic error from the previous code which said that if
# we Encountered any 'requires_python' attributes, basically only create a
# single result no matter how many we resolved. This should fix
# a majority of the remaining non-deterministic resolution issues.
if any(requires_python.startswith(op) for op in Specifier._operators.keys()):
# We are checking first if we have leading specifier operator
# if not, we can assume we should be doing a == comparison
specifierset = list(SpecifierSet(requires_python))
# for multiple specifiers, the correct way to represent that in
# a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
marker_key = Variable('python_version')
markers = []
for spec in specifierset:
operator, val = spec._spec
operator = Op(operator)
val = Value(val)
markers.append(''.join([marker_key.serialize(), operator.serialize(), val.serialize()]))
marker_str = ' and '.join(markers)
# The best way to add markers to a requirement is to make a separate requirement
# with only markers on it, and then to transfer the object istelf
marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker
result.remove(ireq)
ireq.req.marker = marker_to_add
result.add(ireq)
self._dependencies_cache[ireq] = result
reqset.cleanup_files()
return set(self._dependencies_cache[ireq])
class PyPIRepository(BaseRepository):
DEFAULT_INDEX_URL = PyPI.simple_url
"""
The PyPIRepository will use the provided Finder instance to lookup
packages. Typically, it looks up packages on PyPI (the default implicit
config), but any other PyPI mirror can be used if index_urls is
changed/configured on the Finder.
"""
def __init__(self, pip_options, session, use_json=False):
self.session = session
self.use_json = use_json
self.pip_options = pip_options
self.wheel_cache = WheelCache(PIPENV_CACHE_DIR, pip_options.format_control)
index_urls = [pip_options.index_url] + pip_options.extra_index_urls
if pip_options.no_index:
index_urls = []
self.finder = PackageFinder(
find_links=pip_options.find_links,
index_urls=index_urls,
trusted_hosts=pip_options.trusted_hosts,
allow_all_prereleases=pip_options.pre,
process_dependency_links=pip_options.process_dependency_links,
session=self.session,
)
# Caches
# stores project_name => InstallationCandidate mappings for all
# versions reported by PyPI, so we only have to ask once for each
# project
self._available_candidates_cache = {}
# stores InstallRequirement => list(InstallRequirement) mappings
# of all secondary dependencies for the given requirement, so we
# only have to go to disk once for each requirement
self._dependencies_cache = {}
self._json_dep_cache = {}
# stores *full* path + fragment => sha256
self._hash_cache = HashCache(session=session)
# Setup file paths
self.freshen_build_caches()
self._download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'pkgs'))
self._wheel_download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'wheels'))
def freshen_build_caches(self):
"""
Start with fresh build/source caches. Will remove any old build
caches from disk automatically.
"""
self._build_dir = TemporaryDirectory(fs_str('build'))
self._source_dir = TemporaryDirectory(fs_str('source'))
@property
def build_dir(self):
return self._build_dir.name
@property
def source_dir(self):
return self._source_dir.name
def clear_caches(self):
rmtree(self._download_dir, ignore_errors=True)
rmtree(self._wheel_download_dir, ignore_errors=True)
def find_all_candidates(self, req_name):
if req_name not in self._available_candidates_cache:
candidates = self.finder.find_all_candidates(req_name)
self._available_candidates_cache[req_name] = candidates
return self._available_candidates_cache[req_name]
def find_best_match(self, ireq, prereleases=None):
"""
Returns a Version object that indicates the best match for the given
InstallRequirement according to the external repository.
"""
if ireq.editable:
return ireq # return itself as the best match
all_candidates = clean_requires_python(self.find_all_candidates(ireq.name))
candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
try:
matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates),
prereleases=prereleases)
except TypeError:
matching_versions = [candidate.version for candidate in all_candidates]
# Reuses pip's internal candidate sort key to sort
matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
if not matching_candidates:
raise NoCandidateFound(ireq, all_candidates, self.finder)
best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key)
# Turn the candidate into a pinned InstallRequirement
new_req = make_install_requirement(
best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint
)
# KR TODO: Marker here?
return new_req
def get_json_dependencies(self, ireq):
if not (is_pinned_requirement(ireq)):
raise TypeError('Expected pinned InstallRequirement, got {}'.format(ireq))
def gen(ireq):
if self.DEFAULT_INDEX_URL not in self.finder.index_urls:
return
url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name)
releases = self.session.get(url).json()['releases']
matches = [
r for r in releases
if '=={0}'.format(r) == str(ireq.req.specifier)
]
if not matches:
return
release_requires = self.session.get(
'https://pypi.org/pypi/{0}/{1}/json'.format(
ireq.req.name, matches[0],
),
).json()
try:
requires_dist = release_requires['info']['requires_dist']
except KeyError:
return
for requires in requires_dist:
i = InstallRequirement.from_line(requires)
if 'extra' not in repr(i.markers):
yield i
try:
if ireq not in self._json_dep_cache:
self._json_dep_cache[ireq] = [g for g in gen(ireq)]
return set(self._json_dep_cache[ireq])
except Exception:
return set()
def get_dependencies(self, ireq):
json_results = set()
if self.use_json:
try:
json_results = self.get_json_dependencies(ireq)
except TypeError:
json_results = set()
legacy_results = self.get_legacy_dependencies(ireq)
json_results.update(legacy_results)
return json_results
def get_legacy_dependencies(self, ireq):
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
They indicate the secondary dependencies for the given requirement.
"""
if not (ireq.editable or is_pinned_requirement(ireq)):
raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq))
if ireq not in self._dependencies_cache:
if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)):
# No download_dir for locally available editable requirements.
# If a download_dir is passed, pip will unnecessarely
# archive the entire source directory
download_dir = None
elif ireq.link and not ireq.link.is_artifact:
# No download_dir for VCS sources. This also works around pip
# using git-checkout-index, which gets rid of the .git dir.
download_dir = None
else:
download_dir = self._download_dir
if not os.path.isdir(download_dir):
os.makedirs(download_dir)
if not os.path.isdir(self._wheel_download_dir):
os.makedirs(self._wheel_download_dir)
# Collect setup_requires info from local eggs.
# Do this after we call the preparer on these reqs to make sure their
# egg info has been created
setup_requires = {}
dist = None
if ireq.editable:
try:
dist = ireq.get_dist()
except InstallationError:
ireq.run_egg_info()
dist = ireq.get_dist()
except (TypeError, ValueError, AttributeError):
pass
else:
if dist.has_metadata('requires.txt'):
setup_requires = self.finder.get_extras_links(
dist.get_metadata_lines('requires.txt')
)
try:
# Pip 9 and below
reqset = RequirementSet(
self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
ignore_installed=True,
ignore_compatibility=False,
wheel_cache=self.wheel_cache,
)
result = reqset._prepare_file(
self.finder,
ireq,
ignore_requires_python=True
)
except TypeError:
# Pip >= 10 (new resolver!)
preparer = RequirementPreparer(
build_dir=self.build_dir,
src_dir=self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
progress_bar='off',
build_isolation=False
)
reqset = RequirementSet()
ireq.is_direct = True
reqset.add_requirement(ireq)
self.resolver = PipResolver(
preparer=preparer,
finder=self.finder,
session=self.session,
upgrade_strategy="to-satisfy-only",
force_reinstall=True,
ignore_dependencies=False,
ignore_requires_python=True,
ignore_installed=True,
isolated=False,
wheel_cache=self.wheel_cache,
use_user_site=False,
ignore_compatibility=False
)
self.resolver.resolve(reqset)
result = set(reqset.requirements.values())
# HACK: Sometimes the InstallRequirement doesn't properly get
# these values set on it during the resolution process. It's
# difficult to pin down what is going wrong. This fixes things.
if not getattr(ireq, 'version', None):
try:
dist = ireq.get_dist() if not dist else None
ireq.version = ireq.get_dist().version
except (ValueError, OSError, TypeError, AttributeError) as e:
pass
if not getattr(ireq, 'project_name', None):
try:
ireq.project_name = dist.project_name if dist else None
except (ValueError, TypeError) as e:
pass
if not getattr(ireq, 'req', None):
try:
ireq.req = dist.as_requirement() if dist else None
except (ValueError, TypeError) as e:
pass
# Convert setup_requires dict into a somewhat usable form.
if setup_requires:
for section in setup_requires:
python_version = section
not_python = not (section.startswith('[') and ':' in section)
# This is for cleaning up :extras: formatted markers
# by adding them to the results of the resolver
# since any such extra would have been returned as a result anyway
for value in setup_requires[section]:
# This is a marker.
if value.startswith('[') and ':' in value:
python_version = value[1:-1]
not_python = False
# Strip out other extras.
if value.startswith('[') and ':' not in value:
not_python = True
if ':' not in value:
try:
if not not_python:
result = result + [InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';'))]
# Anything could go wrong here -- can't be too careful.
except Exception:
pass
# this section properly creates 'python_version' markers for cross-python
# virtualenv creation and for multi-python compatibility.
requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else self.resolver.requires_python
if requires_python:
marker_str = ''
# This corrects a logic error from the previous code which said that if
# we Encountered any 'requires_python' attributes, basically only create a
# single result no matter how many we resolved. This should fix
# a majority of the remaining non-deterministic resolution issues.
if any(requires_python.startswith(op) for op in Specifier._operators.keys()):
# We are checking first if we have leading specifier operator
# if not, we can assume we should be doing a == comparison
specifierset = list(SpecifierSet(requires_python))
# for multiple specifiers, the correct way to represent that in
# a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
marker_key = Variable('python_version')
markers = []
for spec in specifierset:
operator, val = spec._spec
operator = Op(operator)
val = Value(val)
markers.append(''.join([marker_key.serialize(), operator.serialize(), val.serialize()]))
marker_str = ' and '.join(markers)
# The best way to add markers to a requirement is to make a separate requirement
# with only markers on it, and then to transfer the object istelf
marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker
result.remove(ireq)
ireq.req.marker = marker_to_add
result.add(ireq)
self._dependencies_cache[ireq] = result
reqset.cleanup_files()
return set(self._dependencies_cache[ireq])
def get_hashes(self, ireq):
"""
Given an InstallRequirement, return a set of hashes that represent all
of the files for a given requirement. Editable requirements return an
empty set. Unpinned requirements raise a TypeError.
"""
if ireq.editable:
return set()
if not is_pinned_requirement(ireq):
raise TypeError(
"Expected pinned requirement, got {}".format(ireq))
# We need to get all of the candidates that match our current version
# pin, these will represent all of the files that could possibly
# satisfy this constraint.
### Modification -- this is much more efficient....
### modification again -- still more efficient
matching_candidates = (
c for c in clean_requires_python(self.find_all_candidates(ireq.name))
if c.version in ireq.specifier
)
# candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version)
# matching_versions = list(
# ireq.specifier.filter((candidate.version for candidate in all_candidates)))
# matching_candidates = candidates_by_version[matching_versions[0]]
return {
self._hash_cache.get_hash(candidate.location)
for candidate in matching_candidates
}
@contextmanager
def allow_all_wheels(self):
"""
Monkey patches pip.Wheel to allow wheels from all platforms and Python versions.
This also saves the candidate cache and set a new one, or else the results from the
previous non-patched calls will interfere.
"""
def _wheel_supported(self, tags=None):
# Ignore current platform. Support everything.
return True
def _wheel_support_index_min(self, tags=None):
# All wheels are equal priority for sorting.
return 0
original_wheel_supported = Wheel.supported
original_support_index_min = Wheel.support_index_min
original_cache = self._available_candidates_cache
Wheel.supported = _wheel_supported
Wheel.support_index_min = _wheel_support_index_min
self._available_candidates_cache = {}
try:
yield
finally:
Wheel.supported = original_wheel_supported
Wheel.support_index_min = original_support_index_min
self._available_candidates_cache = original_cache
class PyPIRepository(BaseRepository):
DEFAULT_INDEX_URL = PyPI.simple_url
"""
The PyPIRepository will use the provided Finder instance to lookup
packages. Typically, it looks up packages on PyPI (the default implicit
config), but any other PyPI mirror can be used if index_urls is
changed/configured on the Finder.
"""
def __init__(self, pip_options, session, use_json=False):
self.session = session
self.use_json = use_json
self.pip_options = pip_options
self.wheel_cache = WheelCache(PIPENV_CACHE_DIR,
pip_options.format_control)
index_urls = [pip_options.index_url] + pip_options.extra_index_urls
if pip_options.no_index:
index_urls = []
self.finder = PackageFinder(
find_links=pip_options.find_links,
index_urls=index_urls,
trusted_hosts=pip_options.trusted_hosts,
allow_all_prereleases=pip_options.pre,
process_dependency_links=pip_options.process_dependency_links,
session=self.session,
)
# Caches
# stores project_name => InstallationCandidate mappings for all
# versions reported by PyPI, so we only have to ask once for each
# project
self._available_candidates_cache = {}
# stores InstallRequirement => list(InstallRequirement) mappings
# of all secondary dependencies for the given requirement, so we
# only have to go to disk once for each requirement
self._dependencies_cache = {}
self._json_dep_cache = {}
# stores *full* path + fragment => sha256
self._hash_cache = HashCache(session=session)
# Setup file paths
self.freshen_build_caches()
self._download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'pkgs'))
self._wheel_download_dir = fs_str(
os.path.join(PIPENV_CACHE_DIR, 'wheels'))
def freshen_build_caches(self):
"""
Start with fresh build/source caches. Will remove any old build
caches from disk automatically.
"""
self._build_dir = TemporaryDirectory(fs_str('build'))
self._source_dir = TemporaryDirectory(fs_str('source'))
@property
def build_dir(self):
return self._build_dir.name
@property
def source_dir(self):
return self._source_dir.name
def clear_caches(self):
rmtree(self._download_dir, ignore_errors=True)
rmtree(self._wheel_download_dir, ignore_errors=True)
def find_all_candidates(self, req_name):
if req_name not in self._available_candidates_cache:
candidates = self.finder.find_all_candidates(req_name)
self._available_candidates_cache[req_name] = candidates
return self._available_candidates_cache[req_name]
def find_best_match(self, ireq, prereleases=None):
"""
Returns a Version object that indicates the best match for the given
InstallRequirement according to the external repository.
"""
if ireq.editable:
return ireq # return itself as the best match
all_candidates = clean_requires_python(
self.find_all_candidates(ireq.name))
candidates_by_version = lookup_table(all_candidates,
key=lambda c: c.version,
unique=True)
try:
matching_versions = ireq.specifier.filter(
(candidate.version for candidate in all_candidates),
prereleases=prereleases)
except TypeError:
matching_versions = [
candidate.version for candidate in all_candidates
]
# Reuses pip's internal candidate sort key to sort
matching_candidates = [
candidates_by_version[ver] for ver in matching_versions
]
if not matching_candidates:
raise NoCandidateFound(ireq, all_candidates, self.finder)
best_candidate = max(matching_candidates,
key=self.finder._candidate_sort_key)
# Turn the candidate into a pinned InstallRequirement
new_req = make_install_requirement(best_candidate.project,
best_candidate.version,
ireq.extras,
ireq.markers,
constraint=ireq.constraint)
# KR TODO: Marker here?
return new_req
def get_json_dependencies(self, ireq):
if not (is_pinned_requirement(ireq)):
raise TypeError(
'Expected pinned InstallRequirement, got {}'.format(ireq))
def gen(ireq):
if self.DEFAULT_INDEX_URL not in self.finder.index_urls:
return
url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name)
releases = self.session.get(url).json()['releases']
matches = [
r for r in releases
if '=={0}'.format(r) == str(ireq.req.specifier)
]
if not matches:
return
release_requires = self.session.get(
'https://pypi.org/pypi/{0}/{1}/json'.format(
ireq.req.name,
matches[0],
), ).json()
try:
requires_dist = release_requires['info']['requires_dist']
except KeyError:
return
for requires in requires_dist:
i = InstallRequirement.from_line(requires)
if 'extra' not in repr(i.markers):
yield i
try:
if ireq not in self._json_dep_cache:
self._json_dep_cache[ireq] = [g for g in gen(ireq)]
return set(self._json_dep_cache[ireq])
except Exception:
return set()
def get_dependencies(self, ireq):
json_results = set()
if self.use_json:
try:
json_results = self.get_json_dependencies(ireq)
except TypeError:
json_results = set()
legacy_results = self.get_legacy_dependencies(ireq)
json_results.update(legacy_results)
return json_results
def get_legacy_dependencies(self, ireq):
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
They indicate the secondary dependencies for the given requirement.
"""
if not (ireq.editable or is_pinned_requirement(ireq)):
raise TypeError(
'Expected pinned or editable InstallRequirement, got {}'.
format(ireq))
if ireq not in self._dependencies_cache:
if ireq.editable and (ireq.source_dir
and os.path.exists(ireq.source_dir)):
# No download_dir for locally available editable requirements.
# If a download_dir is passed, pip will unnecessarely
# archive the entire source directory
download_dir = None
elif ireq.link and not ireq.link.is_artifact:
# No download_dir for VCS sources. This also works around pip
# using git-checkout-index, which gets rid of the .git dir.
download_dir = None
else:
download_dir = self._download_dir
if not os.path.isdir(download_dir):
os.makedirs(download_dir)
if not os.path.isdir(self._wheel_download_dir):
os.makedirs(self._wheel_download_dir)
# Collect setup_requires info from local eggs.
# Do this after we call the preparer on these reqs to make sure their
# egg info has been created
setup_requires = {}
dist = None
if ireq.editable:
try:
dist = ireq.get_dist()
except InstallationError:
ireq.run_egg_info()
dist = ireq.get_dist()
except (TypeError, ValueError, AttributeError):
pass
else:
if dist.has_metadata('requires.txt'):
setup_requires = self.finder.get_extras_links(
dist.get_metadata_lines('requires.txt'))
try:
# Pip 9 and below
reqset = RequirementSet(
self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
ignore_installed=True,
ignore_compatibility=False,
wheel_cache=self.wheel_cache,
)
result = reqset._prepare_file(self.finder,
ireq,
ignore_requires_python=True)
except TypeError:
# Pip >= 10 (new resolver!)
preparer = RequirementPreparer(
build_dir=self.build_dir,
src_dir=self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
progress_bar='off',
build_isolation=False)
reqset = RequirementSet()
ireq.is_direct = True
reqset.add_requirement(ireq)
self.resolver = PipResolver(preparer=preparer,
finder=self.finder,
session=self.session,
upgrade_strategy="to-satisfy-only",
force_reinstall=True,
ignore_dependencies=False,
ignore_requires_python=True,
ignore_installed=True,
isolated=False,
wheel_cache=self.wheel_cache,
use_user_site=False,
ignore_compatibility=False)
self.resolver.resolve(reqset)
result = set(reqset.requirements.values())
# HACK: Sometimes the InstallRequirement doesn't properly get
# these values set on it during the resolution process. It's
# difficult to pin down what is going wrong. This fixes things.
if not getattr(ireq, 'version', None):
try:
dist = ireq.get_dist() if not dist else None
ireq.version = ireq.get_dist().version
except (ValueError, OSError, TypeError, AttributeError) as e:
pass
if not getattr(ireq, 'project_name', None):
try:
ireq.project_name = dist.project_name if dist else None
except (ValueError, TypeError) as e:
pass
if not getattr(ireq, 'req', None):
try:
ireq.req = dist.as_requirement() if dist else None
except (ValueError, TypeError) as e:
pass
# Convert setup_requires dict into a somewhat usable form.
if setup_requires:
for section in setup_requires:
python_version = section
not_python = not (section.startswith('[')
and ':' in section)
# This is for cleaning up :extras: formatted markers
# by adding them to the results of the resolver
# since any such extra would have been returned as a result anyway
for value in setup_requires[section]:
# This is a marker.
if value.startswith('[') and ':' in value:
python_version = value[1:-1]
not_python = False
# Strip out other extras.
if value.startswith('[') and ':' not in value:
not_python = True
if ':' not in value:
try:
if not not_python:
result = result + [
InstallRequirement.from_line(
"{0}{1}".format(
value, python_version).replace(
':', ';'))
]
# Anything could go wrong here -- can't be too careful.
except Exception:
pass
# this section properly creates 'python_version' markers for cross-python
# virtualenv creation and for multi-python compatibility.
requires_python = reqset.requires_python if hasattr(
reqset, 'requires_python') else self.resolver.requires_python
if requires_python:
marker_str = ''
# This corrects a logic error from the previous code which said that if
# we Encountered any 'requires_python' attributes, basically only create a
# single result no matter how many we resolved. This should fix
# a majority of the remaining non-deterministic resolution issues.
if any(
requires_python.startswith(op)
for op in Specifier._operators.keys()):
# We are checking first if we have leading specifier operator
# if not, we can assume we should be doing a == comparison
specifierset = list(SpecifierSet(requires_python))
# for multiple specifiers, the correct way to represent that in
# a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
marker_key = Variable('python_version')
markers = []
for spec in specifierset:
operator, val = spec._spec
operator = Op(operator)
val = Value(val)
markers.append(''.join([
marker_key.serialize(),
operator.serialize(),
val.serialize()
]))
marker_str = ' and '.join(markers)
# The best way to add markers to a requirement is to make a separate requirement
# with only markers on it, and then to transfer the object istelf
marker_to_add = Requirement(
'fakepkg; {0}'.format(marker_str)).marker
result.remove(ireq)
ireq.req.marker = marker_to_add
result.add(ireq)
self._dependencies_cache[ireq] = result
reqset.cleanup_files()
return set(self._dependencies_cache[ireq])
def get_hashes(self, ireq):
"""
Given an InstallRequirement, return a set of hashes that represent all
of the files for a given requirement. Editable requirements return an
empty set. Unpinned requirements raise a TypeError.
"""
if ireq.editable:
return set()
if not is_pinned_requirement(ireq):
raise TypeError("Expected pinned requirement, got {}".format(ireq))
# We need to get all of the candidates that match our current version
# pin, these will represent all of the files that could possibly
# satisfy this constraint.
### Modification -- this is much more efficient....
### modification again -- still more efficient
matching_candidates = (
c
for c in clean_requires_python(self.find_all_candidates(ireq.name))
if c.version in ireq.specifier)
# candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version)
# matching_versions = list(
# ireq.specifier.filter((candidate.version for candidate in all_candidates)))
# matching_candidates = candidates_by_version[matching_versions[0]]
return {
self._hash_cache.get_hash(candidate.location)
for candidate in matching_candidates
}
@contextmanager
def allow_all_wheels(self):
"""
Monkey patches pip.Wheel to allow wheels from all platforms and Python versions.
This also saves the candidate cache and set a new one, or else the results from the
previous non-patched calls will interfere.
"""
def _wheel_supported(self, tags=None):
# Ignore current platform. Support everything.
return True
def _wheel_support_index_min(self, tags=None):
# All wheels are equal priority for sorting.
return 0
original_wheel_supported = Wheel.supported
original_support_index_min = Wheel.support_index_min
original_cache = self._available_candidates_cache
Wheel.supported = _wheel_supported
Wheel.support_index_min = _wheel_support_index_min
self._available_candidates_cache = {}
try:
yield
finally:
Wheel.supported = original_wheel_supported
Wheel.support_index_min = original_support_index_min
self._available_candidates_cache = original_cache
def run(self, options, args):
cmdoptions.check_install_build_global(options)
upgrade_strategy = "to-satisfy-only"
if options.upgrade:
upgrade_strategy = options.upgrade_strategy
if options.build_dir:
options.build_dir = os.path.abspath(options.build_dir)
cmdoptions.check_dist_restriction(options, check_target=True)
if options.python_version:
python_versions = [options.python_version]
else:
python_versions = None
options.src_dir = os.path.abspath(options.src_dir)
install_options = options.install_options or []
if options.use_user_site:
if options.prefix_path:
raise CommandError(
"Can not combine '--user' and '--prefix' as they imply "
"different installation locations")
if virtualenv_no_global():
raise InstallationError(
"Can not perform a '--user' install. User site-packages "
"are not visible in this virtualenv.")
install_options.append('--user')
install_options.append('--prefix=')
target_temp_dir = TempDirectory(kind="target")
if options.target_dir:
options.ignore_installed = True
options.target_dir = os.path.abspath(options.target_dir)
if (os.path.exists(options.target_dir)
and not os.path.isdir(options.target_dir)):
raise CommandError(
"Target path exists but is not a directory, will not "
"continue.")
# Create a target directory for using with the target option
target_temp_dir.create()
install_options.append('--home=' + target_temp_dir.path)
global_options = options.global_options or []
with self._build_session(options) as session:
finder = self._build_package_finder(
options=options,
session=session,
platform=options.platform,
python_versions=python_versions,
abi=options.abi,
implementation=options.implementation,
)
build_delete = (not (options.no_clean or options.build_dir))
wheel_cache = WheelCache(options.cache_dir, options.format_control)
if options.cache_dir and not check_path_owner(options.cache_dir):
logger.warning(
"The directory '%s' or its parent directory is not owned "
"by the current user and caching wheels has been "
"disabled. check the permissions and owner of that "
"directory. If executing pip with sudo, you may want "
"sudo's -H flag.",
options.cache_dir,
)
options.cache_dir = None
with RequirementTracker() as req_tracker, TempDirectory(
options.build_dir, delete=build_delete,
kind="install") as directory:
requirement_set = RequirementSet(
require_hashes=options.require_hashes,
check_supported_wheels=not options.target_dir,
)
try:
self.populate_requirement_set(requirement_set, args,
options, finder, session,
self.name, wheel_cache)
preparer = RequirementPreparer(
build_dir=directory.path,
src_dir=options.src_dir,
download_dir=None,
wheel_download_dir=None,
progress_bar=options.progress_bar,
build_isolation=options.build_isolation,
req_tracker=req_tracker,
)
resolver = Resolver(
preparer=preparer,
finder=finder,
session=session,
wheel_cache=wheel_cache,
use_user_site=options.use_user_site,
upgrade_strategy=upgrade_strategy,
force_reinstall=options.force_reinstall,
ignore_dependencies=options.ignore_dependencies,
ignore_requires_python=options.ignore_requires_python,
ignore_installed=options.ignore_installed,
isolated=options.isolated_mode,
use_pep517=options.use_pep517)
resolver.resolve(requirement_set)
protect_pip_from_modification_on_windows(
modifying_pip=requirement_set.has_requirement("pip"))
# Consider legacy and PEP517-using requirements separately
legacy_requirements = []
pep517_requirements = []
for req in requirement_set.requirements.values():
if req.use_pep517:
pep517_requirements.append(req)
else:
legacy_requirements.append(req)
# We don't build wheels for legacy requirements if we
# don't have wheel installed or we don't have a cache dir
try:
import wheel # noqa: F401
build_legacy = bool(options.cache_dir)
except ImportError:
build_legacy = False
wb = WheelBuilder(
finder,
preparer,
wheel_cache,
build_options=[],
global_options=[],
)
# Always build PEP 517 requirements
build_failures = wb.build(pep517_requirements,
session=session,
autobuilding=True)
if build_legacy:
# We don't care about failures building legacy
# requirements, as we'll fall through to a direct
# install for those.
wb.build(legacy_requirements,
session=session,
autobuilding=True)
# If we're using PEP 517, we cannot do a direct install
# so we fail here.
if build_failures:
raise InstallationError(
"Could not build wheels for {} which use"
" PEP 517 and cannot be installed directly".format(
", ".join(r.name for r in build_failures)))
to_install = resolver.get_installation_order(
requirement_set)
# Consistency Checking of the package set we're installing.
should_warn_about_conflicts = (
not options.ignore_dependencies
and options.warn_about_conflicts)
if should_warn_about_conflicts:
self._warn_about_conflicts(to_install)
# Don't warn about script install locations if
# --target has been specified
warn_script_location = options.warn_script_location
if options.target_dir:
warn_script_location = False
installed = install_given_reqs(
to_install,
install_options,
global_options,
root=options.root_path,
home=target_temp_dir.path,
prefix=options.prefix_path,
pycompile=options.compile,
warn_script_location=warn_script_location,
use_user_site=options.use_user_site,
)
lib_locations = get_lib_location_guesses(
user=options.use_user_site,
home=target_temp_dir.path,
root=options.root_path,
prefix=options.prefix_path,
isolated=options.isolated_mode,
)
working_set = pkg_resources.WorkingSet(lib_locations)
reqs = sorted(installed, key=operator.attrgetter('name'))
items = []
for req in reqs:
item = req.name
try:
installed_version = get_installed_version(
req.name, working_set=working_set)
if installed_version:
item += '-' + installed_version
except Exception:
pass
items.append(item)
installed = ' '.join(items)
if installed:
logger.info('Successfully installed %s', installed)
except EnvironmentError as error:
show_traceback = (self.verbosity >= 1)
message = create_env_error_message(
error,
show_traceback,
options.use_user_site,
)
logger.error(message, exc_info=show_traceback)
return ERROR
except PreviousBuildDirError:
options.no_clean = True
raise
finally:
# Clean up
if not options.no_clean:
requirement_set.cleanup_files()
wheel_cache.cleanup()
if options.target_dir:
self._handle_target_dir(options.target_dir, target_temp_dir,
options.upgrade)
return requirement_set
def run(self, options, args):
cmdoptions.check_install_build_global(options)
index_urls = [options.index_url] + options.extra_index_urls
if options.no_index:
logger.debug('Ignoring indexes: %s', ','.join(index_urls))
index_urls = []
if options.build_dir:
options.build_dir = os.path.abspath(options.build_dir)
options.src_dir = os.path.abspath(options.src_dir)
with self._build_session(options) as session:
finder = self._build_package_finder(options, session)
build_delete = (not (options.no_clean or options.build_dir))
wheel_cache = WheelCache(options.cache_dir, options.format_control)
with TempDirectory(
options.build_dir, delete=build_delete, kind="wheel"
) as directory:
requirement_set = RequirementSet(
require_hashes=options.require_hashes,
)
try:
self.populate_requirement_set(
requirement_set, args, options, finder, session,
self.name, wheel_cache
)
preparer = RequirementPreparer(
build_dir=directory.path,
src_dir=options.src_dir,
download_dir=None,
wheel_download_dir=options.wheel_dir,
progress_bar=options.progress_bar,
build_isolation=options.build_isolation,
)
resolver = Resolver(
preparer=preparer,
finder=finder,
session=session,
wheel_cache=wheel_cache,
use_user_site=False,
upgrade_strategy="to-satisfy-only",
force_reinstall=False,
ignore_dependencies=options.ignore_dependencies,
ignore_requires_python=options.ignore_requires_python,
ignore_installed=True,
isolated=options.isolated_mode,
)
resolver.resolve(requirement_set)
# build wheels
wb = WheelBuilder(
finder, preparer, wheel_cache,
build_options=options.build_options or [],
global_options=options.global_options or [],
no_clean=options.no_clean,
)
wheels_built_successfully = wb.build(
requirement_set.requirements.values(), session=session,
)
if not wheels_built_successfully:
raise CommandError(
"Failed to build one or more wheels"
)
except PreviousBuildDirError:
options.no_clean = True
raise
finally:
if not options.no_clean:
requirement_set.cleanup_files()
wheel_cache.cleanup()
def get_legacy_dependencies(self, ireq):
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
They indicate the secondary dependencies for the given requirement.
"""
if not (ireq.editable or is_pinned_requirement(ireq)):
raise TypeError(
'Expected pinned or editable InstallRequirement, got {}'.
format(ireq))
if ireq not in self._dependencies_cache:
if ireq.editable and (ireq.source_dir
and os.path.exists(ireq.source_dir)):
# No download_dir for locally available editable requirements.
# If a download_dir is passed, pip will unnecessarely
# archive the entire source directory
download_dir = None
elif ireq.link and not ireq.link.is_artifact:
# No download_dir for VCS sources. This also works around pip
# using git-checkout-index, which gets rid of the .git dir.
download_dir = None
else:
download_dir = self._download_dir
if not os.path.isdir(download_dir):
os.makedirs(download_dir)
if not os.path.isdir(self._wheel_download_dir):
os.makedirs(self._wheel_download_dir)
# Collect setup_requires info from local eggs.
# Do this after we call the preparer on these reqs to make sure their
# egg info has been created
setup_requires = {}
dist = None
if ireq.editable:
try:
dist = ireq.get_dist()
except InstallationError:
ireq.run_egg_info()
dist = ireq.get_dist()
except (TypeError, ValueError, AttributeError):
pass
else:
if dist.has_metadata('requires.txt'):
setup_requires = self.finder.get_extras_links(
dist.get_metadata_lines('requires.txt'))
try:
# Pip 9 and below
reqset = RequirementSet(
self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
ignore_installed=True,
ignore_compatibility=False,
wheel_cache=self.wheel_cache,
)
result = reqset._prepare_file(self.finder,
ireq,
ignore_requires_python=True)
except TypeError:
# Pip >= 10 (new resolver!)
preparer = RequirementPreparer(
build_dir=self.build_dir,
src_dir=self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
progress_bar='off',
build_isolation=False)
reqset = RequirementSet()
ireq.is_direct = True
reqset.add_requirement(ireq)
self.resolver = PipResolver(preparer=preparer,
finder=self.finder,
session=self.session,
upgrade_strategy="to-satisfy-only",
force_reinstall=True,
ignore_dependencies=False,
ignore_requires_python=True,
ignore_installed=True,
isolated=False,
wheel_cache=self.wheel_cache,
use_user_site=False,
ignore_compatibility=False)
self.resolver.resolve(reqset)
result = set(reqset.requirements.values())
# HACK: Sometimes the InstallRequirement doesn't properly get
# these values set on it during the resolution process. It's
# difficult to pin down what is going wrong. This fixes things.
if not getattr(ireq, 'version', None):
try:
dist = ireq.get_dist() if not dist else None
ireq.version = ireq.get_dist().version
except (ValueError, OSError, TypeError, AttributeError) as e:
pass
if not getattr(ireq, 'project_name', None):
try:
ireq.project_name = dist.project_name if dist else None
except (ValueError, TypeError) as e:
pass
if not getattr(ireq, 'req', None):
try:
ireq.req = dist.as_requirement() if dist else None
except (ValueError, TypeError) as e:
pass
# Convert setup_requires dict into a somewhat usable form.
if setup_requires:
for section in setup_requires:
python_version = section
not_python = not (section.startswith('[')
and ':' in section)
# This is for cleaning up :extras: formatted markers
# by adding them to the results of the resolver
# since any such extra would have been returned as a result anyway
for value in setup_requires[section]:
# This is a marker.
if value.startswith('[') and ':' in value:
python_version = value[1:-1]
not_python = False
# Strip out other extras.
if value.startswith('[') and ':' not in value:
not_python = True
if ':' not in value:
try:
if not not_python:
result = result + [
InstallRequirement.from_line(
"{0}{1}".format(
value, python_version).replace(
':', ';'))
]
# Anything could go wrong here -- can't be too careful.
except Exception:
pass
# this section properly creates 'python_version' markers for cross-python
# virtualenv creation and for multi-python compatibility.
requires_python = reqset.requires_python if hasattr(
reqset, 'requires_python') else self.resolver.requires_python
if requires_python:
marker_str = ''
# This corrects a logic error from the previous code which said that if
# we Encountered any 'requires_python' attributes, basically only create a
# single result no matter how many we resolved. This should fix
# a majority of the remaining non-deterministic resolution issues.
if any(
requires_python.startswith(op)
for op in Specifier._operators.keys()):
# We are checking first if we have leading specifier operator
# if not, we can assume we should be doing a == comparison
specifierset = list(SpecifierSet(requires_python))
# for multiple specifiers, the correct way to represent that in
# a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
marker_key = Variable('python_version')
markers = []
for spec in specifierset:
operator, val = spec._spec
operator = Op(operator)
val = Value(val)
markers.append(''.join([
marker_key.serialize(),
operator.serialize(),
val.serialize()
]))
marker_str = ' and '.join(markers)
# The best way to add markers to a requirement is to make a separate requirement
# with only markers on it, and then to transfer the object istelf
marker_to_add = Requirement(
'fakepkg; {0}'.format(marker_str)).marker
result.remove(ireq)
ireq.req.marker = marker_to_add
result.add(ireq)
self._dependencies_cache[ireq] = result
reqset.cleanup_files()
return set(self._dependencies_cache[ireq])
def run(self, options, args):
cmdoptions.check_install_build_global(options)
index_urls = [options.index_url] + options.extra_index_urls
if options.no_index:
logger.debug('Ignoring indexes: %s', ','.join(index_urls))
index_urls = []
if options.build_dir:
options.build_dir = os.path.abspath(options.build_dir)
options.src_dir = os.path.abspath(options.src_dir)
with self._build_session(options) as session:
finder = self._build_package_finder(options, session)
build_delete = (not (options.no_clean or options.build_dir))
wheel_cache = WheelCache(options.cache_dir, options.format_control)
with TempDirectory(options.build_dir,
delete=build_delete,
kind="wheel") as directory:
requirement_set = RequirementSet(
require_hashes=options.require_hashes, )
try:
self.populate_requirement_set(requirement_set, args,
options, finder, session,
self.name, wheel_cache)
preparer = RequirementPreparer(
build_dir=directory.path,
src_dir=options.src_dir,
download_dir=None,
wheel_download_dir=options.wheel_dir,
progress_bar=options.progress_bar,
build_isolation=options.build_isolation,
)
resolver = Resolver(
preparer=preparer,
finder=finder,
session=session,
wheel_cache=wheel_cache,
use_user_site=False,
upgrade_strategy="to-satisfy-only",
force_reinstall=False,
ignore_dependencies=options.ignore_dependencies,
ignore_requires_python=options.ignore_requires_python,
ignore_installed=True,
isolated=options.isolated_mode,
)
resolver.resolve(requirement_set)
# build wheels
wb = WheelBuilder(
finder,
preparer,
wheel_cache,
build_options=options.build_options or [],
global_options=options.global_options or [],
no_clean=options.no_clean,
)
wheels_built_successfully = wb.build(
requirement_set.requirements.values(),
session=session,
)
if not wheels_built_successfully:
raise CommandError(
"Failed to build one or more wheels")
except PreviousBuildDirError:
options.no_clean = True
raise
finally:
if not options.no_clean:
requirement_set.cleanup_files()
wheel_cache.cleanup()
def run(self, options, args):
options.ignore_installed = True
# editable doesn't really make sense for `pip download`, but the bowels
# of the RequirementSet code require that property.
options.editables = []
if options.python_version:
python_versions = [options.python_version]
else:
python_versions = None
dist_restriction_set = any([
options.python_version,
options.platform,
options.abi,
options.implementation,
])
binary_only = FormatControl(set(), {':all:'})
no_sdist_dependencies = (options.format_control != binary_only
and not options.ignore_dependencies)
if dist_restriction_set and no_sdist_dependencies:
raise CommandError(
"When restricting platform and interpreter constraints using "
"--python-version, --platform, --abi, or --implementation, "
"either --no-deps must be set, or --only-binary=:all: must be "
"set and --no-binary must not be set (or must be set to "
":none:).")
options.src_dir = os.path.abspath(options.src_dir)
options.download_dir = normalize_path(options.download_dir)
ensure_dir(options.download_dir)
with self._build_session(options) as session:
finder = self._build_package_finder(
options=options,
session=session,
platform=options.platform,
python_versions=python_versions,
abi=options.abi,
implementation=options.implementation,
)
build_delete = (not (options.no_clean or options.build_dir))
if options.cache_dir and not check_path_owner(options.cache_dir):
logger.warning(
"The directory '%s' or its parent directory is not owned "
"by the current user and caching wheels has been "
"disabled. check the permissions and owner of that "
"directory. If executing pip with sudo, you may want "
"sudo's -H flag.",
options.cache_dir,
)
options.cache_dir = None
with TempDirectory(options.build_dir,
delete=build_delete,
kind="download") as directory:
requirement_set = RequirementSet(
require_hashes=options.require_hashes, )
self.populate_requirement_set(requirement_set, args, options,
finder, session, self.name, None)
preparer = RequirementPreparer(
build_dir=directory.path,
src_dir=options.src_dir,
download_dir=options.download_dir,
wheel_download_dir=None,
progress_bar=options.progress_bar,
build_isolation=options.build_isolation,
)
resolver = Resolver(
preparer=preparer,
finder=finder,
session=session,
wheel_cache=None,
use_user_site=False,
upgrade_strategy="to-satisfy-only",
force_reinstall=False,
ignore_dependencies=options.ignore_dependencies,
ignore_requires_python=False,
ignore_installed=True,
isolated=options.isolated_mode,
)
resolver.resolve(requirement_set)
downloaded = ' '.join([
req.name for req in requirement_set.successfully_downloaded
])
if downloaded:
logger.info('Successfully downloaded %s', downloaded)
# Clean up
if not options.no_clean:
requirement_set.cleanup_files()
return requirement_set
def run(self, options, args):
options.ignore_installed = True
# editable doesn't really make sense for `pip download`, but the bowels
# of the RequirementSet code require that property.
options.editables = []
if options.python_version:
python_versions = [options.python_version]
else:
python_versions = None
cmdoptions.check_dist_restriction(options)
options.src_dir = os.path.abspath(options.src_dir)
options.download_dir = normalize_path(options.download_dir)
ensure_dir(options.download_dir)
with self._build_session(options) as session:
finder = self._build_package_finder(
options=options,
session=session,
platform=options.platform,
python_versions=python_versions,
abi=options.abi,
implementation=options.implementation,
)
build_delete = (not (options.no_clean or options.build_dir))
if options.cache_dir and not check_path_owner(options.cache_dir):
logger.warning(
"The directory '%s' or its parent directory is not owned "
"by the current user and caching wheels has been "
"disabled. check the permissions and owner of that "
"directory. If executing pip with sudo, you may want "
"sudo's -H flag.",
options.cache_dir,
)
options.cache_dir = None
with RequirementTracker() as req_tracker, TempDirectory(
options.build_dir, delete=build_delete, kind="download"
) as directory:
requirement_set = RequirementSet(
require_hashes=options.require_hashes,
)
self.populate_requirement_set(
requirement_set,
args,
options,
finder,
session,
self.name,
None
)
preparer = RequirementPreparer(
build_dir=directory.path,
src_dir=options.src_dir,
download_dir=options.download_dir,
wheel_download_dir=None,
progress_bar=options.progress_bar,
build_isolation=options.build_isolation,
req_tracker=req_tracker,
)
resolver = Resolver(
preparer=preparer,
finder=finder,
session=session,
wheel_cache=None,
use_user_site=False,
upgrade_strategy="to-satisfy-only",
force_reinstall=False,
ignore_dependencies=options.ignore_dependencies,
ignore_requires_python=False,
ignore_installed=True,
isolated=options.isolated_mode,
)
resolver.resolve(requirement_set)
downloaded = ' '.join([
req.name for req in requirement_set.successfully_downloaded
])
if downloaded:
logger.info('Successfully downloaded %s', downloaded)
# Clean up
if not options.no_clean:
requirement_set.cleanup_files()
return requirement_set
def resolve_reqs(self,
download_dir,
ireq,
wheel_cache,
setup_requires={},
dist=None):
results = None
setup_requires = {}
dist = None
ireq.isolated = False
ireq._wheel_cache = wheel_cache
try:
from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer
except ImportError:
# Pip 9 and below
reqset = RequirementSet(
self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
ignore_installed=True,
ignore_compatibility=False,
wheel_cache=wheel_cache)
results = reqset._prepare_file(self.finder,
ireq,
ignore_requires_python=True)
else:
# pip >= 10
preparer_kwargs = {
'build_dir': self.build_dir,
'src_dir': self.source_dir,
'download_dir': download_dir,
'wheel_download_dir': self._wheel_download_dir,
'progress_bar': 'off',
'build_isolation': False
}
resolver_kwargs = {
'finder': self.finder,
'session': self.session,
'upgrade_strategy': "to-satisfy-only",
'force_reinstall': True,
'ignore_dependencies': False,
'ignore_requires_python': True,
'ignore_installed': True,
'isolated': False,
'wheel_cache': wheel_cache,
'use_user_site': False,
'ignore_compatibility': False
}
resolver = None
preparer = None
with RequirementTracker() as req_tracker:
# Pip 18 uses a requirement tracker to prevent fork bombs
if req_tracker:
preparer_kwargs['req_tracker'] = req_tracker
preparer = RequirementPreparer(**preparer_kwargs)
resolver_kwargs['preparer'] = preparer
reqset = RequirementSet()
ireq.is_direct = True
# reqset.add_requirement(ireq)
resolver = PipResolver(**resolver_kwargs)
resolver.require_hashes = False
results = resolver._resolve_one(reqset, ireq)
cleanup_fn = getattr(reqset, "cleanup_files", None)
if cleanup_fn is not None:
try:
cleanup_fn()
except OSError:
pass
if ireq.editable and (not ireq.source_dir
or not os.path.exists(ireq.source_dir)):
if ireq.editable:
self._source_dir = TemporaryDirectory(fs_str("source"))
ireq.ensure_has_source_dir(self.source_dir)
if ireq.editable and (ireq.source_dir
and os.path.exists(ireq.source_dir)):
# Collect setup_requires info from local eggs.
# Do this after we call the preparer on these reqs to make sure their
# egg info has been created
from pipenv.utils import chdir
with chdir(ireq.setup_py_dir):
try:
from setuptools.dist import distutils
dist = distutils.core.run_setup(ireq.setup_py)
except InstallationError:
ireq.run_egg_info()
except (TypeError, ValueError, AttributeError):
pass
if not dist:
try:
dist = ireq.get_dist()
except (ImportError, ValueError, TypeError,
AttributeError):
pass
if ireq.editable and dist:
setup_requires = getattr(dist, "extras_require", None)
if not setup_requires:
setup_requires = {
"setup_requires": getattr(dist, "setup_requires", None)
}
if not getattr(ireq, 'req', None):
try:
ireq.req = dist.as_requirement() if dist else None
except (ValueError, TypeError) as e:
pass
# Convert setup_requires dict into a somewhat usable form.
if setup_requires:
for section in setup_requires:
python_version = section
not_python = not (section.startswith('[')
and ':' in section)
# This is for cleaning up :extras: formatted markers
# by adding them to the results of the resolver
# since any such extra would have been returned as a result anyway
for value in setup_requires[section]:
# This is a marker.
if value.startswith('[') and ':' in value:
python_version = value[1:-1]
not_python = False
# Strip out other extras.
if value.startswith('[') and ':' not in value:
not_python = True
if ':' not in value:
try:
if not not_python:
results.add(
InstallRequirement.from_line(
"{0}{1}".format(
value, python_version).replace(
':', ';')))
# Anything could go wrong here -- can't be too careful.
except Exception:
pass
# this section properly creates 'python_version' markers for cross-python
# virtualenv creation and for multi-python compatibility.
requires_python = reqset.requires_python if hasattr(
reqset, 'requires_python') else resolver.requires_python
if requires_python:
marker_str = ''
# This corrects a logic error from the previous code which said that if
# we Encountered any 'requires_python' attributes, basically only create a
# single result no matter how many we resolved. This should fix
# a majority of the remaining non-deterministic resolution issues.
if any(
requires_python.startswith(op)
for op in Specifier._operators.keys()):
# We are checking first if we have leading specifier operator
# if not, we can assume we should be doing a == comparison
specifierset = SpecifierSet(requires_python)
# for multiple specifiers, the correct way to represent that in
# a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
from passa.internals.specifiers import cleanup_pyspecs
marker_str = str(
Marker(" and ".join(
dedup([
"python_version {0[0]} '{0[1]}'".format(spec)
for spec in cleanup_pyspecs(specifierset)
]))))
# The best way to add markers to a requirement is to make a separate requirement
# with only markers on it, and then to transfer the object istelf
marker_to_add = Requirement(
'fakepkg; {0}'.format(marker_str)).marker
if ireq in results:
results.remove(ireq)
print(marker_to_add)
ireq.req.marker = marker_to_add
results = set(results) if results else set()
return results, ireq
class PyPIRepository(BaseRepository):
DEFAULT_INDEX_URL = PyPI.simple_url
"""
The PyPIRepository will use the provided Finder instance to lookup
packages. Typically, it looks up packages on PyPI (the default implicit
config), but any other PyPI mirror can be used if index_urls is
changed/configured on the Finder.
"""
def __init__(self, pip_options, session, use_json=False):
self.session = session
self.use_json = use_json
self.pip_options = pip_options
self.wheel_cache = WheelCache(PIPENV_CACHE_DIR,
pip_options.format_control)
index_urls = [pip_options.index_url] + pip_options.extra_index_urls
if pip_options.no_index:
index_urls = []
self.finder = PackageFinder(
find_links=pip_options.find_links,
index_urls=index_urls,
trusted_hosts=pip_options.trusted_hosts,
allow_all_prereleases=pip_options.pre,
process_dependency_links=pip_options.process_dependency_links,
session=self.session,
)
# Caches
# stores project_name => InstallationCandidate mappings for all
# versions reported by PyPI, so we only have to ask once for each
# project
self._available_candidates_cache = {}
# stores InstallRequirement => list(InstallRequirement) mappings
# of all secondary dependencies for the given requirement, so we
# only have to go to disk once for each requirement
self._dependencies_cache = {}
self._json_dep_cache = {}
# stores *full* path + fragment => sha256
self._hash_cache = HashCache(session=session)
# Setup file paths
self.freshen_build_caches()
self._download_dir = fs_str(os.path.join(PIPENV_CACHE_DIR, 'pkgs'))
self._wheel_download_dir = fs_str(
os.path.join(PIPENV_CACHE_DIR, 'wheels'))
def freshen_build_caches(self):
"""
Start with fresh build/source caches. Will remove any old build
caches from disk automatically.
"""
self._build_dir = TemporaryDirectory(fs_str('build'))
self._source_dir = TemporaryDirectory(fs_str('source'))
@property
def build_dir(self):
return self._build_dir.name
@property
def source_dir(self):
return self._source_dir.name
def clear_caches(self):
rmtree(self._download_dir, ignore_errors=True)
rmtree(self._wheel_download_dir, ignore_errors=True)
def find_all_candidates(self, req_name):
if req_name not in self._available_candidates_cache:
candidates = self.finder.find_all_candidates(req_name)
self._available_candidates_cache[req_name] = candidates
return self._available_candidates_cache[req_name]
def find_best_match(self, ireq, prereleases=None):
"""
Returns a Version object that indicates the best match for the given
InstallRequirement according to the external repository.
"""
if ireq.editable:
return ireq # return itself as the best match
py_version = parse_version(
os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3])))
all_candidates = []
for c in self.find_all_candidates(ireq.name):
if c.requires_python:
# Old specifications had people setting this to single digits
# which is effectively the same as '>=digit,<digit+1'
if c.requires_python.isdigit():
c.requires_python = '>={0},<{1}'.format(
c.requires_python,
int(c.requires_python) + 1)
try:
specifier_set = SpecifierSet(c.requires_python)
except InvalidSpecifier:
pass
else:
if not specifier_set.contains(py_version):
continue
all_candidates.append(c)
candidates_by_version = lookup_table(all_candidates,
key=lambda c: c.version,
unique=True)
try:
matching_versions = ireq.specifier.filter(
(candidate.version for candidate in all_candidates),
prereleases=prereleases)
except TypeError:
matching_versions = [
candidate.version for candidate in all_candidates
]
# Reuses pip's internal candidate sort key to sort
matching_candidates = [
candidates_by_version[ver] for ver in matching_versions
]
if not matching_candidates:
raise NoCandidateFound(ireq, all_candidates, self.finder)
best_candidate = max(matching_candidates,
key=self.finder._candidate_sort_key)
# Turn the candidate into a pinned InstallRequirement
new_req = make_install_requirement(best_candidate.project,
best_candidate.version,
ireq.extras,
ireq.markers,
constraint=ireq.constraint)
# KR TODO: Marker here?
return new_req
def get_json_dependencies(self, ireq):
if not (is_pinned_requirement(ireq)):
raise TypeError(
'Expected pinned InstallRequirement, got {}'.format(ireq))
def gen(ireq):
if self.DEFAULT_INDEX_URL not in self.finder.index_urls:
return
url = 'https://pypi.org/pypi/{0}/json'.format(ireq.req.name)
releases = self.session.get(url).json()['releases']
matches = [
r for r in releases
if '=={0}'.format(r) == str(ireq.req.specifier)
]
if not matches:
return
release_requires = self.session.get(
'https://pypi.org/pypi/{0}/{1}/json'.format(
ireq.req.name,
matches[0],
), ).json()
try:
requires_dist = release_requires['info']['requires_dist']
except KeyError:
return
for requires in requires_dist:
i = InstallRequirement.from_line(requires)
if 'extra' not in repr(i.markers):
yield i
try:
if ireq not in self._json_dep_cache:
self._json_dep_cache[ireq] = [g for g in gen(ireq)]
return set(self._json_dep_cache[ireq])
except Exception:
return set()
def get_dependencies(self, ireq):
json_results = set()
if self.use_json:
try:
json_results = self.get_json_dependencies(ireq)
except TypeError:
json_results = set()
legacy_results = self.get_legacy_dependencies(ireq)
json_results.update(legacy_results)
return json_results
def get_legacy_dependencies(self, ireq):
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
They indicate the secondary dependencies for the given requirement.
"""
if not (ireq.editable or is_pinned_requirement(ireq)):
raise TypeError(
'Expected pinned or editable InstallRequirement, got {}'.
format(ireq))
if ireq not in self._dependencies_cache:
if ireq.editable and (ireq.source_dir
and os.path.exists(ireq.source_dir)):
# No download_dir for locally available editable requirements.
# If a download_dir is passed, pip will unnecessarely
# archive the entire source directory
download_dir = None
elif ireq.link and not ireq.link.is_artifact:
# No download_dir for VCS sources. This also works around pip
# using git-checkout-index, which gets rid of the .git dir.
download_dir = None
else:
download_dir = self._download_dir
if not os.path.isdir(download_dir):
os.makedirs(download_dir)
if not os.path.isdir(self._wheel_download_dir):
os.makedirs(self._wheel_download_dir)
try:
# Pip < 9 and below
reqset = RequirementSet(
self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
ignore_installed=True,
ignore_compatibility=False,
wheel_cache=self.wheel_cache,
)
result = reqset._prepare_file(self.finder,
ireq,
ignore_requires_python=True)
except TypeError:
# Pip >= 10 (new resolver!)
preparer = RequirementPreparer(
build_dir=self.build_dir,
src_dir=self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
progress_bar='off',
build_isolation=False)
reqset = RequirementSet()
ireq.is_direct = True
reqset.add_requirement(ireq)
self.resolver = PipResolver(preparer=preparer,
finder=self.finder,
session=self.session,
upgrade_strategy="to-satisfy-only",
force_reinstall=False,
ignore_dependencies=False,
ignore_requires_python=True,
ignore_installed=True,
isolated=False,
wheel_cache=self.wheel_cache,
use_user_site=False,
ignore_compatibility=False)
self.resolver.resolve(reqset)
result = reqset.requirements.values()
# Collect setup_requires info from local eggs.
# Do this after we call the preparer on these reqs to make sure their
# egg info has been created
setup_requires = {}
if ireq.editable:
try:
dist = ireq.get_dist()
if dist.has_metadata('requires.txt'):
setup_requires = self.finder.get_extras_links(
dist.get_metadata_lines('requires.txt'))
# HACK: Sometimes the InstallRequirement doesn't properly get
# these values set on it during the resolution process. It's
# difficult to pin down what is going wrong. This fixes things.
ireq.version = dist.version
ireq.project_name = dist.project_name
ireq.req = dist.as_requirement()
except (TypeError, ValueError):
pass
# Convert setup_requires dict into a somewhat usable form.
if setup_requires:
for section in setup_requires:
python_version = section
not_python = not (section.startswith('[')
and ':' in section)
for value in setup_requires[section]:
# This is a marker.
if value.startswith('[') and ':' in value:
python_version = value[1:-1]
not_python = False
# Strip out other extras.
if value.startswith('[') and ':' not in value:
not_python = True
if ':' not in value:
try:
if not not_python:
result = result + [
InstallRequirement.from_line(
"{0}{1}".format(
value, python_version).replace(
':', ';'))
]
# Anything could go wrong here — can't be too careful.
except Exception:
pass
requires_python = reqset.requires_python if hasattr(
reqset, 'requires_python') else self.resolver.requires_python
if requires_python:
marker = 'python_version=="{0}"'.format(
requires_python.replace(' ', ''))
new_req = InstallRequirement.from_line('{0}; {1}'.format(
str(ireq.req), marker))
result = [new_req]
self._dependencies_cache[ireq] = result
reqset.cleanup_files()
return set(self._dependencies_cache[ireq])
def get_hashes(self, ireq):
"""
Given an InstallRequirement, return a set of hashes that represent all
of the files for a given requirement. Editable requirements return an
empty set. Unpinned requirements raise a TypeError.
"""
if ireq.editable:
return set()
if not is_pinned_requirement(ireq):
raise TypeError("Expected pinned requirement, got {}".format(ireq))
# We need to get all of the candidates that match our current version
# pin, these will represent all of the files that could possibly
# satisfy this constraint.
all_candidates = self.find_all_candidates(ireq.name)
candidates_by_version = lookup_table(all_candidates,
key=lambda c: c.version)
matching_versions = list(
ireq.specifier.filter(
(candidate.version for candidate in all_candidates)))
matching_candidates = candidates_by_version[matching_versions[0]]
return {
self._hash_cache.get_hash(candidate.location)
for candidate in matching_candidates
}
@contextmanager
def allow_all_wheels(self):
"""
Monkey patches pip.Wheel to allow wheels from all platforms and Python versions.
This also saves the candidate cache and set a new one, or else the results from the
previous non-patched calls will interfere.
"""
def _wheel_supported(self, tags=None):
# Ignore current platform. Support everything.
return True
def _wheel_support_index_min(self, tags=None):
# All wheels are equal priority for sorting.
return 0
original_wheel_supported = Wheel.supported
original_support_index_min = Wheel.support_index_min
original_cache = self._available_candidates_cache
Wheel.supported = _wheel_supported
Wheel.support_index_min = _wheel_support_index_min
self._available_candidates_cache = {}
try:
yield
finally:
Wheel.supported = original_wheel_supported
Wheel.support_index_min = original_support_index_min
self._available_candidates_cache = original_cache
def get_legacy_dependencies(self, ireq):
"""
Given a pinned or an editable InstallRequirement, returns a set of
dependencies (also InstallRequirements, but not necessarily pinned).
They indicate the secondary dependencies for the given requirement.
"""
if not (ireq.editable or is_pinned_requirement(ireq)):
raise TypeError(
'Expected pinned or editable InstallRequirement, got {}'.
format(ireq))
if ireq not in self._dependencies_cache:
if ireq.editable and (ireq.source_dir
and os.path.exists(ireq.source_dir)):
# No download_dir for locally available editable requirements.
# If a download_dir is passed, pip will unnecessarely
# archive the entire source directory
download_dir = None
elif ireq.link and not ireq.link.is_artifact:
# No download_dir for VCS sources. This also works around pip
# using git-checkout-index, which gets rid of the .git dir.
download_dir = None
else:
download_dir = self._download_dir
if not os.path.isdir(download_dir):
os.makedirs(download_dir)
if not os.path.isdir(self._wheel_download_dir):
os.makedirs(self._wheel_download_dir)
try:
# Pip < 9 and below
reqset = RequirementSet(
self.build_dir,
self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
session=self.session,
ignore_installed=True,
ignore_compatibility=False,
wheel_cache=self.wheel_cache,
)
result = reqset._prepare_file(self.finder,
ireq,
ignore_requires_python=True)
except TypeError:
# Pip >= 10 (new resolver!)
preparer = RequirementPreparer(
build_dir=self.build_dir,
src_dir=self.source_dir,
download_dir=download_dir,
wheel_download_dir=self._wheel_download_dir,
progress_bar='off',
build_isolation=False)
reqset = RequirementSet()
ireq.is_direct = True
reqset.add_requirement(ireq)
self.resolver = PipResolver(preparer=preparer,
finder=self.finder,
session=self.session,
upgrade_strategy="to-satisfy-only",
force_reinstall=False,
ignore_dependencies=False,
ignore_requires_python=True,
ignore_installed=True,
isolated=False,
wheel_cache=self.wheel_cache,
use_user_site=False,
ignore_compatibility=False)
self.resolver.resolve(reqset)
result = reqset.requirements.values()
# Collect setup_requires info from local eggs.
# Do this after we call the preparer on these reqs to make sure their
# egg info has been created
setup_requires = {}
if ireq.editable:
try:
dist = ireq.get_dist()
if dist.has_metadata('requires.txt'):
setup_requires = self.finder.get_extras_links(
dist.get_metadata_lines('requires.txt'))
# HACK: Sometimes the InstallRequirement doesn't properly get
# these values set on it during the resolution process. It's
# difficult to pin down what is going wrong. This fixes things.
ireq.version = dist.version
ireq.project_name = dist.project_name
ireq.req = dist.as_requirement()
except (TypeError, ValueError):
pass
# Convert setup_requires dict into a somewhat usable form.
if setup_requires:
for section in setup_requires:
python_version = section
not_python = not (section.startswith('[')
and ':' in section)
for value in setup_requires[section]:
# This is a marker.
if value.startswith('[') and ':' in value:
python_version = value[1:-1]
not_python = False
# Strip out other extras.
if value.startswith('[') and ':' not in value:
not_python = True
if ':' not in value:
try:
if not not_python:
result = result + [
InstallRequirement.from_line(
"{0}{1}".format(
value, python_version).replace(
':', ';'))
]
# Anything could go wrong here — can't be too careful.
except Exception:
pass
requires_python = reqset.requires_python if hasattr(
reqset, 'requires_python') else self.resolver.requires_python
if requires_python:
marker = 'python_version=="{0}"'.format(
requires_python.replace(' ', ''))
new_req = InstallRequirement.from_line('{0}; {1}'.format(
str(ireq.req), marker))
result = [new_req]
self._dependencies_cache[ireq] = result
reqset.cleanup_files()
return set(self._dependencies_cache[ireq])
def run(self, options, args):
cmdoptions.check_install_build_global(options)
upgrade_strategy = "to-satisfy-only"
if options.upgrade:
upgrade_strategy = options.upgrade_strategy
if options.build_dir:
options.build_dir = os.path.abspath(options.build_dir)
options.src_dir = os.path.abspath(options.src_dir)
install_options = options.install_options or []
if options.use_user_site:
if options.prefix_path:
raise CommandError(
"Can not combine '--user' and '--prefix' as they imply "
"different installation locations")
if virtualenv_no_global():
raise InstallationError(
"Can not perform a '--user' install. User site-packages "
"are not visible in this virtualenv.")
install_options.append('--user')
install_options.append('--prefix=')
target_temp_dir = TempDirectory(kind="target")
if options.target_dir:
options.ignore_installed = True
options.target_dir = os.path.abspath(options.target_dir)
if (os.path.exists(options.target_dir)
and not os.path.isdir(options.target_dir)):
raise CommandError(
"Target path exists but is not a directory, will not "
"continue.")
# Create a target directory for using with the target option
target_temp_dir.create()
install_options.append('--home=' + target_temp_dir.path)
global_options = options.global_options or []
with self._build_session(options) as session:
finder = self._build_package_finder(options, session)
build_delete = (not (options.no_clean or options.build_dir))
wheel_cache = WheelCache(options.cache_dir, options.format_control)
if options.cache_dir and not check_path_owner(options.cache_dir):
logger.warning(
"The directory '%s' or its parent directory is not owned "
"by the current user and caching wheels has been "
"disabled. check the permissions and owner of that "
"directory. If executing pip with sudo, you may want "
"sudo's -H flag.",
options.cache_dir,
)
options.cache_dir = None
with TempDirectory(options.build_dir,
delete=build_delete,
kind="install") as directory:
requirement_set = RequirementSet(
require_hashes=options.require_hashes, )
try:
self.populate_requirement_set(requirement_set, args,
options, finder, session,
self.name, wheel_cache)
preparer = RequirementPreparer(
build_dir=directory.path,
src_dir=options.src_dir,
download_dir=None,
wheel_download_dir=None,
progress_bar=options.progress_bar,
build_isolation=options.build_isolation,
)
resolver = Resolver(
preparer=preparer,
finder=finder,
session=session,
wheel_cache=wheel_cache,
use_user_site=options.use_user_site,
upgrade_strategy=upgrade_strategy,
force_reinstall=options.force_reinstall,
ignore_dependencies=options.ignore_dependencies,
ignore_requires_python=options.ignore_requires_python,
ignore_installed=options.ignore_installed,
isolated=options.isolated_mode,
)
resolver.resolve(requirement_set)
# If caching is disabled or wheel is not installed don't
# try to build wheels.
if wheel and options.cache_dir:
# build wheels before install.
wb = WheelBuilder(
finder,
preparer,
wheel_cache,
build_options=[],
global_options=[],
)
# Ignore the result: a failed wheel will be
# installed from the sdist/vcs whatever.
wb.build(requirement_set.requirements.values(),
session=session,
autobuilding=True)
to_install = resolver.get_installation_order(
requirement_set)
# Consistency Checking of the package set we're installing.
should_warn_about_conflicts = (
not options.ignore_dependencies
and options.warn_about_conflicts)
if should_warn_about_conflicts:
self._warn_about_conflicts(to_install)
# Don't warn about script install locations if
# --target has been specified
warn_script_location = options.warn_script_location
if options.target_dir:
warn_script_location = False
installed = install_given_reqs(
to_install,
install_options,
global_options,
root=options.root_path,
home=target_temp_dir.path,
prefix=options.prefix_path,
pycompile=options.compile,
warn_script_location=warn_script_location,
use_user_site=options.use_user_site,
)
possible_lib_locations = get_lib_location_guesses(
user=options.use_user_site,
home=target_temp_dir.path,
root=options.root_path,
prefix=options.prefix_path,
isolated=options.isolated_mode,
)
reqs = sorted(installed, key=operator.attrgetter('name'))
items = []
for req in reqs:
item = req.name
try:
installed_version = get_installed_version(
req.name, possible_lib_locations)
if installed_version:
item += '-' + installed_version
except Exception:
pass
items.append(item)
installed = ' '.join(items)
if installed:
logger.info('Successfully installed %s', installed)
except EnvironmentError as error:
show_traceback = (self.verbosity >= 1)
message = create_env_error_message(
error,
show_traceback,
options.use_user_site,
)
logger.error(message, exc_info=show_traceback)
return ERROR
except PreviousBuildDirError:
options.no_clean = True
raise
finally:
# Clean up
if not options.no_clean:
requirement_set.cleanup_files()
wheel_cache.cleanup()
if options.target_dir:
self._handle_target_dir(options.target_dir, target_temp_dir,
options.upgrade)
return requirement_set
def run(self, options, args):
cmdoptions.check_install_build_global(options)
upgrade_strategy = "to-satisfy-only"
if options.upgrade:
upgrade_strategy = options.upgrade_strategy
if options.build_dir:
options.build_dir = os.path.abspath(options.build_dir)
options.src_dir = os.path.abspath(options.src_dir)
install_options = options.install_options or []
if options.use_user_site:
if options.prefix_path:
raise CommandError(
"Can not combine '--user' and '--prefix' as they imply "
"different installation locations"
)
if virtualenv_no_global():
raise InstallationError(
"Can not perform a '--user' install. User site-packages "
"are not visible in this virtualenv."
)
install_options.append('--user')
install_options.append('--prefix=')
target_temp_dir = TempDirectory(kind="target")
if options.target_dir:
options.ignore_installed = True
options.target_dir = os.path.abspath(options.target_dir)
if (os.path.exists(options.target_dir) and not
os.path.isdir(options.target_dir)):
raise CommandError(
"Target path exists but is not a directory, will not "
"continue."
)
# Create a target directory for using with the target option
target_temp_dir.create()
install_options.append('--home=' + target_temp_dir.path)
global_options = options.global_options or []
with self._build_session(options) as session:
finder = self._build_package_finder(options, session)
build_delete = (not (options.no_clean or options.build_dir))
wheel_cache = WheelCache(options.cache_dir, options.format_control)
if options.cache_dir and not check_path_owner(options.cache_dir):
logger.warning(
"The directory '%s' or its parent directory is not owned "
"by the current user and caching wheels has been "
"disabled. check the permissions and owner of that "
"directory. If executing pip with sudo, you may want "
"sudo's -H flag.",
options.cache_dir,
)
options.cache_dir = None
with TempDirectory(
options.build_dir, delete=build_delete, kind="install"
) as directory:
requirement_set = RequirementSet(
require_hashes=options.require_hashes,
)
try:
self.populate_requirement_set(
requirement_set, args, options, finder, session,
self.name, wheel_cache
)
preparer = RequirementPreparer(
build_dir=directory.path,
src_dir=options.src_dir,
download_dir=None,
wheel_download_dir=None,
progress_bar=options.progress_bar,
build_isolation=options.build_isolation,
)
resolver = Resolver(
preparer=preparer,
finder=finder,
session=session,
wheel_cache=wheel_cache,
use_user_site=options.use_user_site,
upgrade_strategy=upgrade_strategy,
force_reinstall=options.force_reinstall,
ignore_dependencies=options.ignore_dependencies,
ignore_requires_python=options.ignore_requires_python,
ignore_installed=options.ignore_installed,
isolated=options.isolated_mode,
)
resolver.resolve(requirement_set)
# If caching is disabled or wheel is not installed don't
# try to build wheels.
if wheel and options.cache_dir:
# build wheels before install.
wb = WheelBuilder(
finder, preparer, wheel_cache,
build_options=[], global_options=[],
)
# Ignore the result: a failed wheel will be
# installed from the sdist/vcs whatever.
wb.build(
requirement_set.requirements.values(),
session=session, autobuilding=True
)
to_install = resolver.get_installation_order(
requirement_set
)
# Consistency Checking of the package set we're installing.
should_warn_about_conflicts = (
not options.ignore_dependencies and
options.warn_about_conflicts
)
if should_warn_about_conflicts:
self._warn_about_conflicts(to_install)
# Don't warn about script install locations if
# --target has been specified
warn_script_location = options.warn_script_location
if options.target_dir:
warn_script_location = False
installed = install_given_reqs(
to_install,
install_options,
global_options,
root=options.root_path,
home=target_temp_dir.path,
prefix=options.prefix_path,
pycompile=options.compile,
warn_script_location=warn_script_location,
use_user_site=options.use_user_site,
)
possible_lib_locations = get_lib_location_guesses(
user=options.use_user_site,
home=target_temp_dir.path,
root=options.root_path,
prefix=options.prefix_path,
isolated=options.isolated_mode,
)
reqs = sorted(installed, key=operator.attrgetter('name'))
items = []
for req in reqs:
item = req.name
try:
installed_version = get_installed_version(
req.name, possible_lib_locations
)
if installed_version:
item += '-' + installed_version
except Exception:
pass
items.append(item)
installed = ' '.join(items)
if installed:
logger.info('Successfully installed %s', installed)
except EnvironmentError as error:
show_traceback = (self.verbosity >= 1)
message = create_env_error_message(
error, show_traceback, options.use_user_site,
)
logger.error(message, exc_info=show_traceback)
return ERROR
except PreviousBuildDirError:
options.no_clean = True
raise
finally:
# Clean up
if not options.no_clean:
requirement_set.cleanup_files()
wheel_cache.cleanup()
if options.target_dir:
self._handle_target_dir(
options.target_dir, target_temp_dir, options.upgrade
)
return requirement_set
def run(self, options, args):
options.ignore_installed = True
# editable doesn't really make sense for `pip download`, but the bowels
# of the RequirementSet code require that property.
options.editables = []
if options.python_version:
python_versions = [options.python_version]
else:
python_versions = None
dist_restriction_set = any([
options.python_version,
options.platform,
options.abi,
options.implementation,
])
binary_only = FormatControl(set(), {':all:'})
no_sdist_dependencies = (
options.format_control != binary_only and
not options.ignore_dependencies
)
if dist_restriction_set and no_sdist_dependencies:
raise CommandError(
"When restricting platform and interpreter constraints using "
"--python-version, --platform, --abi, or --implementation, "
"either --no-deps must be set, or --only-binary=:all: must be "
"set and --no-binary must not be set (or must be set to "
":none:)."
)
options.src_dir = os.path.abspath(options.src_dir)
options.download_dir = normalize_path(options.download_dir)
ensure_dir(options.download_dir)
with self._build_session(options) as session:
finder = self._build_package_finder(
options=options,
session=session,
platform=options.platform,
python_versions=python_versions,
abi=options.abi,
implementation=options.implementation,
)
build_delete = (not (options.no_clean or options.build_dir))
if options.cache_dir and not check_path_owner(options.cache_dir):
logger.warning(
"The directory '%s' or its parent directory is not owned "
"by the current user and caching wheels has been "
"disabled. check the permissions and owner of that "
"directory. If executing pip with sudo, you may want "
"sudo's -H flag.",
options.cache_dir,
)
options.cache_dir = None
with TempDirectory(
options.build_dir, delete=build_delete, kind="download"
) as directory:
requirement_set = RequirementSet(
require_hashes=options.require_hashes,
)
self.populate_requirement_set(
requirement_set,
args,
options,
finder,
session,
self.name,
None
)
preparer = RequirementPreparer(
build_dir=directory.path,
src_dir=options.src_dir,
download_dir=options.download_dir,
wheel_download_dir=None,
progress_bar=options.progress_bar,
build_isolation=options.build_isolation,
)
resolver = Resolver(
preparer=preparer,
finder=finder,
session=session,
wheel_cache=None,
use_user_site=False,
upgrade_strategy="to-satisfy-only",
force_reinstall=False,
ignore_dependencies=options.ignore_dependencies,
ignore_requires_python=False,
ignore_installed=True,
isolated=options.isolated_mode,
)
resolver.resolve(requirement_set)
downloaded = ' '.join([
req.name for req in requirement_set.successfully_downloaded
])
if downloaded:
logger.info('Successfully downloaded %s', downloaded)
# Clean up
if not options.no_clean:
requirement_set.cleanup_files()
return requirement_set